From 78942011dc030faa80a4f6a1907b78044d4c8fbf Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Fri, 5 Jul 2024 16:14:20 +1000 Subject: [PATCH 1/7] fix: Fixing Cohort project state --- Makefile | 6 ++--- .../src/cfg/config-cohort-sdk.ts | 12 +++++++--- .../src/cfg/config-db-pool.ts | 22 ++++++++++++------- .../package-lock.json | 2 +- cohort_banking_replicator_js/package.json | 2 +- .../src/cfg/config-db-pool.ts | 22 ++++++++++++------- .../src/cfg/config-kafka.ts | 12 +++++++--- .../examples/cohort_banking_with_sdk.rs | 22 ++++++++++--------- packages/cohort_sdk_js/README.md | 2 +- .../cohort_sdk_js/scripts/bundle-talos.sh | 2 +- scripts/cohort/data-generator.py | 2 +- 11 files changed, 65 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index b9c106e5..a2555b6c 100644 --- a/Makefile +++ b/Makefile @@ -165,14 +165,12 @@ cohort_banking.run_replicator_rust: ## cohort_banking.run_initiator_js: 🧪 Executes load test through Cohort Initiator implemented in JS cohort_banking.run_initiator_load_test_in_js: $(call pp,running "Cohort Initiator" implemented in JS...) - cd ./cohort_banking_initiator_js - npm start -- $(args) + cd ./cohort_banking_initiator_js && npm start -- $(args) ## cohort_banking.run_replicator_js: 🧪 Runs Replicator JS app cohort_banking.run_replicator_js: $(call pp,running "Cohort Replicator" implemented in JS...) - cd ./cohort_banking_replicator_js - npm start + cd ./cohort_banking_replicator_js && npm start ## dev.histogram_decision_timeline_from_kafka: 🧪 Reads all decisions from kafka and prints processing timeline as csv dev.histogram_decision_timeline_from_kafka: diff --git a/cohort_banking_initiator_js/src/cfg/config-cohort-sdk.ts b/cohort_banking_initiator_js/src/cfg/config-cohort-sdk.ts index de728555..76abd093 100644 --- a/cohort_banking_initiator_js/src/cfg/config-cohort-sdk.ts +++ b/cohort_banking_initiator_js/src/cfg/config-cohort-sdk.ts @@ -1,5 +1,11 @@ import { JsInitiatorConfig } from "@kindredgroup/cohort_sdk_client" +const getParam = (param: string, defaultValue: string): string => { + const v = process.env[param] + if (v) return v + return defaultValue +} + const SDK_CONFIG: JsInitiatorConfig = { backoffOnConflict: { minMs: 2, @@ -22,7 +28,7 @@ const SDK_CONFIG: JsInitiatorConfig = { timeoutMs: 60_000, kafka: { brokers: ["127.0.0.1:9092"], - topic: "dev.ksp.certification", + topic: getParam("KAFKA_TOPIC", "dev.ksp.certification"), clientId: "cohort-js", groupId: "cohort-js", producerSendTimeoutMs: 10, @@ -31,8 +37,8 @@ const SDK_CONFIG: JsInitiatorConfig = { consumerConfigOverrides: { "enable.auto.commit": "false" }, - username: "", - password: "", + username: getParam("KAFKA_USERNAME", ""), + password: getParam("KAFKA_PASSWORD", ""), }, } diff --git a/cohort_banking_initiator_js/src/cfg/config-db-pool.ts b/cohort_banking_initiator_js/src/cfg/config-db-pool.ts index e4758b4c..7b3f2445 100644 --- a/cohort_banking_initiator_js/src/cfg/config-db-pool.ts +++ b/cohort_banking_initiator_js/src/cfg/config-db-pool.ts @@ -1,15 +1,21 @@ import { PoolConfig } from "pg" +const getParam = (param: string, defaultValue: string): string => { + const v = process.env[param] + if (v) return v + return defaultValue +} + const DB_CONFIG: PoolConfig = { application_name: "cohort_banking_js", - keepAlive: true, - host: "127.0.0.1", - port: 5432, - database: "talos-sample-cohort-dev", - user: "postgres", - password: "admin", - max: 100, - min: 100, + keepAlive: true, + host: getParam("COHORT_PG_HOST", "127.0.0.1"), + port: parseInt(getParam("COHORT_PG_PORT", "5432")), + database: getParam("COHORT_PG_DATABASE", "talos-sample-cohort-dev"), + user: getParam("COHORT_PG_USER", "postgres"), + password: getParam("COHORT_PG_PASSWORD", "admin"), + max: parseInt(getParam("COHORT_PG_POOL_SIZE", "100")), + min: parseInt(getParam("COHORT_PG_POOL_SIZE", "100")), } export { DB_CONFIG } \ No newline at end of file diff --git a/cohort_banking_replicator_js/package-lock.json b/cohort_banking_replicator_js/package-lock.json index 4d268e4e..31e080ee 100644 --- a/cohort_banking_replicator_js/package-lock.json +++ b/cohort_banking_replicator_js/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "cohort_banking_replicator_js", "version": "0.0.1", - "license": "UNLICENSED", + "license": "MIT", "dependencies": { "@kindredgroup/cohort_sdk_client": "^0.0.1-0ed84cd", "pg": "^8.11.3", diff --git a/cohort_banking_replicator_js/package.json b/cohort_banking_replicator_js/package.json index dde9b6a7..74bc0f81 100644 --- a/cohort_banking_replicator_js/package.json +++ b/cohort_banking_replicator_js/package.json @@ -19,7 +19,7 @@ "typescript": "^5.1.6" }, "dependencies": { - "@kindredgroup/cohort_sdk_client": "^0.0.1-0ed84cd", + "@kindredgroup/cohort_sdk_client": "^0.2.10", "pg": "^8.11.3", "winston": "^3.10.0" } diff --git a/cohort_banking_replicator_js/src/cfg/config-db-pool.ts b/cohort_banking_replicator_js/src/cfg/config-db-pool.ts index a016476a..97699586 100644 --- a/cohort_banking_replicator_js/src/cfg/config-db-pool.ts +++ b/cohort_banking_replicator_js/src/cfg/config-db-pool.ts @@ -1,15 +1,21 @@ import { PoolConfig } from "pg" +const getParam = (param: string, defaultValue: string): string => { + const v = process.env[param] + if (v) return v + return defaultValue +} + const DB_CONFIG: PoolConfig = { application_name: "cohort_banking_replicator_js", - keepAlive: true, - host: "127.0.0.1", - port: 5432, - database: "talos-sample-cohort-dev", - user: "postgres", - password: "admin", - max: 100, - min: 100, + keepAlive: true, + host: getParam("COHORT_PG_HOST", "127.0.0.1"), + port: parseInt(getParam("COHORT_PG_PORT", "5432")), + database: getParam("COHORT_PG_DATABASE", "talos-sample-cohort-dev"), + user: getParam("COHORT_PG_USER", "postgres"), + password: getParam("COHORT_PG_PASSWORD", "admin"), + max: parseInt(getParam("COHORT_PG_POOL_SIZE", "100")), + min: parseInt(getParam("COHORT_PG_POOL_SIZE", "100")), } export { DB_CONFIG } \ No newline at end of file diff --git a/cohort_banking_replicator_js/src/cfg/config-kafka.ts b/cohort_banking_replicator_js/src/cfg/config-kafka.ts index d6a21afd..4a49b58e 100644 --- a/cohort_banking_replicator_js/src/cfg/config-kafka.ts +++ b/cohort_banking_replicator_js/src/cfg/config-kafka.ts @@ -1,16 +1,22 @@ import { JsKafkaConfig } from "@kindredgroup/cohort_sdk_client" +const getParam = (param: string, defaultValue: string): string => { + const v = process.env[param] + if (v) return v + return defaultValue +} + const kafkaConfig: JsKafkaConfig = { brokers: ["127.0.0.1:9092"], - topic: "dev.ksp.certification", + topic: getParam("KAFKA_TOPIC", "dev.ksp.certification"), clientId: "cohort-replicator-js", groupId: "cohort-replicator-js", producerSendTimeoutMs: 10, logLevel: "info", producerConfigOverrides: {}, consumerConfigOverrides: { "enable.auto.commit": "false" }, - username: "", - password: "", + username: getParam("KAFKA_USERNAME", ""), + password: getParam("KAFKA_PASSWORD", ""), } export { kafkaConfig as KAFKA_CONFIG } \ No newline at end of file diff --git a/examples/cohort_banking_with_sdk/examples/cohort_banking_with_sdk.rs b/examples/cohort_banking_with_sdk/examples/cohort_banking_with_sdk.rs index 85d57789..432b1384 100644 --- a/examples/cohort_banking_with_sdk/examples/cohort_banking_with_sdk.rs +++ b/examples/cohort_banking_with_sdk/examples/cohort_banking_with_sdk.rs @@ -35,8 +35,8 @@ struct LaunchParams { metric_print_raw: bool, } -/// Connects to database, to kafka certification topic as talso agent and as cohort replicator. -/// Generates some number of banking transactions and passes then all to Talos for certification. +/// Connects to database, to kafka certification topic as talos agent and as cohort replicator. +/// Generates some number of banking transactions and passes them all to Talos for certification. /// Once all transactions have been processed, it prints some output metrics to console. /// Metric logging is set to WARN. If RUST_LOG is set to stricter than WARN then no metrics will be printed. /// Preprequisite: @@ -110,14 +110,16 @@ async fn main() -> Result<(), String> { }, }; - let db_config = DatabaseConfig { - pool_size: 100, - user: "postgres".into(), - password: "admin".into(), - host: "127.0.0.1".into(), - port: "5432".into(), - database: "talos-sample-cohort-dev".into(), - }; + let db_config = DatabaseConfig::from_env(Some("COHORT"))?; + + // let db_config = DatabaseConfig { + // pool_size: 100, + // user: "postgres".into(), + // password: "admin".into(), + // host: "127.0.0.1".into(), + // port: "5432".into(), + // database: "talos-sample-cohort-dev".into(), + // }; let printer = MetricsToStringPrinter::new(params.threads, params.metric_print_raw, ScalingConfig { ratios: params.scaling_config }); let (tx_metrics, rx_metrics) = tokio::sync::watch::channel("".to_string()); diff --git a/packages/cohort_sdk_js/README.md b/packages/cohort_sdk_js/README.md index 4f7da5f7..11f1bcd6 100644 --- a/packages/cohort_sdk_js/README.md +++ b/packages/cohort_sdk_js/README.md @@ -6,7 +6,7 @@ When publishing it is being packaged as __incomplete__ NPM module and published The content of NPM module published to repository is missing `index.js`, `index.d.ts` and _native library_ with SDK implementation. All these missig files are generated by [NAPI-RS](https://napi.rs/) during **package install time**. -When `cohort_sdk_js` NPM package is installed via `npm -i` the "postinstall" hook is invoking `cohort_sdk_js/scripts/postinstall.sh` which in turn compiles Rust code for your architecture, then generates JS code and moves files to their indended destinations so that we end up with finished working NPM package. All this happens on the client machine under `./node_modules/@kindredgroup/cohort_sdk_js`. The build process is verbose if installation is invoked with additional flag: `npm -i @kindredgroup/cohort_sdk_js@SOME_VERSION --forground-scripts`, otherwise native compilation happens in the background. +When `cohort_sdk_js` NPM package is installed via `npm -i` the "postinstall" hook is invoking `cohort_sdk_js/scripts/postinstall.sh` which in turn compiles Rust code for your architecture, then generates JS code and moves files to their indended destinations so that we end up with finished working NPM package. All this happens on the client machine under `./node_modules/@kindredgroup/cohort_sdk_js`. The build process is verbose if installation is invoked with additional flag: `npm -i @kindredgroup/cohort_sdk_js@SOME_VERSION --foreground-scripts`, otherwise native compilation happens in the background. As such, for prerequisites of installing this package we require that Rust development environment is setup on the user machine. diff --git a/packages/cohort_sdk_js/scripts/bundle-talos.sh b/packages/cohort_sdk_js/scripts/bundle-talos.sh index eaad0e5f..7fc9dd7b 100755 --- a/packages/cohort_sdk_js/scripts/bundle-talos.sh +++ b/packages/cohort_sdk_js/scripts/bundle-talos.sh @@ -17,11 +17,11 @@ cp -r \ ../talos_certifier_adapters \ ../talos_cohort_replicator \ ../talos_common_utils \ + ../talos_metrics \ ../talos_rdkafka_utils \ ../talos_suffix \ ../cohort_sdk \ ../logger \ - ../metrics \ $TARGET_DIR/packages/ cp -r \ diff --git a/scripts/cohort/data-generator.py b/scripts/cohort/data-generator.py index 5662e89b..47be9c08 100755 --- a/scripts/cohort/data-generator.py +++ b/scripts/cohort/data-generator.py @@ -1,4 +1,4 @@ -#!python3 +#!/opt/homebrew/bin/python3 import sys import math From bbd70b783360328e0e0df139edca29d102a4987c Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 8 Jul 2024 12:53:50 +1000 Subject: [PATCH 2/7] Set toolchan version to 1.73.0 --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index b7ea63c2..687b7fff 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "stable" #1.68 +channel = "1.73.0" #1.68 From ad8c50c326a2a3f7be7cf615b04a243be01687fd Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 8 Jul 2024 13:01:45 +1000 Subject: [PATCH 3/7] Set toolchan version to 1.73.0 --- .github/workflows/build.yml | 4 ++-- .github/workflows/coverage.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0f2a233e..276c0816 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@1.73.0 with: - toolchain: stable + toolchain: 1.73.0 - run: scripts/ubuntu-setup.sh - run: rustup component add rustfmt clippy - name: cargo build diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7084a51c..68d739f8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/rust-toolchain@1.73.0 with: - toolchain: nightly + toolchain: 1.73.0 - run: scripts/ubuntu-setup.sh - run: rustup component add llvm-tools-preview - run: curl -L https://github.com/mozilla/grcov/releases/latest/download/grcov-x86_64-unknown-linux-gnu.tar.bz2 | tar jxf - From 1fe6486e734a0ca6c8c095c6a84ba905062da99a Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 8 Jul 2024 13:11:08 +1000 Subject: [PATCH 4/7] Set toolchan version to 1.73.0 --- scripts/coverage-report.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/coverage-report.sh b/scripts/coverage-report.sh index f409b0d0..b55b706c 100755 --- a/scripts/coverage-report.sh +++ b/scripts/coverage-report.sh @@ -16,11 +16,11 @@ export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Co excludes="cohort_sdk_js" echo "Compiling $app_name" -cargo +nightly build +cargo +1.73.0 build echo "Testing $app_name" export LLVM_PROFILE_FILE="${app_name}-%p-%m.profraw" -cargo +nightly test --tests --workspace --exclude ${excludes} # don't run doctests +cargo +1.73.0 test --tests --workspace --exclude ${excludes} # don't run doctests rm ccov.zip 2> /dev/null || true zip -0 ccov.zip `find . \( -name "*.gc*" \) -print | grep -v ${excludes}` @@ -38,7 +38,7 @@ grcov ccov.zip -s . --llvm --ignore-not-existing --ignore "/*" --excl-start "\\ rm ccov.zip # Re-run tests with JSON output -cargo +nightly test --tests --workspace --exclude ${excludes} -- -Z unstable-options --format json --report-time > coverage/test-report.json +cargo +1.73.0 test --tests --workspace --exclude ${excludes} -- -Z unstable-options --format json --report-time > coverage/test-report.json if [ "$1" == "--open" ]; then index="file://$(pwd)/${base_dir}/../coverage/index.html" From e55f45306316144e723a0d1ba603f37364bdf6ae Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 8 Jul 2024 13:48:32 +1000 Subject: [PATCH 5/7] Set toolchan version to 1.73.0 --- .github/workflows/coverage.yml | 4 ++-- scripts/coverage-report.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 68d739f8..228f0604 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,9 +8,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@1.73.0 + - uses: dtolnay/rust-toolchain@nightly-2023-10-03 with: - toolchain: 1.73.0 + toolchain: nightly-2023-10-03 - run: scripts/ubuntu-setup.sh - run: rustup component add llvm-tools-preview - run: curl -L https://github.com/mozilla/grcov/releases/latest/download/grcov-x86_64-unknown-linux-gnu.tar.bz2 | tar jxf - diff --git a/scripts/coverage-report.sh b/scripts/coverage-report.sh index b55b706c..7edd9581 100755 --- a/scripts/coverage-report.sh +++ b/scripts/coverage-report.sh @@ -16,11 +16,11 @@ export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Co excludes="cohort_sdk_js" echo "Compiling $app_name" -cargo +1.73.0 build +cargo +nightly-2023-10-03 build echo "Testing $app_name" export LLVM_PROFILE_FILE="${app_name}-%p-%m.profraw" -cargo +1.73.0 test --tests --workspace --exclude ${excludes} # don't run doctests +cargo +nightly-2023-10-03 test --tests --workspace --exclude ${excludes} # don't run doctests rm ccov.zip 2> /dev/null || true zip -0 ccov.zip `find . \( -name "*.gc*" \) -print | grep -v ${excludes}` @@ -38,7 +38,7 @@ grcov ccov.zip -s . --llvm --ignore-not-existing --ignore "/*" --excl-start "\\ rm ccov.zip # Re-run tests with JSON output -cargo +1.73.0 test --tests --workspace --exclude ${excludes} -- -Z unstable-options --format json --report-time > coverage/test-report.json +cargo +nightly-2023-10-03 test --tests --workspace --exclude ${excludes} -- -Z unstable-options --format json --report-time > coverage/test-report.json if [ "$1" == "--open" ]; then index="file://$(pwd)/${base_dir}/../coverage/index.html" From 65040feb09b1b4c0ccbbf6621f17b8515f2268fb Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 8 Jul 2024 13:54:56 +1000 Subject: [PATCH 6/7] Set toolchan version to 1.73.0 --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 228f0604..cba969f1 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@nightly-2023-10-03 + - uses: dtolnay/rust-toolchain@1.73.0 with: toolchain: nightly-2023-10-03 - run: scripts/ubuntu-setup.sh From 5d5dc5d62edf3bf9f50a238df60a8c8478e6a919 Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 8 Jul 2024 14:01:17 +1000 Subject: [PATCH 7/7] Set toolchan version to 1.73.0 --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index cba969f1..212c9a22 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@1.73.0 + - uses: dtolnay/rust-toolchain@master with: toolchain: nightly-2023-10-03 - run: scripts/ubuntu-setup.sh