From 7d95d1fba0ae6a3201ce5b6380fe12107a24e61c Mon Sep 17 00:00:00 2001 From: Marek Fedorovic Date: Mon, 9 Oct 2023 16:48:08 +1100 Subject: [PATCH] chore: Add README.md content to initiator and replicator JS --- .env.example | 34 +++---- cohort_banking_initiator_js/README.md | 122 ++++++++++++++++++++++++- cohort_banking_replicator_js/README.md | 15 ++- 3 files changed, 149 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index 0c175510..c857df0f 100644 --- a/.env.example +++ b/.env.example @@ -21,30 +21,23 @@ PG_HOST=$LOCAL_HOST_IP PG_PORT=5432 PG_USER=admin PG_PASSWORD=admin -PG_DATABASE=talos_certifier +PG_DATABASE=talos-certifier-dev ADMIN_PG_USER=admin ADMIN_PG_PASSWORD=admin ADMIN_PG_DATABASE=postgres -# Mock - +# CONFIG +RUST_LOG="warn" DB_MOCK=false CERTIFIER_MOCK=false -# CONFIG -RUST_LOG=debug - # Cohort configs AGENT_NAME="Sample Cohort" COHORT_NAME="Sample Cohort" -COHORT_WORKLOAD_DURATION=30 - -AGENT_BUFFER_SIZE=10000 -AGENT_TIMEOUT_MS=5000 - -KAFKA_FETCH_WAIT_MAX_MS=6000 -KAFKA_MESSAGE_TIMEOUT_MS=15000 -KAFKA_ENQUEUE_TIMEOUT_MS=10 +AGENT_BUFFER_SIZE=10000000 +# Timeout used by synchronous rdkafka publisher. +AGENT_KAFKA_PRODUCER_SEND_TIMEOUT_MS=10 +AGENT_TIMEOUT_MS=20000 KAFKA_LOG_LEVEL=info COHORT_PG_HOST=127.0.0.1 @@ -52,19 +45,20 @@ COHORT_PG_PORT=5432 COHORT_PG_USER=postgres COHORT_PG_PASSWORD=admin COHORT_PG_DATABASE=talos-sample-cohort-dev -COHORT_PG_POOL_SIZE=10 +COHORT_ADMIN_PG_DATABASE=postgres +COHORT_PG_POOL_SIZE=100 # Replicator and Statemap Installer Services -REPLICATOR_COMMIT_FREQ_MS=10000 -REPLICATOR_ENABLE_STATS=false +REPLICATOR_KAFKA_COMMIT_FREQ_MS=10000 +REPLICATOR_ENABLE_STATS=true REPLICATOR_CHANNEL_SIZE=100000 -REPLICATOR_SUFFIX_CAPACITY=100000 +REPLICATOR_SUFFIX_CAPACITY=800000 REPLICATOR_SUFFIX_PRUNE_THRESHOLD=1 +STATEMAP_QUEUE_ENABLE_STATS=true STATEMAP_QUEUE_CLEANUP_FREQUENCY_MS=10000 -STATEMAP_INSTALLER_THREAD_POOL=10 - +STATEMAP_INSTALLER_THREAD_POOL=100 # For example banking cohort BANK_REPLICATOR_KAFKA_GROUP_ID="talos-replicator-dev" diff --git a/cohort_banking_initiator_js/README.md b/cohort_banking_initiator_js/README.md index 83a116fc..120da7b2 100644 --- a/cohort_banking_initiator_js/README.md +++ b/cohort_banking_initiator_js/README.md @@ -1 +1,121 @@ -# Cohort Initiator JS App \ No newline at end of file +# Cohort Initiator JS App + +## About +This is an example project demonstrating how to use Cohort SDK to communicate with Talos Certifier. + +It is one of many components in Talos ecosystem. Section below describes the functionality of this application and how it depends on other components. + +Project is implemented in TypeScript and used the external library `cohort_sdk_client`, which in turn depends on `cohort_sdk_js`. The `cohort_sdk_js` is not desiged to be used directly, but only via its wrapper `cohort_sdk_client`. Both `cohort_sdk_*` libraries are shipped as NPM modules and hosted under `@kinderedgroup` in GitHub Packages NPM repository. While they both are NPM modules and can be installed via `npm install`, the installation of `cohort_sdk_js` has one extra step to compile native code on developer's machine. Please read more about it in the `cohort_sdk_js` package. + +Cohort Initiator JS is TypeScript app. When launched it connects to Postgres database and to Kafka broker, sends some number of certification requests to Talos Certifier. Once all requests are processed the app prints basic performance metrics and shuts down. + +## Usage + +1. Start Postgres database server +1. Start Kafka broker +1. Prepare .env config file +1. Start Talos Certifier +1. Generate sample data file +1. Create DB schema for cohort +1. Insert sample data to cohort DB +1. Build Cohort Replicator JS +1. Run Cohort Replicator JS +1. Build Cohort Initiator JS +1. Run Cohort Initiator JS + +### Setup your machine + +#### Install development ecosystem for Rust +Refer to https://www.rust-lang.org/tools/install + +Check: +``` +cargo --version && rustc --version && rustup --version + +cargo 1.72.0 (103a7ff2e 2023-08-15) +rustc 1.72.0 (5680fa18f 2023-08-23) +rustup 1.26.0 (5af9b9484 2023-04-05) +info: This is the version for the rustup toolchain manager, not the rustc compiler. +info: The currently active `rustc` version is `rustc 1.72.0 (5680fa18f 2023-08-23)` +``` + +#### Start Postgres server + +``` +sudo su postgres + +# Locate your Postgres installation (could be under "/Library/PostgreSQL/13/"), pg_ctl and data directory (could be "/Library/PostgreSQL/13/data"). +# find ~ -name pg_ctl + +./bin/pg_ctl -D ./data + +``` + +#### Prepare .env config file +``` +cp ./.env.example ./.env + +``` +Read the content of `.env` and adjsut connectivity details to Postgres and to Kafka + +#### Start Talos Certifier + +Please refer to Talos Certifier readme. + +#### Generate data file + +Generate data JSON file containing some number of random banking transactions. + +``` +cd $TALOS/ + +accounts_file=../10k-accounts.json +./scripts/cohort/data-generator.py --action gen-initial --count 10000 --out $accounts_file + +Generating 10000 accounts into: +../10k-accounts.json +...completed 1000 of 10000 +... +...completed 10000 of 10000 +``` + +#### Prepare DB + +``` +cd $TALOS/ +# Create database schema +make withenv RECIPE=cohort_banking.create_db +make withenv RECIPE=cohort_banking.migrate_db + +# Populate database +make withenv RECIPE=cohort_banking.preload_db args="--accounts $accounts_file" +``` + + +### Building Cohort Replicator and Initiator apps + +``` +cd $TALOS/cohort_banking_initiator_js +# This might take some time while native JS bindings are being compiled +npm ci + +cd $TALOS/cohort_banking_replicator_js +# This might take some time while native JS bindings are being compiled +npm ci +``` + +### Running Cohort Replicator +``` +cd $TALOS/cohort_banking_replicator_js + +# This app is the application server. It does not stop until process is terminated. +npm start +``` + +### Running Cohort Initiator +``` +cd $TALOS/cohort_banking_initiator_js + +# Genrate 1000 transation requests at the rate of 500 TPS +npm start count=1000 rate=500 +``` \ No newline at end of file diff --git a/cohort_banking_replicator_js/README.md b/cohort_banking_replicator_js/README.md index 0028f514..21c32bd7 100644 --- a/cohort_banking_replicator_js/README.md +++ b/cohort_banking_replicator_js/README.md @@ -1 +1,14 @@ -# Cohort Replicator JS App \ No newline at end of file +# Cohort Replicator JS App + +## About +This is an example project demonstrating how to create Talos Replicator component using Cohort SDK + +It is one of many components in Talos ecosystem. Section below describes the functionality of this application and how it depends on other components. + +Project is implemented in TypeScript and used the external library `cohort_sdk_client`, which in turn depends on `cohort_sdk_js`. The `cohort_sdk_js` is not desiged to be used directly, but only via its wrapper `cohort_sdk_client`. Both `cohort_sdk_*` libraries are shipped as NPM modules and hosted under `@kinderedgroup` in GitHub Packages NPM repository. While they both are NPM modules and can be installed via `npm install`, the installation of `cohort_sdk_js` has one extra step to compile native code on developer's machine. Please read more about it in the `cohort_sdk_js` package. + +Cohort Replicator JS is TypeScript app. When launched it connects to Postgres database and to Kafka broker, then listens to incoming messages (these are certification requests and decisions) then updates local database of Cohort Initiator. Cohort Initiator JS and Cohort Replicator JS share the same database. + +## Usage + +Please refer to readme of Cohort Initiator JS app. \ No newline at end of file