feature: Publish cohort SDK NPMs with bundled rust src (#85)

* feat: Publish sdk to GH Actions Packages

* feat: Added CI build step for Initiator JS

* feat: Added CI build step for Replicator JS

* fix: Use latest rdkafka

* chore: Rename build job

* chore: Add README.md content to initiator and replicator JS

* fix: Review comments, update readme.

* chore: Update readme.

* chore: Update comments.

* chore: Add comments

.env.example

@@ -21,50 +21,44 @@ PG_HOST=$LOCAL_HOST_IP
 PG_PORT=5432
 PG_USER=admin
 PG_PASSWORD=admin
-PG_DATABASE=talos_certifier
+PG_DATABASE=talos-certifier-dev
 ADMIN_PG_USER=admin
 ADMIN_PG_PASSWORD=admin
 ADMIN_PG_DATABASE=postgres
 
-# Mock
-
+# CONFIG
+RUST_LOG="warn"
 DB_MOCK=false
 CERTIFIER_MOCK=false
 
-# CONFIG
-RUST_LOG=debug
-
 # Cohort configs
 AGENT_NAME="Sample Cohort"
 COHORT_NAME="Sample Cohort"
-COHORT_WORKLOAD_DURATION=30
-
-AGENT_BUFFER_SIZE=10000
-AGENT_TIMEOUT_MS=5000
-
-KAFKA_FETCH_WAIT_MAX_MS=6000
-KAFKA_MESSAGE_TIMEOUT_MS=15000
-KAFKA_ENQUEUE_TIMEOUT_MS=10
+AGENT_BUFFER_SIZE=10000000
+# Timeout used by synchronous rdkafka publisher.
+AGENT_KAFKA_PRODUCER_SEND_TIMEOUT_MS=10
+AGENT_TIMEOUT_MS=20000
 KAFKA_LOG_LEVEL=info
 
 COHORT_PG_HOST=127.0.0.1
 COHORT_PG_PORT=5432
 COHORT_PG_USER=postgres
 COHORT_PG_PASSWORD=admin
 COHORT_PG_DATABASE=talos-sample-cohort-dev
-COHORT_PG_POOL_SIZE=10
+COHORT_ADMIN_PG_DATABASE=postgres
+COHORT_PG_POOL_SIZE=100
 
 # Replicator and Statemap Installer Services
-REPLICATOR_COMMIT_FREQ_MS=10000
-REPLICATOR_ENABLE_STATS=false
+REPLICATOR_KAFKA_COMMIT_FREQ_MS=10000
+REPLICATOR_ENABLE_STATS=true
 REPLICATOR_CHANNEL_SIZE=100000
 
-REPLICATOR_SUFFIX_CAPACITY=100000
+REPLICATOR_SUFFIX_CAPACITY=800000
 REPLICATOR_SUFFIX_PRUNE_THRESHOLD=1
 
+STATEMAP_QUEUE_ENABLE_STATS=true
 STATEMAP_QUEUE_CLEANUP_FREQUENCY_MS=10000
-STATEMAP_INSTALLER_THREAD_POOL=10
-
+STATEMAP_INSTALLER_THREAD_POOL=100
 
 # For example banking cohort
 BANK_REPLICATOR_KAFKA_GROUP_ID="talos-replicator-dev"

.github/workflows/build.yml

@@ -15,4 +15,4 @@ jobs:
             - run: rustup component add rustfmt clippy
             - name: cargo build
               run: cargo build --release
-            - run: scripts/pre-commit-checks.sh
+            - run: scripts/pre-commit-checks.sh

.github/workflows/master.yml

@@ -10,7 +10,3 @@ jobs:
         uses: ./.github/workflows/build.yml
     coverage:
         uses: ./.github/workflows/coverage.yml
-
-
-
-

.github/workflows/npm.yml

@@ -0,0 +1,120 @@
+name: cohort_sdk_js
+on:
+    workflow_call:
+    #push:
+env:
+    DEBUG: "napi:*"
+    OPENSSL_DIR: /usr/local/Cellar/[email protected]/1.1.1w
+    SASL2_DIR: /usr/local/Cellar/cyrus-sasl/2.1.28_2
+    LDFLAGS: "-L/usr/local/Cellar/[email protected]/1.1.1w/lib -L/usr/local/opt/cyrus-sasl/lib"
+    CPPFLAGS: "-I/usr/local/Cellar/[email protected]/1.1.1w/include -I/usr/local/opt/cyrus-sasl/include"
+jobs:
+    npm:
+        strategy:
+            fail-fast: false
+            matrix:
+                settings:
+                  - host: macos-latest
+                    target: x86_64-apple-darwin
+                    jsPackageHome: packages/cohort_sdk_js
+                    finalPackageName: packages/cohort_sdk_js/cohort_sdk_js.darwin-x64.node
+                    customiseHostCommand: echo 'No customizations are needed'
+                    buildCommand: npm run build -- --target x86_64-apple-darwin
+
+                  - host: macos-latest
+                    target: aarch64-apple-darwin
+                    jsPackageHome: packages/cohort_sdk_js
+                    finalPackageName: packages/cohort_sdk_js/cohort_sdk_js.darwin-arm64.node
+                    customiseHostCommand: |-
+                      touch ~/.cargo/config
+                      cat <<EOT >> ~/.cargo/config
+                      [target.aarch64-apple-darwin]
+                      rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+                      EOT
+                      echo "~/.cargo/config"
+                      cat ~/.cargo/config
+                    buildCommand: |-
+                      npm run build -- --target aarch64-apple-darwin
+
+                  - host: windows-latest
+                    target: x86_64-pc-windows-msvc
+                    jsPackageHome: packages\\cohort_sdk_js
+                    finalPackageName: packages\cohort_sdk_js\cohort_sdk_js.*.node
+                    customiseHostCommand: ECHO 'No customizations are needed'
+                    # customiseHostCommand: |-
+                    #   choco install openssl --version=1.1.1.2100
+                    #   ECHO "C:\Program Files\OpenSSL\="
+                    #   ls -l "C:\\Program Files\\OpenSSL"
+                    #   setx OPENSSL_DIR "C:\\Program Files\\OpenSSL"
+                    buildCommand: npm run build -- --target x86_64-pc-windows-msvc
+
+                  - host: ubuntu-latest
+                    target: x86_64-unknown-linux-gnu
+                    jsPackageHome: packages/cohort_sdk_js
+                    finalPackageName: packages/cohort_sdk_js/cohort_sdk_js.linux-x64-gnu.node
+                    customiseHostCommand: sudo apt-get update -y && sudo apt-get install -y libsasl2-dev
+                    buildCommand: npm run build -- --target x86_64-unknown-linux-gnu
+
+        name: stable - ${{ matrix.settings.target }} - node@16
+        runs-on: ${{ matrix.settings.host }}
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup node
+              uses: actions/setup-node@v3
+              with:
+                  node-version: 16
+                  check-latest: true
+
+            - name: Install
+              uses: actions-rs/toolchain@v1
+              with:
+                  profile: minimal
+                  override: true
+                  toolchain: stable
+                  target: ${{ matrix.settings.target }}
+
+            - name: Cache cargo
+              uses: actions/cache@v3
+              with:
+                path: |
+                  ~/.cargo/registry/index/
+                  ~/.cargo/registry/cache/
+                  ~/.cargo/git/db/
+                  .cargo-cache
+                  target/
+                key: ${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
+
+            - name: Customise host image
+              shell: bash
+              run: ${{ matrix.settings.customiseHostCommand }}
+
+            - name: Show current directory name
+              shell: bash
+              run: pwd
+
+            - name: List content before build
+              shell: bash
+              run: ls -l && ls -l ${{ matrix.settings.jsPackageHome }}
+
+            # - name: Cache cargo TODO deal with caching
+            - name: Install node dependencies
+              run: cd ${{ matrix.settings.jsPackageHome }} && pwd && ls -l . && npm install -verbose
+
+            # - name: Cargo build
+            #   run: cargo build
+
+            - name: NAPI-RS build
+              shell: bash
+              run: cd ${{ matrix.settings.jsPackageHome }} && ${{ matrix.settings.buildCommand }}
+
+            - name: List content after build
+              shell: bash
+              run: ls -l && ls -l ${{ matrix.settings.jsPackageHome }}
+
+            - name: Upload built binary
+              uses: actions/upload-artifact@v3
+              with:
+                  name: bindings-${{ matrix.settings.target }}
+                  path: ${{ matrix.settings.finalPackageName }}
+                  if-no-files-found: error

.github/workflows/npm_publish_light.yml

@@ -0,0 +1,132 @@
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Creates and publishes @kindredgroup/cohort_sdk_js, @kindredgroup/cohort_sdk_client
+# NPM packages. The "cohort_sdk_js" is bundled together with Talos sources so that users
+# could build it locally.
+# Performs a basic build test by installing "cohort_sdk_client" into
+# "cohort_banking_initiator_js", "cohort_banking_replicator_js" apps and then builds apps.
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+name: Build and Publish Cohort SDK NPMs
+on:
+    #workflow_call:
+    push:
+env:
+    SDK_JS_PACKAGE_NAME: "@kindredgroup/cohort_sdk_js"
+    SDK_CLIENT_PACKAGE_NAME: "@kindredgroup/cohort_sdk_client"
+jobs:
+    npm:
+        strategy:
+            fail-fast: false
+            matrix:
+                settings:
+                  - host: macos-latest
+                    dirSdkJs: packages/cohort_sdk_js
+                    dirSdkJsClient: cohort_sdk_client
+                    dirCohortInitiator: cohort_banking_initiator_js
+                    dirCohortReplicator: cohort_banking_replicator_js
+
+        name: Publish SDK NPS with bundled Rust for node@16
+        runs-on: ${{ matrix.settings.host }}
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Setup node
+              uses: actions/setup-node@v3
+              with:
+                  node-version: 16
+                  check-latest: true
+                  registry-url: "https://npm.pkg.github.com"
+
+            - name: Customise host image
+              shell: bash
+              run: |-
+                echo "NPM_CONFIG_USERCONFIG=$NPM_CONFIG_USERCONFIG"
+                cat $NPM_CONFIG_USERCONFIG
+                echo ""
+
+            - name: Setup version for ${{ matrix.settings.dirSdkJs }} NPM
+              id: sdk-js-version
+              shell: bash
+              run: |-
+                echo "Building SDK JS Library"
+                cd ${{ matrix.settings.dirSdkJs }}
+                pwd
+                ls -l
+                npm ci --foreground-scripts
+                ../../scripts/github-actions-ci/set-npm-dev-version.sh ${{ env.SDK_JS_PACKAGE_NAME }} "SDK_JS_PACKAGE_VERSION"
+                echo ""
+
+            - name: Build and Publish ${{ matrix.settings.dirSdkJs }}
+              shell: bash
+              env:
+                  NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |-
+                cd ${{ matrix.settings.dirSdkJs }}
+                pwd
+                ls -l
+                echo "Current version is"
+                npm version
+                npm publish --foreground-scripts
+                echo ""
+
+            - name: Setup version for ${{ matrix.settings.dirSdkJsClient }} NPM
+              id: sdk-client-version
+              shell: bash
+              env:
+                  SDK_JS_PACKAGE_VERSION: ${{ steps.sdk-js-version.outputs.SDK_JS_PACKAGE_VERSION }}
+                  NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |-
+                echo "Building SDK Client"
+                cd ${{ matrix.settings.dirSdkJsClient }}
+                pwd
+                ls -lah
+                rm package-lock.json || true
+                echo "D: npm install $SDK_JS_PACKAGE_NAME@$SDK_JS_PACKAGE_VERSION --foreground-scripts"
+                npm install "$SDK_JS_PACKAGE_NAME@$SDK_JS_PACKAGE_VERSION" --foreground-scripts
+                ../scripts/github-actions-ci/set-npm-dev-version.sh ${{ env.SDK_CLIENT_PACKAGE_NAME }} "SDK_CLIENT_PACKAGE_VERSION"
+                echo ""
+
+            - name: Build and Publish ${{ matrix.settings.dirSdkJsClient }} NPM
+              shell: bash
+              env:
+                  NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |-
+                cd ${{ matrix.settings.dirSdkJsClient }}
+                pwd
+                ls -l
+                echo "Current version is"
+                npm version
+                npm ci --foreground-scripts
+                npm run build --foreground-scripts
+                npm publish --foreground-scripts
+                echo ""
+
+            - name: Build ${{ matrix.settings.dirCohortInitiator }}
+              env:
+                  SDK_CLIENT_PACKAGE_VERSION: ${{ steps.sdk-client-version.outputs.SDK_CLIENT_PACKAGE_VERSION }}
+                  NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  SKIP_NAPI_RS_STEP: true # this will re-use already bundled native lib
+              shell: bash
+              run: |-
+                cd ${{ matrix.settings.dirCohortInitiator }}
+                pwd
+                ls -l
+                rm package-lock.json || true
+                echo "D: npm install $SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION --foreground-scripts"
+                npm install "$SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION" --foreground-scripts
+                npm run build --foreground-scripts
+
+            - name: Build ${{ matrix.settings.dirCohortReplicator }}
+              env:
+                  SDK_CLIENT_PACKAGE_VERSION: ${{ steps.sdk-client-version.outputs.SDK_CLIENT_PACKAGE_VERSION }}
+                  NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  SKIP_NAPI_RS_STEP: true # this will re-use already bundled native lib
+              shell: bash
+              run: |-
+                cd ${{ matrix.settings.dirCohortInitiator }}
+                pwd
+                ls -l
+                rm package-lock.json || true
+                echo "D: npm install $SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION --foreground-scripts"
+                npm install "$SDK_CLIENT_PACKAGE_NAME@$SDK_CLIENT_PACKAGE_VERSION" --foreground-scripts
+                npm run build --foreground-scripts