chore(release): pull release/v1.100.0 into main

.github/workflows/raise-pr-for-constants.yml

@@ -33,10 +33,10 @@ jobs:
         id: compare_files
         run: |
           if cmp -s "${{ env.SRC_FILE }}" "dest-repo/${{ env.DEST_PATH }}"; then
-            echo "No changes detected." > "result.txt"
+            echo "No changes detected."
             echo "pr_required=false" >> $GITHUB_OUTPUT
           else
-            echo "Changes detected." > "result.txt"
+            echo "Changes detected."
             echo "pr_required=true" >> $GITHUB_OUTPUT
           fi
 
@@ -58,20 +58,21 @@ jobs:
           GH_TOKEN: ${{ secrets.PAT }}
         run: |
           cd dest-repo
-          EXISTING_PR=$(gh pr list --head ${{ env.BRANCH_NAME }} --json number --jq ".[0].number")
+          EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number --jq ".[0].number")
           if [ -z "$EXISTING_PR" ]; then
-            gh pr create \
-              --title "fix: update destination constants" \
-              --body "$(cat <<EOF
-                This PR updates the destination constants file.
+          # Create the PR body content in a variable
+          PR_BODY="This PR updates the destination constants file.
 
-                **Changes:**
-                - Updated \`Destinations.ts\` with latest constants
+          **Changes:**
+          - Updated \`Destinations.ts\` with latest constants
 
-                NOTE: This PR was automatically generated by GitHub Actions.
-                EOF
-                )" \
-              --label "automated,dependencies"
+          **NOTE:** This PR was automatically generated by GitHub Actions."
+
+          # Create a new PR
+          gh pr create \
+            --title "fix: update destination constants" \
+            --body "$PR_BODY" \
+            --label "github_actions,dependencies"
           else
             echo "PR already exists: $EXISTING_PR"
           fi

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.100.0](https://github.com/rudderlabs/rudder-config-schema/compare/v1.99.1...v1.100.0) (2024-12-13)
+
+
+### Features
+
+* add tags for sdk source types ([#1841](https://github.com/rudderlabs/rudder-config-schema/issues/1841)) ([1f4e1c5](https://github.com/rudderlabs/rudder-config-schema/commit/1f4e1c5edb6fb95be21e62d5671427a91a371a50))
+* onboard topsort destination ([#1842](https://github.com/rudderlabs/rudder-config-schema/issues/1842)) ([2c683fe](https://github.com/rudderlabs/rudder-config-schema/commit/2c683fec0ac8d8518096413c6d2b018a1e7b4d1e))
+
+
+### Bug Fixes
+
+* pr body in constants pr workflow ([83d7b26](https://github.com/rudderlabs/rudder-config-schema/commit/83d7b2674f2b4bc209331d1dcd2b3c3bacafb4ab))
+* pr body in constants pr workflow ([3d227f7](https://github.com/rudderlabs/rudder-config-schema/commit/3d227f7126c96df9bf64fbd9587ac5d8f19fe17c))
+* pr labels in constants pr workflow ([e8c5167](https://github.com/rudderlabs/rudder-config-schema/commit/e8c5167cad5f9792e2621ddc7695fe85b7aa8fb4))
+
 ### [1.99.1](https://github.com/rudderlabs/rudder-config-schema/compare/v1.99.0...v1.99.1) (2024-12-12)
 
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudder-config-schema",
-  "version": "1.99.1",
+  "version": "1.100.0",
   "description": "",
   "main": "src/index.ts",
   "private": true,

scripts/deployToDB.py

@@ -8,41 +8,67 @@
 from constants import CONFIG_DIR
 
 
+ALL_SELECTORS = ["destination", "source", "wht-lib-project"]
+
+
 def get_command_line_arguments():
     parser = argparse.ArgumentParser(description="Script to deploy config files to DB.")
     parser.add_argument("control_plane_url", nargs="?", help="Control plane URL")
     parser.add_argument("username", nargs="?", help="Control plane admin username")
     parser.add_argument("password", nargs="?", help="Control plane admin password")
+    parser.add_argument(
+        "selector",
+        nargs="?",
+        help="Specify destination, source or wht-lib-project",
+        default=None,
+    )
+    parser.add_argument(
+        "item_name", nargs="?", help="Specific item name to update.", default=None
+    )
 
     args = parser.parse_args()
 
     control_plane_url = args.control_plane_url or os.getenv("CONTROL_PLANE_URL")
     username = args.username or os.getenv("API_USER")
     password = args.password or os.getenv("API_PASSWORD")
+    selector = args.selector or os.getenv("SELECTOR")
+    item_name = args.item_name or os.getenv("ITEM_NAME")
 
-    missing_args = []
+    invalid_args = []
 
     if control_plane_url is None:
-        missing_args.append(
-            "1st positional argument or CONTROL_PLANE_URL environment variable"
+        invalid_args.append(
+            "1st positional argument or CONTROL_PLANE_URL environment variable is missing"
         )
     if username is None:
-        missing_args.append("2nd positional argument or API_USER environment variable")
+        invalid_args.append(
+            "2nd positional argument or API_USER environment variable is missing"
+        )
     if password is None:
-        missing_args.append(
-            "3rd positional argument or API_PASSWORD environment variable"
+        invalid_args.append(
+            "3rd positional argument or API_PASSWORD environment variable is missing"
         )
+    if selector is None:
+        SELECTORS = ALL_SELECTORS
+    elif selector not in ALL_SELECTORS:
+        invalid_args.append(
+            "4th positional argument or SELECTOR environment variable is invalid"
+        )
+    else:
+        SELECTORS = [selector]
 
-    if missing_args:
-        print("Error: Missing the following arguments or environment variables:")
-        for arg in missing_args:
+    if invalid_args:
+        print("Error: The following arguments or environment variables are invalid:")
+        for arg in invalid_args:
             print(arg)
         sys.exit(1)
 
-    return control_plane_url, username, password
+    return control_plane_url, username, password, SELECTORS, item_name
 
 
-CONTROL_PLANE_URL, USERNAME, PASSWORD = get_command_line_arguments()
+CONTROL_PLANE_URL, USERNAME, PASSWORD, SELECTORS, ITEM_NAME = (
+    get_command_line_arguments()
+)
 
 # CONSTANTS
 HEADER = {"Content-Type": "application/json"}
@@ -139,11 +165,16 @@ def update_config(data_diff, selector):
     return json.dumps(results, indent=2)
 
 
-def update_diff_db(selector):
+def update_diff_db(selector, item_name=None):
     final_report = []
 
     ## data sets
-    current_items = os.listdir(f"./{CONFIG_DIR}/{selector}s")
+    if item_name:
+        current_items = [item_name]
+    else:
+        current_items = os.listdir(f"./{CONFIG_DIR}/{selector}s")
+
+    print(f"Current items: {current_items}")
 
     for item in current_items:
         # check if item is a directory
@@ -203,47 +234,18 @@ def get_stale_data(selector, report):
 
 
 if __name__ == "__main__":
-    print("\n")
-    print("#" * 50)
-    print("Running Destination Definitions Updates")
-    dest_final_report = update_diff_db("destination")
-
-    print("\n")
-    print("#" * 50)
-    print("Destination Definition Update Report")
-    print(get_formatted_json(dest_final_report))
-
-    print("\n")
-    print("#" * 50)
-    print("Stale Destinations Report")
-    print(get_formatted_json(get_stale_data("destination", dest_final_report)))
-
-    print("\n")
-    print("#" * 50)
-    print("Running Source Definitions Updates")
-    src_final_report = update_diff_db("source")
-
-    print("\n")
-    print("#" * 50)
-    print("Source Definition Update Report")
-    print(get_formatted_json(src_final_report))
-
-    print("\n")
-    print("#" * 50)
-    print("Stale Sources Report")
-    print(get_formatted_json(get_stale_data("source", src_final_report)))
-
-    print("\n")
-    print("#" * 50)
-    print("Running Wht Lib Project Definitions Updates")
-    wht_final_report = update_diff_db("wht-lib-project")
-
-    print("\n")
-    print("#" * 50)
-    print("Wht Lib Project Definition Update Report")
-    print(get_formatted_json(wht_final_report))
-
-    print("\n")
-    print("#" * 50)
-    print("Stale Wht Lib Projects Report")
-    print(get_formatted_json(get_stale_data("wht-lib-project", wht_final_report)))
+    for selector in SELECTORS:
+        print("\n")
+        print("#" * 50)
+        print("Running {} Definitions Updates".format(selector.capitalize()))
+        final_report = update_diff_db(selector, ITEM_NAME)
+
+        print("\n")
+        print("#" * 50)
+        print("{} Definition Update Report".format(selector.capitalize()))
+        print(get_formatted_json(final_report))
+
+        print("\n")
+        print("#" * 50)
+        print("Stale {}s Report".format(selector.capitalize()))
+        print(get_formatted_json(get_stale_data(selector, final_report)))

scripts/listJsonMapperDestinations.js

@@ -0,0 +1,48 @@
+/* eslint-disable no-console */
+const fs = require('fs');
+const path = require('path');
+
+const destinationsDir = path.join(__dirname, '../src/configurations/destinations');
+
+function findJsonMapperDestinations() {
+  try {
+    if (!fs.existsSync(destinationsDir)) {
+      throw new Error(`Destinations directory not found: ${destinationsDir}`);
+    }
+    return fs
+      .readdirSync(destinationsDir)
+      .map((destination) => {
+        try {
+          const destinationsFilePath = path.join(destinationsDir, destination, 'db-config.json');
+          if (!fs.existsSync(destinationsFilePath)) {
+            console.warn(`Skipping ${destination}: Missing configuration file`);
+            return null;
+          }
+          const destinationsContent = fs.readFileSync(destinationsFilePath, 'utf8');
+          const destinationDefinition = JSON.parse(destinationsContent);
+          if (!destinationDefinition.name) {
+            console.warn(`Skipping ${destination}: Missing name`);
+            return null;
+          }
+          return {
+            name: destinationDefinition.name,
+            config: destinationDefinition.config,
+          };
+        } catch (err) {
+          console.error(`Error processing ${destination}:`, err.message);
+          return null;
+        }
+      })
+      .filter(Boolean)
+      .filter(
+        (destination) =>
+          !destination.config?.disableJsonMapper && !destination.config?.supportsVisualMapper,
+      )
+      .map((destination) => destination.name);
+  } catch (err) {
+    console.error('Failed to process destinations:', err.message);
+    return [];
+  }
+}
+
+console.log(findJsonMapperDestinations().join('\n'));