diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 480a9a6..8da6f49 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -2,7 +2,11 @@ name: Integration Test
 
 on:
   workflow_dispatch:
-    ## The tests take a long time to run and can be potentially quite costly, so set it to run manually
+    inputs:
+      baseFolderPath:
+        description: 'Base folder path for diffs'
+        required: true
+        default: 'diffs'
 
 permissions:
   contents: read
@@ -10,6 +14,10 @@ permissions:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      matrix:
+        dynamicCoordinates: [true, false]
     defaults:
       run:
         working-directory: ./tools/integration
@@ -29,7 +37,16 @@ jobs:
         run: npm test
         
       - name: Trigger harvest and verify completion
-        run: npm run e2e-test-harvest
+        run: DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-harvest
 
       - name: Verify service functions
-        run: npm run e2e-test-service
+        run: DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-service
+
+      - name: Generate structured diffs
+        run: npm run definitions-diff ${{ github.event.inputs.baseFolderPath }}
+
+      - name: Upload diffs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffs-${{ matrix.dynamicCoordinates == 'true' && 'dynamic' || 'static' }}
+          path: ${{ github.event.inputs.baseFolderPath }}
diff --git a/tools/harvester-forwarding/.funcignore b/tools/harvester-forwarding/.funcignore
new file mode 100644
index 0000000..d5b3b4a
--- /dev/null
+++ b/tools/harvester-forwarding/.funcignore
@@ -0,0 +1,10 @@
+*.js.map
+*.ts
+.git*
+.vscode
+__azurite_db*__.json
+__blobstorage__
+__queuestorage__
+local.settings.json
+test
+tsconfig.json
\ No newline at end of file
diff --git a/tools/harvester-forwarding/.gitignore b/tools/harvester-forwarding/.gitignore
new file mode 100644
index 0000000..01774db
--- /dev/null
+++ b/tools/harvester-forwarding/.gitignore
@@ -0,0 +1,99 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# next.js build output
+.next
+
+# nuxt.js build output
+.nuxt
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TypeScript output
+dist
+out
+
+# Azure Functions artifacts
+bin
+obj
+appsettings.json
+local.settings.json
+
+# Azurite artifacts
+__blobstorage__
+__queuestorage__
+__azurite_db*__.json
\ No newline at end of file
diff --git a/tools/harvester-forwarding/host.json b/tools/harvester-forwarding/host.json
new file mode 100644
index 0000000..90e13e1
--- /dev/null
+++ b/tools/harvester-forwarding/host.json
@@ -0,0 +1,16 @@
+{
+  "version": "2.0",
+  "functionTimeout": "00:10:00",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  }
+}
\ No newline at end of file
diff --git a/tools/harvester-forwarding/package-lock.json b/tools/harvester-forwarding/package-lock.json
new file mode 100644
index 0000000..06a23d8
--- /dev/null
+++ b/tools/harvester-forwarding/package-lock.json
@@ -0,0 +1,317 @@
+{
+  "name": "harvester-forwarding",
+  "version": "1.0.0",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "harvester-forwarding",
+      "version": "1.0.0",
+      "dependencies": {
+        "@azure/functions": "^4.0.0",
+        "mongodb": "^6.8.0"
+      },
+      "devDependencies": {}
+    },
+    "node_modules/@azure/functions": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/@azure/functions/-/functions-4.5.1.tgz",
+      "integrity": "sha512-ikiw1IrM2W9NlQM3XazcX+4Sq3XAjZi4eeG22B5InKC2x5i7MatGF2S/Gn1ACZ+fEInwu+Ru9J8DlnBv1/hIvg==",
+      "dependencies": {
+        "cookie": "^0.6.0",
+        "long": "^4.0.0",
+        "undici": "^5.13.0"
+      },
+      "engines": {
+        "node": ">=18.0"
+      }
+    },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
+      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@mongodb-js/saslprep": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.9.tgz",
+      "integrity": "sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==",
+      "dependencies": {
+        "sparse-bitfield": "^3.0.3"
+      }
+    },
+    "node_modules/@types/webidl-conversions": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
+      "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="
+    },
+    "node_modules/@types/whatwg-url": {
+      "version": "11.0.5",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz",
+      "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==",
+      "dependencies": {
+        "@types/webidl-conversions": "*"
+      }
+    },
+    "node_modules/bson": {
+      "version": "6.8.0",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-6.8.0.tgz",
+      "integrity": "sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==",
+      "engines": {
+        "node": ">=16.20.1"
+      }
+    },
+    "node_modules/cookie": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+      "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/long": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+      "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
+    },
+    "node_modules/memory-pager": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg=="
+    },
+    "node_modules/mongodb": {
+      "version": "6.8.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.8.0.tgz",
+      "integrity": "sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==",
+      "dependencies": {
+        "@mongodb-js/saslprep": "^1.1.5",
+        "bson": "^6.7.0",
+        "mongodb-connection-string-url": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=16.20.1"
+      },
+      "peerDependencies": {
+        "@aws-sdk/credential-providers": "^3.188.0",
+        "@mongodb-js/zstd": "^1.1.0",
+        "gcp-metadata": "^5.2.0",
+        "kerberos": "^2.0.1",
+        "mongodb-client-encryption": ">=6.0.0 <7",
+        "snappy": "^7.2.2",
+        "socks": "^2.7.1"
+      },
+      "peerDependenciesMeta": {
+        "@aws-sdk/credential-providers": {
+          "optional": true
+        },
+        "@mongodb-js/zstd": {
+          "optional": true
+        },
+        "gcp-metadata": {
+          "optional": true
+        },
+        "kerberos": {
+          "optional": true
+        },
+        "mongodb-client-encryption": {
+          "optional": true
+        },
+        "snappy": {
+          "optional": true
+        },
+        "socks": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/mongodb-connection-string-url": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.1.tgz",
+      "integrity": "sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==",
+      "dependencies": {
+        "@types/whatwg-url": "^11.0.2",
+        "whatwg-url": "^13.0.0"
+      }
+    },
+    "node_modules/punycode": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/sparse-bitfield": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
+      "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+      "dependencies": {
+        "memory-pager": "^1.0.2"
+      }
+    },
+    "node_modules/tr46": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz",
+      "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==",
+      "dependencies": {
+        "punycode": "^2.3.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/undici": {
+      "version": "5.28.4",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
+      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
+      "dependencies": {
+        "@fastify/busboy": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.0"
+      }
+    },
+    "node_modules/webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/whatwg-url": {
+      "version": "13.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-13.0.0.tgz",
+      "integrity": "sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==",
+      "dependencies": {
+        "tr46": "^4.1.1",
+        "webidl-conversions": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    }
+  },
+  "dependencies": {
+    "@azure/functions": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/@azure/functions/-/functions-4.5.1.tgz",
+      "integrity": "sha512-ikiw1IrM2W9NlQM3XazcX+4Sq3XAjZi4eeG22B5InKC2x5i7MatGF2S/Gn1ACZ+fEInwu+Ru9J8DlnBv1/hIvg==",
+      "requires": {
+        "cookie": "^0.6.0",
+        "long": "^4.0.0",
+        "undici": "^5.13.0"
+      }
+    },
+    "@fastify/busboy": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz",
+      "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA=="
+    },
+    "@mongodb-js/saslprep": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.9.tgz",
+      "integrity": "sha512-tVkljjeEaAhCqTzajSdgbQ6gE6f3oneVwa3iXR6csiEwXXOFsiC6Uh9iAjAhXPtqa/XMDHWjjeNH/77m/Yq2dw==",
+      "requires": {
+        "sparse-bitfield": "^3.0.3"
+      }
+    },
+    "@types/webidl-conversions": {
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz",
+      "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA=="
+    },
+    "@types/whatwg-url": {
+      "version": "11.0.5",
+      "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz",
+      "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==",
+      "requires": {
+        "@types/webidl-conversions": "*"
+      }
+    },
+    "bson": {
+      "version": "6.8.0",
+      "resolved": "https://registry.npmjs.org/bson/-/bson-6.8.0.tgz",
+      "integrity": "sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ=="
+    },
+    "cookie": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+      "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw=="
+    },
+    "long": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
+      "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
+    },
+    "memory-pager": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
+      "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg=="
+    },
+    "mongodb": {
+      "version": "6.8.0",
+      "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.8.0.tgz",
+      "integrity": "sha512-HGQ9NWDle5WvwMnrvUxsFYPd3JEbqD3RgABHBQRuoCEND0qzhsd0iH5ypHsf1eJ+sXmvmyKpP+FLOKY8Il7jMw==",
+      "requires": {
+        "@mongodb-js/saslprep": "^1.1.5",
+        "bson": "^6.7.0",
+        "mongodb-connection-string-url": "^3.0.0"
+      }
+    },
+    "mongodb-connection-string-url": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.1.tgz",
+      "integrity": "sha512-XqMGwRX0Lgn05TDB4PyG2h2kKO/FfWJyCzYQbIhXUxz7ETt0I/FqHjUeqj37irJ+Dl1ZtU82uYyj14u2XsZKfg==",
+      "requires": {
+        "@types/whatwg-url": "^11.0.2",
+        "whatwg-url": "^13.0.0"
+      }
+    },
+    "punycode": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
+      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="
+    },
+    "sparse-bitfield": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
+      "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+      "requires": {
+        "memory-pager": "^1.0.2"
+      }
+    },
+    "tr46": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz",
+      "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==",
+      "requires": {
+        "punycode": "^2.3.0"
+      }
+    },
+    "undici": {
+      "version": "5.28.4",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz",
+      "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
+      "requires": {
+        "@fastify/busboy": "^2.0.0"
+      }
+    },
+    "webidl-conversions": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
+      "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g=="
+    },
+    "whatwg-url": {
+      "version": "13.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-13.0.0.tgz",
+      "integrity": "sha512-9WWbymnqj57+XEuqADHrCJ2eSXzn8WXIW/YSGaZtb2WKAInQ6CHfaUUcTyyver0p8BDg5StLQq8h1vtZuwmOig==",
+      "requires": {
+        "tr46": "^4.1.1",
+        "webidl-conversions": "^7.0.0"
+      }
+    }
+  }
+}
diff --git a/tools/harvester-forwarding/package.json b/tools/harvester-forwarding/package.json
new file mode 100644
index 0000000..1810498
--- /dev/null
+++ b/tools/harvester-forwarding/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "harvester-forwarding",
+  "version": "1.0.0",
+  "description": "",
+  "scripts": {
+    "start": "func start",
+    "test": "echo \"No tests yet...\""
+  },
+  "dependencies": {
+    "@azure/functions": "^4.0.0",
+    "mongodb": "^6.8.0"
+  },
+  "devDependencies": {},
+  "main": "src/{index.js,functions/*.js}"
+}
\ No newline at end of file
diff --git a/tools/harvester-forwarding/src/functions/getRecentDefinitions.js b/tools/harvester-forwarding/src/functions/getRecentDefinitions.js
new file mode 100644
index 0000000..4cc1a71
--- /dev/null
+++ b/tools/harvester-forwarding/src/functions/getRecentDefinitions.js
@@ -0,0 +1,152 @@
+const { app } = require("@azure/functions");
+const { MongoClient } = require("mongodb");
+
+app.http("getRecentDefinitions", {
+  methods: ["GET"],
+  authLevel: "anonymous",
+  handler: async (request, context) => {
+    context.log(`Http function processed request for url "${request.url}"`);
+
+    try {
+      const daysBack = parseInt(request.query.get("days"));
+      if (!daysBack || isNaN(daysBack) || daysBack < 1 || daysBack > 30) {
+        return { status: 400, body: "days must be a number between 1 and 30" };
+      }
+      const limitPerTypeAndDay = parseInt(request.query.get("limit"));
+      if (
+        !limitPerTypeAndDay ||
+        isNaN(limitPerTypeAndDay) ||
+        limitPerTypeAndDay < 1 ||
+        limitPerTypeAndDay > 100
+      ) {
+        return {
+          status: 400,
+          body: "limitPerType must be a number between 1 and 100",
+        };
+      }
+
+      return await getData(context, daysBack, limitPerTypeAndDay);
+    } catch (error) {
+      context.log(`Error: ${error.message}`);
+      return { status: 500, body: `An error occurred: ${error.message}` };
+    }
+  },
+});
+
+async function getData(context, days, limitPerType) {
+  const connectionString = process.env.COSMOSDB_CONNECTION_STRING;
+  const dbName = process.env.COSMOSDB_DATABASE_NAME;
+  const collectionName = process.env.COSMOSDB_COLLECTION_NAME;
+
+  const groupLimit = 30;
+
+  if (!connectionString) {
+    return { status: 500, body: "Database connection string not configured" };
+  }
+
+  try {
+    const client = await MongoClient.connect(connectionString);
+    const db = client.db(dbName);
+    const collection = db.collection(collectionName);
+
+    const aggregationPipeline = [
+      // Limit the date range to the last N days
+      {
+        $match: {
+          "_meta.updated": {
+            $gt: new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString(),
+          },
+        },
+      },
+    
+      // Project necessary fields and add a day field
+      {
+        $project: {
+          _id: 1,
+          "coordinates.type": 1,
+          "_meta.updated": 1,
+          day: {
+            $dateToString: {
+              format: "%Y-%m-%d",
+              date: { $toDate: "$_meta.updated" },
+            },
+          },
+        },
+      },
+    
+      // Group by type and day
+      {
+        $group: {
+          _id: {
+            type: "$coordinates.type",
+            day: "$day",
+          },
+          documents: {
+            $push: {
+              _id: "$_id",
+              updated: "$_meta.updated",
+            },
+          },
+        },
+      },
+    
+      // Slice to get only the first limitPerType documents per day
+      {
+        $project: {
+          _id: 1,
+          documents: { $slice: ["$documents", limitPerType] },
+        },
+      },
+    
+      // Group by type to combine all days
+      {
+        $group: {
+          _id: "$_id.type",
+          documents: { $push: "$documents" },
+        },
+      },
+    
+      // Flatten the documents array and limit to limitPerType * days
+      {
+        $project: {
+          documents: {
+            $slice: [
+              { $reduce: {
+                input: "$documents",
+                initialValue: [],
+                in: { $concatArrays: ["$$value", "$$this"] }
+              }},
+              limitPerType * days
+            ]
+          }
+        }
+      },
+    
+      // Optional: Limit the number of types returned
+      { $limit: groupLimit },
+    ];
+
+    // Use this aggregation pipeline in your Azure Function
+    const result = await collection
+      .aggregate(aggregationPipeline, {
+        maxTimeMS: 120000, // 2 minutes timeout
+        allowDiskUse: true, // Allow using disk for large aggregations
+      })
+      .toArray();
+
+    await client.close();
+
+    return {
+      status: 200,
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(result.flatMap((r) => r.documents.map((d) => d._id)))
+    };
+      
+       
+  } catch (error) {
+    context.log(`CosmosDB Error: ${error.message}`);
+    return { status: 500, body: `An error occurred: ${error.message}` };
+  }
+}
diff --git a/tools/harvester-forwarding/src/index.js b/tools/harvester-forwarding/src/index.js
new file mode 100644
index 0000000..0c7432e
--- /dev/null
+++ b/tools/harvester-forwarding/src/index.js
@@ -0,0 +1,5 @@
+const { app } = require('@azure/functions');
+
+app.setup({
+    enableHttpStream: true,
+});
diff --git a/tools/harvester-forwarding/src/test/compareRequest.json b/tools/harvester-forwarding/src/test/compareRequest.json
new file mode 100644
index 0000000..0fb18e4
--- /dev/null
+++ b/tools/harvester-forwarding/src/test/compareRequest.json
@@ -0,0 +1,224 @@
+{
+    "productionDoc":{
+        "described": {
+          "releaseDate": "2021-12-12",
+          "urls": {
+            "registry": "https://pkg.go.dev/github.com/docker/cli",
+            "version": "https://pkg.go.dev/github.com/docker/cli@v20.10.12+incompatible",
+            "download": "https://proxy.golang.org/github.com/docker/cli/@v/v20.10.12+incompatible.zip"
+          },
+          "hashes": {
+            "sha1": "d6a524762d88242708dac60e0f61485876348a5c",
+            "sha256": "d52fcb80bdcd3eeaebd25e231769b2780149a39d85ad2b42923132426a0486fb"
+          },
+          "files": 1228,
+          "tools": [
+            "clearlydefined/1.2.0",
+            "licensee/9.14.0",
+            "scancode/30.3.0"
+          ],
+          "toolScore": {
+            "total": 100,
+            "date": 30,
+            "source": 70
+          },
+          "sourceLocation": {
+            "type": "go",
+            "provider": "golang",
+            "namespace": "github.com%2Fdocker",
+            "name": "cli",
+            "revision": "v20.10.12+incompatible",
+            "url": "https://pkg.go.dev/github.com/docker/cli@v20.10.12+incompatible"
+          },
+          "score": {
+            "total": 100,
+            "date": 30,
+            "source": 70
+          }
+        },
+        "licensed": {
+          "declared": "Apache-2.0",
+          "toolScore": {
+            "total": 45,
+            "declared": 30,
+            "discovered": 0,
+            "consistency": 0,
+            "spdx": 15,
+            "texts": 0
+          },
+          "facets": {
+            "core": {
+              "attribution": {
+                "unknown": 1222,
+                "parties": [
+                  "Copyright 2010 The Go Authors",
+                  "Copyright (c) 2013, Felix Riedel",
+                  "Copyright 2012-2017 Docker, Inc.",
+                  "Copyright 2013-2017 Docker, Inc.",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors",
+                  "Copyright (c) 2004, 2006 The Linux Foundation and its contributors"
+                ]
+              },
+              "discovered": {
+                "unknown": 1224,
+                "expressions": [
+                  "Apache-2.0",
+                  "BSD-3-Clause",
+                  "NOASSERTION"
+                ]
+              },
+              "files": 1228
+            }
+          },
+          "score": {
+            "total": 45,
+            "declared": 30,
+            "discovered": 0,
+            "consistency": 0,
+            "spdx": 15,
+            "texts": 0
+          }
+        },
+        "coordinates": {
+          "type": "go",
+          "provider": "golang",
+          "namespace": "github.com%2Fdocker",
+          "name": "cli",
+          "revision": "v20.10.12+incompatible"
+        },
+        "_meta": {
+          "schemaVersion": "1.6.1",
+          "updated": "2022-06-02T01:36:50.040Z"
+        },
+        "scores": {
+          "effective": 72,
+          "tool": 72
+        }
+      },
+    "stagingDoc":{
+      "described": {
+        "releaseDate": "2021-12-12",
+        "urls": {
+          "registry": "https://pkg.go.dev/github.com/docker/cli",
+          "version": "https://pkg.go.dev/github.com/docker/cli@v20.10.12+incompatible",
+          "download": "https://proxy.golang.org/github.com/docker/cli/@v/v20.10.12+incompatible.zip"
+        },
+        "hashes": {
+          "sha1": "d6a524762d88242708dac60e0f61485876348a5c",
+          "sha256": "d52fcb80bdcd3eeaebd25e231769b2780149a39d85ad2b42923132426a0486fb"
+        },
+        "files": 1228,
+        "tools": [
+          "clearlydefined/1.2.1",
+          "reuse/3.2.1",
+          "licensee/9.18.1",
+          "scancode/32.3.0"
+        ],
+        "toolScore": {
+          "total": 100,
+          "date": 30,
+          "source": 70
+        },
+        "sourceLocation": {
+          "type": "go",
+          "provider": "golang",
+          "namespace": "github.com%2fdocker",
+          "name": "cli",
+          "revision": "v20.10.12+incompatible",
+          "url": "https://pkg.go.dev/github.com/docker/cli@v20.10.12+incompatible"
+        },
+        "score": {
+          "total": 100,
+          "date": 30,
+          "source": 70
+        }
+      },
+      "licensed": {
+        "declared": "Apache-2.0",
+        "toolScore": {
+          "total": 45,
+          "declared": 30,
+          "discovered": 0,
+          "consistency": 0,
+          "spdx": 15,
+          "texts": 0
+        },
+        "facets": {
+          "core": {
+            "attribution": {
+              "unknown": 1222,
+              "parties": [
+                "Copyright 2010 The Go Authors",
+                "Copyright (c) 2013, Felix Riedel",
+                "Copyright 2012-2017 Docker, Inc.",
+                "Copyright 2013-2017 Docker, Inc.",
+                "Copyright (c) 2004, 2006 The Linux Foundation and its contributors"
+              ]
+            },
+            "discovered": {
+              "unknown": 1221,
+              "expressions": [
+                "Apache-2.0",
+                "BSD-3-Clause",
+                "LicenseRef-scancode-dco-1.1"
+              ]
+            },
+            "files": 1228
+          }
+        },
+        "score": {
+          "total": 45,
+          "declared": 30,
+          "discovered": 0,
+          "consistency": 0,
+          "spdx": 15,
+          "texts": 0
+        }
+      },
+      "coordinates": {
+        "type": "go",
+        "provider": "golang",
+        "namespace": "github.com%2fdocker",
+        "name": "cli",
+        "revision": "v20.10.12+incompatible"
+      },
+      "_meta": {
+        "schemaVersion": "1.7.0",
+        "updated": "2024-09-13T12:20:43.514Z"
+      },
+      "scores": {
+        "effective": 72,
+        "tool": 72
+      }
+    },
+    "ignoredKeys":["_meta", "licensed.score", "licensed.toolScore", "described.score", "described.toolScore"]
+}
\ No newline at end of file
diff --git a/tools/integration/lib/compareDefinitions.js b/tools/integration/lib/compareDefinitions.js
new file mode 100644
index 0000000..74495db
--- /dev/null
+++ b/tools/integration/lib/compareDefinitions.js
@@ -0,0 +1,218 @@
+// (c) Copyright 2024, GitHub and ClearlyDefined contributors. Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+
+function compareDocuments(staging, production, ignoredKeys, path = '') {
+  let differences = {}
+  let overallResults = []
+
+  staging = staging === undefined ? {} : staging
+  production = production === undefined ? {} : production
+
+  const keys = new Set([...Object.keys(staging), ...Object.keys(production)])
+
+  for (let key of keys) {
+    const currentPath = path ? `${path}.${key}` : key
+
+    if (shouldIgnore(currentPath, ignoredKeys)) {
+      continue
+    }
+
+    const stagingValue = staging[key]
+    const productionValue = production[key]
+
+    const comparison = compareValues(stagingValue, productionValue, ignoredKeys, currentPath)
+    overallResults.push(comparison.result)
+
+    if (comparison.result !== 'parity' && comparison.differences) {
+      for (let [resultType, diffs] of Object.entries(comparison.differences)) {
+        if (!differences[resultType]) {
+          differences[resultType] = []
+        }
+        differences[resultType] = differences[resultType].concat(diffs)
+      }
+    }
+  }
+
+  const overallResult = aggregateOverallResult(overallResults)
+
+  return {
+    overallResult,
+    differences
+  }
+}
+
+function compareValues(val1, val2, ignoredKeys, path) {
+  val1 = val1 === undefined ? null : val1
+  val2 = val2 === undefined ? null : val2
+
+  // Check for null values first
+  if (val1 === null && val2 === null) {
+    return { result: 'parity' }
+  }
+
+  if (val1 === null) {
+    if (isEmpty(val2)) {
+      return { result: 'parity' }
+    } else {
+      return handleLargeArrays(null, val2, path, 'regression')
+    }
+  }
+
+  if (val2 === null) {
+    if (isEmpty(val1)) {
+      return { result: 'parity' }
+    } else {
+      return handleLargeArrays(val1, null, path, 'improvement')
+    }
+  }
+
+  const type1 = getType(val1)
+  const type2 = getType(val2)
+
+  if (type1 !== type2) {
+    return handleLargeArrays(val1, val2, path, 'inconclusive')
+  }
+
+  if (type1 === 'string') {
+    if (val1.toLowerCase() === val2.toLowerCase()) {
+      return { result: 'parity' }
+    } else {
+      return handleLargeArrays(val1, val2, path, 'inconclusive')
+    }
+  }
+
+  if (['number', 'boolean'].includes(type1)) {
+    if (val1 === val2) {
+      return { result: 'parity' }
+    } else {
+      return handleLargeArrays(val1, val2, path, 'inconclusive')
+    }
+  }
+
+  if (type1 === 'array') {
+    return compareArrays(val1, val2, path)
+  }
+
+  if (type1 === 'object') {
+    const objectComparison = compareDocuments(val1, val2, ignoredKeys, path)
+    const overallResult = objectComparison.overallResult
+
+    if (Object.keys(objectComparison.differences).length === 0) {
+      return { result: 'parity' }
+    } else {
+      return {
+        result: overallResult,
+        differences: objectComparison.differences
+      }
+    }
+  }
+
+  return handleLargeArrays(val1, val2, path, 'inconclusive')
+}
+
+function handleLargeArrays(val1, val2, path, result) {
+  const MAX_ELEMENTS = 10
+  let diff = {}
+
+  if (Array.isArray(val1) && Array.isArray(val2)) {
+    const set1 = new Set(val1.map(item => (typeof item === 'string' ? item.toLowerCase() : JSON.stringify(item))))
+    const set2 = new Set(val2.map(item => (typeof item === 'string' ? item.toLowerCase() : JSON.stringify(item))))
+
+    if (isSuperset(set1, set2)) {
+      const addedElements = [...set1].filter(x => !set2.has(x))
+      diff.addedElements = addedElements.slice(0, MAX_ELEMENTS)
+      result = 'improvement'
+    } else if (isSubset(set1, set2)) {
+      const missingElements = [...set2].filter(x => !set1.has(x))
+      diff.missingElements = missingElements.slice(0, MAX_ELEMENTS)
+      result = 'regression'
+    } else {
+      diff.staging = val1.slice(0, MAX_ELEMENTS)
+      diff.production = val2.slice(0, MAX_ELEMENTS)
+      result = 'inconclusive'
+    }
+
+    if (val1.length > MAX_ELEMENTS || val2.length > MAX_ELEMENTS) {
+      diff.truncated = true
+    }
+  } else {
+    diff.staging = Array.isArray(val1) ? val1.slice(0, MAX_ELEMENTS) : val1
+    diff.production = Array.isArray(val2) ? val2.slice(0, MAX_ELEMENTS) : val2
+    if ((Array.isArray(val1) && val1.length > MAX_ELEMENTS) || (Array.isArray(val2) && val2.length > MAX_ELEMENTS)) {
+      diff.truncated = true
+    }
+  }
+
+  return {
+    result: result,
+    differences: {
+      [result]: [
+        {
+          field: path,
+          diff: diff
+        }
+      ]
+    }
+  }
+}
+
+function compareArrays(arr1, arr2, path) {
+  if (JSON.stringify(arr1).toLowerCase() === JSON.stringify(arr2).toLowerCase()) {
+    return { result: 'parity' }
+  }
+
+  return handleLargeArrays(arr1, arr2, path, 'inconclusive')
+}
+
+function shouldIgnore(path, ignoredKeys) {
+  return ignoredKeys.some(prefix => path.startsWith(prefix))
+}
+
+function isEmpty(value) {
+  if (value === null || value === undefined) return true
+  if (Array.isArray(value)) return value.length === 0
+  if (typeof value === 'object') return Object.keys(value).length === 0
+  return false
+}
+
+function isSuperset(setA, setB) {
+  for (let elem of setB) {
+    if (!setA.has(elem)) {
+      return false
+    }
+  }
+  return true
+}
+
+function isSubset(setA, setB) {
+  for (let elem of setA) {
+    if (!setB.has(elem)) {
+      return false
+    }
+  }
+  return true
+}
+
+function aggregateOverallResult(results) {
+  if (results.includes('regression')) {
+    return 'regression'
+  }
+
+  if (results.includes('inconclusive')) {
+    return 'inconclusive'
+  }
+
+  if (results.includes('improvement')) {
+    return 'improvement'
+  }
+
+  return 'parity'
+}
+
+function getType(value) {
+  if (value === null) return 'null'
+  if (Array.isArray(value)) return 'array'
+  return typeof value
+}
+
+module.exports = { compareDocuments }
diff --git a/tools/integration/package.json b/tools/integration/package.json
index c2420d5..2eea4fc 100644
--- a/tools/integration/package.json
+++ b/tools/integration/package.json
@@ -12,7 +12,8 @@
     "eslint": "eslint .",
     "eslint:fix": "eslint . --fix",
     "prettier:check": "prettier . --check",
-    "prettier:write": "prettier . --write"
+    "prettier:write": "prettier . --write",
+    "definitions-diff": "node test/integration/definitionDiff.js"
   },
   "author": "",
   "license": "MIT",
diff --git a/tools/integration/test/integration/definitionDiff.js b/tools/integration/test/integration/definitionDiff.js
new file mode 100644
index 0000000..05e2f82
--- /dev/null
+++ b/tools/integration/test/integration/definitionDiff.js
@@ -0,0 +1,54 @@
+const fs = require('fs').promises
+const path = require('path')
+const { callFetch } = require('../../lib/fetch')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents } = require('./testConfig')
+const { compareDocuments } = require('../../lib/compareDefinitions')
+
+async function main() {
+  const baseFolderPath = process.argv[2]
+  if (!baseFolderPath) {
+    console.error('Error: Base folder path is required as an argument.')
+    process.exit(1)
+  }
+
+  try {
+    const components = await getComponents()
+    console.info(`Testing definitions for ${JSON.stringify(components)}`)
+    await Promise.all(components.map(coordinates => fetchAndCompareDefinition(coordinates, baseFolderPath)))
+  } catch (error) {
+    console.error('Error:', error)
+  }
+}
+
+async function fetchAndCompareDefinition(coordinates, baseFolderPath) {
+  const [recomputedDef, expectedDef] = await Promise.all([
+    getDefinition(devApiBaseUrl, coordinates, true),
+    getDefinition(prodApiBaseUrl, coordinates)
+  ])
+  const diff = compareDocuments(recomputedDef, expectedDef, [
+    '_meta',
+    'licensed.score',
+    'licensed.toolScore',
+    'described.score',
+    'described.toolScore'
+  ])
+  await saveDiffToFile(coordinates, diff, baseFolderPath)
+  return diff
+}
+
+async function getDefinition(apiBaseUrl, coordinates, reCompute = false) {
+  reCompute = apiBaseUrl === devApiBaseUrl && reCompute
+  let url = `${apiBaseUrl}/definitions/${coordinates}`
+  if (reCompute) url += '?force=true'
+  return await callFetch(url).then(r => r.json())
+}
+
+async function saveDiffToFile(coordinates, diff, baseFolderPath) {
+  const dirPath = path.join(baseFolderPath, coordinates)
+  const filePath = path.join(dirPath, 'diff.json')
+  await fs.mkdir(dirPath, { recursive: true })
+  await fs.writeFile(filePath, JSON.stringify(diff, null, 2), 'utf8')
+}
+
+// Run the main function
+main()
diff --git a/tools/integration/test/integration/e2e-test-service/attachmentTest.js b/tools/integration/test/integration/e2e-test-service/attachmentTest.js
index 295cc82..8b7ad5e 100644
--- a/tools/integration/test/integration/e2e-test-service/attachmentTest.js
+++ b/tools/integration/test/integration/e2e-test-service/attachmentTest.js
@@ -2,15 +2,16 @@
 // SPDX-License-Identifier: MIT
 
 const { callFetch } = require('../../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('../testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const { strictEqual } = require('assert')
 
-describe('Validation attachments between dev and prod', function () {
+describe('Validation attachments between dev and prod', async function () {
   this.timeout(definition.timeout * 2)
 
   //Rest a bit to avoid overloading the servers
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
 
+  const components = await getComponents()
   components.forEach(coordinates => {
     it(`should have the same attachement as prod for ${coordinates}`, () => fetchAndCompareAttachments(coordinates))
   })
diff --git a/tools/integration/test/integration/e2e-test-service/definitionTest.js b/tools/integration/test/integration/e2e-test-service/definitionTest.js
index a5fdd79..c5ab248 100644
--- a/tools/integration/test/integration/e2e-test-service/definitionTest.js
+++ b/tools/integration/test/integration/e2e-test-service/definitionTest.js
@@ -4,7 +4,7 @@
 const { omit, isEqual, pick } = require('lodash')
 const { deepStrictEqual, strictEqual, ok } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('../testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')
 
@@ -14,19 +14,21 @@ describe('Validation definitions between dev and prod', function () {
   //Rest a bit to avoid overloading the servers
   afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
 
-  describe('Validation between dev and prod', function () {
+  describe('Validation between dev and prod', async function () {
     before(() => {
-      loadFixtures().forEach(([url, definition]) =>
+      loadFixtures().forEach(([url, definition]) => {
         nock(prodApiBaseUrl, { allowUnmocked: true }).get(url).reply(200, definition)
-      )
+      })
     })
-
+    const components = await getComponents()
+    console.info(`Testing definitions for ${JSON.stringify(components)}`)
     components.forEach(coordinates => {
       it(`should return the same definition as prod for ${coordinates}`, () => fetchAndCompareDefinition(coordinates))
     })
   })
 
-  describe('Validate on dev', function () {
+  describe('Validate on dev', async function () {
+    const components = await getComponents()
     const coordinates = components[0]
 
     describe('Search definitions', function () {
diff --git a/tools/integration/test/integration/e2e-test-service/noticeTest.js b/tools/integration/test/integration/e2e-test-service/noticeTest.js
index 36d4765..d366b1a 100755
--- a/tools/integration/test/integration/e2e-test-service/noticeTest.js
+++ b/tools/integration/test/integration/e2e-test-service/noticeTest.js
@@ -3,11 +3,11 @@
 
 const { deepStrictEqual } = require('assert')
 const { callFetch, buildPostOpts } = require('../../../lib/fetch')
-const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('../testConfig')
+const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
 const nock = require('nock')
 const fs = require('fs')
 
-describe('Validate notice files between dev and prod', function () {
+describe('Validate notice files between dev and prod', async function () {
   this.timeout(definition.timeout)
 
   //Rest a bit to avoid overloading the servers
@@ -20,7 +20,7 @@ describe('Validate notice files between dev and prod', function () {
         .reply(200, notice)
     })
   })
-
+  const components = await getComponents()
   components.forEach(coordinates => {
     it(`should return the same notice as prod for ${coordinates}`, () => fetchAndCompareNotices(coordinates))
   })
diff --git a/tools/integration/test/integration/harvestTest.js b/tools/integration/test/integration/harvestTest.js
index 48c4b16..ac258e6 100644
--- a/tools/integration/test/integration/harvestTest.js
+++ b/tools/integration/test/integration/harvestTest.js
@@ -1,7 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const { components, devApiBaseUrl, harvest } = require('./testConfig')
+const { getComponents, devApiBaseUrl, harvest } = require('./testConfig')
 const Poller = require('../../lib/poller')
 const Harvester = require('../../lib/harvester')
 const { strictEqual } = require('assert')
@@ -10,7 +10,9 @@ describe('Tests for harvesting different components', function () {
   it('should verify all harvests are complete', async function () {
     this.timeout(harvest.timeout)
     console.time('Harvest Test')
-    const status = await harvestTillCompletion(components)
+    const recentDefinitions = await getComponents()
+    console.info(`Recent definitions: ${recentDefinitions}`)
+    const status = await harvestTillCompletion(recentDefinitions)
     for (const [coordinates, isHarvested] of status) {
       strictEqual(isHarvested, true, `Harvest for ${coordinates} is not complete`)
     }
diff --git a/tools/integration/test/integration/testConfig.js b/tools/integration/test/integration/testConfig.js
index fa50b3b..3604986 100644
--- a/tools/integration/test/integration/testConfig.js
+++ b/tools/integration/test/integration/testConfig.js
@@ -1,5 +1,7 @@
 // (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
+const fs = require('fs').promises
+const path = require('path')
 
 const devApiBaseUrl = 'https://dev-api.clearlydefined.io'
 const prodApiBaseUrl = 'https://api.clearlydefined.io'
@@ -11,7 +13,7 @@ const pollingMaxTime = 1000 * 60 * 60 // 60 minutes
 const harvestTools = ['licensee', 'reuse', 'scancode']
 
 //Components to test
-const components = [
+const componentsStatic = [
   'pypi/pypi/-/platformdirs/4.2.0', //Keep this as the first element to test, it is relatively small
   'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16',
   'maven/mavengoogle/android.arch.lifecycle/common/1.0.1',
@@ -32,10 +34,47 @@ const components = [
   // 'sourcearchive/mavencentral/org.apache.httpcomponents/httpcore/4.1' // Dev and prod have different license and scores. See https://github.com/clearlydefined/crawler/issues/533
 ]
 
+function shouldUseDynamicComponents() {
+  // check for environment variable DYNAMIC_COORDINATES, if it is set to true, use dynamic components
+  return process.env.DYNAMIC_COORDINATES === 'true'
+}
+
+async function getComponents() {
+  if (shouldUseDynamicComponents()) {
+    console.info('Using dynamic components')
+    return componentsDynamic()
+  } else {
+    console.info('Using static components')
+    return Promise.resolve(componentsStatic)
+  }
+}
+
+const componentsDynamic = async () => {
+  const filePath = path.join(__dirname, 'recentDefinitions.json')
+
+  try {
+    // Check if the file exists
+    await fs.access(filePath)
+    // Read the file contents
+    const data = await fs.readFile(filePath, 'utf8')
+    console.info('Read dynamic components from disk')
+    return JSON.parse(data)
+  } catch (err) {
+    // If the file doesn't exist, fetch the data and save it to disk
+    const response = await fetch(
+      'https://cosmos-query-function-app.azurewebsites.net/api/getrecentdefinitions?days=1&limit=1'
+    )
+    const data = await response.json()
+    await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf8')
+    console.info('Read dynamic components from remote')
+    return data
+  }
+}
+
 module.exports = {
   devApiBaseUrl,
   prodApiBaseUrl,
-  components,
+  getComponents,
   harvest: {
     poll: { interval: pollingInterval, maxTime: pollingMaxTime }, // for each component
     tools: harvestTools,