Merge branch 'main' of https://github.com/biothings/biothings_explorer …

…into parallel-creative
biothings · Oct 11, 2024 · 3af1f8b · 3af1f8b
2 parents 3ee93de + b4753eb
commit 3af1f8b
Show file tree

Hide file tree

Showing 10 changed files with 568 additions and 495 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -185,6 +185,33 @@
       ],
       "stopOnEntry": true,
       "console": "integratedTerminal"
-    }
+    },
+    {
+      "name": "Build w/ Cache and Jaeger",
+      "type": "pwa-node",
+      "preLaunchTask": "start redis",
+      "runtimeArgs": [
+        "run",
+        "debug-otel",
+      ],
+      "cwd": "${workspaceFolder}",
+      "runtimeExecutable": "pnpm",
+      "restart": false,
+      "env": {
+        "REDIS_HOST": "localhost",
+        "REDIS_PORT": "6379",
+        "DEBUG": "biomedical-id-resolver,bte*",
+        "SMARTAPI_SYNC": "false",
+        "RESULT_CACHING": "true",
+        "JAEGER_HOST": "localhost",
+        "JAEGER_PORT": "4318",
+      },
+      "request": "launch",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "stopOnEntry": true,
+      "console": "integratedTerminal"
+    },
   ]
 }
diff --git a/deploy/values.yaml b/deploy/values.yaml
@@ -78,8 +78,8 @@ affinity:
       topologyKey: "kubernetes.io/hostname"
 resources:
   requests:
-    memory: 50Gi
-    cpu: 10000m
+    memory: 25Gi
+    cpu: 7000m
   limits:
-    memory: 58Gi
-    cpu: 14000m 
+    memory: 35Gi
+    cpu: 8000m 
diff --git a/package.json b/package.json
@@ -17,16 +17,18 @@
     "format:fix": "prettier --write .",
     "lint": "eslint . --ext .ts,.js",
     "lint:fix": "pnpm lint --fix",
-    "test": "turbo run --parallel test",
-    "test-cov": "turbo run --parallel test-cov",
+    "test": "turbo run --parallel test --ui tui",
+    "test-cov": "turbo run --parallel test-cov --ui tui",
     "siege-local": "artillery run -e local --output report_local.json --config performance-test/config.yaml performance-test/scenarios/creative.yaml",
     "smartapi_sync": "node ./scripts/smartapi_sync.js",
     "get_rev": "./scripts/get_rev.sh",
     "basic-start": "node .",
     "start": "./scripts/start_server.sh",
     "stop": "./scripts/stop_server.sh",
     "watch": "nodemon -e js,mjs,json,ts --watch './packages/**/src/*' './packages/**/data/*' --exec 'pnpm run build && node .'",
-    "debug": "DEBUG=biomedical-id-resolver,bte* pnpm run watch"
+    "watch-otel": "nodemon -e js,mjs,json,ts --watch './packages/**/src/*' './packages/**/data/*' --exec 'pnpm run build && node --require ./packages/bte-server/built/controllers/opentelemetry.js .'",
+    "debug": "DEBUG=biomedical-id-resolver,bte* pnpm run watch",
+    "debug-otel": "DEBUG=biomedical-id-resolver,bte* pnpm run watch-otel"
   },
   "engines": {
     "node": ">=18.0.0",
@@ -71,22 +73,22 @@
     "rimraf": "^3.0.2",
     "supertest": "^6.3.3",
     "supports-color": "^7.2.0",
-    "turbo": "^1.13.0",
+    "turbo": "^2.0.14",
     "typescript": "5.2.2"
   },
   "dependencies": {
     "@bull-board/express": "^5.9.1",
     "@sentry/node": "^7.74.1",
     "@sentry/profiling-node": "^1.2.1",
     "axios": "^0.28.0",
-    "body-parser": "^1.20.2",
+    "body-parser": "^1.20.3",
     "bull": "^4.11.4",
     "compression": "^1.7.4",
     "connect-history-api-fallback": "^2.0.0",
     "cors": "^2.8.5",
     "debug": "^4.3.4",
     "dotenv": "^8.6.0",
-    "express": "^4.18.2",
+    "express": "^4.20.0",
     "express-rate-limit": "^5.5.1",
     "express-winston": "^4.2.0",
     "helmet": "^4.6.0",
@@ -111,5 +113,6 @@
     "hooks": {
       "commit-msg": "commitlint -E HUSKY_GIT_PARAMS"
     }
-  }
+  },
+  "packageManager": "[email protected]"
 }
diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts
@@ -13,11 +13,15 @@ export interface QueryQueue {
   bte_query_queue_by_team: Queue;
 }
 
+export interface recordCount {
+  [handler_index: number]: number;
+}
+
 export interface QueryInformation {
   queryGraph: TrapiQueryGraph;
   isCreativeMode?: boolean;
   creativeTemplate?: string;
-  totalRecords?: number;
+  totalRecords?: recordCount;
   jobID?: string;
   callback_url?: string;
 }

diff --git a/packages/types/src/misc.ts b/packages/types/src/misc.ts
@@ -4,6 +4,8 @@ export type APIDefinition = {
   name: string; // Must match name on SmartAPI registry
   infores?: string; // infores of API
   primarySource?: boolean;
+  includeFlipped?: boolean; // Automatically generate flipped MetaEdges (TRAPI only)
+  timeout?: number;
 } & ({ id: string } | { infores: string });
 
 export interface APIList {

diff --git a/packages/types/src/tasks.ts b/packages/types/src/tasks.ts
@@ -6,7 +6,7 @@ import {
   TrapiSchema,
   TrapiWorkflow,
 } from "./trapi";
-import {
+import MetaKG, {
   SmartAPIKGOperationObject,
   SmartAPIQueryResult,
 } from "@biothings-explorer/smartapi-kg";
@@ -17,6 +17,7 @@ export interface QueryOptions {
   submitter?: string;
   smartAPIID?: string;
   teamName?: string;
+  dryrun_pathfinder?: boolean;
   dryrun?: boolean;
   caching?: boolean; // from request url query values
 }
@@ -31,16 +32,20 @@ export interface QueryHandlerOptions extends QueryOptions {
   resolveOutputIDs?: boolean;
   EDGE_ATTRIBUTES_USED_IN_RECORD_HASH?: string[];
   smartapi?: SmartAPIQueryResult;
+  skipPfocr?: boolean;
+  handlerIndex?: number;
 }
 
 export interface QueueData {
   queryGraph: TrapiQueryGraph;
-  options: QueryOptions;
+  options: QueryHandlerOptions;
   workflow?: TrapiWorkflow[];
   callback_url?: string;
   smartAPIID?: string;
   teamName?: string;
   route: string;
+  traceparent?: string;
+  tracestate?: string;
 }
 
 export interface QueryParams {

diff --git a/packages/types/src/trapi.ts b/packages/types/src/trapi.ts
@@ -67,12 +67,12 @@ export interface TrapiAttribute {
   value_type_id?: string;
   attribute_source?: string | null;
   value_url?: string | null;
-  attributes?: TrapiAttribute;
+  attributes?: TrapiAttribute[];
   [additionalProperties: string]:
   | string
   | string[]
   | null
-  | TrapiAttribute
+  | TrapiAttribute[]
   | number
   | number[];
 }
@@ -97,12 +97,12 @@ export interface TrapiAttributeConstraint {
 export interface TrapiNodeBinding {
   id: string;
   query_id?: string;
-  attributes?: TrapiAttribute[];
+  attributes: TrapiAttribute[];
 }
 
 export interface TrapiEdgeBinding {
   id: string;
-  attributes?: TrapiAttribute[];
+  attributes: TrapiAttribute[];
 }
 
 export interface TrapiAnalysis {
@@ -118,7 +118,7 @@ export interface TrapiAnalysis {
 
 export interface TrapiAuxiliaryGraph {
   edges: string[];
-  attributes?: TrapiAttribute[];
+  attributes: TrapiAttribute[];
 }
 
 export interface TrapiPfocrFigure {