feat: filter by deleted, retry overview, update cache policy

hyperledger-iroha/iroha-2-docs#497
soramitsu · Jul 23, 2024 · fa3fa40 · fa3fa40
1 parent 453d082
commit fa3fa40
Show file tree

Hide file tree

Showing 6 changed files with 317 additions and 123 deletions.
diff --git a/deno.lock b/deno.lock
diff --git a/deps.ts b/deps.ts
@@ -1,10 +1,10 @@
 export * as log from "https://deno.land/[email protected]/log/mod.ts";
 export { getLogger } from "https://deno.land/[email protected]/log/mod.ts";
-export { ms } from "https://deno.land/x/[email protected]/ms.ts";
 export * as oak from "https://deno.land/x/[email protected]/mod.ts";
 export { default as oakLogger } from "https://deno.land/x/[email protected]/mod.ts";
 export { match, P } from "https://esm.sh/[email protected]";
 export { sortBy } from "https://deno.land/[email protected]/collections/mod.ts";
+export { retry } from "https://deno.land/x/[email protected]/mod.ts";
 
 export async function dotenvLoad() {
   await import("https://deno.land/[email protected]/dotenv/load.ts");

diff --git a/src/agent.ts b/src/agent.ts
diff --git a/src/aggregate.ts b/src/aggregate.ts
@@ -83,9 +83,12 @@ async function getTestCaseData(
 ): Promise<MetaMap> {
   const entries = await Promise.all(
     testCaseIdList.map(async (id): Promise<[number, TestCaseMeta][]> => {
-		await delay(50000);
-      const custom_fields = await api.getTestCaseCustomFields(id);
-      const map = customFieldsToMap(custom_fields);
+      const { deleted, customFields } = await api.getTestCaseOverview(id);
+      if (deleted) {
+        logger().debug({ msg: "ignore deleted test case", id });
+        return [];
+      }
+      const map = customFieldsToMap(customFields);
 
       if (!map.has(CUSTOM_FIELD_SDK)) {
         logger().warning({
@@ -121,26 +124,14 @@ async function getTestCaseData(
 function customFieldsToMap(
   input: ApiTestCaseCustomFieldData[],
 ): Map<string, CustomFieldData> {
-  if (!input) {
-    logger().warning({
-          msg: `Missing test!`
-        })
-    return new Map(null);
-  } else {
   const entries = input
     .map((x) => ({
       id: x.customField.id,
       name: x.customField.name,
       value: x.name,
     }))
     .map((x): [string, CustomFieldData] => [x.name, x]);
-    return new Map(entries);
-}
-
-}
-
-function delay(ms: number) {
-    return new Promise( resolve => setTimeout(resolve, ms) );
+  return new Map(entries);
 }
 
 function aggregateStories(
@@ -216,4 +207,4 @@ function pickResults<T extends ApiTestCaseResult>(
         map.has(item.testCaseId) ? map : map.set(item.testCaseId, item),
       new Map<number, T>(),
     );
-}
+}
diff --git a/src/api.ts b/src/api.ts
@@ -1,6 +1,4 @@
-import { getLogger } from "../deps.ts";
-
-const ENABLE_CACHE = false;
+import { getLogger, retry } from "../deps.ts";
 
 const logger = () => getLogger("api");
 
@@ -33,6 +31,12 @@ export interface ApiTestCaseCustomFieldData {
 
 export type ApiCustomFieldKind = "Story" | "Permission" | "SDK Test Id" | "SDK";
 
+export interface ApiTestCaseOverview {
+  deleted: boolean;
+  customFields: ApiTestCaseCustomFieldData[];
+  // there are other fields too, ignore
+}
+
 export default class Api {
   #config: Config;
 
@@ -58,58 +62,64 @@ export default class Api {
       content: ApiTestCaseResult[];
     }
 
-    return cacheData("cache/test-results.json", () => {
-      const URL = `${this.baseUrl}/api/rs/testresult/__search?` +
-        new URLSearchParams({
-          projectId: "1",
-          rql:
-            `not cf["SDK"] is null and createdDate > ${params.createdDateAfter.getTime()}`,
-          page: "0",
-          size: "999999",
-          sort: "created_date,DESC",
-        });
-      logger().debug({ msg: "Request", URL }); 
-      return fetch(URL, {
-        headers: this.commonHeaders(),
-      })
-        .then((x) => x.json() as Promise<Response>)
-        .then((x) => x.content)
-        .then((result) => {
-          logger().debug({ msg: "Found test cases", result });
-          return result;
-        });
-    });
+    const URL = `${this.baseUrl}/api/rs/testresult/__search?` +
+      new URLSearchParams({
+        projectId: "1",
+        rql:
+          `not cf["SDK"] is null and createdDate > ${params.createdDateAfter.getTime()}`,
+        page: "0",
+        size: "999999",
+        sort: "created_date,DESC",
+      });
+    logger().debug({ msg: "Request", URL });
+    return fetch(URL, {
+      headers: this.commonHeaders(),
+    })
+      .then((x) => x.json() as Promise<Response>)
+      .then((x) => x.content)
+      .then((result) => {
+        logger().debug({ msg: "Found test cases", result });
+        return result;
+      });
   }
 
-  public async getTestCaseCustomFields(
+  public async getTestCaseOverview(
     id: number,
-  ): Promise<ApiTestCaseCustomFieldData[]> {
-    return cacheData(`cache/test-case-${id}.json`, () => {		
-     logger().debug({ msg: "Loading test case custom fields", id });
-      return fetch(`${this.baseUrl}/api/rs/testcase/${id}/cfv`, {
-        headers: this.commonHeaders(),
-      })
-       .then ((x) => {
-          if (x.status !== 200) {
-          logger().error({ msg: "Failed to load test case custom fields", id, status: x.status });
-        }
-          return x.text();
+  ): Promise<ApiTestCaseOverview> {
+    logger().debug({ msg: "Loading test case custom fields", id });
+
+    const result = await retry(
+      () =>
+        fetch(`${this.baseUrl}/api/rs/testcase/${id}/overview`, {
+          headers: this.commonHeaders(),
         })
-        .then((text) => {
-        try {
-          const json = JSON.parse(text); // Attempt to parse the text as JSON
-          logger().debug({ msg: "Test case custom fields", id, data: json });
-          return json;
-        } catch (error) {
-          logger().error({ msg: "Invalid JSON response", id, error: error.message });
-          return;
-        }
-      })
-       .then((x) => {
-        logger().debug({ msg: "Test case custom fields", id, data: x });
-         return x;
-       });
-    });
+          .then((x) => {
+            if (x.status !== 200) {
+              logger().error({
+                msg: "Failed to load test case custom fields",
+                id,
+                status: x.status,
+              });
+            }
+            return x.text();
+          })
+          .then((text) => {
+            try {
+              const json = JSON.parse(text) as ApiTestCaseOverview; // Attempt to parse the text as JSON
+              return json;
+            } catch (error) {
+              logger().error({
+                msg: "Invalid JSON response",
+                id,
+                error: error.message,
+              });
+              throw error;
+            }
+          }),
+      { maxTry: 3 },
+    );
+
+    return result;
   }
 
   private commonHeaders() {
@@ -119,18 +129,3 @@ export default class Api {
     });
   }
 }
-
-async function cacheData<T>(file: string, fn: () => Promise<T>): Promise<T> {
-  if (!ENABLE_CACHE) return fn();
-
-  try {
-    const content = await Deno.readTextFile(file).then((x) => JSON.parse(x));
-    logger().debug({ msg: "Loaded cached", file });
-    return content;
-  } catch {
-    const data = await fn();
-    await Deno.writeTextFile(file, JSON.stringify(data));
-    logger().debug({ msg: "Written cache", file });
-    return data;
-  }
-}
diff --git a/src/main.ts b/src/main.ts
@@ -1,9 +1,8 @@
 import Api from "./api.ts";
-import { getMatrix } from "./aggregate.ts";
+import { getMatrix, Matrix } from "./aggregate.ts";
 import * as web from "./web.ts";
 import { get as getConfig } from "./config.ts";
-import Agent from "./agent.ts";
-import { log, match, ms, P } from "../deps.ts";
+import { log, match, P } from "../deps.ts";
 
 const CONFIG = {
   apiToken: getConfig("ALLURE_API_TOKEN"),
@@ -41,22 +40,36 @@ const api = new Api({
   baseUrl: CONFIG.allureBaseUrl,
 });
 
-const agent = new Agent(async () => {
-  try {
-    log.info("Getting matrix");
-    const data = await getMatrix(api);
-    log.info("Getting matrix complete");
-    return data;
-  } catch (err) {
-    log.error("Getting matrix failed");
-    console.error(err);
-    throw err;
+// 24 hours
+const CACHE_TTL = 60_000 * 24;
+
+class State {
+  #data: null | { matrix: Matrix; lastUpdated: Date } = null;
+  #promise: null | Promise<Matrix> = null;
+
+  getMatrix(): Promise<Matrix> {
+    if (this.#promise) return this.#promise;
+    const now = Date.now();
+    if (!this.#data || this.#data.lastUpdated.getTime() + CACHE_TTL < now) {
+      log.debug("No data/cache is stale, reloading matrix");
+      this.#promise = getMatrix(api).then((matrix) => {
+        this.#data = { matrix, lastUpdated: new Date() };
+        return matrix;
+      }).finally(() => {
+        this.#promise = null;
+      });
+      return this.#promise;
+    }
+    log.debug("Using cached data");
+    return Promise.resolve(this.#data.matrix);
   }
-}, ms("2h") as number);
+}
+
+const state = new State();
 
 await web.run({
   port: CONFIG.port,
   provider: {
-    getMatrix: () => agent.get(),
+    getMatrix: () => state.getMatrix(),
   },
 });