refactor: add new QB parser

libs/base/types/src/lib/fs.ts

@@ -0,0 +1,3 @@
+type ReadFile = (path: string, string: "utf-8") => Promise<string>;
+
+export type MiniFs = { readFile: ReadFile };

libs/base/types/src/lib/mdx.ts

@@ -0,0 +1,3 @@
+export type MdxDocument = {
+  content: string;
+};

libs/core/question-bank/executors/arrange/executor.ts

@@ -0,0 +1,96 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { makeMap } from "@chair-flight/base/utils";
+import { getAllFiles } from "../../src/executors/get-all-files";
+import { getPaths } from "../../src/executors/get-paths";
+import {
+  arrangeAnnexes,
+  arrangeQuestions,
+} from "../../src/executors/question-bank-arrange";
+import { connectQuestionBank } from "../../src/executors/question-bank-connect";
+import {
+  readAllCoursesFromFs,
+  readAllLosFromFs,
+  readAllAnnexesFromFs,
+  readAllQuestionsFromFs,
+  readAllSubjectsFromFs,
+  readAllDocsFromFs,
+} from "../../src/executors/question-bank-read";
+import { questionBankValidation } from "../../src/schemas/question-bank-validation-schema";
+import type { ExecutorContext } from "@nx/devkit";
+
+type ExecutorOptions = Record<string, never>;
+
+const runExecutor = async (_: ExecutorOptions, context: ExecutorContext) => {
+  const { contentFolder, subjectsJson, coursesJson, losJson, projectName } =
+    getPaths({
+      context,
+    });
+
+  const questionTemplates = await readAllQuestionsFromFs(contentFolder);
+  const docs = await readAllDocsFromFs(contentFolder);
+  const annexes = await readAllAnnexesFromFs(contentFolder, projectName);
+  const learningObjectives = await readAllLosFromFs(losJson);
+  const courses = await readAllCoursesFromFs(coursesJson);
+  const subjects = await readAllSubjectsFromFs(subjectsJson);
+
+  const mediaMap = makeMap(
+    [...(await getAllFiles(contentFolder, ".jpg"))],
+    (p) => p.split("/").pop()?.split(".")[0] ?? "",
+    (p) => p,
+  );
+
+  connectQuestionBank({
+    questionTemplates,
+    docs,
+    annexes,
+    learningObjectives,
+    courses,
+    subjects,
+  });
+
+  questionBankValidation.parse({
+    questionTemplates,
+    docs,
+    annexes,
+    learningObjectives,
+    subjects,
+    courses,
+  });
+
+  const annexFiles = arrangeAnnexes({ annexes, docs });
+  const questionFiles = arrangeQuestions({ questionTemplates, docs });
+
+  await Promise.all(
+    Object.values(annexFiles).map(({ fileName, annexes }) =>
+      fs.writeFile(fileName, JSON.stringify(annexes, null, 2)),
+    ),
+  );
+
+  await Promise.all(
+    Object.values(questionFiles).map(({ fileName, questions }) =>
+      fs.writeFile(fileName, JSON.stringify(questions, null, 2)),
+    ),
+  );
+
+  await Promise.all(
+    Object.values(annexFiles).flatMap(({ annexes, fileName }) =>
+      annexes.map((annex) => {
+        const origin = mediaMap[annex.id];
+        const folderName = fileName.replaceAll("annexes.json", "annexes");
+        const destination = path.join(
+          folderName,
+          `${annex.id}.${annex.format}`,
+        );
+        if (!mediaMap[annex.id]) return Promise.resolve(undefined);
+        return fs.rename(origin, destination);
+      }),
+    ),
+  );
+
+  return {
+    success: true,
+  };
+};
+
+export default runExecutor;

libs/core/question-bank/executors/arrange/schema.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "http://json-schema.org/schema",
+  "version": 2,
+  "title": "Arrange",
+  "description": "Read the question bank, and resort question and annexes in the best possible folder",
+  "type": "object",
+  "properties": {},
+  "required": []
+}

libs/core/question-bank/src/executors/get-all-files.ts

@@ -0,0 +1,31 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+
+export const getAllFiles = async (
+  relativePath: string,
+  targetFileName: string,
+) => {
+  const result: string[] = [];
+  const stack: string[] = [relativePath];
+
+  while (stack.length > 0) {
+    const currentPath = stack.pop();
+    if (!currentPath) continue;
+
+    const files = await fs.readdir(currentPath);
+
+    for (const file of files) {
+      const newPath = path.join(currentPath, file);
+
+      if ((await fs.stat(newPath)).isDirectory()) {
+        stack.push(newPath);
+      }
+
+      if (file.endsWith(targetFileName)) {
+        result.push(newPath);
+      }
+    }
+  }
+
+  return result.sort();
+};

libs/core/question-bank/src/executors/get-paths.ts

@@ -0,0 +1,53 @@
+import * as path from "node:path";
+import type { ExecutorContext } from "@nx/devkit";
+
+export const getPaths = ({ context }: { context: ExecutorContext }) => {
+  const projects = context.workspace?.projects ?? {};
+  const nextProjectName = "next-app";
+  /** i.e.: `content-question-bank-atpl` */
+  const projectName = context.projectName ?? "";
+  /** i.e.: `libs/content/content-question-bank-atpl` */
+  const contentRoot = projects[projectName]?.root ?? "";
+  /** i.e.: `libs/content/content-question-bank-atpl/annexes` */
+  const annexesFolder = path.join(contentRoot, "annexes");
+  /** i.e.: `libs/content/content-question-bank-atpl/content` */
+  const contentFolder = path.join(contentRoot, "content");
+  /** i.e.: `libs/content/content-question-bank-atpl/flashcards` */
+  const flashcardsFolder = path.join(contentRoot, "flashcards");
+  /** i.e.: `apps/next-app */
+  const outputProject = projects[nextProjectName]?.root ?? "";
+  /** i.e.: `apps/next-app/public/content/content-question-bank-atpl` */
+  const outputDir = path.join(outputProject, "public", "content", projectName);
+
+  return {
+    projectName,
+    annexesFolder,
+    contentFolder,
+    flashcardsFolder,
+
+    /** i.e.: `libs/content/question-bank-atpl/content/subjects.json` */
+    subjectsJson: path.join(contentFolder, "subjects.json"),
+    /** i.e.: `libs/content/content-question-bank-atpl/content/courses.json` */
+    coursesJson: path.join(contentFolder, "courses.json"),
+    /** i.e.: `libs/content/content-question-bank-atpl/content/learning-objectives.json` */
+    losJson: path.join(contentFolder, "learning-objectives.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl` */
+    outputDir: outputDir,
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/questions.json` */
+    outputQuestionsJson: path.join(outputDir, "questions.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/annexes.json` */
+    outputAnnexesJson: path.join(outputDir, "annexes.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/docs.json` */
+    outputDocsJson: path.join(outputDir, "docs.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/subjects.json` */
+    outputSubjectsJson: path.join(outputDir, "subjects.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/courses.json` */
+    outputCoursesJson: path.join(outputDir, "courses.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/learningObjectives.json` */
+    outputLosJson: path.join(outputDir, "learningObjectives.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/flashcards.json` */
+    outputFlashcardsJson: path.join(outputDir, "flashcards.json"),
+    /** i.e.: `apps/next-app/public/content/content-question-bank-atpl/media` */
+    outputMediaDir: path.join(outputDir, "media"),
+  };
+};

libs/core/question-bank/src/executors/parse-learning-objectives-xlsx.ts

@@ -0,0 +1,92 @@
+import * as XLSX from "xlsx";
+import type {
+  LearningObjective,
+  LearningObjectiveId,
+} from "../types/question-bank-types";
+
+type QuestionBankLearningObjectiveJson = Omit<
+  LearningObjective,
+  "questions" | "nestedQuestions"
+>;
+
+const courseNames: Record<string, string> = {
+  "ATPL(A)": "ATPL_A",
+  "CPL(A)": "CPL_A",
+  "ATPL(H)/IR": "ATPL_H_IR",
+  "ATPL(H)/VFR": "ATPL_H_VFR",
+  "CPL(H)": "CPL_H",
+  IR: "IR",
+  "CBIR(A)": "CBIR_A",
+};
+
+const intentionallyLeftBlankPattern = /Intentionally left blank/i;
+
+export const parseLearningObjectivesXlsx = async ({
+  xlsxPath,
+}: {
+  xlsxPath: string;
+}): Promise<QuestionBankLearningObjectiveJson[]> => {
+  const workbook = XLSX.readFile(xlsxPath);
+  const sheetNames = workbook.SheetNames;
+  const learningObjectives = sheetNames
+    .slice(2)
+    .flatMap<QuestionBankLearningObjectiveJson>((name) => {
+      const sheet = XLSX.utils.sheet_to_json(workbook.Sheets[name]) as Record<
+        string,
+        string
+      >[];
+      return sheet.map<QuestionBankLearningObjectiveJson>((row) => {
+        const text = (row["2020 syllabus text"] ?? "")
+          .replaceAll(":", ":\n- ")
+          .replaceAll(";", ";\n- ")
+          .replace(/\b[A-Z]+\b/g, (match) => {
+            return match.charAt(0) + match.slice(1).toLowerCase();
+          })
+          .split("Remark:")[0];
+        const id =
+          row["2020 syllabus reference"]
+            ?.replaceAll(".00", "")
+            ?.replaceAll(" 00", "")
+            ?.trim() ?? "";
+
+        const rawParentId = id.split(".").slice(0, -1).join(".");
+        const parentId = rawParentId === "071" ? "070" : rawParentId;
+
+        return {
+          id,
+          parentId,
+          courses: Object.keys(courseNames)
+            .filter((item) => row[item])
+            .map((k) => courseNames[k]),
+          questions: [],
+          text,
+          learningObjectives: [],
+          // some sources are just 0 (?)... ignore those!
+          source: row["Source / Comment"] || "",
+        };
+      });
+    })
+    .filter((lo) => {
+      if (intentionallyLeftBlankPattern.test(lo.text)) return false;
+      if (lo.id === "") return false;
+      if (lo.id.split(".").length === 1) return false;
+      return true;
+    });
+
+  const learningObjectivesMap = learningObjectives.reduce(
+    (sum, lo) => {
+      sum[lo.id] = lo;
+      return sum;
+    },
+    {} as Record<LearningObjectiveId, QuestionBankLearningObjectiveJson>,
+  );
+
+  // link learning objectives to one another:
+  learningObjectives.forEach((lo) => {
+    const parentId = lo.id.split(".").slice(0, -1).join(".");
+    const parent = learningObjectivesMap[parentId];
+    if (parent) parent.learningObjectives.push(lo.id);
+  });
+
+  return learningObjectives;
+};

libs/core/question-bank/src/executors/question-bank-arrange.ts

@@ -0,0 +1,76 @@
+import { makeMap } from "@chair-flight/base/utils";
+import type {
+  Annex,
+  Doc,
+  QuestionTemplate,
+} from "../types/question-bank-types";
+
+export const arrangeQuestions = ({
+  questionTemplates,
+  docs,
+}: {
+  questionTemplates: QuestionTemplate[];
+  docs: Doc[];
+}) => {
+  const questionMap = makeMap(
+    docs,
+    (doc) => doc.id,
+    (doc) => ({
+      fileName: doc.fileName.replace("page.md", "questions.json"),
+      questions: [] as QuestionTemplate[],
+    }),
+  );
+
+  return questionTemplates.reduce((sum, question) => {
+    if (!sum[question.doc]) {
+      throw new Error(`Question ${question.id} has no doc`);
+    }
+
+    const cleanQuestion = {
+      id: question.id,
+      relatedQuestions: question.relatedQuestions.sort(),
+      externalIds: question.externalIds.sort(),
+      annexes: question.annexes,
+      learningObjectives: question.learningObjectives.sort(),
+      explanation: question.explanation,
+      variant: question.variant,
+      doc: question.doc,
+      subjects: question.subjects.sort(),
+      srcLocation: sum[question.doc].fileName,
+    };
+    sum[question.doc].questions.push(cleanQuestion);
+    return sum;
+  }, questionMap);
+};
+
+export const arrangeAnnexes = ({
+  annexes,
+  docs,
+}: {
+  annexes: Annex[];
+  docs: Doc[];
+}) => {
+  const annexMap = makeMap(
+    docs,
+    (doc) => doc.id,
+    (doc) => ({
+      fileName: doc.fileName.replace("page.md", "annexes.json"),
+      annexes: [] as Pick<Annex, "id" | "description" | "format">[],
+    }),
+  );
+
+  return annexes.reduce((sum, annex) => {
+    if (!sum[annex.doc]) {
+      throw new Error(`Annex ${annex.id} has no doc`);
+    }
+
+    const cleanAnnex = {
+      id: annex.id,
+      description: annex.description,
+      format: annex.format,
+    };
+
+    sum[annex.doc].annexes.push(cleanAnnex);
+    return sum;
+  }, annexMap);
+};