Check MDX frontmatter (#96)

* Check MDX frontmatter

* set up secret

* test .mdx

* updated descriptions and new script

* one more mdx

* mdxDir

* test

* added promises

* long descriptions generated based on descriptions not the articles

* removed description genetarion

* limit changed

* limit changed

* Update .github/scripts/check-mdx-frontmatter.js

Co-authored-by: Shubham Shukla <[email protected]>
Signed-off-by: Kate Sv <[email protected]>

---------

Signed-off-by: Kate Sv <[email protected]>
Co-authored-by: Shubham Shukla <[email protected]>

.github/scripts/check-mdx-frontmatter.js

@@ -0,0 +1,109 @@
+const fs = require("fs").promises;
+const path = require("path");
+const matter = require("gray-matter");
+
+const mdxDir = path.join(__dirname, "../../fern/pages");
+const filePattern = /\.mdx$/;
+
+// Counters
+let totalFilesChecked = 0;
+let totalFilesValid = 0;
+let totalFilesInvalid = 0;
+
+// List of folders to exclude (relative to mdxDir)
+const excludedFolders = ["-ARCHIVE-", "api-reference", "llm-university"];
+
+function shouldExcludeFolder(dirPath) {
+  return excludedFolders.some((excludedFolder) => {
+    return path.relative(mdxDir, dirPath).startsWith(excludedFolder);
+  });
+}
+
+async function shouldExcludeFile(filePath) {
+  try {
+    const fileContent = await fs.readFile(filePath, "utf8");
+    const { data } = matter(fileContent);
+    return data.hidden === true;
+  } catch (error) {
+    console.error(`Error reading file "${filePath}":`, error);
+    return false; // In case of error, don't exclude the file
+  }
+}
+
+async function checkDescriptionLength(filePath) {
+  totalFilesChecked++;
+  const fileContent = await fs.readFile(filePath, "utf8");
+  const { data } = matter(fileContent);
+
+  if (!data.description) {
+    console.log(`File "${filePath}" is missing a description.`);
+    totalFilesInvalid++;
+    return false;
+  }
+
+  const descriptionLength = data.description.length;
+
+  if (descriptionLength < 50 || descriptionLength > 160) {
+    console.log(
+      `File "${filePath}" has an invalid description length: ${descriptionLength} characters.`
+    );
+    totalFilesInvalid++;
+    return false;
+  }
+
+  totalFilesValid++;
+  return true;
+}
+
+async function checkMDXFiles(dirPath) {
+  let allFilesValid = true;
+  const files = await fs.readdir(dirPath);
+
+  for (const file of files) {
+    const fullPath = path.join(dirPath, file);
+    const stat = await fs.lstat(fullPath);
+
+    if (stat.isDirectory()) {
+      if (shouldExcludeFolder(fullPath)) {
+        console.log(`Skipping excluded directory: ${fullPath}`);
+        continue;
+      }
+      const isValid = await checkMDXFiles(fullPath);
+      if (!isValid) {
+        allFilesValid = false;
+      }
+    } else if (filePattern.test(file)) {
+      if (await shouldExcludeFile(fullPath)) {
+        console.log(`Skipping excluded file: ${fullPath}`);
+        continue;
+      }
+      const isValid = await checkDescriptionLength(fullPath);
+      if (!isValid) {
+        allFilesValid = false;
+      }
+    }
+  }
+
+  return allFilesValid;
+}
+
+(async () => {
+  const allFilesValid = await checkMDXFiles(mdxDir);
+
+  // Summary report
+  console.log(`\nSummary Report:`);
+  console.log(`Total files checked: ${totalFilesChecked}`);
+  console.log(`Total valid files: ${totalFilesValid}`);
+  console.log(`Total invalid files: ${totalFilesInvalid}`);
+
+  if (!allFilesValid) {
+    console.error(
+      "Some files have invalid or missing descriptions. Meta description needing to be 50-160 characters"
+    );
+    process.exit(1); // Fail if any file is invalid
+  } else {
+    console.log(
+      "All files have a valid description length in the frontmatter."
+    );
+  }
+})();

.github/workflows/check-mdx-frontmatter.yml

@@ -0,0 +1,29 @@
+name: check-mdx-frontmatter
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'fern/pages/**/*.mdx'
+      - 'fern/pages/**/**/*.mdx'
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install Dependencies
+        shell: bash
+        run: pnpm install
+
+      - name: Run MDX frontmatter check
+        run: node .github/scripts/check-mdx-frontmatter.js

fern/pages/responsible-use/security.mdx

@@ -7,4 +7,4 @@ createdAt: "Wed Sep 14 2022 22:54:55 GMT+0000 (Coordinated Universal Time)"
 updatedAt: "Mon Oct 23 2023 14:40:59 GMT+0000 (Coordinated Universal Time)"
 type: "link"
 link_url: "https://cohere.ai/security"
----
+---

package.json

@@ -17,6 +17,7 @@
     "tailwindcss": "^3.4.4"
   },
   "dependencies": {
+    "gray-matter": "^4.0.3",
     "fern-api": "^0.41.16",
     "react": "^18.3.1"
   }