Merge branch 'main' into feedback-fixes-round-1

Signed-off-by: trentfowlercohere <[email protected]>

.github/scripts/check-mdx-frontmatter.js → .github/scripts/check-mdx-frontmatter.cjs

@@ -10,6 +10,9 @@ let totalFilesChecked = 0;
 let totalFilesValid = 0;
 let totalFilesInvalid = 0;
 
+// List of validators to run
+const validators = [checkDescriptionLength, checkTitleLength];
+
 // List of folders to exclude (relative to mdxDir)
 const excludedFolders = ["-ARCHIVE-", "api-reference", "llm-university"];
 
@@ -31,30 +34,62 @@ async function shouldExcludeFile(filePath) {
 }
 
 async function checkDescriptionLength(filePath) {
-  totalFilesChecked++;
   const fileContent = await fs.readFile(filePath, "utf8");
   const { data } = matter(fileContent);
+  const minDescriptionLength = 50;
+  const maxDescriptionLength = 160;
 
   if (!data.description) {
-    console.log(`File "${filePath}" is missing a description.`);
-    totalFilesInvalid++;
+    console.error(`File "${filePath}" is missing a description.`);
     return false;
   }
 
   const descriptionLength = data.description.length;
 
-  if (descriptionLength < 50 || descriptionLength > 160) {
-    console.log(
-      `File "${filePath}" has an invalid description length: ${descriptionLength} characters.`
+  if (descriptionLength < minDescriptionLength || descriptionLength > maxDescriptionLength) {
+    console.error(
+      `File "${filePath}" has an invalid description length: ${descriptionLength} characters. ` +
+      `Description should be between ${minDescriptionLength}-${maxDescriptionLength} characters.`
     );
-    totalFilesInvalid++;
     return false;
   }
 
-  totalFilesValid++;
   return true;
 }
 
+
+async function checkTitleLength(filePath) {
+    // these two files are layout files 
+    // and we don't expect to have title in them
+    const filesToExclude = ["index.mdx", "cookbooks.mdx"];
+
+    const fileContent = await fs.readFile(filePath, "utf8");
+    const { data } = matter(fileContent);
+    const minTitleLength = 30;
+    const maxTitleLength = 60;
+
+    filePath = path.relative(mdxDir, filePath);
+
+    if (!data.title) {
+        if (filesToExclude.includes(filePath)) {
+            return true;
+        }
+        console.error(`File "${filePath}" is missing a title.`);
+        return false;
+    }
+
+    const titleLength = data.title.length;
+    if (titleLength < minTitleLength || titleLength > maxTitleLength) {
+        console.warn(
+            `File "${filePath}" has an invalid title length: ${titleLength} characters. ` +
+            `Title should be between ${minTitleLength}-${maxTitleLength} characters.`
+        );
+        return true;
+    }
+
+    return true;
+}
+
 async function checkMDXFiles(dirPath) {
   let allFilesValid = true;
   const files = await fs.readdir(dirPath);
@@ -77,9 +112,22 @@ async function checkMDXFiles(dirPath) {
         console.log(`Skipping excluded file: ${fullPath}`);
         continue;
       }
-      const isValid = await checkDescriptionLength(fullPath);
+      let isValid = true;
+
+      for (const validate of validators) {
+        const fileIsValid = await validate(fullPath);
+        if (!fileIsValid) {
+            isValid = false; 
+        }
+      }
+
+      totalFilesChecked++;
       if (!isValid) {
         allFilesValid = false;
+        totalFilesInvalid++;
+      }
+      else {
+        totalFilesValid++;
       }
     }
   }
@@ -98,12 +146,12 @@ async function checkMDXFiles(dirPath) {
 
   if (!allFilesValid) {
     console.error(
-      "Some files have invalid or missing descriptions. Meta description needing to be 50-160 characters"
+      "Some files have invalid or missing content."
     );
     process.exit(1); // Fail if any file is invalid
   } else {
     console.log(
-      "All files have a valid description length in the frontmatter."
+      "All files a valid for frontmatter."
     );
   }
 })();

.github/scripts/check_python_code_snippets.py

@@ -0,0 +1,86 @@
+import os
+import re
+from pathlib import Path
+import black
+
+DEFAULT_LINE_LENGTH = 70
+BASE_DIR = Path(__file__).resolve().parent
+MDX_DIR = BASE_DIR / "../../fern/pages"
+FILE_PATTERN = re.compile(r"\.mdx$")
+
+
+def find_files_by_pattern(directory, pattern):
+    """
+    Finds all files in the given directory that match the provided regex pattern.
+    """
+    directory = Path(directory).resolve()
+    if not directory.is_dir():
+        raise ValueError(f"Provided directory {directory} is not valid.")
+    return [f for f in directory.rglob('*') if f.is_file() and pattern.search(f.name)]
+
+
+def format_python_snippets_in_mdx(file_path, line_length=DEFAULT_LINE_LENGTH):
+    """
+    Formats Python code snippets inside MDX files using Black.
+    """
+    black_mode = black.FileMode(line_length=line_length)
+    code_block_pattern = re.compile(r"```python\n(.*?)\n```", re.DOTALL)
+
+    with open(file_path, 'r', encoding='utf-8') as file:
+        original_content = file.read()
+
+    def format_with_black(match):
+        code = match.group(1)
+        formatted_code = black.format_str(code, mode=black_mode)
+        return f"```python\n{formatted_code.strip()}\n```"
+
+    new_content = code_block_pattern.sub(format_with_black, original_content)
+
+    with open(file_path, 'w', encoding='utf-8') as file:
+        file.write(new_content)
+
+    return original_content, new_content
+
+
+def process_mdx_files(directory, file_pattern, line_length=DEFAULT_LINE_LENGTH, check_changes=False):
+    """
+    Processes all MDX files in the directory, formatting Python code snippets.
+
+    Args:
+        directory (Path or str): Path to the directory containing MDX files.
+        file_pattern (re.Pattern): Regex pattern to match MDX files.
+        line_length (int): Line length to use for Black formatting.
+        check_changes (bool): If True, raises an exception if changes are detected.
+    """
+    matching_files = find_files_by_pattern(directory, file_pattern)
+    files_changed = []
+
+    for file_path in matching_files:
+        original_content, new_content = format_python_snippets_in_mdx(file_path, line_length)
+
+        if original_content != new_content:
+            files_changed.append(file_path)
+
+    if check_changes and files_changed:
+        raise RuntimeError(
+            f"The following files were modified during the run:\n"
+            + "\n".join(str(file) for file in files_changed)
+        )
+
+
+if __name__ == "__main__":
+    import sys
+
+    path = sys.argv[1] if len(sys.argv) > 1 else MDX_DIR
+    line_length = int(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_LINE_LENGTH
+    check_changes = os.getenv("CI") == "true"  # Set to True in CI pipeline
+
+    if Path(path).is_dir():
+        process_mdx_files(path, FILE_PATTERN, line_length, check_changes)
+    elif Path(path).is_file():
+        if FILE_PATTERN.search(path):
+            process_mdx_files(Path(path).parent, FILE_PATTERN, line_length, check_changes)
+        else:
+            print("The specified file does not match the MDX pattern.")
+    else:
+        print("Provided path is not valid.")

.github/workflows/check-mdx-frontmatter.yml

@@ -26,4 +26,4 @@ jobs:
         run: pnpm install
 
       - name: Run MDX frontmatter check
-        run: node .github/scripts/check-mdx-frontmatter.js
+        run: node .github/scripts/check-mdx-frontmatter.cjs

.github/workflows/check-python-code-snippets.yml

@@ -0,0 +1,39 @@
+name: check-python-code-snippets
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'fern/pages/**/*.mdx'
+      - 'fern/pages/**/**/*.mdx'
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    permissions: write-all
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        shell: bash
+        run: |
+          pipx install poetry
+
+      - name: Install Project Dependencies with Poetry
+        shell: bash
+        run: |
+          poetry install
+
+      - name: Run Python MDX Snippet Formatter
+        shell: bash
+        env:
+          CI: true
+        run: poetry run python .github/scripts/check_python_code_snippets.py fern/pages

.github/workflows/snippet-ci.yml

@@ -0,0 +1,40 @@
+name: snippet-ci
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - main
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install Dependencies
+        shell: bash
+        run: pnpm install             
+
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: poetry install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install poetry
+          poetry install
+
+      - name: Run snippet tests
+        continue-on-error: true
+        env:
+          CO_API_KEY: ${{ secrets.COHERE_TOKEN }}
+        run: pnpm run --filter snippet-tester test