Add tests for snippets (#215)

* drop api key

* ts -> js

* ts -> js

* js working

* Java fixes

* go works

* Final config

* Add CI

* rename action

* Add package

* Add poetry ci

* poetry run python

* Add categories

* Update utils

* shaps

* fix fix

.github/workflows/snippet-ci.yml

@@ -0,0 +1,40 @@
+name: snippet-ci
+
+on:
+  pull_request: {}
+  push:
+    branches:
+      - main
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
+      - name: Install Dependencies
+        shell: bash
+        run: pnpm install             
+
+      - name: Set up python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: poetry install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install poetry
+          poetry install
+
+      - name: Run snippet tests
+        continue-on-error: true
+        env:
+          CO_API_KEY: ${{ secrets.COHERE_TOKEN }}
+        run: pnpm run --filter snippet-tester test

package.json

@@ -1,6 +1,7 @@
 {
   "name": "cohere-developer-experience",
   "version": "0.0.0",
+  "type": "module",
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "clean": "rm -rf ./fern/dist && rm -rf ./node_modules",
@@ -17,8 +18,9 @@
     "tailwindcss": "^3.4.4"
   },
   "dependencies": {
-    "gray-matter": "^4.0.3",
+    "cohere-ai": "^7.14.0",
     "fern-api": "^0.41.16",
+    "gray-matter": "^4.0.3",
     "react": "^18.3.1"
   }
-}
+}

packages/snippet-tester/babel.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', { targets: { node: 'current' } }],
+        '@babel/preset-typescript',
+    ],
+};
+

packages/snippet-tester/jest.config.ts

@@ -0,0 +1,199 @@
+/**
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/configuration
+ */
+
+import type {Config} from 'jest';
+
+const config: Config = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/private/var/folders/7y/pd12xsnj72x0t4drmr3tt23c0000gp/T/jest_dy",
+
+  // Automatically clear mock calls, instances, contexts and results before every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  // coverageDirectory: undefined,
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // The default configuration for fake timers
+  // fakeTimers: {
+  //   "enableGlobally": false
+  // },
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "mjs",
+  //   "cjs",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "json",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state before every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state and implementation before every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  // testEnvironment: "jest-environment-node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jest-circus/runner",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};
+
+export default config;

packages/snippet-tester/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "snippet-tester",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "jest"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@jest/globals": "^29.7.0",
+    "@pnpm/find-workspace-dir": "^7.0.2",
+    "cohere": "^1.1.1",
+    "cohere-ai": "^7.14.0",
+    "convict": "^6.2.4",
+    "glob": "^11.0.0",
+    "jest": "^29.7.0",
+    "nodemon": "^3.1.7",
+    "ts-node": "^10.9.2"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.25.8",
+    "@babel/preset-env": "^7.25.8",
+    "@babel/preset-typescript": "^7.25.7",
+    "@types/convict": "^6.1.6",
+    "@types/jest": "^29.5.13",
+    "babel-jest": "^29.7.0"
+  }
+}

packages/snippet-tester/src/config.ts

@@ -0,0 +1,43 @@
+import path from "path";
+
+export interface Language {
+    name: string;
+    exec: (file: string) => string;
+    snippetRoot: (base: string) => string;
+    glob: string;
+}
+
+export const config: {
+    languages: Language[];
+} = {
+    "languages": [
+        {
+            "name": "node",
+            "exec": (file: string) => `node ${file}`,
+            "snippetRoot": (base: string) => path.join(base, "./snippets/node"),
+            "glob": "**/*.js"
+        },
+        {
+            "name": "java",
+            "exec": (file: string) => {
+                const classPath = file.replace("app/src/main/java/", "").replace(".java", "").split("/").join(".");
+
+                return `gradle -PmainClass=${classPath} run`
+            },
+            "snippetRoot": (base: string) => path.join(base, "./snippets/java"),
+            "glob": "./app/src/main/java/**/*.java"
+        },
+        {
+            "name": "python",
+            "exec": (file: string) => `poetry run python ${file}`,
+            "snippetRoot": (base: string) => path.join(base, "./snippets/python"),
+            "glob": "**/*.py"   
+        },
+        {
+            "name": "golang",
+            "exec": (file: string) => `go run ${file}`,
+            "snippetRoot": (base: string) => path.join(base, "./snippets/go"),
+            "glob": "**/*.go"
+        }
+    ]
+}