Merge deno-slack-builder into this repo, add support for multiple pro…

…tocols. (#57)

* The config returned by `src/mod.ts` now sets the protocol specifier, message boundary protocol to supported list

* Pull in split out protocol repo, use protocol in get-trigger hook implementation

* use new protocols in check- and install-update hooks, add some additional logging to install-update to flex the new protocol a bit

* Moving deno-slack-builder source code into this repo.

* Move tests from -builder into this repo. Add allow-write and allow-run permissions to test task to run the builder tests. Remove conditional logic from old builder code related to reusing the same Options object for the get-manifest and build hooks; since they no longer share a code path, can instead have specific parameters only relevant to each hook. Added a MockProtocol built on top of deno Spies to easily inspect how the protocol object is used in tests.

* Move protocol imports to standalone package.

* Fix code coverage uploading to codecov since introduction of nicer local coverage reporting

* Fix get-trigger hook implementation, add a basic test for get_trigger.ts.

* Add a test to ensure functions of type=API are not bundled in the build hook.

* Add a few more build hook tests.

* Add another test for get_manifest hook

* Remove extra logging, update tests to stub out Deno.run where applicable.

* Factor hasDefaultExport out into utilities.ts, use it in get_manifest to ensure any imported manifest js/ts files specify a default export.

* Move removing directory to top scope of build hook. Remove extraneous TODO/code.

* Ignore src/test/fixtures from any lint/fmt tasks. Rename hasDefaultExport to getDefaultExport. Added some JSDocs to a few methods (mainly to document that the expectation for arguments are an absolute path name). Add tests for manifest hook createManifest functionality.

* Have get-trigger use getDefaultExport, add tests for get-trigger hook.

* Have get-manifest and get-trigger hooks inspect the returned type from userland trigger/manifest code and ensure they are objects. Add tests ensuring manifest and trigger objects returned from userland are of the correct types.

* Dont add dependency to deno-slack-sdk in manifest test fixture file, just mimic structure of manifest object should be sufficient.

* Remove SDK reference from invalid manifest ts test fixture.

* Set dependent deno-slack-runtime version to the newly-released 0.6.0

* Add --quiet flag to deno-bundle invocation in the build hook, to keep the output clean when invoking via the CLI.

* Refactor validation of functions into a new utility method forEachValidatedManifestFunction, which will be consumed in both get-manifest and build hooks. Move dependency on parsing JSONC into deps.ts, and rename the parse* deps more explicitly to be able to differentiate between parsing of CLI arguments and parsing of JSONC. Rename createManifest to getManifest.

* Add function validation to get-manifest hook. Move function validation tests to utilities tests.

* Clarify functionality in JSdoc.

* Name function validator "validateManifestFunctions". Remove extraneous callback from this helper.

* Remove unnecessary exports from src/mod.ts. Add more integration-y tests to the build hook, ensuring the combination of function source modules and manifest contents together are taken into account to prevent building bad function files.

* Factor out get-manifest hook main module logic into an exported function for easier testing. Added some app fixtures (where both manifest and function files are included) so that the manifest+function validation in get-manifest can be faked out. Added tests for validating manifest+functions in built hook tests.

.github/workflows/deno-ci.yml

@@ -31,9 +31,9 @@ jobs:
         run: deno task test
 
       - name: Generate CodeCov-friendly coverage report
-        run: deno task coverage --lcov --output=codecov.lcov
+        run: deno task coverage
 
       - name: Upload coverage to CodeCov
         uses: codecov/[email protected]
         with:
-          file: ./codecov.lcov
+          file: ./lcov.info

deno.jsonc

@@ -2,7 +2,8 @@
   "$schema": "https://deno.land/x/deno/cli/schemas/config-file.v1.json",
   "fmt": {
     "files": {
-      "include": ["src"]
+      "include": ["src"],
+      "exclude": ["src/tests/fixtures"]
     },
     "options": {
       "semiColons": true,
@@ -15,17 +16,19 @@
   },
   "lint": {
     "files": {
-      "include": ["src"]
+      "include": ["src"],
+      "exclude": ["src/tests/fixtures"]
     }
   },
   "test": {
     "files": {
-      "include": ["src"]
+      "include": ["src"],
+      "exclude": ["src/tests/fixtures"]
     }
   },
   "tasks": {
-    "test": "deno fmt --check && deno lint && deno test --allow-read --allow-net src",
-    "coverage": "deno test --allow-read --allow-net --coverage=.coverage src && deno coverage --exclude=fixtures --exclude=test --lcov --output=lcov.info .coverage && deno run --allow-read https://deno.land/x/[email protected]/cli.ts"
+    "test": "deno fmt --check && deno lint && deno test --allow-read --allow-net --allow-write --allow-run src",
+    "coverage": "deno test --allow-read --allow-net --allow-write --allow-run --coverage=.coverage src && deno coverage --exclude=fixtures --exclude=test --lcov --output=lcov.info .coverage && deno run --allow-read https://deno.land/x/[email protected]/cli.ts"
   },
   "lock": false
 }

src/build.ts

@@ -0,0 +1,134 @@
+import {
+  ensureDir,
+  getProtocolInterface,
+  parseCLIArguments,
+  path,
+} from "./deps.ts";
+import type { Protocol } from "./deps.ts";
+import { cleanManifest, getManifest } from "./get_manifest.ts";
+import { validateManifestFunctions } from "./utilities.ts";
+
+export const validateAndCreateFunctions = async (
+  workingDirectory: string,
+  outputDirectory: string,
+  // deno-lint-ignore no-explicit-any
+  manifest: any,
+  protocol: Protocol,
+) => {
+  // Ensure functions output directory exists
+  const functionsPath = path.join(outputDirectory, "functions");
+  await ensureDir(functionsPath);
+
+  // Ensure manifest and function userland exists and is valid
+  await validateManifestFunctions(
+    workingDirectory,
+    manifest,
+  );
+
+  // Write out functions to disk
+  for (const fnId in manifest.functions) {
+    const fnFilePath = path.join(
+      workingDirectory,
+      manifest.functions[fnId].source_file,
+    );
+    await createFunctionFile(
+      outputDirectory,
+      fnId,
+      fnFilePath,
+      protocol,
+    );
+  }
+};
+
+const createFunctionFile = async (
+  outputDirectory: string,
+  fnId: string,
+  fnFilePath: string,
+  protocol: Protocol,
+) => {
+  const fnFileRelative = path.join("functions", `${fnId}.js`);
+  const fnBundledPath = path.join(outputDirectory, fnFileRelative);
+
+  // We'll default to just using whatever Deno executable is on the path
+  // Ideally we should be able to rely on Deno.execPath() so we make sure to bundle with the same version of Deno
+  // that called this script. This is perhaps a bit overly cautious, so we can look to remove the defaulting here in the future.
+  let denoExecutablePath = "deno";
+  try {
+    denoExecutablePath = Deno.execPath();
+  } catch (e) {
+    protocol.error("Error calling Deno.execPath()", e);
+  }
+
+  try {
+    // call out to deno to handle bundling
+    const p = Deno.run({
+      cmd: [
+        denoExecutablePath,
+        "bundle",
+        "--quiet",
+        fnFilePath,
+        fnBundledPath,
+      ],
+    });
+
+    const status = await p.status();
+    p.close();
+    if (status.code !== 0 || !status.success) {
+      throw new Error(`Error bundling function file: ${fnId}`);
+    }
+  } catch (e) {
+    protocol.error(`Error bundling function file: ${fnId}`);
+    throw e;
+  }
+};
+
+/**
+ * Recursively deletes the specified directory.
+ *
+ * @param directoryPath the directory to delete
+ * @returns true when the directory is deleted or throws unexpected errors
+ */
+async function removeDirectory(directoryPath: string): Promise<boolean> {
+  try {
+    await Deno.remove(directoryPath, { recursive: true });
+    return true;
+  } catch (err) {
+    if (err instanceof Deno.errors.NotFound) {
+      return false;
+    }
+
+    throw err;
+  }
+}
+
+if (import.meta.main) {
+  const protocol = getProtocolInterface(Deno.args);
+
+  // Massage source and output directories
+  let { source, output } = parseCLIArguments(Deno.args);
+  if (!output) output = "dist";
+  const outputDirectory = path.isAbsolute(output)
+    ? output
+    : path.join(Deno.cwd(), output);
+
+  // Clean output dir prior to build
+  await removeDirectory(outputDirectory);
+
+  const workingDirectory = path.isAbsolute(source || "")
+    ? source
+    : path.join(Deno.cwd(), source || "");
+
+  const generatedManifest = await getManifest(Deno.cwd());
+  await validateAndCreateFunctions(
+    workingDirectory,
+    outputDirectory,
+    generatedManifest,
+    protocol,
+  );
+  const prunedManifest = cleanManifest(generatedManifest);
+  const manifestPath = path.join(outputDirectory, "manifest.json");
+  await Deno.writeTextFile(
+    manifestPath,
+    JSON.stringify(prunedManifest, null, 2),
+  );
+}

src/check_update.ts

@@ -3,7 +3,7 @@ import {
   DENO_SLACK_HOOKS,
   DENO_SLACK_SDK,
 } from "./libraries.ts";
-import { JSONValue } from "./deps.ts";
+import { getProtocolInterface, JSONValue } from "./deps.ts";
 import { getJSON } from "./utilities.ts";
 
 const IMPORT_MAP_SDKS = [DENO_SLACK_SDK, DENO_SLACK_API];
@@ -347,6 +347,8 @@ export function createFileErrorMsg(
 
   return fileErrorMsg;
 }
+
 if (import.meta.main) {
-  console.log(JSON.stringify(await checkForSDKUpdates()));
+  const protocol = getProtocolInterface(Deno.args);
+  protocol.respond(JSON.stringify(await checkForSDKUpdates()));
 }

src/deps.ts

@@ -1,3 +1,8 @@
-export { parse } from "https://deno.land/[email protected]/flags/mod.ts";
+export { parse as parseCLIArguments } from "https://deno.land/[email protected]/flags/mod.ts";
 export * as path from "https://deno.land/[email protected]/path/mod.ts";
+export { ensureDir } from "https://deno.land/[email protected]/fs/mod.ts";
+export { parse as parseJSONC } from "https://deno.land/[email protected]/encoding/jsonc.ts";
 export type { JSONValue } from "https://deno.land/[email protected]/encoding/jsonc.ts";
+export { deepMerge } from "https://deno.land/[email protected]/collections/deep_merge.ts";
+export { getProtocolInterface } from "https://deno.land/x/[email protected]/mod.ts";
+export type { Protocol } from "https://deno.land/x/[email protected]/types.ts";

src/dev_deps.ts

@@ -1,8 +1,17 @@
 export {
   assertEquals,
+  assertExists,
   assertRejects,
   assertStringIncludes,
 } from "https://deno.land/[email protected]/testing/asserts.ts";
+export {
+  assertSpyCall,
+  assertSpyCalls,
+  returnsNext,
+  spy,
+  stub,
+} from "https://deno.land/[email protected]/testing/mock.ts";
+export type { Spy } from "https://deno.land/[email protected]/testing/mock.ts";
 export * as mockFetch from "https://deno.land/x/[email protected]/mod.ts";
-
 export * as mockFile from "https://deno.land/x/[email protected]/mod.ts";
+export { MockProtocol } from "https://deno.land/x/[email protected]/mock.ts";

src/flags.ts

@@ -1,11 +1,11 @@
-import { parse } from "./deps.ts";
+import { parseCLIArguments } from "./deps.ts";
 
 const UNSAFELY_IGNORE_CERT_ERRORS_FLAG =
   "sdk-unsafely-ignore-certificate-errors";
 const SLACK_DEV_DOMAIN_FLAG = "sdk-slack-dev-domain";
 
 export const getStartHookAdditionalDenoFlags = (args: string[]): string => {
-  const parsedArgs = parse(args);
+  const parsedArgs = parseCLIArguments(args);
   const extraFlagValue = parsedArgs[SLACK_DEV_DOMAIN_FLAG] ??
     parsedArgs[UNSAFELY_IGNORE_CERT_ERRORS_FLAG] ?? "";
 

src/get_manifest.ts

@@ -0,0 +1,138 @@
+import { deepMerge, getProtocolInterface, path } from "./deps.ts";
+import { getDefaultExport, validateManifestFunctions } from "./utilities.ts";
+
+// Responsible for taking a working directory, and an output directory
+// and placing a manifest.json in the root of the output directory
+
+/**
+ * Returns a merged manifest object from expected files used to represent an application manifest:
+ * `manifest.json`, `manifest.ts` and `manifest.js`. If both a `json` and `ts` _or_ `js` are present,
+ * then first the `json` file will be used as a base object, then the `.ts` or the `.js` file export
+ * will be merged over the `json` file. If a `.ts` file exists, the `.js` will be ignored. Otherwise,
+ * the `.js` file will be merged over the `.json`.
+ * @param {string} cwd - Absolute path to the root of an application.
+ */
+export const getManifest = async (cwd: string) => {
+  let foundManifest = false;
+  // deno-lint-ignore no-explicit-any
+  let manifest: any = {};
+
+  const manifestJSON = await readManifestJSONFile(path.join(
+    cwd,
+    "manifest.json",
+  ));
+  if (manifestJSON !== false) {
+    manifest = deepMerge(manifest, manifestJSON);
+    foundManifest = true;
+  }
+
+  // First check if there's a manifest.ts file
+  const manifestTS = await readImportedManifestFile(
+    path.join(cwd, "manifest.ts"),
+  );
+  if (manifestTS === false) {
+    // Now check for a manifest.js file
+    const manifestJS = await readImportedManifestFile(
+      path.join(cwd, "manifest.js"),
+    );
+    if (manifestJS !== false) {
+      manifest = deepMerge(manifest, manifestJS);
+      foundManifest = true;
+    }
+  } else {
+    manifest = deepMerge(manifest, manifestTS);
+    foundManifest = true;
+  }
+
+  if (!foundManifest) {
+    throw new Error(
+      "Could not find a manifest.json, manifest.ts or manifest.js file",
+    );
+  }
+
+  return manifest;
+};
+
+// Remove any properties in the manifest specific to the tooling that don't belong in the API payloads
+// deno-lint-ignore no-explicit-any
+export const cleanManifest = (manifest: any) => {
+  for (const fnId in manifest.functions) {
+    const fnDef = manifest.functions[fnId];
+    delete fnDef.source_file;
+  }
+
+  return manifest;
+};
+
+/**
+ * Reads and parses an app's `manifest.json` file, and returns its contents. If the file does not exist
+ * or otherwise reading the file fails, returns `false`. If the file contents are invalid JSON, this method
+ * will throw an exception.
+ * @param {string} manifestJSONFilePath - Absolute path to an app's `manifest.json` file.
+ */
+async function readManifestJSONFile(manifestJSONFilePath: string) {
+  // deno-lint-ignore no-explicit-any
+  let manifestJSON: any = {};
+
+  try {
+    const { isFile } = await Deno.stat(manifestJSONFilePath);
+
+    if (!isFile) {
+      return false;
+    }
+  } catch (_e) {
+    return false;
+  }
+
+  const jsonString = await Deno.readTextFile(manifestJSONFilePath);
+  manifestJSON = JSON.parse(jsonString);
+
+  return manifestJSON;
+}
+
+/**
+ * Reads and parses an app's manifest file, and returns its contents. The file is expected to be one that the
+ * deno runtime can import, and one that returns a default export. If the file does not exist otherwise reading
+ * the file fails, returns `false`. If the file does not contain a default export, this method will throw and
+ * exception.
+ * @param {string} filename - Absolute path to an app's manifest file, to be imported by the deno runtime.
+ */
+async function readImportedManifestFile(filename: string) {
+  // Look for manifest.[js|ts] in working directory
+  // - if present, default export should be a manifest json object
+  try {
+    const { isFile } = await Deno.stat(filename);
+
+    if (!isFile) {
+      return false;
+    }
+  } catch (_e) {
+    return false;
+  }
+
+  // `getDefaultExport` will throw if no default export present
+  const manifest = await getDefaultExport(filename);
+  if (typeof manifest != "object") {
+    throw new Error(
+      `Manifest file: ${filename} default export is not an object!`,
+    );
+  }
+  return manifest;
+}
+
+/**
+ * Retrieves a merged application manifest, validates the manifest and all its specified functions,
+ * and cleans up any bits from it not relevant for the Slack manifest APIs.
+ * @param {string} applicationRoot - An absolute path to the application root, which presumably contains manifest files.
+ */
+export async function getValidateAndCleanManifest(applicationRoot: string) {
+  const generatedManifest = await getManifest(applicationRoot);
+  await validateManifestFunctions(applicationRoot, generatedManifest);
+  return cleanManifest(generatedManifest);
+}
+
+if (import.meta.main) {
+  const protocol = getProtocolInterface(Deno.args);
+  const prunedManifest = await getValidateAndCleanManifest(Deno.cwd());
+  protocol.respond(JSON.stringify(prunedManifest));
+}