Merge pull request #749 from snyk/feat/implement-client-plugin-system

feat: add broker plugin system [HYB-516]

config.default.json

@@ -231,6 +231,10 @@
       },
       "required": {
         "GITHUB": "ghe.yourdomain.com",
+        "GITHUB_APP_ID": "APP_ID",
+        "GITHUB_APP_PRIVATE_PEM_PATH": "abc",
+        "GITHUB_APP_CLIENT_ID": "123",
+        "GITHUB_APP_INSTALLATION_ID": "123",
         "BROKER_CLIENT_URL": "https://<broker.client.hostname>:<port>"
       }
     },

config.universal.json.sample

@@ -0,0 +1,15 @@
+{
+  "BROKER_CLIENT_CONFIGURATION": {
+    "common": {
+      "default": {
+        "BROKER_SERVER_URL": "https://broker.pre-prod.snyk.io",
+        "BROKER_HA_MODE_ENABLED": "false",
+        "BROKER_DISPATCHER_BASE_URL": "https://api.pre-prod.snyk.io"
+      },
+      "oauth": {
+        "clientId": "${CLIENT_ID}",
+        "clientSecret": "${CLIENT_SECRET}"
+      }
+    }
+  }
+}

defaultFilters/github-server-app.json

@@ -299,6 +299,16 @@
       }
     ],
     "private": [
+      {
+        "//": "look up repositories installation can access",
+        "method": "GET",
+        "path": "/installation/repositories",
+        "origin": "https://${GITHUB_API}",
+        "auth": {
+          "scheme": "bearer",
+          "token": "${ACCESS_TOKEN}"
+        }
+      },
       {
         "//": "search for user's repos",
         "method": "GET",
@@ -1287,7 +1297,11 @@
         "//": "get details of the repo",
         "method": "GET",
         "path": "/repos/:name/:repo",
-        "origin": "https://${GITHUB_API}"
+        "origin": "https://${GITHUB_API}",
+        "auth": {
+          "scheme": "bearer",
+          "token": "${ACCESS_TOKEN}"
+        }
       },
       {
         "//": "get the details of the commit to determine its SHA",

lib/client/brokerClientPlugins/abstractBrokerPlugin.ts

@@ -0,0 +1,57 @@
+import {
+  HttpResponse,
+  makeRequestToDownstream,
+} from '../../common/http/request';
+import { PostFilterPreparedRequest } from '../../common/relay/prepareRequest';
+import { log as logger } from '../../logs/logger';
+
+export default abstract class BrokerPlugin {
+  abstract pluginCode: string;
+  abstract pluginName: string;
+  abstract description: string;
+  abstract version: string;
+  abstract applicableBrokerTypes: Array<string>;
+  logger;
+  brokerClientConfiguration: Record<string, any>;
+  makeRequestToDownstream: (
+    req: PostFilterPreparedRequest,
+    retries?: any,
+  ) => Promise<HttpResponse>;
+  request?: PostFilterPreparedRequest;
+
+  constructor(brokerClientCfg: Record<string, any>) {
+    this.logger = logger;
+    this.brokerClientConfiguration = brokerClientCfg;
+    this.makeRequestToDownstream = makeRequestToDownstream;
+  }
+
+  getApplicableTypes(): Array<string> {
+    const applicableTypes: Array<string> = [];
+    if (
+      this.applicableBrokerTypes.every((type) =>
+        this.brokerClientConfiguration.supportedBrokerTypes.includes(type),
+      )
+    ) {
+      applicableTypes.push(...this.applicableBrokerTypes);
+    }
+    return applicableTypes;
+  }
+  isDisabled(config): boolean {
+    let isDisabled = false;
+    if (config[`DISABLE_${this.pluginCode}_PLUGIN`]) {
+      logger.info({ plugin: this.pluginName }, `Plugin disabled`);
+      isDisabled = true;
+    }
+    return isDisabled;
+  }
+  abstract isPluginActive(): boolean;
+
+  abstract startUp(connectionConfiguration: Record<string, any>): Promise<void>;
+
+  async preRequest(
+    connectionConfiguration: Record<string, any>,
+    postFilterPreparedRequest: PostFilterPreparedRequest,
+  ): Promise<PostFilterPreparedRequest> {
+    return postFilterPreparedRequest;
+  }
+}

lib/client/brokerClientPlugins/pluginManager.ts

@@ -0,0 +1,129 @@
+import { readdir } from 'fs/promises';
+import { log as logger } from '../../logs/logger';
+import BrokerPlugin from './abstractBrokerPlugin';
+import { existsSync } from 'fs';
+import { PostFilterPreparedRequest } from '../../common/relay/prepareRequest';
+
+export const loadPlugins = async (pluginsFolderPath: string, clientOpts) => {
+  clientOpts.config['plugins'] = new Map<string, unknown>();
+  clientOpts.config.supportedBrokerTypes.forEach((type) => {
+    clientOpts.config.plugins.set(type, []);
+  });
+  try {
+    if (existsSync(pluginsFolderPath)) {
+      const pluginsFiles = await readdir(pluginsFolderPath);
+      for (const pluginFile of pluginsFiles.filter((filename) =>
+        filename.endsWith('.js'),
+      )) {
+        const plugin = await import(`${pluginsFolderPath}/${pluginFile}`);
+        // Passing the config object so we can mutate things like filters instead of READONLY
+        const pluginInstance = new plugin.Plugin(clientOpts.config);
+        const applicableBrokerTypes = pluginInstance.getApplicableTypes();
+        applicableBrokerTypes.forEach((applicableBrokerType) => {
+          if (
+            !pluginInstance.isDisabled(clientOpts.config) &&
+            pluginInstance.isPluginActive()
+          ) {
+            logger.debug({}, `Loading plugin ${pluginInstance.pluginName}`);
+            const configPluginForCurrentType =
+              clientOpts.config.plugins.get(applicableBrokerType);
+            if (
+              configPluginForCurrentType.some(
+                (x) =>
+                  x.pluginCode === pluginInstance.pluginCode ||
+                  x.pluginName === pluginInstance.pluginName,
+              )
+            ) {
+              const errMsg = `Some Plugins have identical name or code.`;
+              logger.error({}, errMsg);
+              throw new Error(errMsg);
+            }
+            configPluginForCurrentType.push(pluginInstance);
+          } else {
+            logger.debug(
+              {},
+              `Skipping plugin ${pluginInstance.pluginName}, not active.`,
+            );
+          }
+        });
+      }
+    }
+    return clientOpts.config['plugins'];
+  } catch (err) {
+    const errMsg = `Error loading plugins from ${pluginsFolderPath}`;
+    logger.error({ err }, `Error loading plugins from ${pluginsFolderPath}`);
+    throw new Error(errMsg);
+  }
+};
+
+export const runStartupPlugins = async (clientOpts) => {
+  const loadedPlugins = clientOpts.config.plugins as Map<
+    string,
+    BrokerPlugin[]
+  >;
+
+  const connectionsKeys = clientOpts.config.connections
+    ? Object.keys(clientOpts.config.connections)
+    : [];
+
+  for (const connectionKey of connectionsKeys) {
+    if (
+      loadedPlugins.has(`${clientOpts.config.connections[connectionKey].type}`)
+    ) {
+      const pluginInstances =
+        loadedPlugins.get(
+          `${clientOpts.config.connections[connectionKey].type}`,
+        ) ?? [];
+      for (let i = 0; i < pluginInstances.length; i++) {
+        await pluginInstances[i].startUp(
+          clientOpts.config.connections[connectionKey],
+        );
+      }
+    }
+  }
+};
+
+export const runPreRequestPlugins = async (
+  clientOpts,
+  connectionIdentifier,
+  pristinePreRequest: PostFilterPreparedRequest,
+) => {
+  let preRequest = pristinePreRequest;
+  const loadedPlugins = clientOpts.config.plugins as Map<
+    string,
+    BrokerPlugin[]
+  >;
+  const connectionsKeys = Object.keys(clientOpts.config.connections);
+  let connectionKey;
+  for (let i = 0; i < connectionsKeys.length; i++) {
+    if (
+      clientOpts.config.connections[connectionsKeys[i]].identifier ==
+      connectionIdentifier
+    ) {
+      connectionKey = connectionsKeys[i];
+      break;
+    }
+  }
+  if (!connectionsKeys.includes(connectionKey)) {
+    const errMsg = `Plugin preRequest: connection ${connectionKey} not found`;
+    logger.error({ connectionKey }, errMsg);
+    throw new Error(errMsg);
+  }
+
+  if (
+    loadedPlugins.has(`${clientOpts.config.connections[connectionKey].type}`)
+  ) {
+    const pluginInstances =
+      loadedPlugins.get(
+        `${clientOpts.config.connections[connectionKey].type}`,
+      ) ?? [];
+    for (let i = 0; i < pluginInstances.length; i++) {
+      preRequest = await pluginInstances[i].preRequest(
+        clientOpts.config.connections[connectionKey],
+        preRequest,
+      );
+    }
+  }
+
+  return preRequest;
+};