Skip to content

Commit

Permalink
Merge pull request #746 from snyk/feat/retrieve-connections-from-backend
Browse files Browse the repository at this point in the history
Feat/retrieve connections from backend [HYB-261]
  • Loading branch information
aarlaud authored Apr 25, 2024
2 parents 0f93982 + 3d63c3c commit 3834320
Show file tree
Hide file tree
Showing 22 changed files with 361 additions and 114 deletions.
1 change: 1 addition & 0 deletions config.default.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"API_VERSION":"2024-02-08~experimental",
"BROKER_SERVER_UNIVERSAL_CONFIG_ENABLED": false,
"SUPPORTED_BROKER_TYPES": [
"apprisk",
Expand Down
File renamed without changes.
60 changes: 60 additions & 0 deletions lib/client/config/remoteConfig.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
import { readFileSync, writeFileSync } from 'fs';
import { makeRequestToDownstream } from '../../common/http/request';
import { PostFilterPreparedRequest } from '../../common/relay/prepareRequest';
import { ClientOpts } from '../../common/types/options';
import { BrokerConnectionApiResponse } from '../types/api';

export const retrieveConnectionsForDeployment = async (
clientOpts: ClientOpts,
universalFilePath: string,
) => {
try {
const tenantId = clientOpts.config.tenantId;
const installId = clientOpts.config.installId;
const deploymentId = clientOpts.config.deploymentId;
const apiVersion = clientOpts.config.apiVersion;
const request: PostFilterPreparedRequest = {
url: `${clientOpts.config.API_BASE_URL}/rest/tenants/${tenantId}/brokers/installs/${installId}/deployments/${deploymentId}/connections?version=${apiVersion}`,
headers: {
'Content-Type': 'application/vnd.api+json',
Authorization: `${clientOpts.accessToken?.authHeader}`,
},
method: 'GET',
};
const connectionsResponse = await makeRequestToDownstream(request);
if (connectionsResponse.statusCode != 200) {
const errorBody = JSON.parse(connectionsResponse.body);
throw new Error(
`${connectionsResponse.statusCode}-${errorBody.error}:${errorBody.error_description}`,
);
}
const connections = JSON.parse(connectionsResponse.body)
.data as BrokerConnectionApiResponse[];
const connectionsObjectForFile = { CONNECTIONS: {} };
for (let i = 0; i < connections.length; i++) {
connectionsObjectForFile.CONNECTIONS[
`${connections[i].attributes.name}`
] = {
...connections[i].attributes.configuration.default,
...connections[i].attributes.configuration.required,
};
connectionsObjectForFile.CONNECTIONS[
`${connections[i].attributes.name}`
].type = connections[i].attributes.type;
connectionsObjectForFile.CONNECTIONS[
`${connections[i].attributes.name}`
].identifier = connections[i].id;
}
const universalConfigFileBuffer = readFileSync(universalFilePath);
const universalConfigFile = JSON.parse(
universalConfigFileBuffer.toString() ?? { CONNECTIONS: {} },
);
universalConfigFile['CONNECTIONS'] = connectionsObjectForFile.CONNECTIONS;
writeFileSync(universalFilePath, JSON.stringify(universalConfigFile));
return;
} catch (err) {
throw new Error(
`${err} - Error retrieving and loading connections from Deployment ID`,
);
}
};
77 changes: 36 additions & 41 deletions lib/client/hooks/startup/processHooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,53 +5,48 @@ import { commitSigningEnabled, commitSigningFilterRules } from '../../scm';
import { HookResults } from '../../types/client';
import { CheckResult } from '../../checks/types';
import { ClientOpts } from '../../../common/types/options';
import { highAvailabilityModeEnabled } from '../../utils/configHelpers';
import { highAvailabilityModeEnabled } from '../../config/configHelpers';

export const validateMinimalConfig = async (
clientOpts: ClientOpts,
): Promise<void> => {
if (
!clientOpts.config.brokerToken &&
!clientOpts.config.universalBrokerEnabled
) {
const brokerToken = clientOpts.config.brokerToken;
// null, undefined, empty, etc.
logger.error(
{ brokerToken },
'[MISSING_BROKER_TOKEN] BROKER_TOKEN is required to successfully identify itself to the server',
);
const error = new ReferenceError(
'BROKER_TOKEN is required to successfully identify itself to the server',
);
error['code'] = 'MISSING_BROKER_TOKEN';
throw error;
}

if (!clientOpts.config.brokerServerUrl) {
const brokerServerUrl = clientOpts.config.brokerServerUrl;
// null, undefined, empty, etc.
logger.error(
{ brokerServerUrl },
'[MISSING_BROKER_SERVER_URL] BROKER_SERVER_URL is required to connect to the broker server',
);
const error = new ReferenceError(
'BROKER_SERVER_URL is required to connect to the broker server',
);
error['code'] = 'MISSING_BROKER_SERVER_URL';
throw error;
}
};

export const processStartUpHooks = async (
clientOpts: ClientOpts,
brokerClientId: string,
): Promise<HookResults> => {
try {
clientOpts.config.API_BASE_URL =
clientOpts.config.API_BASE_URL ??
clientOpts.config.BROKER_DISPATCHER_BASE_URL ??
clientOpts.config.BROKER_SERVER_URL?.replace(
'//broker.',
'//api.',
).replace('//broker2.', '//api.') ??
'https://api.snyk.io';

if (
!clientOpts.config.brokerToken &&
!clientOpts.config.universalBrokerEnabled
) {
const brokerToken = clientOpts.config.brokerToken;
// null, undefined, empty, etc.
logger.error(
{ brokerToken },
'[MISSING_BROKER_TOKEN] BROKER_TOKEN is required to successfully identify itself to the server',
);
const error = new ReferenceError(
'BROKER_TOKEN is required to successfully identify itself to the server',
);
error['code'] = 'MISSING_BROKER_TOKEN';
throw error;
}

if (!clientOpts.config.brokerServerUrl) {
const brokerServerUrl = clientOpts.config.brokerServerUrl;
// null, undefined, empty, etc.
logger.error(
{ brokerServerUrl },
'[MISSING_BROKER_SERVER_URL] BROKER_SERVER_URL is required to connect to the broker server',
);
const error = new ReferenceError(
'BROKER_SERVER_URL is required to connect to the broker server',
);
error['code'] = 'MISSING_BROKER_SERVER_URL';
throw error;
}

// if (!clientOpts.config.BROKER_CLIENT_URL) {
// const proto =
// !clientOpts.config.key && !clientOpts.config.cert ? 'http' : 'https';
Expand Down
92 changes: 67 additions & 25 deletions lib/client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,25 @@ import {
import { healthCheckHandler } from './routesHandler/healthcheckHandler';
import { systemCheckHandler } from './routesHandler/systemCheckHandler';
import { IdentifyingMetadata, Role, WebSocketConnection } from './types/client';
import { processStartUpHooks } from './hooks/startup/processHooks';
import {
processStartUpHooks,
validateMinimalConfig,
} from './hooks/startup/processHooks';
import { forwardHttpRequestOverHttp } from '../common/relay/forwardHttpRequestOverHttp';
import { isWebsocketConnOpen } from './utils/socketHelpers';
import { loadAllFilters } from '../common/filter/filtersAsync';
import { ClientOpts, LoadedClientOpts } from '../common/types/options';
import { websocketConnectionSelectorMiddleware } from './routesHandler/websocketConnectionMiddlewares';
import { getClientConfigMetadata } from './utils/configHelpers';
import { getClientConfigMetadata } from './config/configHelpers';
import { fetchJwt } from './auth/oauth';
import {
CONFIGURATION,
getConfig,
loadBrokerConfig,
} from '../common/config/config';
import { retrieveConnectionsForDeployment } from './config/remoteConfig';
import { handleTerminationSignal } from '../common/utils/signals';
import { cleanUpUniversalFile } from './utils/cleanup';

process.on('uncaughtException', (error) => {
if (error.message == 'read ECONNRESET') {
Expand All @@ -41,10 +52,41 @@ export const main = async (clientOpts: ClientOpts) => {
try {
logger.info({ version }, 'Broker starting in client mode');

const brokerClientId = uuidv4();
logger.info({ brokerClientId }, 'generated broker client id');
const hookResults = await processStartUpHooks(clientOpts, brokerClientId);
clientOpts.config.API_BASE_URL =
clientOpts.config.API_BASE_URL ??
clientOpts.config.BROKER_DISPATCHER_BASE_URL ??
clientOpts.config.BROKER_SERVER_URL?.replace(
'//broker.',
'//api.',
).replace('//broker2.', '//api.') ??
'https://api.snyk.io';

await validateMinimalConfig(clientOpts);

if (
clientOpts.config.brokerClientConfiguration.common.oauth?.clientId &&
clientOpts.config.brokerClientConfiguration.common.oauth?.clientSecret &&
!process.env.SKIP_REMOTE_CONFIG
) {
clientOpts.accessToken = await fetchJwt(
clientOpts.config.API_BASE_URL,
clientOpts.config.brokerClientConfiguration.common.oauth.clientId,
clientOpts.config.brokerClientConfiguration.common.oauth.clientSecret,
);
await retrieveConnectionsForDeployment(
clientOpts,
`${__dirname}/../../../../config.universal.json`,
);
// Reload config with connection
await loadBrokerConfig();
const globalConfig = { config: getConfig() };
clientOpts.config = Object.assign(
{},
clientOpts.config,
globalConfig.config,
) as Record<string, any> as CONFIGURATION;
handleTerminationSignal(cleanUpUniversalFile);
}
const loadedClientOpts: LoadedClientOpts = {
loadedFilters: loadAllFilters(clientOpts.filters, clientOpts.config),
...clientOpts,
Expand All @@ -55,32 +97,26 @@ export const main = async (clientOpts: ClientOpts) => {
throw new Error('Unable to load filters');
}

if (
clientOpts.config.brokerClientConfiguration.common.oauth?.clientId &&
clientOpts.config.brokerClientConfiguration.common.oauth?.clientSecret
) {
loadedClientOpts.accessToken = await fetchJwt(
clientOpts.config.API_BASE_URL,
clientOpts.config.brokerClientConfiguration.common.oauth.clientId,
clientOpts.config.brokerClientConfiguration.common.oauth.clientSecret,
);
}
const brokerClientId = uuidv4();
logger.info({ brokerClientId }, 'generated broker client id');
const hookResults = await processStartUpHooks(clientOpts, brokerClientId);

const globalIdentifyingMetadata: IdentifyingMetadata = {
capabilities: ['post-streams'],
clientId: brokerClientId,
filters: clientOpts.filters,
filters: loadedClientOpts.filters,
preflightChecks: hookResults.preflightCheckResults,
version,
clientConfig: getClientConfigMetadata(clientOpts.config),
role: Role.primary,
};

let websocketConnections: WebSocketConnection[] = [];
if (clientOpts.config.universalBrokerEnabled) {
const integrationsKeys = clientOpts.config.connections
? Object.keys(clientOpts.config.connections)
if (loadedClientOpts.config.universalBrokerEnabled) {
const integrationsKeys = loadedClientOpts.config.connections
? Object.keys(loadedClientOpts.config.connections)
: [];

if (integrationsKeys.length < 1) {
logger.error(
{},
Expand Down Expand Up @@ -110,19 +146,25 @@ export const main = async (clientOpts: ClientOpts) => {
}

// start the local webserver to listen for relay requests
const { app, server } = webserver(clientOpts.config, clientOpts.port);
const { app, server } = webserver(
loadedClientOpts.config,
loadedClientOpts.port,
);

const httpToWsForwarder = forwardHttpRequest(loadedClientOpts);
const httpToAPIForwarder = forwardHttpRequestOverHttp(
loadedClientOpts,
clientOpts.config,
loadedClientOpts.config,
);
// IMPORTANT: defined before relay (`app.all('/*', ...`)
app.get('/health/checks', handleChecksRoute(clientOpts.config));
app.get('/health/checks/:checkId', handleCheckIdsRoutes(clientOpts.config));
app.get('/health/checks', handleChecksRoute(loadedClientOpts.config));
app.get(
'/health/checks/:checkId',
handleCheckIdsRoutes(loadedClientOpts.config),
);

app.get(
clientOpts.config.brokerHealthcheckPath || '/healthcheck',
loadedClientOpts.config.brokerHealthcheckPath || '/healthcheck',
(req, res, next) => {
res.locals.websocketConnections = websocketConnections;
next();
Expand All @@ -135,7 +177,7 @@ export const main = async (clientOpts: ClientOpts) => {
});

app.get(
clientOpts.config.brokerSystemcheckPath || '/systemcheck',
loadedClientOpts.config.brokerSystemcheckPath || '/systemcheck',
(req, res, next) => {
res.locals.clientOpts = loadedClientOpts;
next();
Expand Down
29 changes: 29 additions & 0 deletions lib/client/types/api.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import { ConnectionValidation } from './config';

export interface BrokerConnectionApiResponse {
id: string;
type: string;
attributes: BrokerConnectionAttributes;
}
export interface BrokerConnectionAttributes {
configuration: {
default: {};
required: {};
validations: Array<ConnectionValidation>;
};
name: string;
deployment_id: string;
secrets?: {
primary: {
encrypted: string;
expires_at: string;
nonce: string;
};
secondary: {
encrypted: string;
expires_at: string;
nonce: string;
};
};
type: string;
}
10 changes: 10 additions & 0 deletions lib/client/utils/cleanup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import { readFileSync, writeFileSync } from 'node:fs';

export const cleanUpUniversalFile = (
universalCfgFilePath = `${__dirname}/../../../../config.universal.json`,
) => {
const fileToCleanUpBuffer = readFileSync(universalCfgFilePath);
const fileToCleanUp = JSON.parse(fileToCleanUpBuffer.toString());
delete fileToCleanUp['CONNECTIONS'];
writeFileSync(universalCfgFilePath, JSON.stringify(fileToCleanUp));
};
3 changes: 2 additions & 1 deletion lib/common/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ export const findProjectRoot = (startDir: string): string | null => {
return null;
};

export const loadBrokerConfig = (localConfigForTest?) => {
export const loadBrokerConfig = async (localConfigForTest?) => {
dotenv.config({
path: path.join(process.cwd(), '.env'),
});
Expand Down Expand Up @@ -71,6 +71,7 @@ export const loadBrokerConfig = (localConfigForTest?) => {
config[key] = value.split(',').map((s) => s.trim());
}
}
return;
};

const getConsolidatedConfigForUniversalBroker = (configToConsolidate) => {
Expand Down
11 changes: 11 additions & 0 deletions lib/common/utils/signals.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
export const handleTerminationSignal = (callback: () => void) => {
process.on('SIGINT', () => {
callback();
process.exit(0);
});

process.on('SIGTERM', () => {
callback();
process.exit(0);
});
};
2 changes: 1 addition & 1 deletion lib/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ export const app = async ({ port = 7341, client = false, config }) => {
}

// loading it "manually" simplifies lot testing
loadBrokerConfig();
await loadBrokerConfig();
const globalConfig = getConfig();
const localConfig = Object.assign({}, globalConfig, config) as Record<
string,
Expand Down
Loading

0 comments on commit 3834320

Please sign in to comment.