From f32432f23823cae2cb127d7635f399fc1c9bce27 Mon Sep 17 00:00:00 2001 From: Andrew Thorburn Date: Thu, 1 Sep 2022 16:29:49 +0200 Subject: [PATCH] feat: rename _ARRAY/Array to _POOL/Pool This feature is generally referred to as "token pooling" or "credential pooling", so it seems reasonable to have the suffix of the environment variables be consistent. --- README.md | 22 +++++++------- lib/client/index.js | 12 ++++---- lib/config.js | 30 +++++++++---------- lib/replace-vars.js | 12 ++++---- .../server-client-pooled-credentials.test.js | 3 +- test/functional/systemcheck.test.js | 2 +- test/unit/config.test.ts | 18 +++++------ test/unit/replace-vars.test.ts | 6 ++-- 8 files changed, 53 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 66b5aa5bf..a77160597 100644 --- a/README.md +++ b/README.md @@ -499,16 +499,16 @@ More details can be found here: ### Credential Pooling Under some circumstances it can be desirable to create a "pool" of credentials, e.g., to work around rate-limiting issues. -This can be achieved by creating an environment variable ending in `_ARRAY`, separate each credential with a comma, and +This can be achieved by creating an environment variable ending in `_POOL`, separate each credential with a comma, and the Broker Client will then, when doing variable replacement, look to see if the variable in use has a variant with an -`_ARRAY` suffix, and use the next item in that array if so. For example, if you have set the environment variable +`_POOL` suffix, and use the next item in that array if so. For example, if you have set the environment variable `GITHUB_TOKEN`, but want to provide multiple tokens, you would do this: ```shell -GITHUB_TOKEN_ARRAY=token1, token2, token3 +GITHUB_TOKEN_POOL=token1, token2, token3 ``` -And then the Broker Server would, any time it needed `GITHUB_TOKEN`, instead take an item from the `GITHUB_TOKEN_ARRAY`. +And then the Broker Server would, any time it needed `GITHUB_TOKEN`, instead take an item from the `GITHUB_TOKEN_POOL`. Credentials will be taken in a round-robin fashion, so the first, the second, the third, etc, etc, until it reaches the end and then takes the first one again. @@ -517,7 +517,7 @@ Calling the `/systemcheck` endpoint will validate all credentials, in order, and is the first credential and so on. For example, if you were running the GitHub Client and had this: ```shell -GITHUB_TOKEN_ARRAY=good_token, bad_token +GITHUB_TOKEN_POOL=good_token, bad_token ``` The `/systemcheck` endpoint would return the following, where the first object is for `good_token` and the second for @@ -558,25 +558,25 @@ cases you will need to create multiple accounts with one credential per account. #### Credentials Matrix Generating a Matrix of credentials is not supported. -A "Matrix" in this case is defined as taking two (or more) `_ARRAY`s of length `x` and `y`, and producing one final array +A "Matrix" in this case is defined as taking two (or more) `_POOL`s of length `x` and `y`, and producing one final array of length `x * y`. For example, given an input like: ```shell -USERNAME_ARRAY=u1, u2, u3 -PASSWORD_ARRAY=p1, p2, p3 -CREDENTIALS_ARRAY=$USERNAME:$PASSWORD +USERNAME_POOL=u1, u2, u3 +PASSWORD_POOL=p1, p2, p3 +CREDENTIALS_POOL=$USERNAME:$PASSWORD ``` Matrix support would generate this internally: ```shell -CREDENTIALS_ARRAY=u1:p1,u1:p2,u1:p3,u2:p1,u2:p2,u2:p3,u3:p1,u3:p2,u3:p3 +CREDENTIALS_POOL=u1:p1,u1:p2,u1:p3,u2:p1,u2:p2,u2:p3,u3:p1,u3:p2,u3:p3 ``` Instead, the Broker Client would generate this internally, using only the first array it finds: ```shell -CREDENTIALS_ARRAY=u1:$PASSWORD,u2:$PASSWORD,u3:$PASSWORD +CREDENTIALS_POOL=u1:$PASSWORD,u2:$PASSWORD,u3:$PASSWORD ``` ### Custom approved-listing filter diff --git a/lib/client/index.js b/lib/client/index.js index 6cdd84880..15aae4ce1 100644 --- a/lib/client/index.js +++ b/lib/client/index.js @@ -62,12 +62,12 @@ module.exports = ({port = null, config = {}, filters = {}}) => { } let auths = []; let rawCreds = []; - if (config.brokerClientValidationAuthorizationHeaderArray) { - auths = config.brokerClientValidationAuthorizationHeaderArray; - rawCreds = config.brokerClientValidationAuthorizationHeaderArray.map(credsFromHeader); - } else if (config.brokerClientValidationBasicAuthArray) { - auths = config.brokerClientValidationBasicAuthArray.map(s => `Basic ${Buffer.from(s).toString('base64')}`) - rawCreds = config.brokerClientValidationBasicAuthArray + if (config.brokerClientValidationAuthorizationHeaderPool) { + auths = config.brokerClientValidationAuthorizationHeaderPool; + rawCreds = config.brokerClientValidationAuthorizationHeaderPool.map(credsFromHeader); + } else if (config.brokerClientValidationBasicAuthPool) { + auths = config.brokerClientValidationBasicAuthPool.map(s => `Basic ${Buffer.from(s).toString('base64')}`) + rawCreds = config.brokerClientValidationBasicAuthPool } else if (config.brokerClientValidationAuthorizationHeader) { auths.push(config.brokerClientValidationAuthorizationHeader); rawCreds.push(config.brokerClientValidationAuthorizationHeader); diff --git a/lib/config.js b/lib/config.js index 208bcf78d..cebb79324 100644 --- a/lib/config.js +++ b/lib/config.js @@ -14,35 +14,35 @@ function camelify(res) { } function expandValue(obj, value) { - let arrayFound = undefined; - let keyWithArray = undefined; + let poolFound = undefined; + let keyWithPool = undefined; const variableRegex = /(\\?\$.+?\b)/g; const variableMatcher = value.match(variableRegex); if (variableMatcher) { for (const key of variableMatcher) { - if (key[0] === "$" && obj[(key.slice(1) + "_ARRAY")]) { - keyWithArray = key.slice(1); - arrayFound = (key.slice(1) + "_ARRAY"); + if (key[0] === "$" && obj[(key.slice(1) + "_POOL")]) { + keyWithPool = key.slice(1); + poolFound = (key.slice(1) + "_POOL"); break; } } } - if (arrayFound) { + if (poolFound) { const values = []; - let array; - if (Array.isArray(obj[arrayFound])) { - array = obj[arrayFound]; + let pool; + if (Array.isArray(obj[poolFound])) { + pool = obj[poolFound]; } else { - array = obj[arrayFound].split(",").map(s => s.trim()); - obj[arrayFound] = array; + pool = obj[poolFound].split(",").map(s => s.trim()); + obj[poolFound] = pool; } - for (const o of array) { + for (const o of pool) { values.push(value.replace(variableRegex, (all, key) => { if (key[0] === '$') { const keyToReplace = key.slice(1); - return keyToReplace === keyWithArray ? o : (obj[keyToReplace] || key); + return keyToReplace === keyWithPool ? o : (obj[keyToReplace] || key); } return key; @@ -68,7 +68,7 @@ function expand(obj) { const value = expandValue(obj, obj[key]); if (value && Array.isArray(value)) { // This will get camel-cased later on - obj[(key + "_ARRAY")] = value; + obj[(key + "_POOL")] = value; } else if (value !== obj[key]) { obj[key] = value; } @@ -94,7 +94,7 @@ if (res.caCert) { } for (const [key, value] of Object.entries(res)) { - if ((key.endsWith("Array") || key.endsWith("_ARRAY")) && !Array.isArray(value)) { + if ((key.endsWith("Pool") || key.endsWith("_POOL")) && !Array.isArray(value)) { res[key] = value.split(",").map(s => s.trim()); } } diff --git a/lib/replace-vars.js b/lib/replace-vars.js index 838ab24c2..3b0c72cf2 100644 --- a/lib/replace-vars.js +++ b/lib/replace-vars.js @@ -8,12 +8,12 @@ function replace(input, source) { const key = match.slice(2, -1); // ditch the wrappers let keyName; let idxName; - if (source[(key + "_ARRAY")]) { - keyName = key + "_ARRAY"; - idxName = key + "_ARRAY_IDX"; - } else if (source[(key + "Array")]) { - keyName = key + "Array"; - idxName = key + "ArrayIdx"; + if (source[(key + "_POOL")]) { + keyName = key + "_POOL"; + idxName = key + "_POOL_IDX"; + } else if (source[(key + "Pool")]) { + keyName = key + "Pool"; + idxName = key + "PoolIdx"; } const array = source[keyName]; diff --git a/test/functional/server-client-pooled-credentials.test.js b/test/functional/server-client-pooled-credentials.test.js index 252caaf99..a8fa1d580 100644 --- a/test/functional/server-client-pooled-credentials.test.js +++ b/test/functional/server-client-pooled-credentials.test.js @@ -44,7 +44,8 @@ test('proxy requests originating from behind the broker server with pooled crede process.env.PASSWORD = 'not-used'; process.env.PASSWORD1 = 'aB}#/:%40*1'; process.env.PASSWORD2 = 'aB}#/:%40*2'; - process.env.PASSWORD_ARRAY = '$PASSWORD1, $PASSWORD2'; + process.env.PASSWORD_POOL = '$PASSWORD1, $PASSWORD2'; + process.env.GITHUB_TOKEN_POOL = 'token1, token2'; const client = app.main({ port: port() }); // wait for the client to successfully connect to the server and identify itself diff --git a/test/functional/systemcheck.test.js b/test/functional/systemcheck.test.js index a043be0dd..30fd10acf 100644 --- a/test/functional/systemcheck.test.js +++ b/test/functional/systemcheck.test.js @@ -316,7 +316,7 @@ test('broker client systemcheck endpoint', (t) => { brokerToken: '1234567890', brokerServerUrl: 'http://localhost:12345', brokerClientValidationUrl: 'https://httpbin.org/headers', - brokerClientValidationBasicAuthArray: ['username:password', 'username1:password1'], + brokerClientValidationBasicAuthPool: ['username:password', 'username1:password1'], }, }); diff --git a/test/unit/config.test.ts b/test/unit/config.test.ts index 162af5127..43b90ea56 100644 --- a/test/unit/config.test.ts +++ b/test/unit/config.test.ts @@ -4,11 +4,11 @@ describe('config', () => { const token = (process.env.BROKER_TOKEN = '1234'); const bitbucketTokens = ['1234', '5678']; const githubTokens = ['9012', '3456']; - process.env.BITBUCKET_PASSWORD_ARRAY = '1234, 5678' - process.env.GITHUB_TOKEN_ARRAY = '9012, 3456' + process.env.BITBUCKET_PASSWORD_POOL = '1234, 5678' + process.env.GITHUB_TOKEN_POOL = '9012, 3456' process.env.FOO_BAR = '$FOO/bar'; process.env.GITHUB_USERNAME = 'git' - process.env.GITHUB_PASSWORD_ARRAY = '9012, 3456' + process.env.GITHUB_PASSWORD_POOL = '9012, 3456' process.env.GITHUB_AUTH = 'Basic $GITHUB_USERNAME:$GITHUB_PASSWORD'; const complexToken = (process.env.COMPLEX_TOKEN = '1234$$%#@!$!$@$$$'); @@ -18,11 +18,11 @@ describe('config', () => { expect(config.brokerToken).toEqual(token); expect(config.fooBar).toEqual('bar/bar'); expect(config.complexToken).toEqual(complexToken); - expect(config.bitbucketPasswordArray).toEqual(bitbucketTokens); - expect(config.BITBUCKET_PASSWORD_ARRAY).toEqual(bitbucketTokens); - expect(config.githubTokenArray).toEqual(githubTokens); - expect(config.GITHUB_TOKEN_ARRAY).toEqual(githubTokens); - expect(config.githubAuthArray).toEqual(['Basic git:9012', 'Basic git:3456']); - expect(config.GITHUB_AUTH_ARRAY).toEqual(['Basic git:9012', 'Basic git:3456']); + expect(config.bitbucketPasswordPool).toEqual(bitbucketTokens); + expect(config.BITBUCKET_PASSWORD_POOL).toEqual(bitbucketTokens); + expect(config.githubTokenPool).toEqual(githubTokens); + expect(config.GITHUB_TOKEN_POOL).toEqual(githubTokens); + expect(config.githubAuthPool).toEqual(['Basic git:9012', 'Basic git:3456']); + expect(config.GITHUB_AUTH_POOL).toEqual(['Basic git:9012', 'Basic git:3456']); }); }); diff --git a/test/unit/replace-vars.test.ts b/test/unit/replace-vars.test.ts index eea08c88c..0efd0db55 100644 --- a/test/unit/replace-vars.test.ts +++ b/test/unit/replace-vars.test.ts @@ -60,9 +60,9 @@ describe('replace - with arrays', () => { RES_BODY_URL_SUB: 'http://replac.ed', BROKER_SERVER_URL: 'broker.com', BROKER_TOKEN: 'a-tok-en', - BITBUCKET_PASSWORD_ARRAY: ['1', '2', '3'], - GITHUB_TOKEN_ARRAY: ['1'], - githubTokenArray: ['1'], + BITBUCKET_PASSWORD_POOL: ['1', '2', '3'], + GITHUB_TOKEN_POOL: ['1'], + githubTokenPool: ['1'], }; it('Uses an array if configured - upper case', () => {