From 3a69960e18bd2c51e349a52de1abef326e691545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Fri, 12 Jan 2024 13:46:00 +0000 Subject: [PATCH 1/6] FT: Provide AWS KMS connector for bucket ciphering Implement KMS Client using aws nodejs official client. --- lib/network/index.ts | 1 + lib/network/kms_aws/Client.ts | 203 ++++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 205 insertions(+) create mode 100644 lib/network/kms_aws/Client.ts diff --git a/lib/network/index.ts b/lib/network/index.ts index 96e9a41ec..6eeb2ed34 100644 --- a/lib/network/index.ts +++ b/lib/network/index.ts @@ -11,5 +11,6 @@ export const probe = { ProbeServer }; export { default as RoundRobin } from './RoundRobin'; export { default as kmip } from './kmip'; export { default as kmipClient } from './kmip/Client'; +export { default as awsClient } from './kms_aws/Client'; export * as rpc from './rpc/rpc'; export * as level from './rpc/level-net'; diff --git a/lib/network/kms_aws/Client.ts b/lib/network/kms_aws/Client.ts new file mode 100644 index 000000000..a78b0e211 --- /dev/null +++ b/lib/network/kms_aws/Client.ts @@ -0,0 +1,203 @@ +'use strict'; // eslint-disable-line +/* eslint new-cap: "off" */ + +import errors from '../../errors'; +import * as werelogs from 'werelogs'; +import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, EncryptCommand, DecryptCommand } from "@aws-sdk/client-kms"; +import { AwsCredentialIdentity } from "@smithy/types"; +import assert from 'assert'; + +/** + * Normalize errors according to arsenal definitions + * @param err - an Error instance or a message string + * @returns - arsenal error + * + * @note Copied from the KMIP implementation + */ +function _arsenalError(err: string | Error) { + const messagePrefix = 'AWS_KMS:'; + if (typeof err === 'string') { + return errors.InternalError + .customizeDescription(`${messagePrefix} ${err}`); + } else if ( + err instanceof Error || + // INFO: The second part is here only for Jest, to remove when we'll be + // fully migrated to TS + // @ts-expect-error + (err && typeof err.message === 'string') + ) { + return errors.InternalError + .customizeDescription(`${messagePrefix} ${err.message}`); + } + return errors.InternalError + .customizeDescription(`${messagePrefix} Unspecified error`); +} + +export default class Client { + client: KMSClient; + options: any; + + /** + * Construct a high level KMIP driver suitable for cloudserver + * @param options - Instance options + * @param options.kms_aws - AWS client options + * @param options.kms_aws.region - KMS region + * @param options.kms_aws.endpoint - Endpoint URL of the KMS service + * @param options.kms_aws.ak - Application Key + * @param options.kms_aws.sk - Secret Key + * + * This client also looks in the standard AWS configuration files (https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). + * If no option is passed to this constructor, the client will try to get it from the configuration file. + */ + constructor( + options: { + kms_aws: { + region?: string, + endpoint?: string, + ak?: string, + sk?: string, + } + }, + ) { + let credentials: {credentials: AwsCredentialIdentity} | null = null; + if (options.kms_aws.ak && options.kms_aws.sk) { + credentials = {credentials: { + accessKeyId: options.kms_aws.ak, + secretAccessKey: options.kms_aws.sk, + }}; + } + + this.client = new KMSClient({ + region: options.kms_aws.region, + endpoint: options.kms_aws.endpoint, + ...credentials + }); + } + + /** + * Create a new cryptographic key managed by the server, + * for a specific bucket + * @param bucketName - The bucket name + * @param logger - Werelog logger object + * @param cb - The callback(err: Error, bucketKeyId: String) + */ + createBucketKey(bucketName: string, logger: werelogs.Logger, cb: any) { + logger.debug("AWS KMS: createBucketKey", {bucketName}); + + const command = new CreateKeyCommand({}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::createBucketKey", {err, bucketName}); + cb (error); + } else { + logger.debug("AWS KMS: createBucketKey", {bucketName, KeyMetadata: data?.KeyMetadata}); + cb(null, data?.KeyMetadata?.KeyId); + } + }); + } + + /** + * Destroy a cryptographic key managed by the server, for a specific bucket. + * @param bucketKeyId - The bucket key Id + * @param logger - Werelog logger object + * @param cb - The callback(err: Error) + */ + destroyBucketKey(bucketKeyId: string, logger: werelogs.Logger, cb: any) { + logger.debug("AWS KMS: destroyBucketKey", {bucketKeyId: bucketKeyId}); + + // Schedule a deletion in 7 days (the minimum value on this API) + const command = new ScheduleKeyDeletionCommand({KeyId: bucketKeyId, PendingWindowInDays: 7}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::destroyBucketKey", {err}); + cb (error); + } else { + // Sanity check + if (data?.KeyState != "PendingDeletion") { + const error = _arsenalError("Key is not in PendingDeletion state") + logger.error("AWS_KMS::destroyBucketKey", {err, data}); + cb(error); + } else { + cb(); + } + } + }); + } + + /** + * + * @param cryptoScheme - crypto scheme version number + * @param masterKeyId - key to retrieve master key + * @param plainTextDataKey - data key + * @param logger - werelog logger object + * @param cb - callback + * @callback called with (err, cipheredDataKey: Buffer) + */ + cipherDataKey( + cryptoScheme: number, + masterKeyId: string, + plainTextDataKey: Buffer, + logger: werelogs.Logger, + cb: any, + ) { + logger.debug("AWS KMS: cipherDataKey", {cryptoScheme, masterKeyId}); + + // Only support cryptoScheme v1 + assert.strictEqual (cryptoScheme, 1); + + const command = new EncryptCommand({KeyId: masterKeyId, Plaintext: plainTextDataKey}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::cipherDataKey", {err}); + cb (error); + } else if (!data) { + const error = _arsenalError("cipherDataKey: empty response"); + logger.error("AWS_KMS::cipherDataKey empty reponse"); + cb (error); + } else { + // Convert to a buffer. This allows the wrapper to use .toString("base64") + cb(null, Buffer.from(data.CiphertextBlob!)); + } + }); + } + + /** + * + * @param cryptoScheme - crypto scheme version number + * @param masterKeyId - key to retrieve master key + * @param cipheredDataKey - data key + * @param logger - werelog logger object + * @param cb - callback + * @callback called with (err, plainTextDataKey: Buffer) + */ + decipherDataKey( + cryptoScheme: number, + masterKeyId: string, + cipheredDataKey: Buffer, + logger: werelogs.Logger, + cb: any, + ) { + logger.debug("AWS KMS: decipherDataKey", {cryptoScheme, masterKeyId}); + + // Only support cryptoScheme v1 + assert.strictEqual (cryptoScheme, 1); + + const command = new DecryptCommand({CiphertextBlob: cipheredDataKey}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::decipherDataKey", {err}); + cb (error); + } else if (!data) { + const error = _arsenalError("decipherDataKey: empty response"); + logger.error("AWS_KMS::decipherDataKey empty reponse"); + cb (error); + } else { + cb(null, Buffer.from(data?.Plaintext!)); + } + }); + } +} diff --git a/package.json b/package.json index a4c3c2787..c9e01b930 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ }, "homepage": "https://github.com/scality/Arsenal#readme", "dependencies": { + "@aws-sdk/client-kms": "^3.485.0", "@js-sdsl/ordered-set": "^4.4.2", "@types/async": "^3.2.12", "@types/utf8": "^3.0.1", From d59e41b40bf4ffef99de333d1dbb733d172b5a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Fri, 19 Jan 2024 13:57:11 +0000 Subject: [PATCH 2/6] FT: AWS KMS, implement the generateDataKey method Implement the generateDataKey method that create a datakey and cipher it in 1 operation. The result is the datakey in both plaintext and cipher forms. This new method is detected by cloudserver and used preferentially when available. --- lib/network/kms_aws/Client.ts | 37 ++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/network/kms_aws/Client.ts b/lib/network/kms_aws/Client.ts index a78b0e211..b950d7116 100644 --- a/lib/network/kms_aws/Client.ts +++ b/lib/network/kms_aws/Client.ts @@ -3,7 +3,7 @@ import errors from '../../errors'; import * as werelogs from 'werelogs'; -import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, EncryptCommand, DecryptCommand } from "@aws-sdk/client-kms"; +import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, EncryptCommand, DecryptCommand, GenerateDataKeyCommand, DataKeySpec } from "@aws-sdk/client-kms"; import { AwsCredentialIdentity } from "@smithy/types"; import assert from 'assert'; @@ -126,6 +126,41 @@ export default class Client { }); } + /** + * @param cryptoScheme - crypto scheme version number + * @param masterKeyId - key to retrieve master key + * @param logger - werelog logger object + * @param cb - callback + * @callback called with (err, plainTextDataKey: Buffer, cipheredDataKey: Buffer) + */ + generateDataKey( + cryptoScheme: number, + masterKeyId: string, + logger: werelogs.Logger, + cb: any, + ) { + logger.debug("AWS KMS: generateDataKey", {cryptoScheme, masterKeyId}); + + // Only support cryptoScheme v1 + assert.strictEqual (cryptoScheme, 1); + + const command = new GenerateDataKeyCommand({KeyId: masterKeyId, KeySpec: DataKeySpec.AES_256}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::generateDataKey", {err}); + cb (error); + } else if (!data) { + const error = _arsenalError("generateDataKey: empty response"); + logger.error("AWS_KMS::generateDataKey empty reponse"); + cb (error); + } else { + // Convert to a buffer. This allows the wrapper to use .toString("base64") + cb(null, Buffer.from(data.Plaintext!), Buffer.from(data.CiphertextBlob!)); + } + }); + } + /** * * @param cryptoScheme - crypto scheme version number From 0efbedd1e57d78196400f3e42eaaf57b8b9a8445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Fri, 26 Jan 2024 12:44:49 +0000 Subject: [PATCH 3/6] FT: AWS KMS, add unit tests --- package.json | 1 + tests/functional/kms_aws/highlevel.spec.js | 259 +++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 tests/functional/kms_aws/highlevel.spec.js diff --git a/package.json b/package.json index c9e01b930..483de22d1 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@types/jest": "^27.4.1", "@types/node": "^17.0.21", "@types/xml2js": "^0.4.11", + "aws-sdk-client-mock": "^3.0.1", "eslint": "^8.12.0", "eslint-config-airbnb": "6.2.0", "eslint-config-scality": "scality/Guidelines#7.10.2", diff --git a/tests/functional/kms_aws/highlevel.spec.js b/tests/functional/kms_aws/highlevel.spec.js new file mode 100644 index 000000000..28847fab6 --- /dev/null +++ b/tests/functional/kms_aws/highlevel.spec.js @@ -0,0 +1,259 @@ +'use strict'; // eslint-disable-line strict + +const assert = require('assert'); +const async = require('async'); + +// Mocking official nodejs aws client +const { mockClient } = require('aws-sdk-client-mock') +const { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, GenerateDataKeyCommand, EncryptCommand, DecryptCommand } = require('@aws-sdk/client-kms') + +const arsenalErrors = require ('../../../lib/errors/arsenalErrors'); +const KMS_AWS_Client = require('../../../lib/network/kms_aws/Client').default; + +describe('KMS AWS Client', () => { + const options = { + kms_aws: { + region: "test-region-1", + endpoint: "http://mocked.doesnt.matter" + } + }; + + const kmsClient = new KMS_AWS_Client(options); + + // Mock the AWS client + let mockedAwsClient = mockClient(KMSClient); + + // Mock the send method to allow using callbacks + // Note: we cannot replace the whole aws client with its mocked version + // because the current mocking implementation doesn't support callbacks + // See https://github.com/m-radzikowski/aws-sdk-client-mock/issues/6 + kmsClient.client.send = function (cmd, cbOrOption, cb = null) { + if (! cb && typeof(cbOrOption) === "function") { + cb = cbOrOption; + } + + let mocked_call = mockedAwsClient.send(cmd); + + // mocked_call is undefined when parameters doesn't match those expected. + expect(mocked_call).toBeDefined() + + let got_error = false; + mocked_call.catch((err) => { + got_error = true; + cb(err) + }) + // Order is important here: the catch is done before + // so it doesn't catch exceptions raised by Jest in case of + // test failure + .then((data) => { + // Looks like this "then" statement is alway run after + // a catch. We check for errors handled in the catch to + // avoid a double invocation of the callback. + if (!got_error) { + cb(null, data); + } + }) + } + + beforeEach(() => { + mockedAwsClient.reset(); + }); + + it('should create a new key on bucket creation', done => { + mockedAwsClient.on(CreateKeyCommand).resolves({ + KeyMetadata: { + KeyId: "mocked-kms-key-id" + } + }); + + kmsClient.createBucketKey('plop', logger, (err, bucketKeyId) => { + // Check the result + expect(err).toBeNull(); + expect(bucketKeyId).toEqual("mocked-kms-key-id"); + + // Check that the CreateKey of the aws client have been used to create the key + expect(mockedAwsClient.commandCalls(CreateKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors creating the key on bucket creation', done => { + mockedAwsClient.on(CreateKeyCommand).rejects("Error"); + + kmsClient.createBucketKey('plop', logger, (err, bucketKeyId) => { + // Check the result + expect(bucketKeyId).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the CreateKey of the aws client have been used to create the key + expect(mockedAwsClient.commandCalls(CreateKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should delete an existing key on bucket deletion', done => { + mockedAwsClient.on(ScheduleKeyDeletionCommand, { + KeyId: "mocked-kms-key-id", + PendingWindowInDays: 7 // Should be set to 7 (the minimum accepted value on this operation) + }).resolves({ + KeyId: "mocked-kms-key-id", + KeyState: "PendingDeletion", + PendingWindowInDays: 7 + }); + + kmsClient.destroyBucketKey('mocked-kms-key-id', logger, (err) => { + // Check the result + expect(err).toBeUndefined(); + + // Check that the ScheduleKeyDeletion of the aws client have been invoked + expect(mockedAwsClient.commandCalls(ScheduleKeyDeletionCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors deleting an existing key on bucket deletion', done => { + mockedAwsClient.on(ScheduleKeyDeletionCommand, { + KeyId: "mocked-kms-key-id", + PendingWindowInDays: 7 // Should be set to 7 (the minimum accepted value on this operation) + }).rejects("Error"); + + kmsClient.destroyBucketKey('mocked-kms-key-id', logger, (err) => { + // Check the result + expect(err).toEqual(Error('InternalError')); + + // Check that the ScheduleKeyDeletion of the aws client have been invoked + expect(mockedAwsClient.commandCalls(ScheduleKeyDeletionCommand).length).toEqual(1); + + done(); + }); + }); + + it('should generate a datakey for ciphering', done => { + mockedAwsClient.on(GenerateDataKeyCommand).resolves({ + CiphertextBlob: "encryptedDataKey", + Plaintext: "dataKey", + KeyId: "mocked-kms-key-id" + }); + + kmsClient.generateDataKey(1, "mocked-kms-key-id", logger, (err, plaintextDataKey, cipheredDataKey) => { + // Check the result + expect(err).toBeNull(); + expect(plaintextDataKey).toEqual(Buffer("dataKey")); + expect(cipheredDataKey).toEqual(Buffer("encryptedDataKey")); + + // Check that the the aws client have been used to create the data key + expect(mockedAwsClient.commandCalls(GenerateDataKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors generating a datakey', done => { + mockedAwsClient.on(GenerateDataKeyCommand).rejects("Error"); + + kmsClient.generateDataKey(1, "mocked-kms-key-id", logger, (err, plaintextDataKey, cipheredDataKey) => { + // Check the result + expect(plaintextDataKey).toBeUndefined(); + expect(cipheredDataKey).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the the aws client have been used to create the data key + expect(mockedAwsClient.commandCalls(GenerateDataKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should allow ciphering a datakey', done => { + mockedAwsClient.on(EncryptCommand, { + KeyId: "mocked-kms-key-id", + Plaintext: "dataKey-value", + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined + }).resolves({ + CiphertextBlob: "encryptedDataKey-value", + KeyId: "mocked-kms-key-id" + }); + + kmsClient.cipherDataKey(1, "mocked-kms-key-id", "dataKey-value", logger, (err, cipheredDataKey) => { + // Check the result + expect(err).toBeNull(); + expect(cipheredDataKey).toEqual(Buffer("encryptedDataKey-value")); + + // Check that the Encrypt of the aws client has been used + expect(mockedAwsClient.commandCalls(EncryptCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors ciphering a datakey', done => { + mockedAwsClient.on(EncryptCommand, { + KeyId: "mocked-kms-key-id", + Plaintext: "dataKey-value", + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined + }).rejects("Error"); + + kmsClient.cipherDataKey(1, "mocked-kms-key-id", "dataKey-value", logger, (err, cipheredDataKey) => { + // Check the result + expect(cipheredDataKey).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the Encrypt of the aws client have been used + expect(mockedAwsClient.commandCalls(EncryptCommand).length).toEqual(1); + + done(); + }); + }); + + it('should allow deciphering a datakey', done => { + mockedAwsClient.on(DecryptCommand, { + KeyId: undefined, // Key id is embedded in the CiphertextBlob + CiphertextBlob: "encryptedDataKey-value", + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined + }).resolves({ + Plaintext: "dataKey-value", + KeyId: "mocked-kms-key-id" + }); + + kmsClient.decipherDataKey(1, "mocked-kms-key-id", "encryptedDataKey-value", logger, (err, plainTextDataKey) => { + // Check the result + expect(err).toBeNull(); + expect(plainTextDataKey).toEqual(Buffer("dataKey-value")); + + // Check that the Decrypt of the aws client have been used + expect(mockedAwsClient.commandCalls(DecryptCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors deciphering a datakey', done => { + mockedAwsClient.on(DecryptCommand, { + KeyId: undefined, // Key id is embedded in the CiphertextBlob + CiphertextBlob: "encryptedDataKey-value", + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined + }).rejects("Error"); + + kmsClient.decipherDataKey(1, "mocked-kms-key-id", "encryptedDataKey-value", logger, (err, plainTextDataKey) => { + // Check the result + expect(plainTextDataKey).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the Decrypt of the aws client have been used + expect(mockedAwsClient.commandCalls(DecryptCommand).length).toEqual(1); + + done(); + }); + }); +}); From a44aa81f9f7dcc2f85db6120616d70b72cb37ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Mon, 24 Jun 2024 14:50:54 +0000 Subject: [PATCH 4/6] FT: AWS KMS, fix lint errors --- lib/network/index.ts | 2 +- lib/network/{kms_aws => kmsAWS}/Client.ts | 22 +-- .../{kms_aws => kmsAWS}/highlevel.spec.js | 165 +++++++++--------- 3 files changed, 97 insertions(+), 92 deletions(-) rename lib/network/{kms_aws => kmsAWS}/Client.ts (93%) rename tests/functional/{kms_aws => kmsAWS}/highlevel.spec.js (63%) diff --git a/lib/network/index.ts b/lib/network/index.ts index 6eeb2ed34..d97640a03 100644 --- a/lib/network/index.ts +++ b/lib/network/index.ts @@ -11,6 +11,6 @@ export const probe = { ProbeServer }; export { default as RoundRobin } from './RoundRobin'; export { default as kmip } from './kmip'; export { default as kmipClient } from './kmip/Client'; -export { default as awsClient } from './kms_aws/Client'; +export { default as awsClient } from './kmsAWS/Client'; export * as rpc from './rpc/rpc'; export * as level from './rpc/level-net'; diff --git a/lib/network/kms_aws/Client.ts b/lib/network/kmsAWS/Client.ts similarity index 93% rename from lib/network/kms_aws/Client.ts rename to lib/network/kmsAWS/Client.ts index b950d7116..190975066 100644 --- a/lib/network/kms_aws/Client.ts +++ b/lib/network/kmsAWS/Client.ts @@ -40,18 +40,18 @@ export default class Client { /** * Construct a high level KMIP driver suitable for cloudserver * @param options - Instance options - * @param options.kms_aws - AWS client options - * @param options.kms_aws.region - KMS region - * @param options.kms_aws.endpoint - Endpoint URL of the KMS service - * @param options.kms_aws.ak - Application Key - * @param options.kms_aws.sk - Secret Key + * @param options.kmsAWS - AWS client options + * @param options.kmsAWS.region - KMS region + * @param options.kmsAWS.endpoint - Endpoint URL of the KMS service + * @param options.kmsAWS.ak - Application Key + * @param options.kmsAWS.sk - Secret Key * * This client also looks in the standard AWS configuration files (https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). * If no option is passed to this constructor, the client will try to get it from the configuration file. */ constructor( options: { - kms_aws: { + kmsAWS: { region?: string, endpoint?: string, ak?: string, @@ -60,16 +60,16 @@ export default class Client { }, ) { let credentials: {credentials: AwsCredentialIdentity} | null = null; - if (options.kms_aws.ak && options.kms_aws.sk) { + if (options.kmsAWS.ak && options.kmsAWS.sk) { credentials = {credentials: { - accessKeyId: options.kms_aws.ak, - secretAccessKey: options.kms_aws.sk, + accessKeyId: options.kmsAWS.ak, + secretAccessKey: options.kmsAWS.sk, }}; } this.client = new KMSClient({ - region: options.kms_aws.region, - endpoint: options.kms_aws.endpoint, + region: options.kmsAWS.region, + endpoint: options.kmsAWS.endpoint, ...credentials }); } diff --git a/tests/functional/kms_aws/highlevel.spec.js b/tests/functional/kmsAWS/highlevel.spec.js similarity index 63% rename from tests/functional/kms_aws/highlevel.spec.js rename to tests/functional/kmsAWS/highlevel.spec.js index 28847fab6..7b33f0fe2 100644 --- a/tests/functional/kms_aws/highlevel.spec.js +++ b/tests/functional/kmsAWS/highlevel.spec.js @@ -1,47 +1,52 @@ 'use strict'; // eslint-disable-line strict -const assert = require('assert'); -const async = require('async'); - // Mocking official nodejs aws client -const { mockClient } = require('aws-sdk-client-mock') -const { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, GenerateDataKeyCommand, EncryptCommand, DecryptCommand } = require('@aws-sdk/client-kms') +const { mockClient } = require('aws-sdk-client-mock'); +const { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, GenerateDataKeyCommand, + EncryptCommand, DecryptCommand } = require('@aws-sdk/client-kms'); + +const KmsAWSClient = require('../../../lib/network/kmsAWS/Client').default; -const arsenalErrors = require ('../../../lib/errors/arsenalErrors'); -const KMS_AWS_Client = require('../../../lib/network/kms_aws/Client').default; +const logger = { + info: () => {}, + debug: () => {}, + warn: () => {}, + error: () => {}, +}; describe('KMS AWS Client', () => { const options = { - kms_aws: { - region: "test-region-1", - endpoint: "http://mocked.doesnt.matter" - } + kmsAWS: { + region: 'test-region-1', + endpoint: 'http://mocked.doesnt.matter', + }, }; - const kmsClient = new KMS_AWS_Client(options); + const kmsClient = new KmsAWSClient(options); // Mock the AWS client - let mockedAwsClient = mockClient(KMSClient); + const mockedAwsClient = mockClient(KMSClient); // Mock the send method to allow using callbacks // Note: we cannot replace the whole aws client with its mocked version // because the current mocking implementation doesn't support callbacks // See https://github.com/m-radzikowski/aws-sdk-client-mock/issues/6 - kmsClient.client.send = function (cmd, cbOrOption, cb = null) { - if (! cb && typeof(cbOrOption) === "function") { - cb = cbOrOption; + kmsClient.client.send = function mockedSend(cmd, cbOrOption, cb = null) { + let callback = cb; + if (! cb && typeof(cbOrOption) === 'function') { + callback = cbOrOption; } - let mocked_call = mockedAwsClient.send(cmd); + const mockedCall = mockedAwsClient.send(cmd); - // mocked_call is undefined when parameters doesn't match those expected. - expect(mocked_call).toBeDefined() + // mockedCall is undefined when parameters doesn't match those expected. + expect(mockedCall).toBeDefined(); - let got_error = false; - mocked_call.catch((err) => { - got_error = true; - cb(err) - }) + let gotError = false; + mockedCall.catch((err) => { + gotError = true; + callback(err); + }) // Order is important here: the catch is done before // so it doesn't catch exceptions raised by Jest in case of // test failure @@ -49,11 +54,11 @@ describe('KMS AWS Client', () => { // Looks like this "then" statement is alway run after // a catch. We check for errors handled in the catch to // avoid a double invocation of the callback. - if (!got_error) { - cb(null, data); + if (!gotError) { + callback(null, data); } - }) - } + }); + }; beforeEach(() => { mockedAwsClient.reset(); @@ -62,14 +67,14 @@ describe('KMS AWS Client', () => { it('should create a new key on bucket creation', done => { mockedAwsClient.on(CreateKeyCommand).resolves({ KeyMetadata: { - KeyId: "mocked-kms-key-id" - } + KeyId: 'mocked-kms-key-id', + }, }); - + kmsClient.createBucketKey('plop', logger, (err, bucketKeyId) => { // Check the result expect(err).toBeNull(); - expect(bucketKeyId).toEqual("mocked-kms-key-id"); + expect(bucketKeyId).toEqual('mocked-kms-key-id'); // Check that the CreateKey of the aws client have been used to create the key expect(mockedAwsClient.commandCalls(CreateKeyCommand).length).toEqual(1); @@ -79,8 +84,8 @@ describe('KMS AWS Client', () => { }); it('should handle errors creating the key on bucket creation', done => { - mockedAwsClient.on(CreateKeyCommand).rejects("Error"); - + mockedAwsClient.on(CreateKeyCommand).rejects('Error'); + kmsClient.createBucketKey('plop', logger, (err, bucketKeyId) => { // Check the result expect(bucketKeyId).toBeUndefined(); @@ -95,14 +100,14 @@ describe('KMS AWS Client', () => { it('should delete an existing key on bucket deletion', done => { mockedAwsClient.on(ScheduleKeyDeletionCommand, { - KeyId: "mocked-kms-key-id", - PendingWindowInDays: 7 // Should be set to 7 (the minimum accepted value on this operation) + KeyId: 'mocked-kms-key-id', + PendingWindowInDays: 7, // Should be set to 7 (the minimum accepted value on this operation) }).resolves({ - KeyId: "mocked-kms-key-id", - KeyState: "PendingDeletion", - PendingWindowInDays: 7 + KeyId: 'mocked-kms-key-id', + KeyState: 'PendingDeletion', + PendingWindowInDays: 7, }); - + kmsClient.destroyBucketKey('mocked-kms-key-id', logger, (err) => { // Check the result expect(err).toBeUndefined(); @@ -116,10 +121,10 @@ describe('KMS AWS Client', () => { it('should handle errors deleting an existing key on bucket deletion', done => { mockedAwsClient.on(ScheduleKeyDeletionCommand, { - KeyId: "mocked-kms-key-id", - PendingWindowInDays: 7 // Should be set to 7 (the minimum accepted value on this operation) - }).rejects("Error"); - + KeyId: 'mocked-kms-key-id', + PendingWindowInDays: 7, // Should be set to 7 (the minimum accepted value on this operation) + }).rejects('Error'); + kmsClient.destroyBucketKey('mocked-kms-key-id', logger, (err) => { // Check the result expect(err).toEqual(Error('InternalError')); @@ -133,16 +138,16 @@ describe('KMS AWS Client', () => { it('should generate a datakey for ciphering', done => { mockedAwsClient.on(GenerateDataKeyCommand).resolves({ - CiphertextBlob: "encryptedDataKey", - Plaintext: "dataKey", - KeyId: "mocked-kms-key-id" + CiphertextBlob: 'encryptedDataKey', + Plaintext: 'dataKey', + KeyId: 'mocked-kms-key-id', }); - - kmsClient.generateDataKey(1, "mocked-kms-key-id", logger, (err, plaintextDataKey, cipheredDataKey) => { + + kmsClient.generateDataKey(1, 'mocked-kms-key-id', logger, (err, plaintextDataKey, cipheredDataKey) => { // Check the result expect(err).toBeNull(); - expect(plaintextDataKey).toEqual(Buffer("dataKey")); - expect(cipheredDataKey).toEqual(Buffer("encryptedDataKey")); + expect(plaintextDataKey).toEqual(Buffer.from('dataKey')); + expect(cipheredDataKey).toEqual(Buffer.from('encryptedDataKey')); // Check that the the aws client have been used to create the data key expect(mockedAwsClient.commandCalls(GenerateDataKeyCommand).length).toEqual(1); @@ -152,9 +157,9 @@ describe('KMS AWS Client', () => { }); it('should handle errors generating a datakey', done => { - mockedAwsClient.on(GenerateDataKeyCommand).rejects("Error"); - - kmsClient.generateDataKey(1, "mocked-kms-key-id", logger, (err, plaintextDataKey, cipheredDataKey) => { + mockedAwsClient.on(GenerateDataKeyCommand).rejects('Error'); + + kmsClient.generateDataKey(1, 'mocked-kms-key-id', logger, (err, plaintextDataKey, cipheredDataKey) => { // Check the result expect(plaintextDataKey).toBeUndefined(); expect(cipheredDataKey).toBeUndefined(); @@ -169,20 +174,20 @@ describe('KMS AWS Client', () => { it('should allow ciphering a datakey', done => { mockedAwsClient.on(EncryptCommand, { - KeyId: "mocked-kms-key-id", - Plaintext: "dataKey-value", + KeyId: 'mocked-kms-key-id', + Plaintext: 'dataKey-value', EncryptionContext: undefined, GrantTokens: undefined, - EncryptionAlgorithm: undefined + EncryptionAlgorithm: undefined, }).resolves({ - CiphertextBlob: "encryptedDataKey-value", - KeyId: "mocked-kms-key-id" + CiphertextBlob: 'encryptedDataKey-value', + KeyId: 'mocked-kms-key-id', }); - - kmsClient.cipherDataKey(1, "mocked-kms-key-id", "dataKey-value", logger, (err, cipheredDataKey) => { + + kmsClient.cipherDataKey(1, 'mocked-kms-key-id', 'dataKey-value', logger, (err, cipheredDataKey) => { // Check the result expect(err).toBeNull(); - expect(cipheredDataKey).toEqual(Buffer("encryptedDataKey-value")); + expect(cipheredDataKey).toEqual(Buffer.from('encryptedDataKey-value')); // Check that the Encrypt of the aws client has been used expect(mockedAwsClient.commandCalls(EncryptCommand).length).toEqual(1); @@ -193,14 +198,14 @@ describe('KMS AWS Client', () => { it('should handle errors ciphering a datakey', done => { mockedAwsClient.on(EncryptCommand, { - KeyId: "mocked-kms-key-id", - Plaintext: "dataKey-value", + KeyId: 'mocked-kms-key-id', + Plaintext: 'dataKey-value', EncryptionContext: undefined, GrantTokens: undefined, - EncryptionAlgorithm: undefined - }).rejects("Error"); - - kmsClient.cipherDataKey(1, "mocked-kms-key-id", "dataKey-value", logger, (err, cipheredDataKey) => { + EncryptionAlgorithm: undefined, + }).rejects('Error'); + + kmsClient.cipherDataKey(1, 'mocked-kms-key-id', 'dataKey-value', logger, (err, cipheredDataKey) => { // Check the result expect(cipheredDataKey).toBeUndefined(); expect(err).toEqual(Error('InternalError')); @@ -215,19 +220,19 @@ describe('KMS AWS Client', () => { it('should allow deciphering a datakey', done => { mockedAwsClient.on(DecryptCommand, { KeyId: undefined, // Key id is embedded in the CiphertextBlob - CiphertextBlob: "encryptedDataKey-value", + CiphertextBlob: 'encryptedDataKey-value', EncryptionContext: undefined, GrantTokens: undefined, - EncryptionAlgorithm: undefined + EncryptionAlgorithm: undefined, }).resolves({ - Plaintext: "dataKey-value", - KeyId: "mocked-kms-key-id" + Plaintext: 'dataKey-value', + KeyId: 'mocked-kms-key-id', }); - - kmsClient.decipherDataKey(1, "mocked-kms-key-id", "encryptedDataKey-value", logger, (err, plainTextDataKey) => { + + kmsClient.decipherDataKey(1, 'mocked-kms-key-id', 'encryptedDataKey-value', logger, (err, plainTextDataKey) => { // Check the result expect(err).toBeNull(); - expect(plainTextDataKey).toEqual(Buffer("dataKey-value")); + expect(plainTextDataKey).toEqual(Buffer.from('dataKey-value')); // Check that the Decrypt of the aws client have been used expect(mockedAwsClient.commandCalls(DecryptCommand).length).toEqual(1); @@ -239,13 +244,13 @@ describe('KMS AWS Client', () => { it('should handle errors deciphering a datakey', done => { mockedAwsClient.on(DecryptCommand, { KeyId: undefined, // Key id is embedded in the CiphertextBlob - CiphertextBlob: "encryptedDataKey-value", + CiphertextBlob: 'encryptedDataKey-value', EncryptionContext: undefined, GrantTokens: undefined, - EncryptionAlgorithm: undefined - }).rejects("Error"); - - kmsClient.decipherDataKey(1, "mocked-kms-key-id", "encryptedDataKey-value", logger, (err, plainTextDataKey) => { + EncryptionAlgorithm: undefined, + }).rejects('Error'); + + kmsClient.decipherDataKey(1, 'mocked-kms-key-id', 'encryptedDataKey-value', logger, (err, plainTextDataKey) => { // Check the result expect(plainTextDataKey).toBeUndefined(); expect(err).toEqual(Error('InternalError')); From 4755f8e91fafec4d4439d525b1800cc42ce8aff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Tue, 30 Jul 2024 10:08:06 +0000 Subject: [PATCH 5/6] AWS KMS: TLS configuration --- lib/network/kmsAWS/Client.ts | 40 +++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/network/kmsAWS/Client.ts b/lib/network/kmsAWS/Client.ts index 190975066..6d2719138 100644 --- a/lib/network/kmsAWS/Client.ts +++ b/lib/network/kmsAWS/Client.ts @@ -2,8 +2,11 @@ /* eslint new-cap: "off" */ import errors from '../../errors'; +import { Agent } from "https"; +import { SecureVersion } from "tls"; import * as werelogs from 'werelogs'; import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, EncryptCommand, DecryptCommand, GenerateDataKeyCommand, DataKeySpec } from "@aws-sdk/client-kms"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; import { AwsCredentialIdentity } from "@smithy/types"; import assert from 'assert'; @@ -45,9 +48,17 @@ export default class Client { * @param options.kmsAWS.endpoint - Endpoint URL of the KMS service * @param options.kmsAWS.ak - Application Key * @param options.kmsAWS.sk - Secret Key + * @param options.kmsAWS.tls.rejectUnauthorized - default to true, reject unauthenticated TLS connections (set to false to accept auto-signed certificates, useful in development ONLY) + * @param options.kmsAWS.tls.ca - override CA definition(s) + * @param options.kmsAWS.tls.cert - certificate or list of certificates + * @param options.kmsAWS.tls.minVersion - min TLS version accepted, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) + * @param options.kmsAWS.tls.maxVersion - max TLS version accepted, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) + * @param options.kmsAWS.tls.key - private key or list of private keys * * This client also looks in the standard AWS configuration files (https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). * If no option is passed to this constructor, the client will try to get it from the configuration file. + * + * TLS configuration options are those of nodejs, you can refere to https://nodejs.org/api/tls.html#tlsconnectoptions and https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions */ constructor( options: { @@ -56,9 +67,35 @@ export default class Client { endpoint?: string, ak?: string, sk?: string, + tls?: { + rejectUnauthorized?: boolean, + ca?: [Buffer] | Buffer, + cert?: [Buffer] | Buffer, + minVersion?: string, + maxVersion?: string, + key?: [Buffer] | Buffer, + } } }, ) { + let requestHandler: {requestHandler: NodeHttpHandler} | null = null; + const tlsOpts = options.kmsAWS.tls; + if (tlsOpts) { + const agent = new Agent({ + rejectUnauthorized: tlsOpts?.rejectUnauthorized, + ca: tlsOpts?.ca, + cert: tlsOpts?.cert, + minVersion: tlsOpts?.minVersion, + maxVersion: tlsOpts?.maxVersion, + key: tlsOpts?.key, + }); + + requestHandler = {requestHandler: new NodeHttpHandler({ + httpAgent: agent, + httpsAgent: agent, + })} + } + let credentials: {credentials: AwsCredentialIdentity} | null = null; if (options.kmsAWS.ak && options.kmsAWS.sk) { credentials = {credentials: { @@ -70,7 +107,8 @@ export default class Client { this.client = new KMSClient({ region: options.kmsAWS.region, endpoint: options.kmsAWS.endpoint, - ...credentials + ...credentials, + ...requestHandler }); } From 6d693d9ad27ca9c76f6c1046a5553e97acbd1cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Tue, 11 Jun 2024 07:02:05 +0000 Subject: [PATCH 6/6] FT: AWS KMS, add a readme for configuration --- lib/network/kmsAWS/README.md | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 lib/network/kmsAWS/README.md diff --git a/lib/network/kmsAWS/README.md b/lib/network/kmsAWS/README.md new file mode 100644 index 000000000..e7ec7e79c --- /dev/null +++ b/lib/network/kmsAWS/README.md @@ -0,0 +1,58 @@ +# AWS KMS connector + +Allow to use AWS KMS backend for encryption of objects. It currently only support AK+SK for authentication. +mTLS can also be used to add extra security. + +## Configuration + +Configuration is done using the configuration file or environment variables. A Mix of both can be used, the configuration file takes precedence over environment variables. +Environment variables are the same as the ones used by the AWS CLI prefixed with "KMS_" (in order to scope them to the KMS module). + +The following parameters are supported: + +| config file | env variable | Description +|---------------------|--------------------------------------------------|------------ +| kmsAWS.region | KMS_AWS_REGION or KMS_AWS_DEFAULT_REGION | AWS region tu use +| kmsAWS.endpoint | KMS_AWS_ENDPOINT_URL_KMS or KMS_AWS_ENDPOINT_URL | Endpoint URL +| kmsAWS.ak | KMS_AWS_ACCESS_KEY_ID | Credentials, Access Key +| kmsAWS.sk | KMS_AWS_SECRET_ACCESS_KEY | Credentials, Secret Key +| kmsAWS.tls | | TLS configuration (Object, see below) + +The TLS configuration is can contain the following attributes: + +| config file | Description +|---------------------|-------------------------------------------------- +| rejectUnauthorized | false to disable TLS certificates checks (useful in development, DON'T disable in production) +| minVersion | min TLS version, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) +| maxVersion | max TLS version, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) +| ca | filename or array of filenames containing CA(s) +| cert | filename or array of filenames containing certificate(s) +| key | filename or array of filenames containing private key(s) + +All TLS attributes conform to their nodejs definition. See https://nodejs.org/api/tls.html#tlsconnectoptions and https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions. + +Configuration example: +```json + "kmsAWS": { + "region": "us-east-1", + "endpoint": "https://kms.us-east-1.amazonaws.com", + "ak": "xxxxxxx", + "sk": "xxxxxxx" + }, +``` + +With TLS configuration: +```json + "kmsAWS": { + "region": "us-east-1", + "endpoint": "https://kms.us-east-1.amazonaws.com", + "ak": "xxxxxxx", + "sk": "xxxxxxx", + "tls": { + "rejectUnauthorized": false, + "cert": "mtls.crt.pem", + "key": "mtls.key.pem", + "minVersion": "TLSv1.3" + } + }, +```