From bada542bbb8b46026a903d45b622562e65348e08 Mon Sep 17 00:00:00 2001 From: Dawid Heyman Date: Thu, 21 Nov 2024 13:54:52 +0100 Subject: [PATCH] SNOW-1524258: Implement GCM encryption (#966) --- lib/file_transfer_agent/encrypt_util.js | 393 +++++++++++------- test/integration/testEncrypt.js | 83 ++++ .../file_transfer_agent/encrypt_util_test.js | 11 +- 3 files changed, 320 insertions(+), 167 deletions(-) create mode 100644 test/integration/testEncrypt.js diff --git a/lib/file_transfer_agent/encrypt_util.js b/lib/file_transfer_agent/encrypt_util.js index 3f48b6f17..835634b43 100644 --- a/lib/file_transfer_agent/encrypt_util.js +++ b/lib/file_transfer_agent/encrypt_util.js @@ -12,6 +12,28 @@ const blockSize = parseInt(AES_BLOCK_SIZE / 8); // in bytes const QUERY_STAGE_MASTER_KEY = 'queryStageMasterKey'; const BASE64 = 'base64'; +const DEFAULT_AAD = Buffer.from(''); +const AUTH_TAG_LENGTH_IN_BYTES = 16; + +const AES_CBC = { + cipherName: function (keySizeInBytes) { + return `aes-${keySizeInBytes * 8}-cbc`; + }, + ivSize: 16 +}; + +const AES_ECB = { + cipherName: function (keySizeInBytes) { + return `aes-${keySizeInBytes * 8}-ecb`; + } +}; + +const AES_GCM = { + cipherName: function (keySizeInBytes) { + return `aes-${keySizeInBytes * 8}-gcm`; + }, + ivSize: 12 +}; // Material Descriptor function MaterialDescriptor(smkId, queryId, keySize) { @@ -23,22 +45,17 @@ function MaterialDescriptor(smkId, queryId, keySize) { } // Encryption Material -function EncryptionMetadata(key, iv, matDesc) { +function EncryptionMetadata(key, dataIv, matDesc, keyIv, dataAad, keyAad) { return { 'key': key, - 'iv': iv, - 'matDesc': matDesc + 'iv': dataIv, + 'matDesc': matDesc, + 'keyIv': keyIv, + 'dataAad': dataAad, + 'keyAad': keyAad }; } -function aesCbc(keySizeInBytes) { - return `aes-${keySizeInBytes * 8}-cbc`; -} - -function aesEcb(keySizeInBytes) { - return `aes-${keySizeInBytes * 8}-ecb`; -} - exports.EncryptionMetadata = EncryptionMetadata; function TempFileGenerator() { @@ -97,6 +114,7 @@ function TempFileGenerator() { */ function EncryptUtil(encrypt, filestream, temp) { const crypto = typeof encrypt !== 'undefined' ? encrypt : require('crypto'); + // TODO: SNOW-1814883: Replace 'fs' with 'fs/promises' const fs = typeof filestream !== 'undefined' ? filestream : require('fs'); const tmp = typeof temp !== 'undefined' ? temp : new TempFileGenerator(); @@ -125,206 +143,261 @@ function EncryptUtil(encrypt, filestream, temp) { return newMatDesc; } + function createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv, keyIv = null, dataAad = null, keyAad = null) { + const matDesc = new MaterialDescriptor( + encryptionMaterial.smkId, + encryptionMaterial.queryId, + keySize * 8 + ); + + return new EncryptionMetadata( + encryptedKey.toString(BASE64), + dataIv.toString(BASE64), + matDescToUnicode(matDesc), + keyIv ? keyIv.toString(BASE64) : null, + dataAad ? dataAad.toString(BASE64) : null, + keyAad ? keyAad.toString(BASE64) : null + ); + } + /** - * Encrypt file stream using AES algorithm. - * - * @param {Object} encryptionMaterial - * @param {String} fileStream - * @param {String} tmpDir - * @param {Number} chunkSize - * - * @returns {Object} - */ - this.encryptFileStream = async function (encryptionMaterial, fileStream) { - // Get decoded key from base64 encoded value - const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - const keySize = decodedKey.length; + * Encrypt content using AES-CBC algorithm. + */ + this.encryptFileStream = async function (encryptionMaterial, content) { + return this.encryptDataCBC(encryptionMaterial, content); + }; + + this.encryptDataCBC = function (encryptionMaterial, data) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keySize = decodedKek.length; - // Get secure random bytes with block size - const ivData = getSecureRandom(blockSize); + const dataIv = getSecureRandom(AES_CBC.ivSize); const fileKey = getSecureRandom(keySize); - // Create cipher with file key, AES CBC, and iv data - let cipher = crypto.createCipheriv(aesCbc(keySize), fileKey, ivData); - const encrypted = cipher.update(fileStream); - const final = cipher.final(); - const encryptedData = Buffer.concat([encrypted, final]); + const dataCipher = crypto.createCipheriv(AES_CBC.cipherName(keySize), fileKey, dataIv); + const encryptedData = performCrypto(dataCipher, data); - // Create key cipher with decoded key and AES ECB - cipher = crypto.createCipheriv(aesEcb(keySize), decodedKey, null); + const keyCipher = crypto.createCipheriv(AES_ECB.cipherName(keySize), decodedKek, null); + const encryptedKey = performCrypto(keyCipher, fileKey); - // Encrypt with file key - const encKek = Buffer.concat([ - cipher.update(fileKey), - cipher.final() - ]); + return { + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv), + dataStream: encryptedData + }; + }; - const matDesc = MaterialDescriptor( - encryptionMaterial.smkId, - encryptionMaterial.queryId, - keySize * 8 - ); + //TODO: SNOW-940981: Add proper usage when feature is ready + this.encryptDataGCM = function (encryptionMaterial, data) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keySize = decodedKek.length; - const metadata = EncryptionMetadata( - encKek.toString(BASE64), - ivData.toString(BASE64), - matDescToUnicode(matDesc) - ); + const dataIv = getSecureRandom(AES_GCM.ivSize); + const fileKey = getSecureRandom(keySize); + + const encryptedData = this.encryptGCM(data, fileKey, dataIv, DEFAULT_AAD); + + const keyIv = getSecureRandom(AES_GCM.ivSize); + const encryptedKey = this.encryptGCM(fileKey, decodedKek, keyIv, DEFAULT_AAD); return { - encryptionMetadata: metadata, + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv, keyIv, DEFAULT_AAD, DEFAULT_AAD), dataStream: encryptedData }; }; + + this.encryptGCM = function (data, key, iv, aad) { + const cipher = crypto.createCipheriv(AES_GCM.cipherName(key.length), key, iv, { authTagLength: AUTH_TAG_LENGTH_IN_BYTES }); + if (aad) { + cipher.setAAD(aad); + } + const encryptedData = performCrypto(cipher, data); + return Buffer.concat([encryptedData, cipher.getAuthTag()]); + }; + + this.decryptGCM = function (data, key, iv, aad) { + const decipher = crypto.createDecipheriv(AES_GCM.cipherName(key.length), key, iv, { authTagLength: AUTH_TAG_LENGTH_IN_BYTES }); + if (aad) { + decipher.setAAD(aad); + } + // last 16 bytes of data is the authentication tag + const authTag = data.slice(data.length - AUTH_TAG_LENGTH_IN_BYTES, data.length); + const cipherText = data.slice(0, data.length - AUTH_TAG_LENGTH_IN_BYTES); + decipher.setAuthTag(authTag); + return performCrypto(decipher, cipherText); + }; + /** - * Encrypt file using AES algorithm. - * - * @param {Object} encryptionMaterial - * @param {String} inFileName - * @param {String} tmpDir - * @param {Number} chunkSize - * - * @returns {Object} - */ - this.encryptFile = async function (encryptionMaterial, inFileName, + * Encrypt file using AES algorithm. + */ + this.encryptFile = async function (encryptionMaterial, inputFilePath, tmpDir = null, chunkSize = blockSize * 4 * 1024) { - // Get decoded key from base64 encoded value - const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - const keySize = decodedKey.length; + return await this.encryptFileCBC(encryptionMaterial, inputFilePath, tmpDir, chunkSize); + }; - // Get secure random bytes with block size - const ivData = getSecureRandom(blockSize); - const fileKey = getSecureRandom(keySize); + this.encryptFileCBC = async function (encryptionMaterial, inputFilePath, + tmpDir = null, chunkSize = blockSize * 4 * 1024) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keySize = decodedKek.length; - // Create cipher with file key, AES CBC, and iv data - let cipher = crypto.createCipheriv(aesCbc(keySize), fileKey, ivData); + const dataIv = getSecureRandom(AES_CBC.ivSize); + const fileKey = getSecureRandom(keySize); + const dataCipher = crypto.createCipheriv(AES_CBC.cipherName(keySize), fileKey, dataIv); + const encryptedFilePath = await performFileStreamCrypto(dataCipher, tmpDir, inputFilePath, chunkSize); - // Create temp file - const tmpobj = tmp.fileSync({ dir: tmpDir, prefix: path.basename(inFileName) + '#' }); - const tempOutputFileName = tmpobj.name; - const tempFd = tmpobj.fd; + const keyCipher = crypto.createCipheriv(AES_ECB.cipherName(keySize), decodedKek, null); + const encryptedKey = performCrypto(keyCipher, fileKey); - await new Promise(function (resolve) { - const infile = fs.createReadStream(inFileName, { highWaterMark: chunkSize }); - const outfile = fs.createWriteStream(tempOutputFileName); - - infile.on('data', function (chunk) { - // Encrypt chunk using cipher - const encrypted = cipher.update(chunk); - // Write to temp file - outfile.write(encrypted); - }); - infile.on('close', function () { - outfile.write(cipher.final()); - outfile.close(resolve); - }); - }); + return { + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, keySize, encryptedKey, dataIv), + dataFile: encryptedFilePath + }; + }; - // Create key cipher with decoded key and AES ECB - cipher = crypto.createCipheriv(aesEcb(keySize), decodedKey, null); + //TODO: SNOW-940981: Add proper usage when feature is ready + this.encryptFileGCM = async function (encryptionMaterial, inputFilePath, tmpDir = null) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - // Encrypt with file key - const encKek = Buffer.concat([ - cipher.update(fileKey), - cipher.final() - ]); + const dataIv = getSecureRandom(AES_GCM.ivSize); + const fileKey = getSecureRandom(decodedKek.length); - const matDesc = MaterialDescriptor( - encryptionMaterial.smkId, - encryptionMaterial.queryId, - keySize * 8 - ); + const fileContent = await new Promise((resolve, reject) => { + fs.readFile(inputFilePath, (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); + }); - const metadata = EncryptionMetadata( - encKek.toString(BASE64), - ivData.toString(BASE64), - matDescToUnicode(matDesc) - ); + const encryptedData = this.encryptGCM(fileContent, fileKey, dataIv, DEFAULT_AAD); + const encryptedFilePath = await writeContentToFile(tmpDir, path.basename(inputFilePath) + '#', encryptedData); - // Close temp file - fs.closeSync(tempFd); + const keyIv = getSecureRandom(AES_GCM.ivSize); + const encryptedKey = this.encryptGCM(fileKey, decodedKek, keyIv, DEFAULT_AAD); return { - encryptionMetadata: metadata, - dataFile: tempOutputFileName + encryptionMetadata: createEncryptionMetadata(encryptionMaterial, fileKey.length, encryptedKey, dataIv, keyIv, DEFAULT_AAD, DEFAULT_AAD), + dataFile: encryptedFilePath }; }; /** - * Decrypt file using AES algorithm. - * - * @param {Object} encryptionMaterial - * @param {String} inFileName - * @param {String} tmpDir - * @param {Number} chunkSize - * - * @returns {String} - */ - this.decryptFile = async function (metadata, encryptionMaterial, inFileName, + * Decrypt file using AES algorithm. + */ + this.decryptFile = async function (metadata, encryptionMaterial, inputFilePath, tmpDir = null, chunkSize = blockSize * 4 * 1024) { - // Get key and iv from metadata - const keyBase64 = metadata.key; - const ivBase64 = metadata.iv; + return await this.decryptFileCBC(metadata, encryptionMaterial, inputFilePath, tmpDir, chunkSize); + }; - // Get decoded key from base64 encoded value - const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); - const keySize = decodedKey.length; + this.decryptFileCBC = async function (metadata, encryptionMaterial, inputFilePath, + tmpDir = null, chunkSize = blockSize * 4 * 1024) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keyBytes = new Buffer.from(metadata.key, BASE64); + const ivBytes = new Buffer.from(metadata.iv, BASE64); + const keyDecipher = crypto.createDecipheriv(AES_ECB.cipherName(decodedKek.length), decodedKek, null); + const fileKey = performCrypto(keyDecipher, keyBytes); + + const dataDecipher = crypto.createDecipheriv(AES_CBC.cipherName(fileKey.length), fileKey, ivBytes); + return await performFileStreamCrypto(dataDecipher, tmpDir, inputFilePath, chunkSize); + }; - // Get key bytes and iv bytes from base64 encoded value - const keyBytes = new Buffer.from(keyBase64, BASE64); - const ivBytes = new Buffer.from(ivBase64, BASE64); + //TODO: SNOW-940981: Add proper usage when feature is ready + this.decryptFileGCM = async function (metadata, encryptionMaterial, inputFilePath, tmpDir = null) { + const decodedKek = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64); + const keyBytes = new Buffer.from(metadata.key, BASE64); + const keyIvBytes = new Buffer.from(metadata.keyIv, BASE64); + const dataIvBytes = new Buffer.from(metadata.iv, BASE64); + const dataAadBytes = new Buffer.from(metadata.dataAad, BASE64); + const keyAadBytes = new Buffer.from(metadata.keyAad, BASE64); - // Create temp file - let tempOutputFileName; - let tempFd; - await new Promise((resolve, reject) => { - tmp.file({ dir: tmpDir, prefix: path.basename(inFileName) + '#' }, (err, path, fd) => { + const fileKey = this.decryptGCM(keyBytes, decodedKek, keyIvBytes, keyAadBytes); + + const fileContent = await new Promise((resolve, reject) => { + fs.readFile(inputFilePath, (err, data) => { if (err) { reject(err); + } else { + resolve(data); } - tempOutputFileName = path; - tempFd = fd; - resolve(); }); }); - // Create key decipher with decoded key and AES ECB - let decipher = crypto.createDecipheriv(aesEcb(keySize), decodedKey, null); - const fileKey = Buffer.concat([ - decipher.update(keyBytes), - decipher.final() - ]); - - // Create decipher with file key, iv bytes, and AES CBC - decipher = crypto.createDecipheriv(aesCbc(keySize), fileKey, ivBytes); + const decryptedData = this.decryptGCM(fileContent, fileKey, dataIvBytes, dataAadBytes); + return await writeContentToFile(tmpDir, path.basename(inputFilePath) + '#', decryptedData); + }; + + function performCrypto(cipherOrDecipher, data) { + const encryptedOrDecrypted = cipherOrDecipher.update(data); + const final = cipherOrDecipher.final(); + return Buffer.concat([encryptedOrDecrypted, final]); + } + async function performFileStreamCrypto(cipherOrDecipher, tmpDir, inputFilePath, chunkSize) { + const outputFile = await new Promise((resolve, reject) => { + tmp.file({ dir: tmpDir, prefix: path.basename(inputFilePath) + '#' }, (err, path, fd) => { + if (err) { + reject(err); + } else { + resolve({ path, fd }); + } + }); + }); await new Promise(function (resolve) { - const infile = fs.createReadStream(inFileName, { highWaterMark: chunkSize }); - const outfile = fs.createWriteStream(tempOutputFileName); - - infile.on('data', function (chunk) { - // Dncrypt chunk using decipher - const decrypted = decipher.update(chunk); - // Write to temp file - outfile.write(decrypted); + const inputStream = fs.createReadStream(inputFilePath, { highWaterMark: chunkSize }); + const outputStream = fs.createWriteStream(outputFile.path); + + inputStream.on('data', function (chunk) { + const encrypted = cipherOrDecipher.update(chunk); + outputStream.write(encrypted); }); - infile.on('close', function () { - outfile.write(decipher.final()); - outfile.close(resolve); + inputStream.on('close', function () { + outputStream.write(cipherOrDecipher.final()); + outputStream.close(resolve); }); }); - // Close temp file await new Promise((resolve, reject) => { - fs.close(tempFd, (err) => { + fs.close(outputFile.fd, (err) => { if (err) { - reject(err); + reject(err); + } else { + resolve(); } - resolve(); }); }); + return outputFile.path; + } - return tempOutputFileName; - }; + async function writeContentToFile(tmpDir, prefix, content,) { + const outputFile = await new Promise((resolve, reject) => { + tmp.file({ dir: tmpDir, prefix: prefix }, (err, path, fd) => { + if (err) { + reject(err); + } else { + resolve({ path, fd }); + } + }); + }); + await new Promise((resolve, reject) => { + fs.writeFile(outputFile.path, content, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + await new Promise((resolve, reject) => { + fs.close(outputFile.fd, (err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + return outputFile.path; + } } exports.EncryptUtil = EncryptUtil; diff --git a/test/integration/testEncrypt.js b/test/integration/testEncrypt.js new file mode 100644 index 000000000..3aef8914b --- /dev/null +++ b/test/integration/testEncrypt.js @@ -0,0 +1,83 @@ +const assert = require('assert'); +const path = require('path'); +const os = require('os'); +const fs = require('fs/promises'); +const SnowflakeEncryptionUtil = require('../../lib/file_transfer_agent/encrypt_util').EncryptUtil; + + +describe('Test Encryption/Decryption', function () { + const BASE64 = 'base64'; + const UTF8 = 'utf-8'; + let encryptUtil; + + before(function () { + encryptUtil = new SnowflakeEncryptionUtil(); + }); + + it('GCM - Encrypt and decrypt raw data', function () { + const data = 'abc'; + const iv = Buffer.from('ab1234567890'); + const key = Buffer.from('1234567890abcdef'); + + const encryptedData = encryptUtil.encryptGCM(data, key, iv, null); + assert.strictEqual(encryptedData.toString(BASE64), 'iG+lT4o27hkzj3kblYRzQikLVQ=='); + + const decryptedData = encryptUtil.decryptGCM(encryptedData, key, iv, null); + assert.strictEqual(decryptedData.toString(UTF8), data); + }); + + it('GCM - Encrypt raw data based on encryption material', function () { + const data = 'abc'; + + const encryptionMaterial = { + 'queryStageMasterKey': 'YWJjZGVmMTIzNDU2Nzg5MA==', + 'queryId': 'unused', + 'smkId': '123' + }; + + const { dataStream, encryptionMetadata } = encryptUtil.encryptDataGCM(encryptionMaterial, data); + assert.ok(dataStream); + assert.ok(encryptionMetadata); + + const decodedKek = Buffer.from(encryptionMaterial['queryStageMasterKey'], BASE64); + const keyBytes = new Buffer.from(encryptionMetadata.key, BASE64); + const keyIvBytes = new Buffer.from(encryptionMetadata.keyIv, BASE64); + const dataIvBytes = new Buffer.from(encryptionMetadata.iv, BASE64); + const dataAadBytes = new Buffer.from(encryptionMetadata.dataAad, BASE64); + const keyAadBytes = new Buffer.from(encryptionMetadata.keyAad, BASE64); + + const fileKey = encryptUtil.decryptGCM(keyBytes, decodedKek, keyIvBytes, keyAadBytes); + + const decryptedData = encryptUtil.decryptGCM(dataStream, fileKey, dataIvBytes, dataAadBytes); + assert.strictEqual(decryptedData.toString(UTF8), data); + }); + + it('GCM - Encrypt and decrypt file', async function () { + await encryptAndDecryptFile('gcm', async function (encryptionMaterial, inputFilePath) { + const output = await encryptUtil.encryptFileGCM(encryptionMaterial, inputFilePath, os.tmpdir()); + return await encryptUtil.decryptFileGCM(output.encryptionMetadata, encryptionMaterial, output.dataFile, os.tmpdir()); + }); + }); + + it('CBC - Encrypt and decrypt file', async function () { + await encryptAndDecryptFile('cbc', async function (encryptionMaterial, inputFilePath) { + const output = await encryptUtil.encryptFileCBC(encryptionMaterial, inputFilePath, os.tmpdir()); + return await encryptUtil.decryptFileCBC(output.encryptionMetadata, encryptionMaterial, output.dataFile, os.tmpdir()); + }); + }); + + async function encryptAndDecryptFile(encryptionTypeName, encryptAndDecrypt) { + const data = 'abc'; + const inputFilePath = path.join(os.tmpdir(), `${encryptionTypeName}_file_encryption_test`); + await fs.writeFile(inputFilePath, data); + + const encryptionMaterial = { + 'queryStageMasterKey': 'YWJjZGVmMTIzNDU2Nzg5MA==', + 'queryId': 'unused', + 'smkId': '123' + }; + const decryptedFilePath = await encryptAndDecrypt(encryptionMaterial, inputFilePath, os.tmpdir()); + const decryptedContent = await fs.readFile(decryptedFilePath); + assert.strictEqual(decryptedContent.toString('utf-8'), data); + } +}); \ No newline at end of file diff --git a/test/unit/file_transfer_agent/encrypt_util_test.js b/test/unit/file_transfer_agent/encrypt_util_test.js index 5f7e6a7d2..d8668830c 100644 --- a/test/unit/file_transfer_agent/encrypt_util_test.js +++ b/test/unit/file_transfer_agent/encrypt_util_test.js @@ -93,16 +93,13 @@ describe('Encryption util', function () { } return new createWriteStream; }, - closeSync: function () { - return; + close: function (fd, callback) { + callback(null); } }); mock('temp', { - fileSync: function () { - return { - name: mockTmpName, - fd: 0 - }; + file: function (object, callback) { + callback(null, mockTmpName, 0); }, openSync: function () { return;