From 2617f9982f42ebda636e00726c2aa3326cc27afe Mon Sep 17 00:00:00 2001 From: Traian Anghel Date: Fri, 24 Nov 2023 13:30:33 +0200 Subject: [PATCH] 2.4.0 Add Enhanced Config Service (#190) * Add Enhanced Config Service (#189) * base config service initial commit * included files in the index component * updated package lock file * added unit tests for base config service * fixed env override regex * updated package lock file --------- Co-authored-by: tanghel * updated package lock * replaced references to localhost --------- Co-authored-by: Horia Schiau <6889483+mad2sm0key@users.noreply.github.com> --- package-lock.json | 69 ++++++- packages/common/package.json | 1 + .../src/common/config/base.config.service.ts | 185 ++++++++++++++++++ packages/common/src/common/config/index.ts | 2 + packages/common/src/index.ts | 2 +- .../test/config/base.config.service.spec.ts | 146 ++++++++++++++ 6 files changed, 397 insertions(+), 8 deletions(-) create mode 100644 packages/common/src/common/config/base.config.service.ts create mode 100644 packages/common/src/common/config/index.ts create mode 100644 packages/common/test/config/base.config.service.spec.ts diff --git a/package-lock.json b/package-lock.json index b68dda27..79636f9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2130,6 +2130,43 @@ } } }, + "node_modules/@nestjs/config": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-3.1.1.tgz", + "integrity": "sha512-qu5QlNiJdqQtOsnB6lx4JCXPQ96jkKUsOGd+JXfXwqJqZcOSAq6heNFg0opW4pq4J/VZoNwoo87TNnx9wthnqQ==", + "peer": true, + "dependencies": { + "dotenv": "16.3.1", + "dotenv-expand": "10.0.0", + "lodash": "4.17.21", + "uuid": "9.0.0" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "reflect-metadata": "^0.1.13" + } + }, + "node_modules/@nestjs/config/node_modules/dotenv": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", + "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", + "peer": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/motdotla/dotenv?sponsor=1" + } + }, + "node_modules/@nestjs/config/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "peer": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@nestjs/core": { "version": "10.1.3", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.1.3.tgz", @@ -5727,6 +5764,15 @@ "node": ">=10" } }, + "node_modules/dotenv-expand": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", + "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", + "peer": true, + "engines": { + "node": ">=12" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -9988,9 +10034,9 @@ "dev": true }, "node_modules/nest-winston": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/nest-winston/-/nest-winston-1.9.3.tgz", - "integrity": "sha512-BLDGvbKCeliQNNrNjFRKSbO9hu/a39YL3y+4aU0eTGHh5TgzNMqx5ZscbecKR0fM3/KumMznZdEWmAJBINHl6Q==", + "version": "1.9.4", + "resolved": "https://registry.npmjs.org/nest-winston/-/nest-winston-1.9.4.tgz", + "integrity": "sha512-ilEmHuuYSAI6aMNR120fLBl42EdY13QI9WRggHdEizt9M7qZlmXJwpbemVWKW/tqRmULjSx/otKNQ3GMQbfoUQ==", "dependencies": { "fast-safe-stringify": "^2.1.1" }, @@ -13392,11 +13438,11 @@ } }, "node_modules/winston": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.10.0.tgz", - "integrity": "sha512-nT6SIDaE9B7ZRO0u3UvdrimG0HkB7dSTAgInQnNR2SOPJ4bvq5q79+pXLftKmP52lJGW15+H5MCK0nM9D3KB/g==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.11.0.tgz", + "integrity": "sha512-L3yR6/MzZAOl0DsysUXHVjOwv8mKZ71TrA/41EIduGpOOV5LQVodqN+QdQ6BS6PJ/RdIshZhq84P/fStEZkk7g==", "dependencies": { - "@colors/colors": "1.5.0", + "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.2", "async": "^3.2.3", "is-stream": "^2.0.0", @@ -13442,6 +13488,14 @@ "node": ">= 6.4.0" } }, + "node_modules/winston/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", @@ -13783,6 +13837,7 @@ "peerDependencies": { "@multiversx/sdk-nestjs-monitoring": "^2.0.0", "@nestjs/common": "^10.x", + "@nestjs/config": "^3.x", "@nestjs/core": "^10.x", "@nestjs/swagger": "^7.x" } diff --git a/packages/common/package.json b/packages/common/package.json index a83fcaaa..c5ed052e 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -44,6 +44,7 @@ "peerDependencies": { "@multiversx/sdk-nestjs-monitoring": "^2.0.0", "@nestjs/common": "^10.x", + "@nestjs/config": "^3.x", "@nestjs/core": "^10.x", "@nestjs/swagger": "^7.x" }, diff --git a/packages/common/src/common/config/base.config.service.ts b/packages/common/src/common/config/base.config.service.ts new file mode 100644 index 00000000..e4372e15 --- /dev/null +++ b/packages/common/src/common/config/base.config.service.ts @@ -0,0 +1,185 @@ +import { Injectable } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; + +@Injectable() +export class BaseConfigService { + constructor(protected readonly configService: ConfigService) { } + + get(key: string): T | undefined { + const envOverride = this.tryGetEnvOverride(key); + if (envOverride || envOverride === false) { + return envOverride; + } + + const configValue = this.configService.get(key); + + if (configValue === undefined) { + return undefined; + } + + if (typeof configValue !== 'string') { + return configValue; + } + + return this.tryGetEnvValue(key, configValue); + } + + private tryGetEnvOverride(key: string): T | undefined { + const overridePrefix = 'MVX_OVERRIDE_'; + const envKey = key + // Replace any non-uppercase sequence before an uppercase letter or number with that letter/number prefixed by an underscore + .replace(/([a-z])([A-Z])/g, '$1_$2') + // Handle the scenario where a group of uppercase letters is followed by a lowercase letter + .replace(/([A-Z])([A-Z])([a-z])/g, '$1_$2$3') + // Replace non-alphanumeric characters with underscores + .replace(/[^a-zA-Z0-9]/g, '_') + // Convert the whole string to uppercase + .toUpperCase(); + + const envValue = this.configService.get(overridePrefix + envKey); + if (!envValue) { + return undefined; + } + + const strValue = this.trimPrefix(envValue, 'str:'); + if (strValue) { + return strValue as T; + } + + const numValue = this.trimPrefix(envValue, 'num:'); + if (numValue) { + return this.parseValueAsNumber(numValue) as T; + } + + const boolValue = this.trimPrefix(envValue, 'bool:'); + if (boolValue) { + return this.parseValueAsBoolean(boolValue) as T; + } + + const jsonValue = this.trimPrefix(envValue, 'json:'); + if (jsonValue) { + return this.parseValueAsJson(jsonValue) as T; + } + + const arrStrValue = this.trimPrefix(envValue, 'arr:str:'); + if (arrStrValue) { + return this.parseValueAsArray(arrStrValue, 'str') as T; + } + + const arrNumValue = this.trimPrefix(envValue, 'arr:num:'); + if (arrNumValue) { + return this.parseValueAsArray(arrNumValue, 'num') as T; + } + + return envValue as T; + } + + private trimPrefix(key: string, prefix: string): string | undefined { + if (key.startsWith(prefix)) { + return key.slice(prefix.length); + } + + return undefined; + } + + private tryGetEnvValue(key: string, value: string): T | undefined { + const regex = /\$\{([^}]+)\}/; + const match = value.match(regex); + + if (!match) { + return value as T; + } + + let keyName = match[1]; + const keySegments = keyName.split(':'); + + if (keySegments.length === 0) { + throw new Error(`Could not parse config key ${key}`); + } + + if (keySegments.length === 1) { + return this.configService.get(keyName); + } + + keyName = keySegments.pop() as string; + if (keyName === '') { + throw new Error(`Could not parse config key ${key}`); + } + + const envValue = this.configService.get(keyName); + + if (keySegments[0] === 'arr') { + const expectedType = keySegments.length === 1 ? 'str' : keySegments[1]; + + return this.parseValueAsArray(envValue, expectedType); + } + + if (envValue === undefined) { + return envValue; + } + + return this.parseValue(envValue, keySegments[0]); + } + + private parseValue(value: string, valueType: string): any { + switch (valueType) { + case 'str': + return value; + case 'bool': + return this.parseValueAsBoolean(value); + case 'num': + return this.parseValueAsNumber(value); + case 'json': + return this.parseValueAsJson(value); + default: + throw new Error(`Cannot parse config value ${value} as ${valueType}`); + } + } + + private parseValueAsJson(value: string): any { + try { + return JSON.parse(value); + } catch (error) { + throw new Error(`Could not parse config value ${value} as json`); + } + } + + private parseValueAsNumber(value: string): number { + if (value.trim() === '') { + throw new Error(`Cannot parse config value ${value} as a number`); + } + + if (isNaN(Number(value))) { + throw new Error(`Cannot parse config value ${value} as a number`); + } + + return Number(value); + } + + private parseValueAsBoolean(value: string): boolean { + if (value === 'true') { + return true; + } + + if (value === 'false') { + return false; + } + + throw new Error(`Cannot parse config value ${value} as a boolean`); + } + + private parseValueAsArray(value: string | undefined, valueType: string): any { + if (value === undefined || value === '') { + return []; + } + + const elements = value.split(','); + if (valueType === 'str') { + return elements; + } + + const result = elements.map(element => this.parseValue(element, valueType)); + + return result; + } +} diff --git a/packages/common/src/common/config/index.ts b/packages/common/src/common/config/index.ts new file mode 100644 index 00000000..215ba361 --- /dev/null +++ b/packages/common/src/common/config/index.ts @@ -0,0 +1,2 @@ +export * from './base.config.service'; +export * from './erdnest.config.service'; diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 6e75491f..24c1c581 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -38,7 +38,7 @@ export * from './pipes/parse.token.pipe'; export * from './pipes/parse.token.or.nft.pipe'; export * from './pipes/parse.transaction.hash.pipe'; export * from './common/entities/amount'; -export * from './common/config/erdnest.config.service'; +export * from './common/config'; export * from './common/swappable-settings'; export * from './common/logging/logging.module'; export * from './common/complexity/apply.complexity'; diff --git a/packages/common/test/config/base.config.service.spec.ts b/packages/common/test/config/base.config.service.spec.ts new file mode 100644 index 00000000..58a9f999 --- /dev/null +++ b/packages/common/test/config/base.config.service.spec.ts @@ -0,0 +1,146 @@ +import { ConfigService } from '@nestjs/config'; +import { BaseConfigService } from '../../src/common/config'; + +describe('BaseConfigService', () => { + let baseConfigService: BaseConfigService; + let configServiceMock: jest.Mocked; + + beforeEach(() => { + configServiceMock = { + get: jest.fn(), + } as unknown as jest.Mocked; + + baseConfigService = new BaseConfigService(configServiceMock); + + jest.resetAllMocks() + }); + + it('should be defined', () => { + expect(baseConfigService).toBeDefined() + }) + + it('should retrieve a simple string value from config.yaml', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('stringValue') + + const result = baseConfigService.get('testString'); + expect(result).toBe('stringValue'); + }); + + it('should retrieve a simple number value from config.yaml', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce(42) + + const result = baseConfigService.get('testNumber'); + expect(result).toBe(42); + }); + + it('should retrieve a string value from .env with and without the prefix', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${str:MY_PREFIXED_VALUE}') + configServiceMock.get.mockReturnValueOnce('hello') + + let result = baseConfigService.get('testPrefixed'); + expect(result).toBe('hello'); + + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${MY_UNPREFIXED_VALUE}') + configServiceMock.get.mockReturnValueOnce('world') + + result = baseConfigService.get('testNoPrefix'); + expect(result).toBe('world'); + }); + + it('should retrieve a string value from .env and return a number', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${num:MY_NUMBER}') + configServiceMock.get.mockReturnValueOnce('42') + + const result = baseConfigService.get('testNumber'); + expect(result).toBe(42); + }); + + it('should retrieve a string value from .env and return a boolean', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${bool:MY_TRUE_BOOLEAN}') + configServiceMock.get.mockReturnValueOnce('true') + + let result = baseConfigService.get('testBooleanTrue'); + expect(result).toBe(true); + + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${bool:MY_FALSE_BOOLEAN}') + configServiceMock.get.mockReturnValueOnce('false') + + result = baseConfigService.get('testBooleanFalse'); + expect(result).toBe(false); + }); + + it('should retrieve a string value from .env and return a json', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${json:MY_JSON}') + configServiceMock.get.mockReturnValueOnce('{"hello":"world","my_number":123,"my_bool":false,"nested":{"all":"good"}}') + + const result = baseConfigService.get('testBoolean'); + expect(result).toEqual({ + hello: "world", + my_number: 123, + my_bool: false, + nested: { all: "good" } + }); + }); + + it('should retrieve a string value from .env and return a string array', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${arr:MY_ARRAY}') + configServiceMock.get.mockReturnValueOnce('hello,world') + + let result = baseConfigService.get('testArray'); + expect(result).toEqual(['hello', 'world']); + + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${arr:str:MY_ARRAY}') + configServiceMock.get.mockReturnValueOnce('hello2,world2') + + result = baseConfigService.get('testArray'); + expect(result).toEqual(['hello2', 'world2']); + }); + + it('should retrieve a string value from .env and return a number array', () => { + configServiceMock.get.mockReturnValueOnce(undefined) // no override + configServiceMock.get.mockReturnValueOnce('${arr:num:MY_ARRAY}') + configServiceMock.get.mockReturnValueOnce('123,456') + + const result = baseConfigService.get('testArray'); + expect(result).toEqual([123, 456]); + }); + + it('should retrieve a string value from .env with override', () => { + configServiceMock.get.mockReturnValueOnce('str:overriden value') + + const result = baseConfigService.get('testArray'); + expect(configServiceMock.get).toHaveBeenCalledWith('MVX_OVERRIDE_TEST_ARRAY') + + expect(result).toBe('overriden value'); + }); + + it('should retrieve a number value from .env with override', () => { + configServiceMock.get.mockReturnValueOnce('num:123') + + const result = baseConfigService.get('testArray.nested'); + expect(configServiceMock.get).toHaveBeenCalledWith('MVX_OVERRIDE_TEST_ARRAY_NESTED') + + expect(result).toBe(123); + }); + + it('should retrieve a boolean value from .env with override', () => { + configServiceMock.get.mockReturnValueOnce('bool:false') + + const result = baseConfigService.get('wrappedEGLDIdentifier.nested'); + expect(configServiceMock.get).toHaveBeenCalledWith('MVX_OVERRIDE_WRAPPED_EGLD_IDENTIFIER_NESTED') + + expect(result).toBe(false); + }); + +}); +