diff --git a/README.md b/README.md index 349e63a..0d6e8c9 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,12 @@ You find an example in [`test/template-escape`](test/template-escape). If you want to use for example `$(_name)` as both an external reference and a normal reference, then you add a `\` for the latter resulting in `$(\_name)` for the latter. +If your YARRRML document contains sensitive information such as database credentials, +you can use dollar variables with curly braces and specify a variable in a `.env` file. +For example, use `${DB_PASSWORD}` in the YARRRML document and add a `.env` file with content `DB_PASSWORD=mySecretPassword`. +During generation of an RML document, the variables will be replaced. +If a variable is not found in the current environment, the yarrrml-parser will emit a warning. + If you want the outputted RML to be pretty, please provide the `-p` or `--pretty` parameter. #### yarrrml-generator diff --git a/bin/parser.js b/bin/parser.js index 1cb4bbf..46c8ac4 100755 --- a/bin/parser.js +++ b/bin/parser.js @@ -18,6 +18,9 @@ const watch = require('../lib/watcher.js'); const glob = require('glob'); const Logger = require('../lib/logger'); +// load environment variables from .env into process.env +require('dotenv').config(); + namespaces.ql = 'http://semweb.mmlab.be/ns/ql#'; pkginfo(module, 'version'); diff --git a/lib/abstract-generator.js b/lib/abstract-generator.js index 7ff16b1..e55a1ca 100644 --- a/lib/abstract-generator.js +++ b/lib/abstract-generator.js @@ -48,8 +48,35 @@ class AbstractGenerator { let json; + // replace found variables with the values from .env + let populatedYarrrml = yarrrml; + let reVars = /.*?(\${.*?}).*?/gm; + let usedEnvVariables = yarrrml.matchAll(reVars); + let nonFoundVars = []; + for(let match of usedEnvVariables) { + let found = match[1]; + // extract var name, e.g. DB_HOST from ${DB_HOST} so we can compare with env variables + let varName = found.substring(2, (found.length - 1) ); + + if(varName in process.env) { + /* split-join solution because 'replaceAll' might not exist and 'replace' needs a global flag + * which is problematic because then a regex has to be used instead of a string + * and then our ${} variables are not recognized + * see Stackoverlow: https://stackoverflow.com/a/542305 + */ + populatedYarrrml = populatedYarrrml.split(found).join(process.env[varName]); + + } else { + nonFoundVars.push(varName); + } + } + + if(nonFoundVars.length > 0) { + Logger.warn(`No value set for the following used environment variables: ${nonFoundVars}`); + } + try { - json = YAML.parse(yarrrml); + json = YAML.parse(populatedYarrrml); } catch (e) { e.code = 'INVALID_YAML'; e.file = file diff --git a/lib/rml-generator.test.js b/lib/rml-generator.test.js index 92c5e54..67c79cd 100644 --- a/lib/rml-generator.test.js +++ b/lib/rml-generator.test.js @@ -75,6 +75,48 @@ describe('YARRRML to RML', function () { work('template-escape/mapping.yml', 'template-escape/mapping.rml.ttl', done, {includeMetadata: false}); }); + + + describe('environment variables', function() { + let envCache; + + beforeEach(() => { + // cache current environment to reset after tests + envCache = Object.assign({}, process.env); + }); + + afterEach(() => { + // reset environment + process.env = envCache; + }); + + it('works for expanded environment variables', function(done) { + process.env.DB_USER = "dbUser"; + process.env.DB_PASSWORD = "dbPassword"; + process.env.DB_HOST = "myHost"; + process.env.DB_PORT = "myPort"; + process.env.DB = "myDB"; + work('env-variables/env-replaced/mapping.yml', 'env-variables/env-replaced/mapping.rml.ttl', done, {includeMetadata: true}); + }); + + it('has warning if no environment variable was replaced', () => { + const y2r = new convertYAMLtoRML(); + + y2r.convert(fs.readFileSync(path.resolve(__dirname, '../test/env-variables/env-not-defined-warning-all/mapping.yml'), 'utf8')); + assert.strictEqual(y2r.getLogger().has('warning'), true); + assert.strictEqual(y2r.getLogger().getAll().length, 1); + }); + + it('has warning if only a few environment variables were not replaced', () => { + process.env.DB_HOST = "myHost"; + process.env.DB_PORT = "myPort"; + const y2r = new convertYAMLtoRML(); + y2r.convert(fs.readFileSync(path.resolve(__dirname, '../test/env-variables/env-not-defined-warning/mapping.yml'), 'utf8')); + assert.strictEqual(y2r.getLogger().has('warning'), true); + assert.strictEqual(y2r.getLogger().getAll().length, 1); + }); + }) + describe('between our worlds rules', function () { it('anime', function (done) { work('betweenourworlds/anime/mapping.yarrrml', 'betweenourworlds/anime/mapping.rml.ttl', done, {includeMetadata: false}); diff --git a/package-lock.json b/package-lock.json index cf1f5e9..aef81eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2494,6 +2494,7 @@ } } }, +>>>>>>> origin/development "dependencies": { "@graphy/content.nq.read": { "version": "4.3.3", @@ -3144,6 +3145,11 @@ "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", "dev": true }, + "dotenv": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", + "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -4149,6 +4155,19 @@ "tweetnacl": "~0.14.0" } }, +<<<<<<< HEAD + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, +======= +>>>>>>> origin/development "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -4157,6 +4176,18 @@ "safe-buffer": "~5.1.0" } }, +<<<<<<< HEAD + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, +======= +>>>>>>> origin/development "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", diff --git a/package.json b/package.json index 4252f21..0fa308f 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ ], "license": "MIT", "dependencies": { + "dotenv": "^10.0.0", "@rdfjs/serializer-jsonld-ext": "^2.0.0", "commander": "^8.3.0", "extend": "^3.0.2", diff --git a/test/env-variables/env-not-defined-warning-all/mapping.rml.ttl b/test/env-variables/env-not-defined-warning-all/mapping.rml.ttl new file mode 100644 index 0000000..cc53c17 --- /dev/null +++ b/test/env-variables/env-not-defined-warning-all/mapping.rml.ttl @@ -0,0 +1,51 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . + +:rules_000 a void:Dataset; + void:exampleResource :map_myMapping_000. +:map_myMapping_000 rml:logicalSource :source_000. +:source_000 a rml:LogicalSource; + rml:source :database_000; + rml:query "SELECT val1, val2 FROM tab". +:database_000 a d2rq:Database; + d2rq:jdbcDSN "//${DB_HOST}:${DB_PORT}/${DB}"; + d2rq:jdbcDriver "org.postgresql.Driver"; + d2rq:username "${DB_USER}"; + d2rq:password "${DB_PASSWORD}". +:source_000 rml:referenceFormulation ql:CSV. +:map_myMapping_000 a rr:TriplesMap; + rdfs:label "myMapping". +:s_000 a rr:SubjectMap. +:map_myMapping_000 rr:subjectMap :s_000. +:s_000 rr:template "http://example.org/ns#entity_{val1}". +:pom_000 a rr:PredicateObjectMap. +:map_myMapping_000 rr:predicateObjectMap :pom_000. +:pm_000 a rr:PredicateMap. +:pom_000 rr:predicateMap :pm_000. +:pm_000 rr:constant rdf:type. +:pom_000 rr:objectMap :om_000. +:om_000 a rr:ObjectMap; + rr:constant "http://example.org/ns#Thing"; + rr:termType rr:IRI. +:pom_001 a rr:PredicateObjectMap. +:map_myMapping_000 rr:predicateObjectMap :pom_001. +:pm_001 a rr:PredicateMap. +:pom_001 rr:predicateMap :pm_001. +:pm_001 rr:constant ex:title. +:pom_001 rr:objectMap :om_001. +:om_001 a rr:ObjectMap; + rml:reference "val2"; + rr:termType rr:Literal; + rml:languageMap :language_000. +:language_000 rr:constant "en". diff --git a/test/env-variables/env-not-defined-warning-all/mapping.yml b/test/env-variables/env-not-defined-warning-all/mapping.yml new file mode 100644 index 0000000..8826543 --- /dev/null +++ b/test/env-variables/env-not-defined-warning-all/mapping.yml @@ -0,0 +1,22 @@ +prefixes: + ex: "http://example.org/ns#" + +variables: + credentials: &credentials + username: ${DB_USER} + password: ${DB_PASSWORD} + +mappings: + + myMapping: + sources: + - access: //${DB_HOST}:${DB_PORT}/${DB} + type: postgresql + credentials: *credentials + queryFormulation: sql2008 + referenceFormulation: csv + query: SELECT val1, val2 FROM tab + s: ex:entity_$(val1) + po: + - [a, ex:Thing] + - [ex:title, $(val2), en~lang] diff --git a/test/env-variables/env-not-defined-warning/mapping.rml.ttl b/test/env-variables/env-not-defined-warning/mapping.rml.ttl new file mode 100644 index 0000000..a0340d6 --- /dev/null +++ b/test/env-variables/env-not-defined-warning/mapping.rml.ttl @@ -0,0 +1,51 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . + +:rules_000 a void:Dataset; + void:exampleResource :map_myMapping_000. +:map_myMapping_000 rml:logicalSource :source_000. +:source_000 a rml:LogicalSource; + rml:source :database_000; + rml:query "SELECT val1, val2 FROM tab". +:database_000 a d2rq:Database; + d2rq:jdbcDSN "//myHost:myPort/${DB}"; + d2rq:jdbcDriver "org.postgresql.Driver"; + d2rq:username "${DB_USER}"; + d2rq:password "${DB_PASSWORD}". +:source_000 rml:referenceFormulation ql:CSV. +:map_myMapping_000 a rr:TriplesMap; + rdfs:label "myMapping". +:s_000 a rr:SubjectMap. +:map_myMapping_000 rr:subjectMap :s_000. +:s_000 rr:template "http://example.org/ns#entity_{val1}". +:pom_000 a rr:PredicateObjectMap. +:map_myMapping_000 rr:predicateObjectMap :pom_000. +:pm_000 a rr:PredicateMap. +:pom_000 rr:predicateMap :pm_000. +:pm_000 rr:constant rdf:type. +:pom_000 rr:objectMap :om_000. +:om_000 a rr:ObjectMap; + rr:constant "http://example.org/ns#Thing"; + rr:termType rr:IRI. +:pom_001 a rr:PredicateObjectMap. +:map_myMapping_000 rr:predicateObjectMap :pom_001. +:pm_001 a rr:PredicateMap. +:pom_001 rr:predicateMap :pm_001. +:pm_001 rr:constant ex:title. +:pom_001 rr:objectMap :om_001. +:om_001 a rr:ObjectMap; + rml:reference "val2"; + rr:termType rr:Literal; + rml:languageMap :language_000. +:language_000 rr:constant "en". diff --git a/test/env-variables/env-not-defined-warning/mapping.yml b/test/env-variables/env-not-defined-warning/mapping.yml new file mode 100644 index 0000000..8826543 --- /dev/null +++ b/test/env-variables/env-not-defined-warning/mapping.yml @@ -0,0 +1,22 @@ +prefixes: + ex: "http://example.org/ns#" + +variables: + credentials: &credentials + username: ${DB_USER} + password: ${DB_PASSWORD} + +mappings: + + myMapping: + sources: + - access: //${DB_HOST}:${DB_PORT}/${DB} + type: postgresql + credentials: *credentials + queryFormulation: sql2008 + referenceFormulation: csv + query: SELECT val1, val2 FROM tab + s: ex:entity_$(val1) + po: + - [a, ex:Thing] + - [ex:title, $(val2), en~lang] diff --git a/test/env-variables/env-replaced/mapping.rml.ttl b/test/env-variables/env-replaced/mapping.rml.ttl new file mode 100644 index 0000000..a0179f7 --- /dev/null +++ b/test/env-variables/env-replaced/mapping.rml.ttl @@ -0,0 +1,51 @@ +@prefix rr: . +@prefix rdf: . +@prefix rdfs: . +@prefix fnml: . +@prefix fno: . +@prefix d2rq: . +@prefix void: . +@prefix dc: . +@prefix foaf: . +@prefix rml: . +@prefix ql: . +@prefix : . +@prefix ex: . + +:rules_000 a void:Dataset; + void:exampleResource :map_myMapping_000. +:map_myMapping_000 rml:logicalSource :source_000. +:source_000 a rml:LogicalSource; + rml:source :database_000; + rml:query "SELECT val1, val2 FROM tab". +:database_000 a d2rq:Database; + d2rq:jdbcDSN "//myHost:myPort/myDB"; + d2rq:jdbcDriver "org.postgresql.Driver"; + d2rq:username "dbUser"; + d2rq:password "dbPassword". +:source_000 rml:referenceFormulation ql:CSV. +:map_myMapping_000 a rr:TriplesMap; + rdfs:label "myMapping". +:s_000 a rr:SubjectMap. +:map_myMapping_000 rr:subjectMap :s_000. +:s_000 rr:template "http://example.org/ns#entity_{val1}". +:pom_000 a rr:PredicateObjectMap. +:map_myMapping_000 rr:predicateObjectMap :pom_000. +:pm_000 a rr:PredicateMap. +:pom_000 rr:predicateMap :pm_000. +:pm_000 rr:constant rdf:type. +:pom_000 rr:objectMap :om_000. +:om_000 a rr:ObjectMap; + rr:constant "http://example.org/ns#Thing"; + rr:termType rr:IRI. +:pom_001 a rr:PredicateObjectMap. +:map_myMapping_000 rr:predicateObjectMap :pom_001. +:pm_001 a rr:PredicateMap. +:pom_001 rr:predicateMap :pm_001. +:pm_001 rr:constant ex:title. +:pom_001 rr:objectMap :om_001. +:om_001 a rr:ObjectMap; + rml:reference "val2"; + rr:termType rr:Literal; + rml:languageMap :language_000. +:language_000 rr:constant "en". diff --git a/test/env-variables/env-replaced/mapping.yml b/test/env-variables/env-replaced/mapping.yml new file mode 100644 index 0000000..8826543 --- /dev/null +++ b/test/env-variables/env-replaced/mapping.yml @@ -0,0 +1,22 @@ +prefixes: + ex: "http://example.org/ns#" + +variables: + credentials: &credentials + username: ${DB_USER} + password: ${DB_PASSWORD} + +mappings: + + myMapping: + sources: + - access: //${DB_HOST}:${DB_PORT}/${DB} + type: postgresql + credentials: *credentials + queryFormulation: sql2008 + referenceFormulation: csv + query: SELECT val1, val2 FROM tab + s: ex:entity_$(val1) + po: + - [a, ex:Thing] + - [ex:title, $(val2), en~lang]