diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js index 2bbec6519..2445fad8b 100644 --- a/lib/connection/connection_config.js +++ b/lib/connection/connection_config.js @@ -5,6 +5,7 @@ const os = require('os'); const url = require('url'); const Util = require('../util'); +const ProxyUtil = require('../proxy_util'); const Errors = require('../errors'); const ConnectionConstants = require('../constants/connection_constants'); const path = require('path'); @@ -222,7 +223,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) { protocol: proxyProtocol, noProxy: noProxy }; - Util.validateProxy(proxy); + ProxyUtil.validateProxy(proxy); } const serviceName = options.serviceName; diff --git a/lib/file_transfer_agent/azure_util.js b/lib/file_transfer_agent/azure_util.js index 496b0aaee..507dfce4e 100644 --- a/lib/file_transfer_agent/azure_util.js +++ b/lib/file_transfer_agent/azure_util.js @@ -6,6 +6,9 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata; const FileHeader = require('./file_util').FileHeader; const expandTilde = require('expand-tilde'); const resultStatus = require('./file_util').resultStatus; +const ProxyUtil = require('../proxy_util'); +const { isBypassProxy } = require('../http/node'); +const Logger = require('../logger'); const EXPIRED_TOKEN = 'ExpiredToken'; @@ -26,7 +29,7 @@ function AzureLocation(containerName, path) { * @returns {Object} * @constructor */ -function AzureUtil(azure, filestream) { +function AzureUtil(connectionConfig, azure, filestream) { const AZURE = typeof azure !== 'undefined' ? azure : require('@azure/storage-blob'); const fs = typeof filestream !== 'undefined' ? filestream : require('fs'); @@ -42,11 +45,20 @@ function AzureUtil(azure, filestream) { const sasToken = stageCredentials['AZURE_SAS_TOKEN']; const account = stageInfo['storageAccount']; - + const connectionString = `https://${account}.blob.core.windows.net${sasToken}`; + let proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'Azure Util'); + if (proxy && !isBypassProxy(proxy, connectionString)) { + proxy = ProxyUtil.getAzureProxy(proxy); + Logger.getInstance().debug(`The destination host is: ${ProxyUtil.getHostFromURL(connectionString)} and the proxy host is: ${proxy.host}`); + } + ProxyUtil.hideEnvironmentProxy(); const blobServiceClient = new AZURE.BlobServiceClient( - `https://${account}.blob.core.windows.net${sasToken}` + connectionString, null, + { + proxyOptions: proxy, + } ); - + ProxyUtil.restoreEnvironmentProxy(); return blobServiceClient; }; @@ -203,7 +215,7 @@ function AzureUtil(azure, filestream) { blobContentEncoding: 'UTF-8', blobContentType: 'application/octet-stream' } - }); + }); } catch (err) { if (err['statusCode'] === 403 && detectAzureTokenExpireError(err)) { meta['lastError'] = err; @@ -215,7 +227,6 @@ function AzureUtil(azure, filestream) { } return; } - meta['dstFileSize'] = meta['uploadSize']; meta['resultStatus'] = resultStatus.UPLOADED; }; @@ -262,7 +273,6 @@ function AzureUtil(azure, filestream) { } return; } - meta['resultStatus'] = resultStatus.DOWNLOADED; }; @@ -282,5 +292,4 @@ function AzureUtil(azure, filestream) { errstr.includes('Server failed to authenticate the request.'); } } - module.exports = AzureUtil; diff --git a/lib/file_transfer_agent/remote_storage_util.js b/lib/file_transfer_agent/remote_storage_util.js index 0657788a1..ffc599ec8 100644 --- a/lib/file_transfer_agent/remote_storage_util.js +++ b/lib/file_transfer_agent/remote_storage_util.js @@ -44,7 +44,7 @@ function RemoteStorageUtil(connectionConfig) { if (type === 'S3') { return new SnowflakeS3Util(connectionConfig); } else if (type === 'AZURE') { - return new SnowflakeAzureUtil(); + return new SnowflakeAzureUtil(connectionConfig); } else if (type === 'GCS') { return new SnowflakeGCSUtil(); } else { diff --git a/lib/file_transfer_agent/s3_util.js b/lib/file_transfer_agent/s3_util.js index 088d780b2..0369ebfc5 100644 --- a/lib/file_transfer_agent/s3_util.js +++ b/lib/file_transfer_agent/s3_util.js @@ -7,9 +7,7 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata; const FileHeader = require('./file_util').FileHeader; const expandTilde = require('expand-tilde'); const getProxyAgent = require('../http/node').getProxyAgent; -const Util = require('../util'); -const Logger = require('../logger'); -const GlobalConfig = require('../global_config'); +const ProxyUtil = require('../proxy_util'); const AMZ_IV = 'x-amz-iv'; const AMZ_KEY = 'x-amz-key'; @@ -72,13 +70,7 @@ function S3Util(connectionConfig, s3, filestream) { useAccelerateEndpoint: useAccelerateEndpoint }; - let proxy = connectionConfig.getProxy(); - if (!proxy && GlobalConfig.isEnvProxyActive()) { - proxy = Util.getProxyFromEnv(); - if (proxy) { - Logger.getInstance().debug(`S3 Util loads the proxy info from the environment variable host: ${proxy.host}`); - } - } + const proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'S3 Util'); if (proxy) { const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), SNOWFLAKE_S3_DESTINATION); config.requestHandler = new NodeHttpHandler({ diff --git a/lib/http/node.js b/lib/http/node.js index 04e30b7c4..b334b8a4b 100644 --- a/lib/http/node.js +++ b/lib/http/node.js @@ -3,6 +3,7 @@ */ const Util = require('../util'); +const ProxyUtil = require('../proxy_util'); const Base = require('./base'); const HttpsAgent = require('../agent/https_ocsp_agent'); const HttpsProxyAgent = require('../agent/https_proxy_agent'); @@ -50,7 +51,7 @@ function getFromCacheOrCreate(agentClass, options, agentId) { Logger.getInstance().trace(`Create and add to cache new agent ${agentId}`); // detect and log PROXY envvar + agent proxy settings - const compareAndLogEnvAndAgentProxies = Util.getCompareAndLogEnvAndAgentProxies(agentOptions); + const compareAndLogEnvAndAgentProxies = ProxyUtil.getCompareAndLogEnvAndAgentProxies(agentOptions); Logger.getInstance().debug(`Proxy settings used in requests:${compareAndLogEnvAndAgentProxies.messages}`); // if there's anything to warn on (e.g. both envvar + agent proxy used, and they are different) // log warnings on them @@ -100,13 +101,8 @@ function isBypassProxy(proxy, destination) { * @inheritDoc */ NodeHttpClient.prototype.getAgent = function (parsedUrl, proxy, mock) { - if (!proxy && GlobalConfig.isEnvProxyActive()) { - const isHttps = parsedUrl.protocol === 'https:'; - proxy = Util.getProxyFromEnv(isHttps); - if (proxy) { - Logger.getInstance().debug(`Load the proxy info from the environment variable host: ${proxy.host} in getAgent`); - } - } + const isHttps = parsedUrl.protocol === 'https:'; + proxy = ProxyUtil.getProxy(proxy, 'Node HTTP Client', isHttps); return getProxyAgent(proxy, parsedUrl, parsedUrl.href, mock); }; @@ -124,7 +120,7 @@ function getProxyAgent(proxyOptions, parsedUrl, destination, mock) { } } - const destHost = Util.getHostFromURL(destination); + const destHost = ProxyUtil.getHostFromURL(destination); Logger.getInstance().debug(`The destination host is: ${destHost}`); const agentId = createAgentId(agentOptions.protocol, agentOptions.hostname, destHost, agentOptions.keepAlive); @@ -157,4 +153,4 @@ function getAgentCacheSize() { return httpsAgentCache.size; } -module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize }; \ No newline at end of file +module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize, isBypassProxy }; \ No newline at end of file diff --git a/lib/proxy_util.js b/lib/proxy_util.js new file mode 100644 index 000000000..50a198b1d --- /dev/null +++ b/lib/proxy_util.js @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + +const Logger = require('./logger'); +const Errors = require('./errors'); +const Util = require('./util'); +const GlobalConfig = require('./global_config'); +const ErrorCodes = Errors.codes; +/** +* remove http:// or https:// from the input, e.g. used with proxy URL +* @param input +* @returns {string} +*/ +exports.removeScheme = function (input) { + return input.toString().replace(/(^\w+:|^)\/\//, ''); +}; + +/** + * Try to get the PROXY environmental variables + * On Windows, envvar name is case-insensitive, but on *nix, it's case-sensitive + * + * Compare them with the proxy specified on the Connection, if any + * Return with the log constructed from the components detection and comparison + * If there's something to warn the user about, return that too + * + * @param the agentOptions object from agent creation + * @returns {object} + */ +exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { + const envProxy = {}; + const logMessages = { 'messages': '', 'warnings': '' }; + envProxy.httpProxy = process.env.http_proxy || process.env.HTTP_PROXY; + envProxy.httpsProxy = process.env.https_proxy || process.env.HTTPS_PROXY; + envProxy.noProxy = process.env.no_proxy || process.env.NO_PROXY; + + envProxy.logHttpProxy = envProxy.httpProxy ? + 'HTTP_PROXY: ' + envProxy.httpProxy : 'HTTP_PROXY: '; + envProxy.logHttpsProxy = envProxy.httpsProxy ? + 'HTTPS_PROXY: ' + envProxy.httpsProxy : 'HTTPS_PROXY: '; + envProxy.logNoProxy = envProxy.noProxy ? + 'NO_PROXY: ' + envProxy.noProxy : 'NO_PROXY: '; + + // log PROXY envvars + if (envProxy.httpProxy || envProxy.httpsProxy) { + logMessages.messages = logMessages.messages + ' // PROXY environment variables: ' + + `${envProxy.logHttpProxy} ${envProxy.logHttpsProxy} ${envProxy.logNoProxy}.`; + } + + // log proxy config on Connection, if any set + if (agentOptions.host) { + const proxyHostAndPort = agentOptions.host + ':' + agentOptions.port; + const proxyProtocolHostAndPort = agentOptions.protocol ? + ' protocol=' + agentOptions.protocol + ' proxy=' + proxyHostAndPort + : ' proxy=' + proxyHostAndPort; + const proxyUsername = agentOptions.user ? ' user=' + agentOptions.user : ''; + logMessages.messages = logMessages.messages + ` // Proxy configured in Agent:${proxyProtocolHostAndPort}${proxyUsername}`; + + // check if both the PROXY envvars and Connection proxy config is set + // generate warnings if they are, and are also different + if (envProxy.httpProxy && + this.removeScheme(envProxy.httpProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { + logMessages.warnings = logMessages.warnings + ` Using both the HTTP_PROXY (${envProxy.httpProxy})` + + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` + + ' If you experience connectivity issues, try unsetting one of them.'; + } + if (envProxy.httpsProxy && + this.removeScheme(envProxy.httpsProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { + logMessages.warnings = logMessages.warnings + ` Using both the HTTPS_PROXY (${envProxy.httpsProxy})` + + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` + + ' If you experience connectivity issues, try unsetting one of them.'; + } + } + logMessages.messages = logMessages.messages ? logMessages.messages : ' none.'; + + return logMessages; +}; + +exports.validateProxy = function (proxy) { + const { host, port, noProxy, user, password } = proxy; + // check for missing proxyHost + Errors.checkArgumentExists(Util.exists(host), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_HOST); + + // check for invalid proxyHost + Errors.checkArgumentValid(Util.isString(host), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_HOST); + + // check for missing proxyPort + Errors.checkArgumentExists(Util.exists(port), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PORT); + + // check for invalid proxyPort + Errors.checkArgumentValid(Util.isNumber(port), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PORT); + + if (Util.exists(noProxy)) { + // check for invalid noProxy + Errors.checkArgumentValid(Util.isString(noProxy), + ErrorCodes.ERR_CONN_CREATE_INVALID_NO_PROXY); + } + + if (Util.exists(user) || Util.exists(password)) { + // check for missing proxyUser + Errors.checkArgumentExists(Util.exists(user), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_USER); + + // check for invalid proxyUser + Errors.checkArgumentValid(Util.isString(user), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_USER); + + // check for missing proxyPassword + Errors.checkArgumentExists(Util.exists(password), + ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PASS); + + // check for invalid proxyPassword + Errors.checkArgumentValid(Util.isString(password), + ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PASS); + + } else { + delete proxy.user; + delete proxy.password; + } +}; + +exports.validateEmptyString = function (value) { + return value !== '' ? value : undefined; +}; + +exports.getProxyFromEnv = function (isHttps = true) { + const protocol = isHttps ? 'https' : 'http'; + let proxyFromEnv = Util.getEnvVar(`${protocol}_proxy`); + if (!proxyFromEnv){ + return null; + } + + Logger.getInstance().debug(`Util.getProxyEnv: Using ${protocol.toUpperCase()}_PROXY from the environment variable`); + if (proxyFromEnv.indexOf('://') === -1) { + Logger.getInstance().info('Util.getProxyEnv: the protocol was missing from the environment proxy. Use the HTTP protocol.'); + proxyFromEnv = 'http' + '://' + proxyFromEnv; + } + proxyFromEnv = new URL(proxyFromEnv); + const proxy = { + host: Util.validateEmptyString(proxyFromEnv.hostname), + port: Number(Util.validateEmptyString(proxyFromEnv.port)), + user: Util.validateEmptyString(proxyFromEnv.username), + password: Util.validateEmptyString(proxyFromEnv.password), + protocol: Util.validateEmptyString(proxyFromEnv.protocol), + noProxy: this.getNoProxyEnv(), + }; + this.validateProxy(proxy); + return proxy; +}; + +exports.getNoProxyEnv = function () { + const noProxy = Util.getEnvVar('no_proxy'); + if (noProxy) { + return noProxy.split(',').join('|'); + } + return undefined; +}; + +exports.getHostFromURL = function (destination) { + if (destination.indexOf('://') === -1) { + destination = 'https' + '://' + destination; + } + + try { + return new URL(destination).hostname; + } catch (err) { + Logger.getInstance().error(`Failed to parse the destination to URL with the error: ${err}. Return destination as the host: ${destination}`); + return destination; + } +}; + +exports.getProxy = function (proxy, fileLocation, isHttps) { + if (!proxy && GlobalConfig.isEnvProxyActive()) { + proxy = this.getProxyFromEnv(isHttps); + if (proxy) { + Logger.getInstance().debug(`${fileLocation} loads the proxy info from the environment variable host: ${proxy.host}`); + } + } + return proxy; +}; + +exports.getAzureProxy = function (proxy) { + const AzureProxy = { + ...proxy, host: `${proxy.protocol}${(proxy.protocol).endsWith(':') ? '' : ':'}//${proxy.host}`, + }; + delete AzureProxy.noProxy; + delete AzureProxy.protocol; + + if (!Util.exists(AzureProxy.user) || !Util.exists(AzureProxy.password)) { + delete AzureProxy.user; + delete proxy.password; + } + return AzureProxy; +}; + +/** + * Currently, there is no way to disable loading the proxy information from the environment path in Azure/blob. + * To control this proxy option on the driver side, A temporary workaround is hide(remove) the environment proxy from the process + * when the client is created (At this time, the client loads the proxy from the environment variables internally). + * After the client is created, restore them with the 'restoreEnvironmentProxy' function. + */ +let envProxyList; +const proxyEnvList = ['http_proxy', 'https_proxy', 'no_proxy']; +exports.hideEnvironmentProxy = function () { + if (GlobalConfig.isEnvProxyActive()) { + return; + } + Logger.getInstance().debug('As the useEnvProxy option is disabled, the proxy environment variables are temporarily hidden during the creation of an Azure client'); + envProxyList = []; + for (const envVar of proxyEnvList) { + saveProxyInfoInList(envVar); + if (!Util.isWindows()) { + saveProxyInfoInList(envVar.toUpperCase()); + } + } +}; + +function saveProxyInfoInList(envVar) { + const proxyEnv = process.env[envVar]; + envProxyList.push(process.env[envVar]); + delete process.env[envVar]; + + if (Util.exists(proxyEnv)) { + Logger.getInstance().debug(`Temporarily exclude ${envVar} from the environment variable value: ${proxyEnv}`); + } else { + Logger.getInstance().debug(`${envVar} was not defined, nothing to do`); + } +} + +exports.restoreEnvironmentProxy = function () { + if (GlobalConfig.isEnvProxyActive()) { + return; + } + + const iterator = envProxyList[Symbol.iterator](); + let nextValue = iterator.next().value; + for (const envVar of proxyEnvList) { + if (Util.exists(nextValue)) { + Logger.getInstance().debug(`The ${envVar} value exists with the value: ${nextValue} Restore back the proxy environment variable values`); + process.env[envVar] = nextValue; + } + nextValue = iterator.next().value; + + if (!Util.isWindows()) { + if (Util.exists(nextValue)) { + Logger.getInstance().debug(`The ${envVar.toUpperCase()} value exists with the value: ${nextValue} Restore back the proxy environment variable values (for Non-Windows machine)`); + process.env[envVar.toUpperCase()] = nextValue; + } + nextValue = iterator.next().value; + } + } + Logger.getInstance().debug('An Azure client has been created. Restore back the proxy environment variable values'); +}; \ No newline at end of file diff --git a/lib/util.js b/lib/util.js index dd1a3be07..cc22322d3 100644 --- a/lib/util.js +++ b/lib/util.js @@ -8,7 +8,6 @@ const os = require('os'); const Logger = require('./logger'); const fs = require('fs'); const Errors = require('./errors'); -const ErrorCodes = Errors.codes; /** * Note: A simple wrapper around util.inherits() for now, but this might change @@ -618,66 +617,6 @@ exports.isCorrectSubdomain = function (value) { return subdomainRegex.test(value); }; -/** - * Try to get the PROXY environmental variables - * On Windows, envvar name is case-insensitive, but on *nix, it's case-sensitive - * - * Compare them with the proxy specified on the Connection, if any - * Return with the log constructed from the components detection and comparison - * If there's something to warn the user about, return that too - * - * @param the agentOptions object from agent creation - * @returns {object} - */ -exports.getCompareAndLogEnvAndAgentProxies = function (agentOptions) { - const envProxy = {}; - const logMessages = { 'messages': '', 'warnings': '' }; - envProxy.httpProxy = process.env.HTTP_PROXY || process.env.http_proxy; - envProxy.httpsProxy = process.env.HTTPS_PROXY || process.env.https_proxy; - envProxy.noProxy = process.env.NO_PROXY || process.env.no_proxy; - - envProxy.logHttpProxy = envProxy.httpProxy ? - 'HTTP_PROXY: ' + envProxy.httpProxy : 'HTTP_PROXY: '; - envProxy.logHttpsProxy = envProxy.httpsProxy ? - 'HTTPS_PROXY: ' + envProxy.httpsProxy : 'HTTPS_PROXY: '; - envProxy.logNoProxy = envProxy.noProxy ? - 'NO_PROXY: ' + envProxy.noProxy : 'NO_PROXY: '; - - // log PROXY envvars - if (envProxy.httpProxy || envProxy.httpsProxy) { - logMessages.messages = logMessages.messages + ' // PROXY environment variables: ' - + `${envProxy.logHttpProxy} ${envProxy.logHttpsProxy} ${envProxy.logNoProxy}.`; - } - - // log proxy config on Connection, if any set - if (agentOptions.host) { - const proxyHostAndPort = agentOptions.host + ':' + agentOptions.port; - const proxyProtocolHostAndPort = agentOptions.protocol ? - ' protocol=' + agentOptions.protocol + ' proxy=' + proxyHostAndPort - : ' proxy=' + proxyHostAndPort; - const proxyUsername = agentOptions.user ? ' user=' + agentOptions.user : ''; - logMessages.messages = logMessages.messages + ` // Proxy configured in Agent:${proxyProtocolHostAndPort}${proxyUsername}`; - - // check if both the PROXY envvars and Connection proxy config is set - // generate warnings if they are, and are also different - if (envProxy.httpProxy && - this.removeScheme(envProxy.httpProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { - logMessages.warnings = logMessages.warnings + ` Using both the HTTP_PROXY (${envProxy.httpProxy})` - + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` - + ' If you experience connectivity issues, try unsetting one of them.'; - } - if (envProxy.httpsProxy && - this.removeScheme(envProxy.httpsProxy).toLowerCase() !== this.removeScheme(proxyHostAndPort).toLowerCase()) { - logMessages.warnings = logMessages.warnings + ` Using both the HTTPS_PROXY (${envProxy.httpsProxy})` - + ` and the proxyHost:proxyPort (${proxyHostAndPort}) settings to connect, but with different values.` - + ' If you experience connectivity issues, try unsetting one of them.'; - } - } - logMessages.messages = logMessages.messages ? logMessages.messages : ' none.'; - - return logMessages; -}; - exports.buildCredentialCacheKey = function (host, username, credType) { if (!host || !username || !credType) { Logger.getInstance().debug('Cannot build the credential cache key because one of host, username, and credType is null'); @@ -710,15 +649,6 @@ exports.checkParametersDefined = function (...parameters) { return parameters.every((element) => element !== undefined && element !== null); }; -/** -* remove http:// or https:// from the input, e.g. used with proxy URL -* @param input -* @returns {string} -*/ -exports.removeScheme = function (input) { - return input.toString().replace(/(^\w+:|^)\/\//, ''); -}; - exports.buildCredentialCacheKey = function (host, username, credType) { if (!host || !username || !credType) { Logger.getInstance().debug('Cannot build the credential cache key because one of host, username, and credType is null'); @@ -810,106 +740,17 @@ exports.getEnvVar = function (variable) { return process.env[variable.toLowerCase()] || process.env[variable.toUpperCase()]; }; -exports.validateProxy = function (proxy) { - const { host, port, noProxy, user, password } = proxy; - // check for missing proxyHost - Errors.checkArgumentExists(this.exists(host), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_HOST); - - // check for invalid proxyHost - Errors.checkArgumentValid(this.isString(host), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_HOST); - - // check for missing proxyPort - Errors.checkArgumentExists(this.exists(port), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PORT); - - // check for invalid proxyPort - Errors.checkArgumentValid(this.isNumber(port), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PORT); - - if (this.exists(noProxy)) { - // check for invalid noProxy - Errors.checkArgumentValid(this.isString(noProxy), - ErrorCodes.ERR_CONN_CREATE_INVALID_NO_PROXY); - } - - if (this.exists(user) || this.exists(password)) { - // check for missing proxyUser - Errors.checkArgumentExists(this.exists(user), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_USER); - - // check for invalid proxyUser - Errors.checkArgumentValid(this.isString(user), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_USER); - - // check for missing proxyPassword - Errors.checkArgumentExists(this.exists(password), - ErrorCodes.ERR_CONN_CREATE_MISSING_PROXY_PASS); - - // check for invalid proxyPassword - Errors.checkArgumentValid(this.isString(password), - ErrorCodes.ERR_CONN_CREATE_INVALID_PROXY_PASS); - - } else { - delete proxy.user; - delete proxy.password; - } -}; - exports.validateEmptyString = function (value) { return value !== '' ? value : undefined; }; -exports.getProxyFromEnv = function (isHttps = true) { - const protocol = isHttps ? 'https' : 'http'; - let proxyFromEnv = this.getEnvVar(`${protocol}_proxy`); - if (!proxyFromEnv){ - return null; - } - - Logger.getInstance().debug(`Util.getProxyEnv: Using ${protocol.toUpperCase()}_PROXY from the environment variable`); - if (proxyFromEnv.indexOf('://') === -1) { - Logger.getInstance().info('Util.getProxyEnv: the protocol was missing from the environment proxy. Use the HTTP protocol.'); - proxyFromEnv = 'http' + '://' + proxyFromEnv; - } - proxyFromEnv = new URL(proxyFromEnv); - const proxy = { - host: this.validateEmptyString(proxyFromEnv.hostname), - port: Number(this.validateEmptyString(proxyFromEnv.port)), - user: this.validateEmptyString(proxyFromEnv.username), - password: this.validateEmptyString(proxyFromEnv.password), - protocol: this.validateEmptyString(proxyFromEnv.protocol), - noProxy: this.getNoProxyEnv(), - }; - this.validateProxy(proxy); - return proxy; -}; - -exports.getNoProxyEnv = function () { - const noProxy = this.getEnvVar('no_proxy'); - if (noProxy) { - return noProxy.split(',').join('|'); - } - return undefined; -}; - -exports.getHostFromURL = function (destination) { - if (destination.indexOf('://') === -1) { - destination = 'https' + '://' + destination; - } - - try { - return new URL(destination).hostname; - } catch (err) { - Logger.getInstance().error(`Failed to parse the destination to URL with the error: ${err}. Return destination as the host: ${destination}`); - return destination; - } -}; - exports.isNotEmptyAsString = function (variable) { if (typeof variable === 'string') { return variable; } return exports.exists(variable); +}; + +exports.isWindows = function () { + return os.platform() === 'win32'; }; \ No newline at end of file diff --git a/test/unit/agent_cache_test.js b/test/unit/agent_cache_test.js index d6a68ae84..fabe534dc 100644 --- a/test/unit/agent_cache_test.js +++ b/test/unit/agent_cache_test.js @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + const GlobalConfig = require('./../../lib/global_config'); const getProxyAgent = require('./../../lib/http/node').getProxyAgent; const getAgentCacheSize = require('./../../lib/http/node').getAgentCacheSize; diff --git a/test/unit/file_transfer_agent/azure_test.js b/test/unit/file_transfer_agent/azure_test.js index fea6661e4..f99c53147 100644 --- a/test/unit/file_transfer_agent/azure_test.js +++ b/test/unit/file_transfer_agent/azure_test.js @@ -16,6 +16,13 @@ describe('Azure client', function () { const mockKey = 'mockKey'; const mockIv = 'mockIv'; const mockMatDesc = 'mockMatDesc'; + const noProxyConnectionConfig = { + getProxy: function () { + return null; + }, + accessUrl: 'http://fakeaccount.snowflakecomputing.com', + }; + let Azure = null; let client = null; @@ -106,7 +113,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); }); it('extract bucket name and path', async function () { @@ -131,7 +138,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - Azure = new SnowflakeAzureUtil(client); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); @@ -146,7 +153,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - const Azure = new SnowflakeAzureUtil(client); + const Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.NOT_FOUND_FILE); @@ -161,7 +168,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - Azure = new SnowflakeAzureUtil(client); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); @@ -176,7 +183,7 @@ describe('Azure client', function () { }, null)); client = require('client'); - Azure = new SnowflakeAzureUtil(client); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client); await Azure.getFileHeader(meta, dataFile); assert.strictEqual(meta['resultStatus'], resultStatus.ERROR); @@ -192,7 +199,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); await Azure.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.UPLOADED); @@ -212,7 +219,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); await Azure.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.RENEW_TOKEN); @@ -232,7 +239,7 @@ describe('Azure client', function () { client = require('client'); filestream = require('filestream'); - Azure = new SnowflakeAzureUtil(client, filestream); + Azure = new SnowflakeAzureUtil(noProxyConnectionConfig, client, filestream); await Azure.uploadFile(dataFile, meta, encryptionMetadata); assert.strictEqual(meta['resultStatus'], resultStatus.NEED_RETRY); diff --git a/test/unit/proxy_util_test.js b/test/unit/proxy_util_test.js new file mode 100644 index 000000000..efc72371b --- /dev/null +++ b/test/unit/proxy_util_test.js @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2015-2024 Snowflake Computing Inc. All rights reserved. + */ + +const ProxyUtil = require('./../../lib/proxy_util'); +const Util = require('./../../lib/util'); +const GlobalConfig = require('../../lib/global_config'); + +const assert = require('assert'); + +describe('ProxyUtil Test - removing http or https from string', () => { + const hostAndPortDone = 'my.pro.xy:8080'; + const ipAndPortDone = '10.20.30.40:8080'; + const somethingEntirelyDifferentDone = 'something ENTIRELY different'; + + [ + { name: 'remove http from url', text: 'http://my.pro.xy:8080', shouldMatch: hostAndPortDone }, + { name: 'remove https from url', text: 'https://my.pro.xy:8080', shouldMatch: hostAndPortDone }, + { name: 'remove http from ip and port', text: 'http://10.20.30.40:8080', shouldMatch: ipAndPortDone }, + { name: 'remove https from ip and port', text: 'https://10.20.30.40:8080', shouldMatch: ipAndPortDone }, + { name: 'dont remove http(s) from hostname and port', text: 'my.pro.xy:8080', shouldMatch: hostAndPortDone }, + { name: 'dont remove http(s) from ip and port', text: '10.20.30.40:8080', shouldMatch: ipAndPortDone }, + { name: 'dont remove http(s) from simple string', text: somethingEntirelyDifferentDone, shouldMatch: somethingEntirelyDifferentDone } + ].forEach(({ name, text, shouldMatch }) => { + it(`${name}`, () => { + assert.deepEqual(ProxyUtil.removeScheme(text), shouldMatch); + }); + }); +}); + +describe('ProxyUtil Test - detecting PROXY envvars and compare with the agent proxy settings', () => { + [ + { + name: 'detect http_proxy envvar, no agent proxy', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: '', + agentOptions: { 'keepalive': true }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: .' + }, { + name: 'detect HTTPS_PROXY envvar, no agent proxy', + isWarn: false, + httpproxy: '', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, no agent proxy', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' + }, { + name: 'detect http_proxy envvar, agent proxy set to an unauthenticated proxy, same as the envvar', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: '', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, same as the envvar', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://10.20.30.40:8080', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://10.20.30.40:8080', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass' }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080 user=PRX' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar, with the protocol set', + isWarn: false, + httpproxy: '10.20.30.40:8080', + HTTPSPROXY: 'http://10.20.30.40:8080', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass', 'protocol': 'http' }, + shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: protocol=http proxy=10.20.30.40:8080 user=PRX' + }, { + // now some WARN level messages + name: 'detect HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, different from the envvar', + isWarn: true, + httpproxy: '', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + }, { + name: 'detect both http_proxy and HTTPS_PROXY envvar, different from each other, agent proxy set to an unauthenticated proxy, different from the envvars', + isWarn: true, + httpproxy: '169.254.169.254:8080', + HTTPSPROXY: 'http://pro.xy:3128', + agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, + shouldLog: ' Using both the HTTP_PROXY (169.254.169.254:8080) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' + } + ].forEach(({ name, isWarn, httpproxy, HTTPSPROXY, agentOptions, shouldLog }) => { + it(`${name}`, () => { + process.env.HTTP_PROXY = httpproxy; + process.env.HTTPS_PROXY = HTTPSPROXY; + + const compareAndLogEnvAndAgentProxies = ProxyUtil.getCompareAndLogEnvAndAgentProxies(agentOptions); + if (!isWarn) { + assert.deepEqual(compareAndLogEnvAndAgentProxies.messages, shouldLog, 'expected log message does not match!'); + } else { + assert.deepEqual(compareAndLogEnvAndAgentProxies.warnings, shouldLog, 'expected warning message does not match!'); + } + }); + }); +}); + +describe('getProxyEnv function test ', function () { + let originalHttpProxy = null; + let originalHttpsProxy = null; + let originalNoProxy = null; + + before(() => { + originalHttpProxy = process.env.HTTP_PROXY; + originalHttpsProxy = process.env.HTTPS_PROXY; + originalNoProxy = process.env.NO_PROXY; + }); + + beforeEach(() => { + delete process.env.HTTP_PROXY; + delete process.env.HTTPS_PROXY; + delete process.env.NO_PROXY; + }); + + after(() => { + originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; + originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; + originalNoProxy ? process.env.NO_PROXY = originalNoProxy : delete process.env.NO_PROXY; + }); + + const testCases = [ + { + name: 'HTTP PROXY without authentication and schema', + isHttps: false, + httpProxy: 'proxy.example.com:8080', + httpsProxy: undefined, + noProxy: '*.amazonaws.com', + result: { + host: 'proxy.example.com', + port: 8080, + protocol: 'http:', + noProxy: '*.amazonaws.com' + } + }, + { + name: 'HTTP PROXY with authentication', + isHttps: false, + httpProxy: 'http://hello:world@proxy.example.com:8080', + httpsProxy: undefined, + noProxy: '*.amazonaws.com,*.my_company.com', + result: { + host: 'proxy.example.com', + user: 'hello', + password: 'world', + port: 8080, + protocol: 'http:', + noProxy: '*.amazonaws.com|*.my_company.com' + } + }, + { + name: 'HTTPS PROXY with authentication without NO proxy', + isHttps: true, + httpsProxy: 'https://user:pass@myproxy.server.com:1234', + result: { + host: 'myproxy.server.com', + user: 'user', + password: 'pass', + port: 1234, + protocol: 'https:', + noProxy: undefined, + }, + }, + { + name: 'HTTPS PROXY with authentication without NO proxy No schema', + isHttps: true, + noProxy: '*.amazonaws.com,*.my_company.com,*.test.com', + httpsProxy: 'myproxy.server.com:1234', + result: { + host: 'myproxy.server.com', + port: 1234, + protocol: 'http:', + noProxy: '*.amazonaws.com|*.my_company.com|*.test.com', + }, + }, + ]; + + testCases.forEach(({ name, isHttps, httpsProxy, httpProxy, noProxy, result }) => { + it(name, function (){ + + if (httpProxy){ + process.env.HTTP_PROXY = httpProxy; + } + if (httpsProxy) { + process.env.HTTPS_PROXY = httpsProxy; + } + if (noProxy) { + process.env.NO_PROXY = noProxy; + } + const proxy = ProxyUtil.getProxyFromEnv(isHttps); + const keys = Object.keys(result); + assert.strictEqual(keys.length, Object.keys(proxy).length); + + for (const key of keys) { + assert.strictEqual(proxy[key], result[key]); + } + }); + }); +}); + +describe('getNoProxyEnv function Test', function () { + let original = null; + + before( function (){ + original = process.env.NO_PROXY; + process.env.NO_PROXY = '*.amazonaws.com,*.my_company.com'; + }); + + after(() => { + process.env.NO_PROXY = original; + }); + + it('test noProxy conversion', function (){ + assert.strictEqual(ProxyUtil.getNoProxyEnv(), '*.amazonaws.com|*.my_company.com'); + }); +}); + +describe('Proxy Util for Azure', function () { + let originalhttpProxy = null; + let originalhttpsProxy = null; + let originalnoProxy = null; + let originalHttpProxy = null; + let originalHttpsProxy = null; + let originalNoProxy = null; + + before(() => { + GlobalConfig.setEnvProxy(false); + originalHttpProxy = process.env.HTTP_PROXY; + originalHttpsProxy = process.env.HTTPS_PROXY; + originalNoProxy = process.env.NO_PROXY; + if (!Util.isWindows()) { + originalhttpProxy = process.env.http_proxy; + originalhttpsProxy = process.env.https_proxy; + originalnoProxy = process.env.no_proxy; + } + }); + + after(() => { + GlobalConfig.setEnvProxy(true); + originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; + originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; + originalNoProxy ? process.env.NO_PROXY = originalNoProxy : delete process.env.NO_PROXY; + if (!Util.isWindows()) { + originalhttpProxy ? process.env['http_proxy'] = originalhttpProxy : delete process.env.http_proxy; + originalhttpsProxy ? process.env['https_proxy'] = originalhttpsProxy : delete process.env.https_proxy; + originalnoProxy ? process.env['no_proxy'] = originalnoProxy : delete process.env.no_proxy; + } + }); + + + it('test hide and restore environment proxy', function () { + const testCases = + { + httpProxy: 'https://user:pass@myproxy.server.com:1234', + httpsProxy: 'https://user:pass@myproxy.server.com:1234', + noProxy: '*.amazonaws.com,*.my_company.com', + HttpProxy: 'https://user:pass@myproxy2.server.com:1234', + HttpsProxy: 'https://user:pass@myproxy2.server.com:1234', + NoProxy: '*.amazonaws2.com,*.my_company2.com', + }; + + process.env.HTTP_PROXY = testCases.HttpProxy; + process.env.HTTPS_PROXY = testCases.HttpsProxy; + process.env.NO_PROXY = testCases.NoProxy; + if (!Util.isWindows()) { + process.env['http_proxy'] = testCases.httpProxy; + process.env['https_proxy'] = testCases.httpsProxy; + process.env['no_proxy'] = testCases.noProxy; + } + + ProxyUtil.hideEnvironmentProxy(); + assert.strictEqual(process.env.HTTP_PROXY, undefined); + assert.strictEqual(process.env.HTTPS_PROXY, undefined); + assert.strictEqual(process.env.NO_PROXY, undefined); + if (!Util.isWindows()) { + assert.strictEqual(process.env['http_proxy'], undefined); + assert.strictEqual(process.env['https_proxy'], undefined); + assert.strictEqual(process.env['no_proxy'], undefined); + } + + ProxyUtil.restoreEnvironmentProxy(); + assert.strictEqual(process.env.HTTP_PROXY, testCases.HttpProxy); + assert.strictEqual(process.env.HTTPS_PROXY, testCases.HttpsProxy); + assert.strictEqual(process.env.NO_PROXY, testCases.NoProxy); + if (!Util.isWindows()) { + assert.strictEqual(process.env.http_proxy, testCases.httpProxy); + assert.strictEqual(process.env.https_proxy, testCases.httpsProxy); + assert.strictEqual(process.env.no_proxy, testCases.noProxy); + } + }); +}); \ No newline at end of file diff --git a/test/unit/util_test.js b/test/unit/util_test.js index feaa48541..e7e7880a6 100644 --- a/test/unit/util_test.js +++ b/test/unit/util_test.js @@ -910,529 +910,308 @@ describe('Util', function () { }); }); - describe('Util Test - removing http or https from string', () => { - const hostAndPortDone = 'my.pro.xy:8080'; - const ipAndPortDone = '10.20.30.40:8080'; - const somethingEntirelyDifferentDone = 'something ENTIRELY different'; + describe('Util test - custom credential manager util functions', function () { + const mockUser = 'mockUser'; + const mockHost = 'mockHost'; + const mockCred = 'mockCred'; - [ - { name: 'remove http from url', text: 'http://my.pro.xy:8080', shouldMatch: hostAndPortDone }, - { name: 'remove https from url', text: 'https://my.pro.xy:8080', shouldMatch: hostAndPortDone }, - { name: 'remove http from ip and port', text: 'http://10.20.30.40:8080', shouldMatch: ipAndPortDone }, - { name: 'remove https from ip and port', text: 'https://10.20.30.40:8080', shouldMatch: ipAndPortDone }, - { name: 'dont remove http(s) from hostname and port', text: 'my.pro.xy:8080', shouldMatch: hostAndPortDone }, - { name: 'dont remove http(s) from ip and port', text: '10.20.30.40:8080', shouldMatch: ipAndPortDone }, - { name: 'dont remove http(s) from simple string', text: somethingEntirelyDifferentDone, shouldMatch: somethingEntirelyDifferentDone } - ].forEach(({ name, text, shouldMatch }) => { - it(`${name}`, () => { - assert.deepEqual(Util.removeScheme(text), shouldMatch); - }); - }); - }); - - describe('Util Test - detecting PROXY envvars and compare with the agent proxy settings', () => { - [ - { - name: 'detect http_proxy envvar, no agent proxy', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: '', - agentOptions: { 'keepalive': true }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: .' - }, { - name: 'detect HTTPS_PROXY envvar, no agent proxy', - isWarn: false, - httpproxy: '', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, no agent proxy', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://pro.xy:3128 NO_PROXY: .' - }, { - name: 'detect http_proxy envvar, agent proxy set to an unauthenticated proxy, same as the envvar', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: '', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, same as the envvar', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://10.20.30.40:8080', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://10.20.30.40:8080', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass' }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: proxy=10.20.30.40:8080 user=PRX' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, agent proxy set to an authenticated proxy, same as the envvar, with the protocol set', - isWarn: false, - httpproxy: '10.20.30.40:8080', - HTTPSPROXY: 'http://10.20.30.40:8080', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080, 'user': 'PRX', 'password': 'proxypass', 'protocol': 'http' }, - shouldLog: ' // PROXY environment variables: HTTP_PROXY: 10.20.30.40:8080 HTTPS_PROXY: http://10.20.30.40:8080 NO_PROXY: . // Proxy configured in Agent: protocol=http proxy=10.20.30.40:8080 user=PRX' - }, { - // now some WARN level messages - name: 'detect HTTPS_PROXY envvar, agent proxy set to an unauthenticated proxy, different from the envvar', - isWarn: true, - httpproxy: '', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' - }, { - name: 'detect both http_proxy and HTTPS_PROXY envvar, different from each other, agent proxy set to an unauthenticated proxy, different from the envvars', - isWarn: true, - httpproxy: '169.254.169.254:8080', - HTTPSPROXY: 'http://pro.xy:3128', - agentOptions: { 'keepalive': true, 'host': '10.20.30.40', 'port': 8080 }, - shouldLog: ' Using both the HTTP_PROXY (169.254.169.254:8080) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them. Using both the HTTPS_PROXY (http://pro.xy:3128) and the proxyHost:proxyPort (10.20.30.40:8080) settings to connect, but with different values. If you experience connectivity issues, try unsetting one of them.' - } - ].forEach(({ name, isWarn, httpproxy, HTTPSPROXY, agentOptions, shouldLog }) => { - it(`${name}`, () => { - process.env.HTTP_PROXY = httpproxy; - process.env.HTTPS_PROXY = HTTPSPROXY; - - const compareAndLogEnvAndAgentProxies = Util.getCompareAndLogEnvAndAgentProxies(agentOptions); - if (!isWarn) { - assert.deepEqual(compareAndLogEnvAndAgentProxies.messages, shouldLog, 'expected log message does not match!'); - } else { - assert.deepEqual(compareAndLogEnvAndAgentProxies.warnings, shouldLog, 'expected warning message does not match!'); - } - }); - }); - - describe('Util test - custom credential manager util functions', function () { - const mockUser = 'mockUser'; - const mockHost = 'mockHost'; - const mockCred = 'mockCred'; - - describe('test function build credential key', function () { - const testCases = [ - { - name: 'when all the parameters are null', - user: null, - host: null, - cred: null, - result: null - }, - { - name: 'when two parameters are null or undefined', - user: mockUser, - host: null, - cred: undefined, - result: null - }, - { - name: 'when one parameter is null', - user: mockUser, - host: mockHost, - cred: undefined, - result: null - }, - { - name: 'when one parameter is undefined', - user: mockUser, - host: undefined, - cred: mockCred, - result: null - }, - { - name: 'when all the parameters are valid', - user: mockUser, - host: mockHost, - cred: mockCred, - result: '{mockHost}:{mockUser}:{SF_NODE_JS_DRIVER}:{mockCred}}' - }, - ]; - testCases.forEach((name, user, host, cred, result) => { - it(`${name}`, function () { - if (!result) { - assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), null); - } else { - assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), result); - } - }); - }); - }); - }); - - describe('test valid custom credential manager', function () { - - function sampleManager() { - this.read = function () {}; - - this.write = function () {}; - - this.remove = function () {}; - } - + describe('test function build credential key', function () { const testCases = [ { - name: 'credential manager is an int', - credentialManager: 123, - result: false, - }, - { - name: 'credential manager is a string', - credentialManager: 'credential manager', - result: false, - }, - { - name: 'credential manager is an array', - credentialManager: ['write', 'read', 'remove'], - result: false, - }, - { - name: 'credential manager is an empty obejct', - credentialManager: {}, - result: false, - }, - { - name: 'credential manager has property, but invalid types', - credentialManager: { - read: 'read', - write: 1234, - remove: [] - }, - result: false, - }, - { - name: 'credential manager has property, but invalid types', - credentialManager: { - read: 'read', - write: 1234, - remove: [] - }, - result: false, - }, - { - name: 'credential manager has two valid properties, but miss one', - credentialManager: { - read: function () { - - }, - write: function () { - - } - }, - result: false, + name: 'when all the parameters are null', + user: null, + host: null, + cred: null, + result: null }, { - name: 'credential manager has two valid properties, but miss one', - credentialManager: new sampleManager(), - result: true, + name: 'when two parameters are null or undefined', + user: mockUser, + host: null, + cred: undefined, + result: null }, - ]; - - for (const { name, credentialManager, result } of testCases) { - it(name, function () { - assert.strictEqual(Util.checkValidCustomCredentialManager(credentialManager), result); - }); - } - }); - - describe('checkParametersDefined function Test', function () { - const testCases = [ { - name: 'all the parameters are null or undefined', - parameters: [null, undefined, null, null], - result: false + name: 'when one parameter is null', + user: mockUser, + host: mockHost, + cred: undefined, + result: null }, { - name: 'one parameter is null', - parameters: ['a', 2, true, null], - result: false + name: 'when one parameter is undefined', + user: mockUser, + host: undefined, + cred: mockCred, + result: null }, { - name: 'all the parameter are existing', - parameters: ['a', 123, ['testing'], {}], - result: true + name: 'when all the parameters are valid', + user: mockUser, + host: mockHost, + cred: mockCred, + result: '{mockHost}:{mockUser}:{SF_NODE_JS_DRIVER}:{mockCred}}' }, ]; - - for (const { name, parameters, result } of testCases) { - it(name, function () { - assert.strictEqual(Util.checkParametersDefined(...parameters), result); - }); - } - }); - }); - - if (os.platform() !== 'win32') { - describe('Util.isFileNotWritableByGroupOrOthers()', function () { - let tempDir = null; - let oldMask = null; - - before(async function () { - tempDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'permission_tests')); - oldMask = process.umask(0o000); - }); - - after(async function () { - await fsPromises.rm(tempDir, { recursive: true, force: true }); - process.umask(oldMask); - }); - - [ - { filePerm: 0o700, isValid: true }, - { filePerm: 0o600, isValid: true }, - { filePerm: 0o500, isValid: true }, - { filePerm: 0o400, isValid: true }, - { filePerm: 0o300, isValid: true }, - { filePerm: 0o200, isValid: true }, - { filePerm: 0o100, isValid: true }, - { filePerm: 0o707, isValid: false }, - { filePerm: 0o706, isValid: false }, - { filePerm: 0o705, isValid: true }, - { filePerm: 0o704, isValid: true }, - { filePerm: 0o703, isValid: false }, - { filePerm: 0o702, isValid: false }, - { filePerm: 0o701, isValid: true }, - { filePerm: 0o770, isValid: false }, - { filePerm: 0o760, isValid: false }, - { filePerm: 0o750, isValid: true }, - { filePerm: 0o740, isValid: true }, - { filePerm: 0o730, isValid: false }, - { filePerm: 0o720, isValid: false }, - { filePerm: 0o710, isValid: true }, - ].forEach(async function ({ filePerm, isValid }) { - it('File with permission: ' + filePerm.toString(8) + ' should be valid=' + isValid, async function () { - const filePath = path.join(tempDir, `file_${filePerm.toString()}`); - await writeFile(filePath, filePerm); - assert.strictEqual(await Util.isFileNotWritableByGroupOrOthers(filePath, fsPromises), isValid); + testCases.forEach((name, user, host, cred, result) => { + it(`${name}`, function () { + if (!result) { + assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), null); + } else { + assert.strictEqual(Util.buildCredentialCacheKey(host, user, cred), result); + } }); }); - - async function writeFile(filePath, mode) { - await fsPromises.writeFile(filePath, '', { encoding: 'utf8', mode: mode }); - } }); - } - - if (os.platform() !== 'win32') { - describe('Util.isFileModeCorrect()', function () { - const tempDir = path.join(os.tmpdir(), 'permission_tests'); - let oldMask = null; - - before(async function () { - await fsPromises.mkdir(tempDir); - oldMask = process.umask(0o000); - }); - - after(async function () { - await fsPromises.rm(tempDir, { recursive: true, force: true }); - process.umask(oldMask); - }); - - [ - { dirPerm: 0o700, expectedPerm: 0o700, isCorrect: true }, - { dirPerm: 0o755, expectedPerm: 0o600, isCorrect: false }, - ].forEach(async function ({ dirPerm, expectedPerm, isCorrect }) { - it('Should return ' + isCorrect + ' when directory permission ' + dirPerm.toString(8) + ' is compared to ' + expectedPerm.toString(8), async function () { - const dirPath = path.join(tempDir, `dir_${dirPerm.toString(8)}`); - await fsPromises.mkdir(dirPath, { mode: dirPerm }); - assert.strictEqual(await Util.isFileModeCorrect(dirPath, expectedPerm, fsPromises), isCorrect); - }); - }); - - [ - { filePerm: 0o700, expectedPerm: 0o700, isCorrect: true }, - { filePerm: 0o755, expectedPerm: 0o600, isCorrect: false }, - ].forEach(async function ({ filePerm, expectedPerm, isCorrect }) { - it('Should return ' + isCorrect + ' when file permission ' + filePerm.toString(8) + ' is compared to ' + expectedPerm.toString(8), async function () { - const dirPath = path.join(tempDir, `file_${filePerm.toString(8)}`); - await fsPromises.appendFile(dirPath, '', { mode: filePerm }); - assert.strictEqual(await Util.isFileModeCorrect(dirPath, expectedPerm, fsPromises), isCorrect); - }); - }); - }); - } + }); - describe('shouldPerformGCPBucket function test', () => { + describe('test valid custom credential manager', function () { + + function sampleManager() { + this.read = function () {}; + + this.write = function () {}; + + this.remove = function () {}; + } + const testCases = [ { - name: 'test - default', - accessToken: 'Token', - forceGCPUseDownscopedCredential: false, - result: true, + name: 'credential manager is an int', + credentialManager: 123, + result: false, }, { - name: 'test - when the disableGCPTokenUplaod is enabled', - accessToken: 'Token', - forceGCPUseDownscopedCredential: true, + name: 'credential manager is a string', + credentialManager: 'credential manager', result: false, }, { - name: 'test - when token is empty but the disableGCPTokenUplaod is enabled', - accessToken: null, - forceGCPUseDownscopedCredential: true, + name: 'credential manager is an array', + credentialManager: ['write', 'read', 'remove'], result: false, }, { - name: 'test - test - when token is empty but the disableGCPTokenUplaod is disabled', - accessToken: null, - forceGCPUseDownscopedCredential: false, + name: 'credential manager is an empty obejct', + credentialManager: {}, result: false, }, - ]; + { + name: 'credential manager has property, but invalid types', + credentialManager: { + read: 'read', + write: 1234, + remove: [] + }, + result: false, + }, + { + name: 'credential manager has property, but invalid types', + credentialManager: { + read: 'read', + write: 1234, + remove: [] + }, + result: false, + }, + { + name: 'credential manager has two valid properties, but miss one', + credentialManager: { + read: function () { - testCases.forEach(({ name, accessToken, forceGCPUseDownscopedCredential, result }) => { - it(name, () => { - process.env.SNOWFLAKE_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL = forceGCPUseDownscopedCredential; - assert.strictEqual(Util.shouldPerformGCPBucket(accessToken), result); - delete process.env.SNOWFLAKE_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL; - }); - }); - }); + }, + write: function () { - describe('getEnvVar function Test', function () { - const testCases = [ - { - name: 'snowflake_env_test', - value: 'mock_value', + } + }, + result: false, }, { - name: 'SNOWFLAKE_ENV_TEST', - value: 'MOCK_VALUE', + name: 'credential manager has two valid properties, but miss one', + credentialManager: new sampleManager(), + result: true, }, ]; - for (const { name, value, } of testCases) { + for (const { name, credentialManager, result } of testCases) { it(name, function () { - process.env[name] = value; - assert.strictEqual(Util.getEnvVar('snowflake_env_test'), value); - assert.strictEqual(Util.getEnvVar('SNOWFLAKE_ENV_TEST'), value); - delete process.env[name]; + assert.strictEqual(Util.checkValidCustomCredentialManager(credentialManager), result); }); } }); - describe('getProxyEnv function test ', function () { - let originalHttpProxy = null; - let originalHttpsProxy = null; - let originalNoProxy = null; - - before(() => { - originalHttpProxy = process.env.HTTP_PROXY; - originalHttpsProxy = process.env.HTTPS_PROXY; - originalNoProxy = process.env.NO_PROXY; - }); - - beforeEach(() => { - delete process.env.HTTP_PROXY; - delete process.env.HTTPS_PROXY; - delete process.env.NO_PROXY; - }); - - after(() => { - originalHttpProxy ? process.env.HTTP_PROXY = originalHttpProxy : delete process.env.HTTP_PROXY; - originalHttpsProxy ? process.env.HTTPS_PROXY = originalHttpsProxy : delete process.env.HTTPS_PROXY; - originalNoProxy ? process.env.NO_PROXY = originalNoProxy : delete process.env.NO_PROXY; - }); - + describe('checkParametersDefined function Test', function () { const testCases = [ { - name: 'HTTP PROXY without authentication and schema', - isHttps: false, - httpProxy: 'proxy.example.com:8080', - httpsProxy: undefined, - noProxy: '*.amazonaws.com', - result: { - host: 'proxy.example.com', - port: 8080, - protocol: 'http:', - noProxy: '*.amazonaws.com' - } - }, - { - name: 'HTTP PROXY with authentication', - isHttps: false, - httpProxy: 'http://hello:world@proxy.example.com:8080', - httpsProxy: undefined, - noProxy: '*.amazonaws.com,*.my_company.com', - result: { - host: 'proxy.example.com', - user: 'hello', - password: 'world', - port: 8080, - protocol: 'http:', - noProxy: '*.amazonaws.com|*.my_company.com' - } + name: 'all the parameters are null or undefined', + parameters: [null, undefined, null, null], + result: false }, { - name: 'HTTPS PROXY with authentication without NO proxy', - isHttps: true, - httpsProxy: 'https://user:pass@myproxy.server.com:1234', - result: { - host: 'myproxy.server.com', - user: 'user', - password: 'pass', - port: 1234, - protocol: 'https:', - noProxy: undefined, - }, + name: 'one parameter is null', + parameters: ['a', 2, true, null], + result: false }, { - name: 'HTTPS PROXY with authentication without NO proxy No schema', - isHttps: true, - noProxy: '*.amazonaws.com,*.my_company.com,*.test.com', - httpsProxy: 'myproxy.server.com:1234', - result: { - host: 'myproxy.server.com', - port: 1234, - protocol: 'http:', - noProxy: '*.amazonaws.com|*.my_company.com|*.test.com', - }, + name: 'all the parameter are existing', + parameters: ['a', 123, ['testing'], {}], + result: true }, ]; + + for (const { name, parameters, result } of testCases) { + it(name, function () { + assert.strictEqual(Util.checkParametersDefined(...parameters), result); + }); + } + }); +}); - testCases.forEach(({ name, isHttps, httpsProxy, httpProxy, noProxy, result }) => { - it(name, function (){ +if (os.platform() !== 'win32') { + describe('Util.isFileNotWritableByGroupOrOthers()', function () { + let tempDir = null; + let oldMask = null; - if (httpProxy){ - process.env.HTTP_PROXY = httpProxy; - } - if (httpsProxy) { - process.env.HTTPS_PROXY = httpsProxy; - } - if (noProxy) { - process.env.NO_PROXY = noProxy; - } - const proxy = Util.getProxyFromEnv(isHttps); - const keys = Object.keys(result); - assert.strictEqual(keys.length, Object.keys(proxy).length); + before(async function () { + tempDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), 'permission_tests')); + oldMask = process.umask(0o000); + }); - for (const key of keys) { - assert.strictEqual(proxy[key], result[key]); - } + after(async function () { + await fsPromises.rm(tempDir, { recursive: true, force: true }); + process.umask(oldMask); + }); + + [ + { filePerm: 0o700, isValid: true }, + { filePerm: 0o600, isValid: true }, + { filePerm: 0o500, isValid: true }, + { filePerm: 0o400, isValid: true }, + { filePerm: 0o300, isValid: true }, + { filePerm: 0o200, isValid: true }, + { filePerm: 0o100, isValid: true }, + { filePerm: 0o707, isValid: false }, + { filePerm: 0o706, isValid: false }, + { filePerm: 0o705, isValid: true }, + { filePerm: 0o704, isValid: true }, + { filePerm: 0o703, isValid: false }, + { filePerm: 0o702, isValid: false }, + { filePerm: 0o701, isValid: true }, + { filePerm: 0o770, isValid: false }, + { filePerm: 0o760, isValid: false }, + { filePerm: 0o750, isValid: true }, + { filePerm: 0o740, isValid: true }, + { filePerm: 0o730, isValid: false }, + { filePerm: 0o720, isValid: false }, + { filePerm: 0o710, isValid: true }, + ].forEach(async function ({ filePerm, isValid }) { + it('File with permission: ' + filePerm.toString(8) + ' should be valid=' + isValid, async function () { + const filePath = path.join(tempDir, `file_${filePerm.toString()}`); + await writeFile(filePath, filePerm); + assert.strictEqual(await Util.isFileNotWritableByGroupOrOthers(filePath, fsPromises), isValid); }); }); + + async function writeFile(filePath, mode) { + await fsPromises.writeFile(filePath, '', { encoding: 'utf8', mode: mode }); + } }); +} - describe('getNoProxyEnv function Test', function () { - let original = null; +if (os.platform() !== 'win32') { + describe('Util.isFileModeCorrect()', function () { + const tempDir = path.join(os.tmpdir(), 'permission_tests'); + let oldMask = null; + + before(async function () { + await fsPromises.mkdir(tempDir); + oldMask = process.umask(0o000); + }); - before( function (){ - original = process.env.NO_PROXY; - process.env.NO_PROXY = '*.amazonaws.com,*.my_company.com'; + after(async function () { + await fsPromises.rm(tempDir, { recursive: true, force: true }); + process.umask(oldMask); }); - after(() => { - process.env.NO_PROXY = original; + [ + { dirPerm: 0o700, expectedPerm: 0o700, isCorrect: true }, + { dirPerm: 0o755, expectedPerm: 0o600, isCorrect: false }, + ].forEach(async function ({ dirPerm, expectedPerm, isCorrect }) { + it('Should return ' + isCorrect + ' when directory permission ' + dirPerm.toString(8) + ' is compared to ' + expectedPerm.toString(8), async function () { + const dirPath = path.join(tempDir, `dir_${dirPerm.toString(8)}`); + await fsPromises.mkdir(dirPath, { mode: dirPerm }); + assert.strictEqual(await Util.isFileModeCorrect(dirPath, expectedPerm, fsPromises), isCorrect); + }); }); - it('test noProxy conversion', function (){ - assert.strictEqual(Util.getNoProxyEnv(), '*.amazonaws.com|*.my_company.com'); + [ + { filePerm: 0o700, expectedPerm: 0o700, isCorrect: true }, + { filePerm: 0o755, expectedPerm: 0o600, isCorrect: false }, + ].forEach(async function ({ filePerm, expectedPerm, isCorrect }) { + it('Should return ' + isCorrect + ' when file permission ' + filePerm.toString(8) + ' is compared to ' + expectedPerm.toString(8), async function () { + const dirPath = path.join(tempDir, `file_${filePerm.toString(8)}`); + await fsPromises.appendFile(dirPath, '', { mode: filePerm }); + assert.strictEqual(await Util.isFileModeCorrect(dirPath, expectedPerm, fsPromises), isCorrect); + }); + }); + }); +} + +describe('shouldPerformGCPBucket function test', () => { + const testCases = [ + { + name: 'test - default', + accessToken: 'Token', + forceGCPUseDownscopedCredential: false, + result: true, + }, + { + name: 'test - when the disableGCPTokenUplaod is enabled', + accessToken: 'Token', + forceGCPUseDownscopedCredential: true, + result: false, + }, + { + name: 'test - when token is empty but the disableGCPTokenUplaod is enabled', + accessToken: null, + forceGCPUseDownscopedCredential: true, + result: false, + }, + { + name: 'test - test - when token is empty but the disableGCPTokenUplaod is disabled', + accessToken: null, + forceGCPUseDownscopedCredential: false, + result: false, + }, + ]; + + testCases.forEach(({ name, accessToken, forceGCPUseDownscopedCredential, result }) => { + it(name, () => { + process.env.SNOWFLAKE_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL = forceGCPUseDownscopedCredential; + assert.strictEqual(Util.shouldPerformGCPBucket(accessToken), result); + delete process.env.SNOWFLAKE_FORCE_GCP_USE_DOWNSCOPED_CREDENTIAL; }); }); }); + +describe('getEnvVar function Test', function () { + const testCases = [ + { + name: 'snowflake_env_test', + value: 'mock_value', + }, + { + name: 'SNOWFLAKE_ENV_TEST', + value: 'MOCK_VALUE', + }, + ]; + + for (const { name, value, } of testCases) { + it(name, function () { + process.env[name] = value; + assert.strictEqual(Util.getEnvVar('snowflake_env_test'), value); + assert.strictEqual(Util.getEnvVar('SNOWFLAKE_ENV_TEST'), value); + delete process.env[name]; + }); + } +}); \ No newline at end of file