SNOW-1781419: when proxy is set in Connection, driver does send traffic through the proxy to S3, but not to Azure blob / GCS bucket (only Snowflake). Works with proxy envvar (Azure)

lib/connection/connection_config.js

@@ -5,6 +5,7 @@
 const os = require('os');
 const url = require('url');
 const Util = require('../util');
+const ProxyUtil = require('../proxy_util');
 const Errors = require('../errors');
 const ConnectionConstants = require('../constants/connection_constants');
 const path = require('path');
@@ -222,7 +223,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
       protocol: proxyProtocol,
       noProxy: noProxy
     };
-    Util.validateProxy(proxy);
+    ProxyUtil.validateProxy(proxy);
   }
 
   const serviceName = options.serviceName;

lib/file_transfer_agent/azure_util.js

@@ -6,6 +6,9 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
 const expandTilde = require('expand-tilde');
 const resultStatus = require('./file_util').resultStatus;
+const ProxyUtil = require('../proxy_util');
+const { isBypassProxy } = require('../http/node');
+const Logger = require('../logger');
 
 const EXPIRED_TOKEN = 'ExpiredToken';
 
@@ -26,7 +29,7 @@ function AzureLocation(containerName, path) {
  * @returns {Object}
  * @constructor
  */
-function AzureUtil(azure, filestream) {
+function AzureUtil(connectionConfig, azure, filestream) {
   const AZURE = typeof azure !== 'undefined' ? azure : require('@azure/storage-blob');
   const fs = typeof filestream !== 'undefined' ? filestream : require('fs');
 
@@ -42,11 +45,20 @@ function AzureUtil(azure, filestream) {
     const sasToken = stageCredentials['AZURE_SAS_TOKEN'];
 
     const account = stageInfo['storageAccount'];
-
+    const connectionString = `https://${account}.blob.core.windows.net${sasToken}`;
+    let proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'Azure Util');
+    if (proxy && !isBypassProxy(proxy, connectionString)) {
+      proxy = ProxyUtil.getAzureProxy(proxy);
+      Logger.getInstance().debug(`The destination host is: ${ProxyUtil.getHostFromURL(connectionString)} and the proxy host is: ${proxy.host}`);
+    }
+    ProxyUtil.hideEnvironmentProxy();
     const blobServiceClient = new AZURE.BlobServiceClient(
-      `https://${account}.blob.core.windows.net${sasToken}`
+      connectionString, null,
+      {
+        proxyOptions: proxy,
+      }
     );
-
+    ProxyUtil.restoreEnvironmentProxy();
     return blobServiceClient;
   };
 
@@ -203,7 +215,7 @@ function AzureUtil(azure, filestream) {
           blobContentEncoding: 'UTF-8',
           blobContentType: 'application/octet-stream'
         }
-      });
+      });    
     } catch (err) {
       if (err['statusCode'] === 403 && detectAzureTokenExpireError(err)) {
         meta['lastError'] = err;
@@ -215,7 +227,6 @@ function AzureUtil(azure, filestream) {
       }
       return;
     }
-
     meta['dstFileSize'] = meta['uploadSize'];
     meta['resultStatus'] = resultStatus.UPLOADED;
   };
@@ -262,7 +273,6 @@ function AzureUtil(azure, filestream) {
       }
       return;
     }
-
     meta['resultStatus'] = resultStatus.DOWNLOADED;
   };
 
@@ -282,5 +292,4 @@ function AzureUtil(azure, filestream) {
       errstr.includes('Server failed to authenticate the request.');
   }
 }
-
 module.exports = AzureUtil;

lib/file_transfer_agent/remote_storage_util.js

@@ -44,7 +44,7 @@ function RemoteStorageUtil(connectionConfig) {
     if (type === 'S3') {
       return new SnowflakeS3Util(connectionConfig);
     } else if (type === 'AZURE') {
-      return new SnowflakeAzureUtil();
+      return new SnowflakeAzureUtil(connectionConfig);
     } else if (type === 'GCS') {
       return new SnowflakeGCSUtil();
     } else {

lib/file_transfer_agent/s3_util.js

@@ -7,9 +7,7 @@ const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
 const expandTilde = require('expand-tilde');
 const getProxyAgent = require('../http/node').getProxyAgent;
-const Util = require('../util');
-const Logger = require('../logger');
-const GlobalConfig = require('../global_config');
+const ProxyUtil = require('../proxy_util');
 
 const AMZ_IV = 'x-amz-iv';
 const AMZ_KEY = 'x-amz-key';
@@ -72,13 +70,7 @@ function S3Util(connectionConfig, s3, filestream) {
       useAccelerateEndpoint: useAccelerateEndpoint
     };
 
-    let proxy = connectionConfig.getProxy();
-    if (!proxy && GlobalConfig.isEnvProxyActive()) {
-      proxy = Util.getProxyFromEnv();
-      if (proxy) {
-        Logger.getInstance().debug(`S3 Util loads the proxy info from the environment variable host: ${proxy.host}`);
-      }
-    }
+    const proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'S3 Util');
     if (proxy) {
       const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), SNOWFLAKE_S3_DESTINATION);
       config.requestHandler = new NodeHttpHandler({

lib/http/node.js

@@ -3,6 +3,7 @@
  */
 
 const Util = require('../util');
+const ProxyUtil = require('../proxy_util');
 const Base = require('./base');
 const HttpsAgent = require('../agent/https_ocsp_agent');
 const HttpsProxyAgent = require('../agent/https_proxy_agent');
@@ -50,7 +51,7 @@ function getFromCacheOrCreate(agentClass, options, agentId) {
     Logger.getInstance().trace(`Create and add to cache new agent ${agentId}`);
 
     // detect and log PROXY envvar + agent proxy settings
-    const compareAndLogEnvAndAgentProxies = Util.getCompareAndLogEnvAndAgentProxies(agentOptions);
+    const compareAndLogEnvAndAgentProxies = ProxyUtil.getCompareAndLogEnvAndAgentProxies(agentOptions);
     Logger.getInstance().debug(`Proxy settings used in requests:${compareAndLogEnvAndAgentProxies.messages}`);
     // if there's anything to warn on (e.g. both envvar + agent proxy used, and they are different)
     // log warnings on them
@@ -100,13 +101,8 @@ function isBypassProxy(proxy, destination) {
  * @inheritDoc
  */
 NodeHttpClient.prototype.getAgent = function (parsedUrl, proxy, mock) {
-  if (!proxy && GlobalConfig.isEnvProxyActive()) {
-    const isHttps = parsedUrl.protocol === 'https:';
-    proxy = Util.getProxyFromEnv(isHttps);
-    if (proxy) {
-      Logger.getInstance().debug(`Load the proxy info from the environment variable host: ${proxy.host} in getAgent`);
-    }
-  }
+  const isHttps = parsedUrl.protocol === 'https:';
+  proxy = ProxyUtil.getProxy(proxy, 'Node HTTP Client', isHttps);
   return getProxyAgent(proxy, parsedUrl, parsedUrl.href, mock);
 };
 
@@ -124,7 +120,7 @@ function getProxyAgent(proxyOptions, parsedUrl, destination, mock) {
     }
   }
 
-  const destHost = Util.getHostFromURL(destination);
+  const destHost = ProxyUtil.getHostFromURL(destination);
   Logger.getInstance().debug(`The destination host is: ${destHost}`);
 
   const agentId = createAgentId(agentOptions.protocol, agentOptions.hostname, destHost, agentOptions.keepAlive);
@@ -157,4 +153,4 @@ function getAgentCacheSize() {
   return httpsAgentCache.size;
 }
 
-module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize };
+module.exports = { NodeHttpClient, getProxyAgent, getAgentCacheSize, isBypassProxy };