SNOW-1820372: (from 947) when proxy is set in Connection, driver does…

… send traffic through the proxy to S3, but not to Azure blob / GCS bucket (only Snowflake). Works with proxy envvar (GCP) (#982)
snowflakedb · Jan 13, 2025 · 3ab8b63 · 3ab8b63
1 parent ea8a0a1
commit 3ab8b63
Show file tree

Hide file tree

Showing 9 changed files with 312 additions and 131 deletions.
diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js
@@ -17,7 +17,6 @@ const levenshtein = require('fastest-levenshtein');
 const RowMode = require('./../constants/row_mode');
 const DataTypes = require('./result/data_types');
 const Logger = require('../logger');
-const LoggingUtil = require('../logger/logging_util');
 const WAIT_FOR_BROWSER_ACTION_TIMEOUT = 120000;
 const DEFAULT_PARAMS =
 [
@@ -513,7 +512,6 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
 
     passcode = options.passcode;
   }
-
 
   if (validateDefaultParameters) {
     for (const [key] of Object.entries(options)) {
@@ -847,21 +845,7 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo) {
   this.describeIdentityAttributes = function () {
     return `host: ${this.host}, account: ${this.account}, accessUrl: ${this.accessUrl}, `
         + `user: ${this.username}, role: ${this.getRole()}, database: ${this.getDatabase()}, `
-        + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + this.describeProxy();
-  };
-
-  /**
-   * @returns {string}
-   */
-  this.describeProxy = function () {
-    const proxy = this.getProxy();
-    if (Util.exists(proxy)) {
-      return `proxyHost: ${proxy.host}, proxyPort: ${proxy.port}, proxyUser: ${proxy.user}, `
-      + `proxyPassword is ${LoggingUtil.describePresence(proxy.password)}, `
-      + `proxyProtocol: ${proxy.protocol}, noProxy: ${proxy.noProxy}`;
-    } else {
-      return 'proxy was not configured';
-    }
+        + `schema: ${this.getSchema()}, warehouse: ${this.getWarehouse()}, ` + ProxyUtil.describeProxy(this.getProxy());
   };
 
   // save config options

diff --git a/lib/file_transfer_agent/azure_util.js b/lib/file_transfer_agent/azure_util.js
@@ -52,7 +52,6 @@ function AzureUtil(connectionConfig, azure, filestream) {
       Logger.getInstance().trace(`Initializing the proxy information for the Azure Client: ${ProxyUtil.describeProxy(proxy)}`);
 
       proxy = ProxyUtil.getAzureProxy(proxy);
-      Logger.getInstance().trace(connectionConfig.describe);
     }
     ProxyUtil.hideEnvironmentProxy();
     const blobServiceClient = new AZURE.BlobServiceClient(

diff --git a/lib/file_transfer_agent/gcs_util.js b/lib/file_transfer_agent/gcs_util.js
@@ -4,8 +4,12 @@
 
 const EncryptionMetadata = require('./encrypt_util').EncryptionMetadata;
 const FileHeader = require('./file_util').FileHeader;
+const getProxyAgent = require('../http/node').getProxyAgent;
+const ProxyUtil = require('../proxy_util');
+const Util = require('../util');
 const { shouldPerformGCPBucket, lstrip } = require('../util');
 
+
 const GCS_METADATA_PREFIX = 'x-goog-meta-';
 const SFC_DIGEST = 'sfc-digest';
 const MATDESC_KEY = 'matdesc';
@@ -19,7 +23,6 @@ const GCS_FILE_HEADER_ENCRYPTION_METADATA = 'gcs-file-header-encryption-metadata
 
 const HTTP_HEADER_CONTENT_ENCODING = 'Content-Encoding';
 const resultStatus = require('./file_util').resultStatus;
-
 const { Storage } = require('@google-cloud/storage');
 
 const EXPIRED_TOKEN = 'ExpiredToken';
@@ -40,17 +43,18 @@ function GCSLocation(bucketName, path) {
 
 /**
  * Creates an GCS utility object.
- *
- * @param {module} httpclient
- * @param {module} filestream
+ * @param {module} connectionConfig
+ * @param {module} httpClient
+ * @param {module} fileStream
  *
  * @returns {Object}
  * @constructor
  */
-function GCSUtil(httpclient, filestream) {
-  const axios = typeof httpclient !== 'undefined' ? httpclient : require('axios');
-  const fs = typeof filestream !== 'undefined' ? filestream : require('fs');
-
+function GCSUtil(connectionConfig, httpClient, fileStream) {
+  let axios = httpClient;
+  const fs = typeof fileStream !== 'undefined' ? fileStream : require('fs');
+  let isProxyEnabled = false;
+
   /**
   * Retrieve the GCS token from the stage info metadata.
   *
@@ -61,7 +65,15 @@ function GCSUtil(httpclient, filestream) {
   this.createClient = function (stageInfo) {
     const stageCredentials = stageInfo['creds'];
     const gcsToken = stageCredentials['GCS_ACCESS_TOKEN'];
-
+    //TODO: SNOW-1789759 the value is hardcoded now, but it should be server driven
+    const isRegionalUrlEnabled = (stageInfo.region).toLowerCase() === 'me-central2' || stageInfo.useRegionalUrl;
+    let endPoint = null;
+    if (stageInfo['endPoint']) {
+      endPoint = stageInfo['endPoint'];
+    } else if (isRegionalUrlEnabled) {
+      endPoint = `storage.${stageInfo.region.toLowerCase()}.rep.googleapis.com`;
+    }
+
     let client;
     if (gcsToken) {
       const interceptors = [];
@@ -72,14 +84,15 @@ function GCSUtil(httpclient, filestream) {
           return requestConfig;
         }
       });
-
-      const endPoint = this.getGCSCustomEndPoint(stageInfo);
-      const storage = endPoint ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
+
+      const storage = Util.exists(endPoint) ? new Storage({ interceptors_: interceptors, apiEndpoint: endPoint }) : new Storage({ interceptors_: interceptors });
       client = { gcsToken: gcsToken, gcsClient: storage };
     } else {
       client = null;
     }
 
+    process.nextTick(() => this.setupHttpClient(endPoint));
+
     return client;
   };
 
@@ -142,14 +155,14 @@ function GCSUtil(httpclient, filestream) {
         let matDescKey;
 
         try {
-          if (shouldPerformGCPBucket(accessToken)) {
+          if (shouldPerformGCPBucket(accessToken) && !isProxyEnabled) {
             const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
             const metadata = await meta['client'].gcsClient
               .bucket(gcsLocation.bucketName)
               .file(gcsLocation.path + filename)
               .getMetadata();
 
+
             digest = metadata[0].metadata[SFC_DIGEST];
             contentLength = metadata[0].size;
             encryptionDataProp = metadata[0].metadata[ENCRYPTIONDATAPROP];
@@ -282,9 +295,9 @@ function GCSUtil(httpclient, filestream) {
     }
 
     try {
-      if (shouldPerformGCPBucket(accessToken)) {
+      if (shouldPerformGCPBucket(accessToken) && !isProxyEnabled) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
+        
         await meta['client'].gcsClient
           .bucket(gcsLocation.bucketName)
           .file(gcsLocation.path + meta['dstFileName'])
@@ -355,9 +368,8 @@ function GCSUtil(httpclient, filestream) {
     let size;
 
     try {
-      if (shouldPerformGCPBucket(accessToken)) {
+      if (shouldPerformGCPBucket(accessToken) && !isProxyEnabled) {
         const gcsLocation = this.extractBucketNameAndPath(meta['stageInfo']['location']);
-
         await meta['client'].gcsClient
           .bucket(gcsLocation.bucketName)
           .file(gcsLocation.path + meta['srcFileName'])
@@ -376,9 +388,7 @@ function GCSUtil(httpclient, filestream) {
         size = metadata[0].size;
       } else {
         let response;
-        await axios({
-          method: 'get',
-          url: downloadUrl,
+        await axios.get(downloadUrl, {
           headers: gcsHeaders,
           responseType: 'stream'
         }).then(async (res) => {
@@ -466,6 +476,25 @@ function GCSUtil(httpclient, filestream) {
     }
     return endPoint;
   };
+
+  this.setupHttpClient = function (endPoint) {
+    if (typeof httpClient === 'undefined') {
+      const proxy = ProxyUtil.getProxy(connectionConfig.getProxy(), 'GCS Util');
+
+      //When http_proxy is enabled, the driver should use Axios for HTTPS requests to avoid relying on HTTP_PROXY in GCS.
+      if (proxy || Util.getEnvVar('http_proxy')) {
+        isProxyEnabled = true;
+        const proxyAgent = getProxyAgent(proxy, new URL(connectionConfig.accessUrl), endPoint || 'storage.googleapis.com');
+        axios = require('axios').create({
+          proxy: false,
+          httpAgent: proxyAgent,
+          httpsAgent: proxyAgent,
+        });
+      } else {
+        axios = require('axios');
+      }
+    }
+  };
 }
 
 module.exports = GCSUtil;
diff --git a/lib/file_transfer_agent/remote_storage_util.js b/lib/file_transfer_agent/remote_storage_util.js
@@ -33,6 +33,8 @@ exports.SnowflakeFileEncryptionMaterial = SnowflakeFileEncryptionMaterial;
  * @constructor
  */
 function RemoteStorageUtil(connectionConfig) {
+  let client = null;
+
   /**
   * Get storage type based on location type.
   *
@@ -41,15 +43,17 @@ function RemoteStorageUtil(connectionConfig) {
   * @returns {Object}
   */
   this.getForStorageType = function (type) {
+    if (client) {
+      return client;
+    }
     if (type === 'S3') {
-      return new SnowflakeS3Util(connectionConfig);
+      client = new SnowflakeS3Util(connectionConfig);
     } else if (type === 'AZURE') {
-      return new SnowflakeAzureUtil(connectionConfig);
+      client = new SnowflakeAzureUtil(connectionConfig);
     } else if (type === 'GCS') {
-      return new SnowflakeGCSUtil();
-    } else {
-      return null;
+      client = new SnowflakeGCSUtil(connectionConfig);
     }
+    return client;
   };
 
   /**