feat: Extend GCP Residency Detection Support (#528)

* feat: Extend GCP Residency Detection Support

* test: update tests

* test: init `gcp-residency` tests

* chore: add temporary debug logs for `windows`

* chore: Remove debug logs for windows

* test: Add some tests for `gcp-residency`

* refactor: use MAC Address to determine GCE residency

* refactor: Re-Add Linux GCE detection

* feat: Extend GCP Serverless Runtime Support

- A refactor + Cloud Run Job support

package.json

@@ -47,6 +47,7 @@
     "@types/mocha": "^9.0.0",
     "@types/ncp": "^2.0.1",
     "@types/node": "^18.0.0",
+    "@types/sinon": "^10.0.13",
     "@types/tmp": "0.2.3",
     "@types/uuid": "^9.0.0",
     "c8": "^7.0.0",
@@ -58,6 +59,7 @@
     "mocha": "^8.0.0",
     "ncp": "^2.0.0",
     "nock": "^13.0.0",
+    "sinon": "^14.0.0",
     "tmp": "^0.2.0",
     "typescript": "^4.6.3",
     "uuid": "^9.0.0"

src/gcp-residency.ts

@@ -0,0 +1,118 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {readFileSync, statSync} from 'fs';
+import {networkInterfaces, platform} from 'os';
+
+/**
+ * Known paths unique to Google Compute Engine Linux instances
+ */
+export const GCE_LINUX_BIOS_PATHS = {
+  BIOS_DATE: '/sys/class/dmi/id/bios_date',
+  BIOS_VENDOR: '/sys/class/dmi/id/bios_vendor',
+};
+
+const GCE_MAC_ADDRESS_REGEX = /^42:01/;
+
+/**
+ * Determines if the process is running on a Google Cloud Serverless environment (Cloud Run or Cloud Functions instance).
+ *
+ * Uses the:
+ * - {@link https://cloud.google.com/run/docs/container-contract#env-vars Cloud Run environment variables}.
+ * - {@link https://cloud.google.com/functions/docs/env-var Cloud Functions environment variables}.
+ *
+ * @returns {boolean} `true` if the process is running on GCP serverless, `false` otherwise.
+ */
+export function isGoogleCloudServerless(): boolean {
+  /**
+   * `CLOUD_RUN_JOB` is used for Cloud Run Jobs
+   * - See {@link https://cloud.google.com/run/docs/container-contract#env-vars Cloud Run environment variables}.
+   *
+   * `FUNCTION_NAME` is used in older Cloud Functions environments:
+   * - See {@link https://cloud.google.com/functions/docs/env-var Python 3.7 and Go 1.11}.
+   *
+   * `K_SERVICE` is used in Cloud Run and newer Cloud Functions environments:
+   * - See {@link https://cloud.google.com/run/docs/container-contract#env-vars Cloud Run environment variables}.
+   * - See {@link https://cloud.google.com/functions/docs/env-var Cloud Functions newer runtimes}.
+   */
+  const isGFEnvironment =
+    process.env.CLOUD_RUN_JOB ||
+    process.env.FUNCTION_NAME ||
+    process.env.K_SERVICE;
+
+  return !!isGFEnvironment;
+}
+
+/**
+ * Determines if the process is running on a Linux Google Compute Engine instance.
+ *
+ * @returns {boolean} `true` if the process is running on Linux GCE, `false` otherwise.
+ */
+export function isGoogleComputeEngineLinux(): boolean {
+  if (platform() !== 'linux') return false;
+
+  try {
+    // ensure this file exist
+    statSync(GCE_LINUX_BIOS_PATHS.BIOS_DATE);
+
+    // ensure this file exist and matches
+    const biosVendor = readFileSync(GCE_LINUX_BIOS_PATHS.BIOS_VENDOR, 'utf8');
+
+    return /Google/.test(biosVendor);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Determines if the process is running on a Google Compute Engine instance with a known
+ * MAC address.
+ *
+ * @returns {boolean} `true` if the process is running on GCE (as determined by MAC address), `false` otherwise.
+ */
+export function isGoogleComputeEngineMACAddress(): boolean {
+  const interfaces = networkInterfaces();
+
+  for (const item of Object.values(interfaces)) {
+    if (!item) continue;
+
+    for (const {mac} of item) {
+      if (GCE_MAC_ADDRESS_REGEX.test(mac)) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Determines if the process is running on a Google Compute Engine instance.
+ *
+ * @returns {boolean} `true` if the process is running on GCE, `false` otherwise.
+ */
+export function isGoogleComputeEngine(): boolean {
+  return isGoogleComputeEngineLinux() || isGoogleComputeEngineMACAddress();
+}
+
+/**
+ * Determines if the process is running on Google Cloud Platform.
+ *
+ * @returns {boolean} `true` if the process is running on GCP, `false` otherwise.
+ */
+export function detectGCPResidency(): boolean {
+  return isGoogleCloudServerless() || isGoogleComputeEngine();
+}

src/index.ts

@@ -8,6 +8,7 @@
 import {GaxiosError, GaxiosOptions, GaxiosResponse, request} from 'gaxios';
 import {OutgoingHttpHeaders} from 'http';
 import jsonBigint = require('json-bigint');
+import {detectGCPResidency} from './gcp-residency';
 
 export const BASE_PATH = '/computeMetadata/v1';
 export const HOST_ADDRESS = 'http://169.254.169.254';
@@ -265,19 +266,35 @@ export function resetIsAvailableCache() {
   cachedIsAvailableResponse = undefined;
 }
 
+/**
+ * A cache for the detected GCP Residency.
+ */
+export let gcpResidencyCache: boolean | null = null;
+
+/**
+ * Sets the detected GCP Residency.
+ * Useful for forcing metadata server detection behavior.
+ *
+ * Set `null` to autodetect the environment (default behavior).
+ */
+export function setGCPResidency(value: boolean | null = null) {
+  gcpResidencyCache = value !== null ? value : detectGCPResidency();
+}
+
 /**
  * Obtain the timeout for requests to the metadata server.
+ *
+ * In certain environments and conditions requests can take longer than
+ * the default timeout to complete. This function will determine the
+ * appropriate timeout based on the environment.
+ *
+ * @returns {number} a request timeout duration in milliseconds.
  */
 export function requestTimeout(): number {
-  // In testing, we were able to reproduce behavior similar to
-  // https://github.com/googleapis/google-auth-library-nodejs/issues/798
-  // by making many concurrent network requests. Requests do not actually fail,
-  // rather they take significantly longer to complete (and we hit our
-  // default 3000ms timeout).
-  //
-  // This logic detects a GCF environment, using the documented environment
-  // variables K_SERVICE and FUNCTION_NAME:
-  // https://cloud.google.com/functions/docs/env-var and, in a GCF environment
-  // eliminates timeouts (by setting the value to 0 to disable).
-  return process.env.K_SERVICE || process.env.FUNCTION_NAME ? 0 : 3000;
+  // Detecting the residency can be resource-intensive. Let's cache the result.
+  if (gcpResidencyCache === null) {
+    gcpResidencyCache = detectGCPResidency();
+  }
+
+  return gcpResidencyCache ? 0 : 3000;
 }

test/gcp-residency.test.ts

@@ -0,0 +1,195 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {strict as assert} from 'assert';
+import * as fs from 'fs';
+import * as os from 'os';
+
+import {beforeEach, describe, it} from 'mocha';
+import {SinonSandbox, createSandbox} from 'sinon';
+
+import * as gcpResidency from '../src/gcp-residency';
+
+const ENVIRONMENT_BACKUP = {...process.env};
+
+describe('gcp-residency', () => {
+  let sandbox: SinonSandbox;
+
+  beforeEach(() => {
+    process.env = {...ENVIRONMENT_BACKUP};
+    sandbox = createSandbox();
+    removeServerlessEnvironmentVariables();
+  });
+
+  afterEach(() => {
+    sandbox.restore();
+  });
+
+  /**
+   * A simple utility for stubbing the networkInterface for GCE emulation.
+   *
+   * @param isGCE determines if the address should begin with `42:01` or not
+   */
+  function setGCENetworkInterface(isGCE = true) {
+    const mac = isGCE ? '42:01:00:00:00:00' : '00:00:00:00:00:00';
+
+    sandbox.stub(os, 'networkInterfaces').returns({
+      'test-interface': [{mac} as os.NetworkInterfaceInfo],
+    });
+  }
+
+  /**
+   * A simple utility for stubbing the platform for GCE emulation.
+   *
+   * @param platform a Node.js platform
+   */
+  function setGCEPlatform(platform: NodeJS.Platform = 'linux') {
+    sandbox.stub(os, 'platform').returns(platform);
+  }
+
+  /**
+   * A simple utility for stubbing the Linux BIOS files for GCE emulation.
+   *
+   * @param isGCE options:
+   *   - set `true` to simulate the files exist and are GCE
+   *   - set `false` for exist, but are not GCE
+   *   - set `null` for simulate ENOENT
+   */
+  function setGCELinuxBios(isGCE: boolean | null) {
+    sandbox.stub(fs, 'statSync').callsFake(path => {
+      assert.equal(path, gcpResidency.GCE_LINUX_BIOS_PATHS.BIOS_DATE);
+
+      return undefined;
+    });
+
+    sandbox.stub(fs, 'readFileSync').callsFake((path, encoding) => {
+      assert.equal(path, gcpResidency.GCE_LINUX_BIOS_PATHS.BIOS_VENDOR);
+      assert.equal(encoding, 'utf8');
+
+      if (isGCE === true) {
+        return 'x Google x';
+      } else if (isGCE === false) {
+        return 'Sandwich Co.';
+      } else {
+        throw new Error("File doesn't exist");
+      }
+    });
+  }
+
+  function removeServerlessEnvironmentVariables() {
+    delete process.env.CLOUD_RUN_JOB;
+    delete process.env.FUNCTION_NAME;
+    delete process.env.K_SERVICE;
+  }
+
+  describe('isGoogleCloudServerless', () => {
+    it('should return `true` if `CLOUD_RUN_JOB` env is set', () => {
+      process.env.CLOUD_RUN_JOB = '1';
+
+      assert(gcpResidency.isGoogleCloudServerless());
+    });
+
+    it('should return `true` if `FUNCTION_NAME` env is set', () => {
+      process.env.FUNCTION_NAME = '1';
+
+      assert(gcpResidency.isGoogleCloudServerless());
+    });
+
+    it('should return `true` if `K_SERVICE` env is set', () => {
+      process.env.K_SERVICE = '1';
+
+      assert(gcpResidency.isGoogleCloudServerless());
+    });
+
+    it('should return `false` if none of the envs are set', () => {
+      assert.equal(gcpResidency.isGoogleCloudServerless(), false);
+    });
+  });
+
+  describe('isGoogleComputeEngine', () => {
+    it('should return `true` if on Linux and has the expected BIOS files', () => {
+      setGCENetworkInterface(false);
+      setGCEPlatform('linux');
+      setGCELinuxBios(true);
+
+      assert.equal(gcpResidency.isGoogleComputeEngine(), true);
+    });
+
+    it('should return `false` if on Linux and the expected BIOS files are not GCE', () => {
+      setGCENetworkInterface(false);
+      setGCEPlatform('linux');
+      setGCELinuxBios(false);
+
+      assert.equal(gcpResidency.isGoogleComputeEngine(), false);
+    });
+
+    it('should return `false` if on Linux and the BIOS files do not exist', () => {
+      setGCENetworkInterface(false);
+      setGCEPlatform('linux');
+      setGCELinuxBios(null);
+
+      assert.equal(gcpResidency.isGoogleComputeEngine(), false);
+    });
+
+    it('should return `true` if the host MAC address begins with `42:01`', () => {
+      setGCENetworkInterface(true);
+      setGCEPlatform('win32');
+      setGCELinuxBios(null);
+
+      assert.equal(gcpResidency.isGoogleComputeEngine(), true);
+    });
+
+    it('should return `false` if the host MAC address does not begin with `42:01` & is not Linux', () => {
+      setGCENetworkInterface(false);
+      setGCEPlatform('win32');
+      setGCELinuxBios(null);
+
+      assert.equal(gcpResidency.isGoogleComputeEngine(), false);
+    });
+  });
+
+  describe('detectGCPResidency', () => {
+    it('should return `true` if `isGoogleCloudServerless`', () => {
+      // `isGoogleCloudServerless` = true
+      process.env.K_SERVICE = '1';
+
+      // `isGoogleComputeEngine` = false
+      setGCENetworkInterface(false);
+
+      assert(gcpResidency.detectGCPResidency());
+    });
+
+    it('should return `true` if `isGoogleComputeEngine`', () => {
+      // `isGoogleCloudServerless` = false
+      removeServerlessEnvironmentVariables();
+
+      // `isGoogleComputeEngine` = true
+      setGCENetworkInterface(true);
+
+      assert(gcpResidency.detectGCPResidency());
+    });
+
+    it('should return `false` !`isGoogleCloudServerless` && !`isGoogleComputeEngine`', () => {
+      // `isGoogleCloudServerless` = false
+      removeServerlessEnvironmentVariables();
+
+      // `isGoogleComputeEngine` = false
+      setGCENetworkInterface(false);
+
+      assert.equal(gcpResidency.detectGCPResidency(), false);
+    });
+  });
+});