fix: retry connections to gce metadata server (#284)

googleapis · Feb 16, 2018 · 7a951e0 · 7a951e0
1 parent 78d6813
commit 7a951e0
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 15 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -24,7 +24,8 @@
     "gtoken": "^2.1.0",
     "jws": "^3.1.4",
     "lodash.isstring": "^4.0.1",
-    "lru-cache": "^4.1.1"
+    "lru-cache": "^4.1.1",
+    "retry-axios": "^0.3.0"
   },
   "devDependencies": {
     "@justinbeckwith/typedoc": "^0.10.1",

diff --git a/src/auth/computeclient.ts b/src/auth/computeclient.ts
@@ -14,21 +14,24 @@
  * limitations under the License.
  */
 
-import {AxiosError, AxiosPromise, AxiosRequestConfig, AxiosResponse} from 'axios';
-import {BASE_PATH, HEADER_NAME, HOST_ADDRESS} from 'gcp-metadata';
-
-import {RequestError} from './../transporters';
+import axios, {AxiosError, AxiosPromise, AxiosRequestConfig, AxiosResponse} from 'axios';
+import * as gcpMetadata from 'gcp-metadata';
+import * as rax from 'retry-axios';
 import {CredentialRequest, Credentials} from './credentials';
 import {GetTokenResponse, OAuth2Client, RefreshOptions} from './oauth2client';
 
 export interface ComputeOptions extends RefreshOptions {}
 
+// Create a scoped axios instance that will retry 3 times by default
+const ax = axios.create();
+rax.attach(ax);
+
 export class Compute extends OAuth2Client {
   /**
    * Google Compute Engine metadata server token endpoint.
    */
   protected static readonly _GOOGLE_OAUTH2_TOKEN_URL =
-      `${BASE_PATH}/instance/service-accounts/default/token`;
+      `${gcpMetadata.BASE_PATH}/instance/service-accounts/default/token`;
 
   /**
    * Google Compute Engine service account credentials.
@@ -61,15 +64,18 @@ export class Compute extends OAuth2Client {
    */
   protected async refreshToken(refreshToken?: string|
                                null): Promise<GetTokenResponse> {
-    const url =
-        this.tokenUrl || `${HOST_ADDRESS}${Compute._GOOGLE_OAUTH2_TOKEN_URL}`;
+    const url = this.tokenUrl ||
+        `${gcpMetadata.HOST_ADDRESS}${Compute._GOOGLE_OAUTH2_TOKEN_URL}`;
     let res: AxiosResponse<CredentialRequest>|null = null;
     // request for new token
     try {
       // TODO: In 2.0, we should remove the ability to configure the tokenUrl,
       // and switch this over to use the gcp-metadata package instead.
-      res = await this.transporter.request<CredentialRequest>(
-          {url, headers: {'Metadata-Flavor': 'Google'}});
+      res = await ax.request<CredentialRequest>({
+        url,
+        headers: {[gcpMetadata.HEADER_NAME]: 'Google'},
+        raxConfig: {noResponseRetries: 3, retry: 3, instance: ax}
+      } as rax.RaxConfig);
     } catch (e) {
       e.message = 'Could not refresh access token.';
       throw e;
@@ -109,7 +115,7 @@ export class Compute extends OAuth2Client {
             e.message = helpfulMessage;
           } else {
             e = new Error(helpfulMessage);
-            (e as RequestError).code = res.status.toString();
+            (e as NodeJS.ErrnoException).code = res.status.toString();
           }
         }
       }

diff --git a/test/test.compute.ts b/test/test.compute.ts
@@ -15,12 +15,10 @@
  */
 
 import * as assert from 'assert';
-import {AxiosRequestConfig} from 'axios';
 import {BASE_PATH, HOST_ADDRESS} from 'gcp-metadata';
 import * as nock from 'nock';
 
 import {Credentials} from '../src/auth/credentials';
-import {GoogleAuth} from '../src/auth/googleauth';
 import {Compute} from '../src/index';
 
 nock.disableNetConnect();
@@ -120,6 +118,42 @@ describe('Compute auth client', () => {
     });
   });
 
+  it('should retry calls to the metadata service if there are network errors',
+     (done) => {
+       const scope =
+           nock(HOST_ADDRESS)
+               .get(tokenPath)
+               .times(2)
+               .replyWithError({code: 'ENOTFOUND'})
+               .get(tokenPath)
+               .reply(200, {access_token: 'abc123', expires_in: 10000});
+       compute.credentials.access_token = 'initial-access-token';
+       compute.credentials.expiry_date = (new Date()).getTime() - 10000;
+       compute.request({url: 'http://foo'}, e => {
+         assert.equal(compute.credentials.access_token, 'abc123');
+         scope.done();
+         done();
+       });
+     });
+
+  it('should retry calls to the metadata service if it returns non-200 errors',
+     (done) => {
+       const scope =
+           nock(HOST_ADDRESS)
+               .get(tokenPath)
+               .times(2)
+               .reply(500)
+               .get(tokenPath)
+               .reply(200, {access_token: 'abc123', expires_in: 10000});
+       compute.credentials.access_token = 'initial-access-token';
+       compute.credentials.expiry_date = (new Date()).getTime() - 10000;
+       compute.request({url: 'http://foo'}, e => {
+         assert.equal(compute.credentials.access_token, 'abc123');
+         scope.done();
+         done();
+       });
+     });
+
   describe('.createScopedRequired', () => {
     it('should return false', () => {
       const c = new Compute();