feat: Enable e2e tracing

OBSERVABILITY.md

@@ -80,7 +80,27 @@ const spanner = new Spanner({
   observabilityOptions: {
     tracerProvider: provider,
     enableExtendedTracing: true,
-  },
+  }
+}),
+```
+
+#### End to end tracing
+
+In addition to client-side tracing, you can opt in for end-to-end tracing. End-to-end tracing helps you understand and debug latency issues that are specific to Spanner. Refer [here](https://cloud.google.com/spanner/docs/tracing-overview) for more information.
+
+You can opt-in by either:
+
+* Setting the environment variable `SPANNER_ENABLE_END_TO_END_TRACING=true` before your application is started
+* In code, setting `enableEndToEndTracing: true` in your SpannerOptions before creating the Cloud Spanner client
+
+```javascript
+const spanner = new Spanner({
+  projectId: projectId,
+  observabilityOptions: {
+    tracerProvider: provider,
+    enableEndToEndTracing: true,
+  }
+}),
 ```
 
 #### OpenTelemetry gRPC instrumentation

observability-test/spanner.ts

@@ -26,6 +26,7 @@ import * as mockInstanceAdmin from '../test/mockserver/mockinstanceadmin';
 import * as mockDatabaseAdmin from '../test/mockserver/mockdatabaseadmin';
 import * as sinon from 'sinon';
 import {Row} from '../src/partial-result-stream';
+import {END_TO_END_TRACING_HEADER} from '../src/common';
 const {
   AlwaysOnSampler,
   NodeTracerProvider,
@@ -1930,3 +1931,59 @@ describe('Traces for ExecuteStream broken stream retries', () => {
     );
   });
 });
+
+describe('End to end tracing headers', () => {
+  let server: grpc.Server;
+  let spanner: Spanner;
+  let spannerMock: mock.MockSpanner;
+  let observabilityOptions: typeof ObservabilityOptions;
+
+  beforeEach(async () => {
+    observabilityOptions = {
+      enableEndToEndTracing: true,
+    };
+
+    const setupResult = await setup(observabilityOptions);
+    spanner = setupResult.spanner;
+    server = setupResult.server;
+    spannerMock = setupResult.spannerMock;
+  });
+
+  afterEach(async () => {
+    spannerMock.resetRequests();
+    spanner.close();
+    server.tryShutdown(() => {});
+  });
+
+  it('run', done => {
+    const instance = spanner.instance('instance');
+    const database = instance.database('database');
+    database.getTransaction((err, tx) => {
+      assert.ifError(err);
+
+      tx!.run('SELECT 1', async () => {
+        tx!.end();
+        let metadataCountWithE2EHeader = 0;
+        let metadataCountWithTraceParent = 0;
+        spannerMock.getMetadata().forEach(metadata => {
+          if (metadata.get(END_TO_END_TRACING_HEADER)[0] !== undefined) {
+            metadataCountWithE2EHeader++;
+            assert.strictEqual(
+              metadata.get(END_TO_END_TRACING_HEADER)[0],
+              'true'
+            );
+          }
+          if (metadata.get('traceparent')[0] !== undefined) {
+            metadataCountWithTraceParent++;
+          }
+        });
+
+        // Batch Create Session request and Select 1 request.
+        assert.strictEqual(spannerMock.getRequests().length, 2);
+        assert.strictEqual(metadataCountWithE2EHeader, 2);
+        assert.strictEqual(metadataCountWithTraceParent, 2);
+        done();
+      });
+    });
+  });
+});

package.json

@@ -60,6 +60,7 @@
     "@google-cloud/promisify": "^4.0.0",
     "@grpc/proto-loader": "^0.7.0",
     "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/core": "^1.27.0",
     "@opentelemetry/context-async-hooks": "^1.26.0",
     "@opentelemetry/semantic-conventions": "^1.25.1",
     "@types/big.js": "^6.0.0",

samples/observability-traces.js

@@ -76,6 +76,7 @@ async function main(
     observabilityOptions: {
       tracerProvider: provider,
       enableExtendedTracing: true,
+      enableEndToEndTracing: true,
     },
   });
 

src/common.ts

@@ -81,10 +81,37 @@ export const CLOUD_RESOURCE_HEADER = 'google-cloud-resource-prefix';
  */
 export const LEADER_AWARE_ROUTING_HEADER = 'x-goog-spanner-route-to-leader';
 
+/*
+ * END TO END TRACING  header.
+ */
+export const END_TO_END_TRACING_HEADER = 'x-goog-spanner-end-to-end-tracing';
+
 /**
  * Add Leader aware routing header to existing header list.
  * @param headers Existing header list.
  */
 export function addLeaderAwareRoutingHeader(headers: {[k: string]: string}) {
   headers[LEADER_AWARE_ROUTING_HEADER] = 'true';
 }
+
+/**
+ * Returns common headers to add.
+ * @param headers Common header list.
+ */
+export function getCommonHeaders(
+  resourceName: string,
+  enableTracing?: boolean
+) {
+  const headers: {[k: string]: string} = {};
+
+  if (
+    process.env.SPANNER_ENABLE_END_TO_END_TRACING === 'true' ||
+    enableTracing
+  ) {
+    headers[END_TO_END_TRACING_HEADER] = 'true';
+  }
+
+  headers[CLOUD_RESOURCE_HEADER] = resourceName;
+
+  return headers;
+}

src/database.ts

@@ -85,12 +85,13 @@
   LongRunningCallback,
   NormalCallback,
   PagedOptionsWithFilter,
   CLOUD_RESOURCE_HEADER,
   PagedResponse,
   RequestCallback,
   ResourceCallback,
   Schema,
   addLeaderAwareRoutingHeader,
+  getCommonHeaders,
 } from './common';
 import {finished, Duplex, Readable, Transform} from 'stream';
 import {PreciseDate} from '@google-cloud/precise-date';
@@ -340,7 +341,7 @@
   formattedName_: string;
   pool_: SessionPoolInterface;
   queryOptions_?: spannerClient.spanner.v1.ExecuteSqlRequest.IQueryOptions;
-  resourceHeader_: {[k: string]: string};
+  commonHeaders_: {[k: string]: string};
   request: DatabaseRequest;
   databaseRole?: string | null;
   labels?: {[k: string]: string} | null;
@@ -471,11 +472,13 @@
       dbName: this.formattedName_,
     };
 
-    this.resourceHeader_ = {
-      [CLOUD_RESOURCE_HEADER]: this.formattedName_,
-    };
     this.request = instance.request;
     this._observabilityOptions = instance._observabilityOptions;
+    this.commonHeaders_ = getCommonHeaders(
+      this.formattedName_,
+      this._observabilityOptions?.enableEndToEndTracing
+    );
+
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     this.requestStream = instance.requestStream as any;
     this.pool_.on('error', this.emit.bind(this, 'error'));
@@ -582,7 +585,7 @@
         method: 'updateDatabase',
         reqOpts,
         gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       callback!
     );
@@ -688,7 +691,7 @@
       sessionCount: count,
     };
 
-    const headers = this.resourceHeader_;
+    const headers = this.commonHeaders_;
     if (this._getSpanner().routeToLeaderEnabled) {
       addLeaderAwareRoutingHeader(headers);
     }
@@ -989,7 +992,7 @@
     reqOpts.session.creatorRole =
       options.databaseRole || this.databaseRole || null;
 
-    const headers = this.resourceHeader_;
+    const headers = this.commonHeaders_;
     if (this._getSpanner().routeToLeaderEnabled) {
       addLeaderAwareRoutingHeader(headers);
     }
@@ -1215,7 +1218,7 @@
           method: 'dropDatabase',
           reqOpts,
           gaxOpts,
-          headers: this.resourceHeader_,
+          headers: this.commonHeaders_,
         },
         callback!
       );
@@ -1447,7 +1450,7 @@
         method: 'getDatabase',
         reqOpts,
         gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       (err, resp) => {
         if (resp) {
@@ -1701,7 +1704,7 @@
         method: 'getDatabaseDdl',
         reqOpts,
         gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       (err, statements, ...args: any[]) => {
@@ -1781,7 +1784,7 @@
         method: 'getIamPolicy',
         reqOpts,
         gaxOpts: options.gaxOptions,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       (err, resp) => {
         callback!(err, resp);
@@ -1920,7 +1923,7 @@
           method: 'listSessions',
           reqOpts,
           gaxOpts,
-          headers: this.resourceHeader_,
+          headers: this.commonHeaders_,
         },
         (err, sessions, nextPageRequest, ...args) => {
           if (err) {
@@ -2013,7 +2016,7 @@
       method: 'listSessionsStream',
       reqOpts,
       gaxOpts,
-      headers: this.resourceHeader_,
+      headers: this.commonHeaders_,
     });
   }
 
@@ -2405,7 +2408,7 @@
         method: 'listDatabaseRoles',
         reqOpts,
         gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       (err, roles, nextPageRequest, ...args) => {
         const nextQuery = nextPageRequest!
@@ -2615,7 +2618,7 @@
         method: 'restoreDatabase',
         reqOpts,
         gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       (err, operation, resp) => {
         if (err) {
@@ -3514,7 +3517,7 @@
             method: 'batchWrite',
             reqOpts,
             gaxOpts,
-            headers: this.resourceHeader_,
+            headers: this.commonHeaders_,
           });
           dataStream
             .once('data', () => (dataReceived = true))
@@ -3796,7 +3799,7 @@
         method: 'setIamPolicy',
         reqOpts,
         gaxOpts: gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       (err, resp) => {
         callback!(err, resp);
@@ -3926,7 +3929,7 @@
         method: 'updateDatabaseDdl',
         reqOpts,
         gaxOpts,
-        headers: this.resourceHeader_,
+        headers: this.commonHeaders_,
       },
       callback!
     );