feat: onboard destination movable ink (#3167)

* feat: onboard destination movable ink

* test: update common.ts

* feat: add batching support

* feat: batching on max size in bytes

* docs: added comments

src/cdk/v2/destinations/movable_ink/config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  MAX_REQUEST_SIZE_IN_BYTES: 13500,
+};

src/cdk/v2/destinations/movable_ink/procWorkflow.yaml

@@ -0,0 +1,72 @@
+bindings:
+  - name: EventType
+    path: ../../../../constants
+  - path: ../../bindings/jsontemplate
+  - name: defaultRequestConfig
+    path: ../../../../v0/util
+  - name: toUnixTimestampInMS
+    path: ../../../../v0/util
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - path: ./utils
+
+steps:
+  - name: messageType
+    template: |
+      .message.type.toLowerCase();
+
+  - name: validateInput
+    template: |
+      let messageType = $.outputs.messageType;
+      $.assert(messageType, "message Type is not present. Aborting");
+      $.assert(messageType in {{$.EventType.([.IDENTIFY,.TRACK])}}, "message type " + messageType + " is not supported");
+      $.assertConfig(.destination.Config.endpoint, "Movable Ink Endpoint is not present. Aborting");
+      $.assertConfig(.destination.Config.accessKey, "Access key is not present . Aborting");
+      $.assertConfig(.destination.Config.accessSecret, "Access Secret is not present. Aborting");
+      $.assert(.message.timestamp ?? .message.originalTimestamp, "Timestamp is not present. Aborting");
+
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      const email = .message.().(
+          {{{{$.getGenericPaths("email")}}}};
+      );
+
+      $.assert(userId ?? email ?? .message.anonymousId, "Either one of userId or email or anonymousId is required. Aborting");
+
+  - name: preparePayload
+    description: Prepare payload for identify and track. This payload schema needs to be configured in the Movable Ink dashboard. Movable Ink will discard any additional fields from the input payload.
+    template: |
+      const userId = .message.().(
+          {{{{$.getGenericPaths("userIdOnly")}}}};
+      );
+      const email = .message.().(
+          {{{{$.getGenericPaths("email")}}}};
+      ); 
+      const timestampInUnix = $.toUnixTimestampInMS(.message.().(
+            {{{{$.getGenericPaths("timestamp")}}}};
+      ));
+      $.context.payload = {
+        ...(.message),
+        userId: userId ?? email,
+        timestamp: timestampInUnix,
+        anonymousId: .message.anonymousId
+      }
+
+  - name: buildResponse
+    description: In batchMode we return payload directly
+    condition: $.batchMode
+    template: |
+      $.context.payload
+    else:
+      name: buildResponseForProcessTransformation
+      template: |
+        const response = $.defaultRequestConfig();
+        response.body.JSON = $.context.payload;
+        response.endpoint = .destination.Config.endpoint;
+        response.method = "POST";
+        response.headers = {
+          "Content-Type": "application/json",
+          "Authorization": "Basic " + $.base64Convertor(.destination.Config.accessKey + ":" + .destination.Config.accessSecret)
+        }
+        response;

src/cdk/v2/destinations/movable_ink/rtWorkflow.yaml

@@ -0,0 +1,74 @@
+bindings:
+  - name: handleRtTfSingleEventError
+    path: ../../../../v0/util/index
+  - path: ./utils
+    exportAll: true
+  - name: base64Convertor
+    path: ../../../../v0/util
+  - name: BatchUtils
+    path: '@rudderstack/workflow-engine'
+  - path: ./config
+
+steps:
+  - name: validateInput
+    template: |
+      $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array")
+
+  - name: transform
+    externalWorkflow:
+      path: ./procWorkflow.yaml
+      bindings:
+        - name: batchMode
+          value: true
+    loopOverInput: true
+
+  - name: successfulEvents
+    template: |
+      $.outputs.transform#idx.output.({
+        "batchedRequest": .,
+        "batched": false,
+        "destination": ^[idx].destination,
+        "metadata": ^[idx].metadata,
+        "statusCode": 200
+      })[]
+
+  - name: failedEvents
+    template: |
+      $.outputs.transform#idx.error.(
+        $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {})
+      )[]
+
+  - name: batchSuccessfulEvents
+    description: Batches the successfulEvents
+    template: |
+      let batches = $.BatchUtils.chunkArrayBySizeAndLength(
+      $.outputs.successfulEvents, {maxSizeInBytes: $.MAX_REQUEST_SIZE_IN_BYTES}).items;
+
+      batches@batch.({
+        "batchedRequest": {
+          "body": {
+            "JSON": {"events": ~r batch.batchedRequest[]},
+            "JSON_ARRAY": {},
+            "XML": {},
+            "FORM": {}
+          },
+          "version": "1",
+          "type": "REST",
+          "method": "POST",
+          "endpoint": batch[0].destination.Config.().(.endpoint),
+          "headers": batch[0].destination.Config.().({
+            "Content-Type": "application/json",
+            "Authorization": "Basic " + $.base64Convertor(.accessKey + ":" + .accessSecret)
+          }),
+          "params": {},
+          "files": {}
+        },
+        "metadata": ~r batch.metadata[],
+        "batched": true,
+        "statusCode": 200,
+        "destination": batch[0].destination
+      })[];
+
+  - name: finalPayload
+    template: |
+      [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents]

src/features.json

@@ -67,6 +67,7 @@
     "THE_TRADE_DESK": true,
     "INTERCOM": true,
     "NINETAILED": true,
+    "MOVABLE_INK": true,
     "KOALA": true
   },
   "regulations": [

test/integrations/destinations/movable_ink/common.ts

@@ -0,0 +1,128 @@
+import { Destination } from '../../../../src/types';
+
+const destType = 'movable_ink';
+const destTypeInUpperCase = 'MOVABLE_INK';
+const displayName = 'Movable Ink';
+const channel = 'web';
+const destination: Destination = {
+  Config: {
+    endpoint: 'https://collector.movableink-dmz.com/behavioral/abc123',
+    accessKey: 'test-access-key',
+    accessSecret: 'test_access_secret',
+  },
+  DestinationDefinition: {
+    DisplayName: displayName,
+    ID: '123',
+    Name: destTypeInUpperCase,
+    Config: { cdkV2Enabled: true },
+  },
+  Enabled: true,
+  ID: '123',
+  Name: destTypeInUpperCase,
+  Transformations: [],
+  WorkspaceID: 'test-workspace-id',
+};
+
+const processorInstrumentationErrorStatTags = {
+  destType: destTypeInUpperCase,
+  errorCategory: 'dataValidation',
+  errorType: 'instrumentation',
+  feature: 'processor',
+  implementation: 'cdkV2',
+  module: 'destination',
+  destinationId: 'default-destinationId',
+  workspaceId: 'default-workspaceId',
+};
+
+const RouterInstrumentationErrorStatTags = {
+  ...processorInstrumentationErrorStatTags,
+  feature: 'router',
+};
+
+const traits = {
+  email: 'test@example.com',
+  firstName: 'John',
+  lastName: 'Doe',
+  phone: '1234567890',
+};
+
+const headers = {
+  'Content-Type': 'application/json',
+  Authorization: 'Basic dGVzdC1hY2Nlc3Mta2V5OnRlc3RfYWNjZXNzX3NlY3JldA==',
+};
+
+const commonProperties = {
+  product_id: '622c6f5d5cf86a4c77358033',
+  sku: '8472-998-0112',
+  categories: [
+    { url: 'https://example1', id: '1' },
+    { url: 'https://example2', id: '2' },
+  ],
+  name: 'Cones of Dunshire',
+  brand: 'Wyatt Games',
+  variant: 'expansion pack',
+  price: 49.99,
+  quantity: 5,
+  coupon: 'PREORDER15',
+  position: 1,
+  url: 'https://www.website.com/product/path',
+  image_url: 'https://www.website.com/product/path.webp',
+};
+
+const customProperties = {
+  key1: 'value1',
+  key2: true,
+  key3: ['value3'],
+  key4: { key5: { key6: 'value6' } },
+};
+
+const trackTestProperties = {
+  'Product Added': { ...commonProperties, ...customProperties },
+  'Product Viewed': { ...commonProperties, ...customProperties },
+  'Order Completed': {
+    checkout_id: '70324a1f0eaf000000000000',
+    order_id: '40684e8f0eaf000000000000',
+    affiliation: 'Vandelay Games',
+    total: 52,
+    subtotal: 45,
+    revenue: 50,
+    shipping: 4,
+    tax: 3,
+    discount: 5,
+    coupon: 'NEWCUST5',
+    currency: 'USD',
+    products: [
+      {
+        product_id: '622c6f5d5cf86a4c77358033',
+        sku: '8472-998-0112',
+        name: 'Cones of Dunshire',
+        price: 40,
+        position: 1,
+        category: 'Games',
+        url: 'https://www.website.com/product/path',
+        image_url: 'https://www.website.com/product/path.jpg',
+      },
+      {
+        product_id: '577c6f5d5cf86a4c7735ba03',
+        sku: '3309-483-2201',
+        name: 'Five Crowns',
+        price: 5,
+        position: 2,
+        category: 'Games',
+      },
+    ],
+  },
+  'Products Searched': { query: 'HDMI cable', url: 'https://www.website.com/product/path' },
+  'Custom event': { ...commonProperties, key1: 'value1', key2: true },
+};
+
+export {
+  destType,
+  channel,
+  destination,
+  processorInstrumentationErrorStatTags,
+  RouterInstrumentationErrorStatTags,
+  traits,
+  headers,
+  trackTestProperties,
+};

test/integrations/destinations/movable_ink/processor/data.ts

@@ -0,0 +1,4 @@
+import { validation } from './validation';
+import { identify } from './identify';
+import { track } from './track';
+export const data = [...identify, ...track, ...validation];

test/integrations/destinations/movable_ink/processor/identify.ts

@@ -0,0 +1,64 @@
+import { ProcessorTestData } from '../../../testTypes';
+import { generateMetadata, transformResultBuilder } from '../../../testUtils';
+import { destType, channel, destination, traits, headers } from '../common';
+
+export const identify: ProcessorTestData[] = [
+  {
+    id: 'MovableInk-identify-test-1',
+    name: destType,
+    description: 'Identify call with traits and anonymousId',
+    scenario: 'Framework+Business',
+    successCriteria:
+      'Response should contain the input payload with few additional mappings configured in transformer and status code should be 200',
+    feature: 'processor',
+    module: 'destination',
+    version: 'v0',
+    input: {
+      request: {
+        body: [
+          {
+            destination,
+            message: {
+              type: 'identify',
+              anonymousId: 'anonId123',
+              traits,
+              integrations: {
+                All: true,
+              },
+              originalTimestamp: '2024-03-04T15:32:56.409Z',
+            },
+            metadata: generateMetadata(1),
+          },
+        ],
+      },
+    },
+    output: {
+      response: {
+        status: 200,
+        body: [
+          {
+            output: transformResultBuilder({
+              method: 'POST',
+              userId: '',
+              endpoint: destination.Config.endpoint,
+              headers,
+              JSON: {
+                type: 'identify',
+                userId: traits.email,
+                anonymousId: 'anonId123',
+                traits,
+                integrations: {
+                  All: true,
+                },
+                originalTimestamp: '2024-03-04T15:32:56.409Z',
+                timestamp: 1709566376409,
+              },
+            }),
+            statusCode: 200,
+            metadata: generateMetadata(1),
+          },
+        ],
+      },
+    },
+  },
+];