new scenario and link support in tests

consumer/consumers/dataplex/scenarios/airflow/config.json

@@ -0,0 +1,40 @@
+{
+  "tests": [
+    {
+      "name":"process",
+      "path": "validation/processes/processes.json",
+      "entity": "process",
+      "tags": {
+        "facets": ["run_event"],
+        "producer": "airflow"
+      }
+    },
+    {
+      "name":"runs",
+      "path": "validation/runs/runs.json",
+      "entity": "run",
+      "tags": {
+        "facets": ["run_event", "processing_engine"],
+        "producer": "airflow"
+      }
+    },
+    {
+      "name":"lineage_events",
+      "path": "validation/lineage_events/lineage_events.json",
+      "entity": "lineage_event",
+      "tags": {
+        "facets": ["run_event"],
+        "producer": "airflow"
+      }
+    },
+    {
+      "name":"links",
+      "path": "validation/links/links.json",
+      "entity": "link",
+      "tags": {
+        "facets": ["run_event"],
+        "producer": "airflow"
+      }
+    }
+  ]
+}

consumer/consumers/dataplex/scenarios/airflow/maintainers.json

@@ -0,0 +1,14 @@
+[
+  {
+    "type": "maintainer",
+    "github-name": "ddebowczyk92",
+    "email": "dominik.debowczyk@getindata.com",
+    "link": ""
+  },
+  {
+    "type": "maintainer",
+    "github-name": "tnazarew",
+    "email": "tomasz.nazarewicz@getindata.com",
+    "link": ""
+  }
+]

consumer/consumers/dataplex/scenarios/airflow/scenario.md

@@ -0,0 +1,7 @@
+# Description
+This test scenario expects that after sending OL events to Dataplex
+
+following entities will be created:
+- 1 Process with `display_name` value `default:cl_i_test_application.execute_insert_into_hive_table.warehouse_t2`
+- 1 Run with `display_name` value `019127df-0850-72bc-b214-b255290588a6`
+- 1 LineageEvent with `start_time` value `2023-04-04T13:21:16.098Z`

consumer/consumers/dataplex/scenarios/airflow/validation/lineage_events/lineage_events.json

@@ -0,0 +1,154 @@
+[
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/1275f3986dc31d314011311e70aacf4a/runs/36a0836e5795f3f12b5a879be6972711/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:05:39.460005Z",
+    "end_time": "2024-11-26T13:05:39.460005Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "bigquery:mock-project.test.upload_cp"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`result.csv`"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/1275f3986dc31d314011311e70aacf4a/runs/36a0836e5795f3f12b5a879be6972711/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:05:34.626595Z",
+    "end_time": "2024-11-26T13:05:34.626595Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "bigquery:mock-project.test.upload"
+        },
+        "target": {
+          "fully_qualified_name": "bigquery:mock-project.test.upload_cp"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/1275f3986dc31d314011311e70aacf4a/runs/36a0836e5795f3f12b5a879be6972711/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:05:29.606867Z",
+    "end_time": "2024-11-26T13:05:29.606867Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`copied.csv`"
+        },
+        "target": {
+          "fully_qualified_name": "bigquery:mock-project.test.upload"
+        }
+      },
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`test.csv`"
+        },
+        "target": {
+          "fully_qualified_name": "bigquery:mock-project.test.upload"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/597c3efb0c87da89121db76aa9a28457/runs/0eecd60ed77cc63e10fc8967bbca8199/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:10:44.750985Z",
+    "end_time": "2024-11-26T13:10:44.750985Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_file.txt`"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/597c3efb0c87da89121db76aa9a28457/runs/0eecd60ed77cc63e10fc8967bbca8199/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:10:44.755944Z",
+    "end_time": "2024-11-26T13:10:44.755944Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`uploaded_data.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_data.txt`"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/597c3efb0c87da89121db76aa9a28457/runs/0eecd60ed77cc63e10fc8967bbca8199/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:10:46.074886Z",
+    "end_time": "2024-11-26T13:10:46.074886Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+        }
+      },
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`uploaded_data.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+        }
+      },
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_data.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+        }
+      },
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_file.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/597c3efb0c87da89121db76aa9a28457/runs/0eecd60ed77cc63e10fc8967bbca8199/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:10:41.007315Z",
+    "end_time": "2024-11-26T13:10:41.007315Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "filesystem:localhost.`files/temp/data.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+        }
+      }
+    ]
+  },
+  {
+    "name": "{{ match(result, 'projects/484892851355/locations/us/processes/597c3efb0c87da89121db76aa9a28457/runs/0eecd60ed77cc63e10fc8967bbca8199/lineageEvents/.*') }}",
+    "start_time": "2024-11-26T13:10:45.874270Z",
+    "end_time": "2024-11-26T13:10:45.874270Z",
+    "links": [
+      {
+        "source": {
+          "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+        },
+        "target": {
+          "fully_qualified_name": "filesystem:localhost.`files/temp/downloaded_file.txt`"
+        }
+      }
+    ]
+  }
+]

consumer/consumers/dataplex/scenarios/airflow/validation/links/links.json

@@ -0,0 +1,134 @@
+[
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "bigquery:mock-project.test.upload_cp"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`result.csv`"
+    },
+    "start_time": "2024-11-26T13:05:39.460005Z",
+    "end_time": "2024-11-26T13:05:39.460005Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`copied.csv`"
+    },
+    "target": {
+      "fully_qualified_name": "bigquery:mock-project.test.upload"
+    },
+    "start_time": "2024-11-26T13:05:29.606867Z",
+    "end_time": "2024-11-26T13:05:29.606867Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`test.csv`"
+    },
+    "target": {
+      "fully_qualified_name": "bigquery:mock-project.test.upload"
+    },
+    "start_time": "2024-11-26T13:05:29.606867Z",
+    "end_time": "2024-11-26T13:05:29.606867Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "bigquery:mock-project.test.upload"
+    },
+    "target": {
+      "fully_qualified_name": "bigquery:mock-project.test.upload_cp"
+    },
+    "start_time": "2024-11-26T13:05:34.626595Z",
+    "end_time": "2024-11-26T13:05:34.626595Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`uploaded_data.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+    },
+    "start_time": "2024-11-26T13:10:46.074886Z",
+    "end_time": "2024-11-26T13:10:46.074886Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_data.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+    },
+    "start_time": "2024-11-26T13:10:46.074886Z",
+    "end_time": "2024-11-26T13:10:46.074886Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_file.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+    },
+    "start_time": "2024-11-26T13:10:46.074886Z",
+    "end_time": "2024-11-26T13:10:46.074886Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`compose_result.txt`"
+    },
+    "start_time": "2024-11-26T13:10:46.074886Z",
+    "end_time": "2024-11-26T13:10:46.074886Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "filesystem:localhost.`files/temp/downloaded_file.txt`"
+    },
+    "start_time": "2024-11-26T13:10:45.874270Z",
+    "end_time": "2024-11-26T13:10:45.874270Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "filesystem:localhost.`files/temp/data.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+    },
+    "start_time": "2024-11-26T13:10:41.007315Z",
+    "end_time": "2024-11-26T13:10:41.007315Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`uploaded_file.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_file.txt`"
+    },
+    "start_time": "2024-11-26T13:10:44.750985Z",
+    "end_time": "2024-11-26T13:10:44.750985Z"
+  },
+  {
+    "name": "{{ match(result, 'projects/.*/locations/us/links/p:.*') }}",
+    "source": {
+      "fully_qualified_name": "gcs:mock-bucket.`uploaded_data.txt`"
+    },
+    "target": {
+      "fully_qualified_name": "gcs:mock-bucket.`copy_of_uploaded_data.txt`"
+    },
+    "start_time": "2024-11-26T13:10:44.755944Z",
+    "end_time": "2024-11-26T13:10:44.755944Z"
+  }
+]

consumer/consumers/dataplex/scenarios/airflow/validation/processes/processes.json

@@ -0,0 +1,17 @@
+[
+  {
+    "name": "projects/484892851355/locations/us/processes/1275f3986dc31d314011311e70aacf4a",
+    "display_name": "{{ match(result, 'airflow:BQ.*') }}",
+    "attributes": {}
+  },
+  {
+    "name": "projects/484892851355/locations/us/processes/8aa5d4068a7fcaf187d2c6f9e0402d3d",
+    "display_name": "{{ match(result, 'airflow:dag.*') }}",
+    "attributes": {}
+  },
+  {
+    "name": "projects/484892851355/locations/us/processes/597c3efb0c87da89121db76aa9a28457",
+    "display_name": "{{ match(result, 'airflow:gcs_hook.*') }}",
+    "attributes": {}
+  }
+]