From 92539d4b9b7d39205956a69ab3868c856f460775 Mon Sep 17 00:00:00 2001 From: "Lennart Kats (databricks)" Date: Mon, 20 Nov 2023 20:25:43 +0100 Subject: [PATCH] Work around DLT issue with `$PYTHONPATH` not being set correctly (#999) ## Changes DLT currently doesn't always set `$PYTHONPATH` correctly (ES-947370). This restores the original workaround to make new pipelines work while that issue is being addressed. The workaround was removed in #832. Manually tested. --- .../resources/{{.project_name}}_pipeline.yml.tmpl | 3 +++ .../template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl | 1 + 2 files changed, 4 insertions(+) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl index 498604f6b6..4b8f74d173 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}_pipeline.yml.tmpl @@ -7,3 +7,6 @@ resources: libraries: - notebook: path: ../src/dlt_pipeline.ipynb + + configuration: + bundle.sourcePath: /Workspace/${workspace.file_path}/src diff --git a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl index 8c85e97ef6..4f50294f6a 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/src/dlt_pipeline.ipynb.tmpl @@ -35,6 +35,7 @@ "# Import DLT and src/{{.project_name}}\n", "import dlt\n", "import sys\n", + "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", "from pyspark.sql.functions import expr\n", "from {{.project_name}} import main" {{else}}