Bugfix/235 all steps bdc errors

src/bdc/steps/__init__.py

@@ -9,6 +9,5 @@
 from .hash_generator import *
 from .preprocess_phonenumbers import *
 from .regionalatlas import *
-from .scrape_address import *
 from .search_offeneregister import *
 from .step import *

src/bdc/steps/regionalatlas.py

@@ -6,7 +6,6 @@
 import geopandas as gpd
 import osmnx
 import pandas as pd
-from geopandas.tools import sjoin
 from pandas import DataFrame
 from tqdm import tqdm
 
@@ -118,13 +117,13 @@ def run(self) -> DataFrame:
 
         tqdm.pandas(desc="Computing Regional Score")
 
-        self.df[self.added_cols[:-1]] = self.df.progress_apply(
+        self.df[self.added_cols[-1:]] = self.df.progress_apply(
             lambda lead: pd.Series(
                 get_lead_hash_generator().hash_check(
                     lead,
                     self.calculate_regional_score,
                     self.name + "_Regional-Score",
-                    self.added_cols[:-1],
+                    self.added_cols[-1:],
                     lead,
                 )
             ),

src/database/leads/local_repository.py

@@ -4,7 +4,6 @@
 import csv
 import json
 import os
-from datetime import datetime
 from pathlib import Path
 
 import joblib
@@ -197,6 +196,11 @@ def fetch_gpt_result(self, file_id, operation_name):
         try:
             with open(json_file_path, "r", encoding="utf-8") as json_file:
                 data = json.load(json_file)
+                if operation_name not in data:
+                    log.info(
+                        f"Data for operation {operation_name} was not found in {json_file_path}"
+                    )
+                    return ""
                 return data[operation_name]
         except:
             log.warning(f"Error loading GPT results from path {json_file_path}.")

src/demo/pipeline_configs/config_sprint09_release.json

@@ -6,10 +6,6 @@
         "name": "AnalyzeEmails",
         "force_refresh": true
       },
-      {
-        "name": "ScrapeAddress",
-        "force_refresh": true
-      },
       {
         "name": "PreprocessPhonenumbers",
         "force_refresh": true

src/demo/pipeline_configs/config_template

@@ -7,10 +7,6 @@
             "name": "AnalyzeEmails",
             "force_refresh": true
         },
-        {
-            "name": "ScrapeAddress",
-            "force_refresh": true
-        },
         {
             "name": "PreprocessPhonenumbers",
             "force_refresh": true

src/demo/pipeline_configs/force_refresh_all_steps.json

@@ -0,0 +1,43 @@
+{
+  "description": "This config runs all steps with force_refresh set to true.",
+  "config": {
+    "steps": [
+      {
+        "name": "HashGenerator",
+        "force_refresh": true
+      },
+      {
+        "name": "AnalyzeEmails",
+        "force_refresh": true
+      },
+      {
+        "name": "PreprocessPhonenumbers",
+        "force_refresh": true
+      },
+      {
+        "name": "GooglePlaces",
+        "force_refresh": true
+      },
+      {
+        "name": "GooglePlacesDetailed",
+        "force_refresh": true
+      },
+      {
+        "name": "GPTReviewSentimentAnalyzer",
+        "force_refresh": true
+      },
+      {
+        "name": "GPTSummarizer",
+        "force_refresh": true
+      },
+      {
+        "name": "SmartReviewInsightsEnhancer",
+        "force_refresh": true
+      },
+      {
+        "name": "RegionalAtlas",
+        "force_refresh": true
+      }
+    ]
+  }
+}

src/demo/pipeline_configs/force_refresh_all_steps.json.license

@@ -0,0 +1,2 @@
+SPDX-License-Identifier: MIT
+SPDX-FileCopyrightText: 2023 Berkay Bozkurt <resitberkaybozkurt@gmail.com>

src/demo/pipeline_configs/regionalatlas_only.json

@@ -0,0 +1,27 @@
+{
+  "description": "This config runs all steps with force_refresh set to true.",
+  "config": {
+    "steps": [
+      {
+        "name": "HashGenerator",
+        "force_refresh": true
+      },
+      {
+        "name": "AnalyzeEmails",
+        "force_refresh": true
+      },
+      {
+        "name": "PreprocessPhonenumbers",
+        "force_refresh": true
+      },
+      {
+        "name": "GooglePlaces",
+        "force_refresh": true
+      },
+      {
+        "name": "RegionalAtlas",
+        "force_refresh": true
+      }
+    ]
+  }
+}

src/demo/pipeline_configs/regionalatlas_only.json.license

@@ -0,0 +1,2 @@
+SPDX-License-Identifier: MIT
+SPDX-FileCopyrightText: 2023 Lucca Baumgärtner <lucca.baumgaertner@fau.de>

src/demo/pipeline_configs/run_all_steps.json

@@ -1,46 +1,42 @@
 {
-  "description": "This config runs all steps with force_refresh set to true.",
+  "description": "This config runs all steps with force_refresh set to false.",
   "config": {
     "steps": [
       {
         "name": "HashGenerator",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "AnalyzeEmails",
-        "force_refresh": true
-      },
-      {
-        "name": "ScrapeAddress",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "PreprocessPhonenumbers",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GooglePlaces",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GooglePlacesDetailed",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GPTReviewSentimentAnalyzer",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "GPTSummarizer",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "SmartReviewInsightsEnhancer",
-        "force_refresh": true
+        "force_refresh": false
       },
       {
         "name": "RegionalAtlas",
-        "force_refresh": true
+        "force_refresh": false
       }
     ]
   }

src/demo/pipeline_utils.py

@@ -17,7 +17,6 @@
     HashGenerator,
     PreprocessPhonenumbers,
     RegionalAtlas,
-    ScrapeAddress,
     SearchOffeneRegister,
     SmartReviewInsightsEnhancer,
 )
@@ -33,14 +32,12 @@
     "GPTSummarizer": GPTSummarizer,
     "PreprocessPhonenumbers": PreprocessPhonenumbers,
     "RegionalAtlas": RegionalAtlas,
-    "ScrapeAddress": ScrapeAddress,
     "SearchOffeneRegister": SearchOffeneRegister,
     "SmartReviewInsightsEnhancer": SmartReviewInsightsEnhancer,
 }
 
 # Please do not write following lists! Use the functions below instead.
 _additional_pipeline_steps = [
-    (ScrapeAddress, "Scrape Address", "(will take a long time)"),
     (SearchOffeneRegister, "Search OffeneRegister", "(will take a long time)"),
     (PreprocessPhonenumbers, "Phone Number Validation", ""),
     (