diff --git a/incident_scraper/__main__.py b/incident_scraper/__main__.py index 329d0cc..f5d653f 100644 --- a/incident_scraper/__main__.py +++ b/incident_scraper/__main__.py @@ -2,7 +2,6 @@ import argparse import logging -import os.path import re from datetime import datetime from typing import Any @@ -13,13 +12,11 @@ from incident_scraper.external.google_logger import init_logger from incident_scraper.external.google_nbd import GoogleNBD from incident_scraper.external.lemmatizer import Lemmatizer -from incident_scraper.external.maroon_google_drive import MaroonGoogleDrive from incident_scraper.models.address_parser import AddressParser from incident_scraper.models.classifier import Classifier from incident_scraper.models.incident import Incident from incident_scraper.scraper.ucpd_scraper import UCPDScraper from incident_scraper.utils.constants import ( - FILE_NAME_INCIDENT_DUMP, INCIDENT_KEY_ADDRESS, INCIDENT_KEY_COMMENTS, INCIDENT_KEY_ID, @@ -119,16 +116,6 @@ def categorize_information(nbd_client: GoogleNBD) -> None: nbd_client.update_list_of_incidents(incidents) -def download_and_upload_records() -> None: - logging.info("Beginning incident download and Google Drive export.") - GoogleNBD().download_all() - if os.path.isfile(FILE_NAME_INCIDENT_DUMP): - MaroonGoogleDrive().upload_file_to_maroon_tech_folder( - FILE_NAME_INCIDENT_DUMP - ) - logging.info("Finished incident download and Google Drive export.") - - def lemmatize_categories(nbd_client: GoogleNBD) -> None: incidents = nbd_client.get_all_incidents() logging.info(f"{len(incidents)} incidents fetched.") diff --git a/incident_scraper/external/maroon_google_drive.py b/incident_scraper/external/maroon_google_drive.py deleted file mode 100644 index 35c2662..0000000 --- a/incident_scraper/external/maroon_google_drive.py +++ /dev/null @@ -1,87 +0,0 @@ -import json -import logging - -from oauth2client.service_account import ServiceAccountCredentials -from pydrive2.auth import GoogleAuth -from pydrive2.drive import GoogleDrive - -from incident_scraper.utils.constants import ( - ENV_GCP_CREDENTIALS, - ENV_GOOGLE_DRIVE_FOLDER_ID, - FILE_TYPE_JSON, -) - - -class MaroonGoogleDrive: - """ - Class that manages interactions with Google Drive. - """ - - SCOPES = ["https://www.googleapis.com/auth/drive"] - - def __init__(self): - auth_client = GoogleAuth() - auth_client.auth_method = "service" - if ENV_GCP_CREDENTIALS.endswith(FILE_TYPE_JSON): - auth_client.credentials = ( - ServiceAccountCredentials.from_json_keyfile_name( - ENV_GCP_CREDENTIALS, - scopes=self.SCOPES, - ) - ) - else: - auth_client.credentials = ( - ServiceAccountCredentials.from_json_keyfile_dict( - json.loads(ENV_GCP_CREDENTIALS), scopes=self.SCOPES - ) - ) - - self.__client = GoogleDrive(auth_client) - logging.debug("Connected to the Chicago Maroon Google Drive.") - - def upload_file_to_maroon_tech_folder(self, file_name: str) -> None: - """ - Upload the parametrized file to the Chicago Maroon's Tech folder. - """ - print(file_name) - logging.debug( - "Starting upload process to the Chicago Maroon's Tech " - f"Google Drive folder for: {file_name}" - ) - - file_id = "" - file_list = self.__client.ListFile( - { - "q": f"'{ENV_GOOGLE_DRIVE_FOLDER_ID}' in parents " - "and trashed=False" - } - ).GetList() - for f in file_list: - if f["title"] == file_name: - file_id = f["id"] - - if file_id: - file = self.__client.CreateFile({"id": file_id}) - logging.debug(f"Updating file: {file_name}.") - else: - file = self.__client.CreateFile( - { - "parents": [ - { - "id": ENV_GOOGLE_DRIVE_FOLDER_ID, - "title": "Tech", - } - ], - "title": file_name, - "mimeType": "text/csv", - } - ) - logging.debug(f"Creating file: {file_name}.") - - file.SetContentFile(file_name) - file.Upload() - - logging.debug( - "Finished upload process to the Chicago Maroon's Tech " - f"Google Drive folder for: {file_name}" - ) diff --git a/incident_scraper/utils/constants.py b/incident_scraper/utils/constants.py index 0d0398d..fd4aa1a 100644 --- a/incident_scraper/utils/constants.py +++ b/incident_scraper/utils/constants.py @@ -20,7 +20,6 @@ # Environment Constants ENV_GCP_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") ENV_GCP_PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT") -ENV_GOOGLE_DRIVE_FOLDER_ID = os.getenv("GOOGLE_DRIVE_FOLDER_ID") ENV_GOOGLE_MAPS_KEY = os.getenv("GOOGLE_MAPS_API_KEY") # File Constants diff --git a/scheduler.py b/scheduler.py index 51f780c..df82f57 100644 --- a/scheduler.py +++ b/scheduler.py @@ -2,10 +2,7 @@ from apscheduler.schedulers.blocking import BlockingScheduler -from incident_scraper.__main__ import ( - download_and_upload_records, - update_records, -) +from incident_scraper.__main__ import update_records scheduler = BlockingScheduler() @@ -18,10 +15,4 @@ def run_scraper(): update_records() -@scheduler.scheduled_job(trigger="cron", day_of_week="sat", hour=17) -def export_to_maroon_google_drive(): - """Export the incidents to Google Drive at the above interval.""" - download_and_upload_records() - - scheduler.start()