Skip to content

Commit

Permalink
Remove Maroon Google Doc upload (#63)
Browse files Browse the repository at this point in the history
## Describe your changes
I removed the Google Doc upload functionality because there is an
up-to-date [download
link](https://chicagomaroon.com/uchicago-police-department-incident-reporter/)
on the main UCPD Incident Reporter page.

## Checklist before requesting a review
- [x] The code runs successfully.

```commandline
(ucpd-incident-scraper-py3.11) michaelp@MacBook-Air-18 ucpd-incident-scraper % make update
python -m incident_scraper update
[nltk_data] Downloading package wordnet to
[nltk_data]     /Users/michaelp/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
Beginning the UCPD Incident scraping process.
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1725854031.884889  478914 check_gcp_environment_no_op.cc:29] ALTS: Platforms other than Linux and Windows are not supported
Finished with the UCPD Incident scraping process.
Waiting up to 5 seconds.
Sent all pending logs.
(ucpd-incident-scraper-py3.11) michaelp@MacBook-Air-18 ucpd-incident-scraper %
```
  • Loading branch information
michplunkett authored Sep 9, 2024
1 parent 52511e2 commit d47dd73
Show file tree
Hide file tree
Showing 4 changed files with 1 addition and 111 deletions.
13 changes: 0 additions & 13 deletions incident_scraper/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import argparse
import logging
import os.path
import re
from datetime import datetime
from typing import Any
Expand All @@ -13,13 +12,11 @@
from incident_scraper.external.google_logger import init_logger
from incident_scraper.external.google_nbd import GoogleNBD
from incident_scraper.external.lemmatizer import Lemmatizer
from incident_scraper.external.maroon_google_drive import MaroonGoogleDrive
from incident_scraper.models.address_parser import AddressParser
from incident_scraper.models.classifier import Classifier
from incident_scraper.models.incident import Incident
from incident_scraper.scraper.ucpd_scraper import UCPDScraper
from incident_scraper.utils.constants import (
FILE_NAME_INCIDENT_DUMP,
INCIDENT_KEY_ADDRESS,
INCIDENT_KEY_COMMENTS,
INCIDENT_KEY_ID,
Expand Down Expand Up @@ -119,16 +116,6 @@ def categorize_information(nbd_client: GoogleNBD) -> None:
nbd_client.update_list_of_incidents(incidents)


def download_and_upload_records() -> None:
logging.info("Beginning incident download and Google Drive export.")
GoogleNBD().download_all()
if os.path.isfile(FILE_NAME_INCIDENT_DUMP):
MaroonGoogleDrive().upload_file_to_maroon_tech_folder(
FILE_NAME_INCIDENT_DUMP
)
logging.info("Finished incident download and Google Drive export.")


def lemmatize_categories(nbd_client: GoogleNBD) -> None:
incidents = nbd_client.get_all_incidents()
logging.info(f"{len(incidents)} incidents fetched.")
Expand Down
87 changes: 0 additions & 87 deletions incident_scraper/external/maroon_google_drive.py

This file was deleted.

1 change: 0 additions & 1 deletion incident_scraper/utils/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
# Environment Constants
ENV_GCP_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
ENV_GCP_PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT")
ENV_GOOGLE_DRIVE_FOLDER_ID = os.getenv("GOOGLE_DRIVE_FOLDER_ID")
ENV_GOOGLE_MAPS_KEY = os.getenv("GOOGLE_MAPS_API_KEY")

# File Constants
Expand Down
11 changes: 1 addition & 10 deletions scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@

from apscheduler.schedulers.blocking import BlockingScheduler

from incident_scraper.__main__ import (
download_and_upload_records,
update_records,
)
from incident_scraper.__main__ import update_records


scheduler = BlockingScheduler()
Expand All @@ -18,10 +15,4 @@ def run_scraper():
update_records()


@scheduler.scheduled_job(trigger="cron", day_of_week="sat", hour=17)
def export_to_maroon_google_drive():
"""Export the incidents to Google Drive at the above interval."""
download_and_upload_records()


scheduler.start()

0 comments on commit d47dd73

Please sign in to comment.