-
Notifications
You must be signed in to change notification settings - Fork 0
/
go.py
110 lines (87 loc) · 4.04 KB
/
go.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
# Simple scraper for APIs with Hydra pagination
#
# This file is part of the Hydra Scraper package.
#
# For the full copyright and license information, please read the
# LICENSE.txt file that was distributed with this source code.
# Import libraries
from sys import argv
# Import script modules
from classes.hydra import *
from classes.beacon import *
from helpers.clean import clean_request
from helpers.config import *
from helpers.fileio import create_folder
from helpers.status import echo_note
# Get request data and create download folder for this job
request = clean_request(argv[1:])
job_folder = config['download_base'] + '/' + request['target_folder']
create_folder(job_folder)
# Set up status messages
status = []
# Set up Hydra API routine if required
if 'lists' in request['download'] or 'beacon' in request['download'] or 'list_triples' in request['download'] or 'list_cgif' in request['download']:
hydra = Hydra(request['target_folder'], request['source_url'], request['content_type'])
# Populate the object, and download each API list if requested
if 'lists' in request['download']:
save_lists = True
else:
save_lists = False
hydra.populate(save_lists, request['resource_url_filter'], request['resource_url_replace'], request['resource_url_replace_with'], request['resource_url_add'])
# Compile a beacon list if requested
if 'beacon' in request['download']:
hydra.save_beacon()
# Compile list triples if requested
if 'list_triples' in request['download']:
hydra.save_triples()
# Compile list CGIF triples if requested
if 'list_cgif' in request['download']:
hydra.save_triples('cgif', 'lists_cgif')
# Add status message
status.extend(hydra.status)
# Mini Hydra routine if Beacon logic is requested but no beacon file is given
elif request['source_file'] == '':
hydra = Hydra(request['target_folder'], request['source_url'], request['content_type'])
hydra.populate(False, request['resource_url_filter'], request['resource_url_replace'], request['resource_url_replace_with'], request['resource_url_add'])
# Mark absence of hydra object if beacon file is present
else:
hydra = False
# Set up beacon file routine if required
if 'resources' in request['download'] or 'resource_triples' in request['download'] or 'resource_cgif' in request['download'] or 'resource_table' in request['download']:
# Use previous resource list if present
if hydra == False:
beacon = Beacon(request['target_folder'], request['content_type'])
else:
beacon = Beacon(request['target_folder'], request['content_type'], hydra.resources)
# Populate the object, and download each resource if requested
if 'resources' in request['download']:
save_resources = True
else:
save_resources = False
beacon.populate(save_resources, request['clean_resource_names'], request['source_file'], request['source_folder'], request['supplement_data_feed'], request['supplement_data_catalog'], request['supplement_data_catalog_publisher'])
# Compile resource triples if requested
if 'resource_triples' in request['download']:
beacon.save_triples()
# Compile resource CGIF triples if requested
if 'resource_cgif' in request['download']:
beacon.save_triples('cgif', 'resources_cgif')
# Compile resource table if requested
if 'resource_table' in request['download']:
beacon.save_table(request['table_data'])
# Add status message
status.extend(beacon.status)
# Compile a report string (success, reason, missing, incompatible)
report = 'Done!'
for entry in status:
if entry['success'] == False:
report = 'Something went wrong!'
for entry in status:
report = report + ' ' + entry['reason']
for entry in status:
if 'missing' in entry:
report = report + '\n\nMissing files:\n- ' + '\n- '.join(entry['missing'])
for entry in status:
if 'incompatible' in entry:
report = report + '\n\nNot compatible:\n- ' + '\n- '.join(entry['incompatible'])
# Provide final report
echo_note('\n' + report + '\n')