-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathpreprocessorsetup.py
173 lines (148 loc) · 8.57 KB
/
preprocessorsetup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
from loghelper import *
import os, sys, argparse, shutil, zipfile, stat
from shutil import ignore_patterns
from future.builtins import input
from preprocess_tools import postgis_manage
from configuration.pathregistry import PathRegistry
from configuration.subregionconfig import SubRegionConfig
def del_rw(action, name, exc):
os.chmod(name, stat.S_IWRITE)
os.remove(name)
def main():
create_script_log(sys.argv[0])
try:
parser = argparse.ArgumentParser(description="sets up external data in working directory for subsequent processes")
parser.add_argument("--pathRegistry", help="path to file registry data")
parser.add_argument("--subRegionConfig", help="path to sub region data")
parser.add_argument("--subRegionNames", help="optional comma delimited "+
"string of sub region names (as defined in "+
"subRegionConfig) to process, if unspecified all "+
"regions will be processed")
parser.add_argument("--spatial", action="store_true", dest="spatial", help="copy spatial files to the working dir")
parser.add_argument("--future", action="store_true", dest="future", help="copys future projection files to the working dir")
parser.add_argument("--aspatial", action="store_true", dest="aspatial", help="copy aspatial files to the working dir")
parser.add_argument("--tools", action="store_true", dest="tools", help="copy tools to the working dir")
parser.add_argument("--postgis", action="store_true", dest="postgis", help="set up postgis credentials/url")
parser.add_argument("--cleanup", action="store_true", dest="cleanup", help="clean up any existing files working dir, and databases first")
parser.set_defaults(spatial=False)
parser.set_defaults(future=False)
parser.set_defaults(aspatial=False)
parser.set_defaults(tools=False)
parser.set_defaults(postgis=False)
args = parser.parse_args()
pathRegistry = PathRegistry(os.path.abspath( args.pathRegistry))
subRegionConfig = SubRegionConfig(
os.path.abspath(args.subRegionConfig),
args.subRegionNames.split(',') if args.subRegionNames else None)
if not args.spatial \
and not args.aspatial \
and not args.tools \
and not args.future \
and not args.postgis:
logging.error("nothing to do")
if args.postgis:
logging.info("postgis setup")
post_gis_variables = [{"name": "PGHOST", "default": "localhost"},
{"name": "PGPORT", "default": "5432" },
{"name": "PGUSER", "default": "postgres"},
{"name": "PGPASSWORD", "default": None}]
print("PostGIS connection variables:")
pg_var_result = {}
for pg_var in post_gis_variables:
input_string = "{v} (push enter for '{d}') : " \
.format(v=pg_var["name"], d=pg_var["default"]) \
if pg_var["default"] is not None else \
"{v} : ".format(v = pg_var["name"])
result = input(input_string)
result = pg_var["default"] if len(result) == 0 and pg_var["default"] else result
pg_var_result[pg_var["name"]] = result
pg_var_result["PGDATABASE"] = "postgres"
connectionVarsPath = pathRegistry.GetPath("PostGIS_Connection_Vars")
if not os.path.exists(os.path.dirname(connectionVarsPath)):
os.makedirs(os.path.dirname(connectionVarsPath))
postgis_manage.save_connection_variables(connectionVarsPath, **pg_var_result)
result = input("test connection? (y/n):")
if result.lower() in ["yes", "y"]:
with postgis_manage.connect(**postgis_manage.get_connection_variables(connectionVarsPath)) as conn:
if conn.closed:
raise RuntimeError("database connection error")
logging.info("db connected sucessfully")
input("any key to continue")
if args.cleanup:
for r in subRegionConfig.GetRegions():
logging.info("dropping working db for {}".format(r["Name"]))
region_path = r["PathName"]
region_postgis_var_path = pathRegistry.GetPath(
"PostGIS_Region_Connection_Vars",
region_path=region_path)
if os.path.exists(region_postgis_var_path):
postgis_manage.drop_working_db(
pathRegistry.GetPath("PostGIS_Connection_Vars"),
region_postgis_var_path)
os.remove(region_postgis_var_path)
if args.aspatial:
src = pathRegistry.GetPath("Source_External_Aspatial_Dir")
dst = pathRegistry.GetPath("External_Aspatial_Dir")
if args.cleanup and os.path.exists(dst):
logging.info("removing dir {}".format(dst))
shutil.rmtree(dst)
logging.info("copying external aspatial data to local working directory")
logging.info("source: {}".format(src))
logging.info("destination: {}".format(dst))
shutil.copytree(src=src, dst=dst)
if args.tools:
toolPathPairs = [("Source_GCBM_Dir", "Local_GCBM_Dir"),
("Source_Recliner2GCBM-x64_Dir", "Local_Recliner2GCBM-x64_Dir"),
("Source_Recliner2GCBM-x86_Dir", "Local_Recliner2GCBM-x86_Dir"),
("Source_lostgis_dir", "Local_lostgis_dir")]
for pair in toolPathPairs:
src = pathRegistry.GetPath(pair[0])
dst = pathRegistry.GetPath(pair[1])
if args.cleanup and os.path.exists(dst):
logging.info("removing dir {}".format(dst))
#del_rw is called on errors to attempt to remove read only files
#(there are a few of them in recent GCBM builds)
shutil.rmtree(dst, onerror=del_rw)
logging.info("copying external tool from {} to {}".format(pair[0],pair[1]))
logging.info("source: {}".format(src))
logging.info("destination: {}".format(dst))
shutil.copytree(src=src,dst=dst)
if args.future:
for region in subRegionConfig.GetRegions():
for sha_scenario in region["SHAScenarios"]:
subdir = os.path.join(*sha_scenario["SubDir"])
src = pathRegistry.GetPath("Source_External_Future_Dir",
sha_future_scenario=subdir)
dst = pathRegistry.GetPath("Future_Dist_Input_Dir",
region_path=region["PathName"],
sha_future_scenario=sha_scenario["Name"])
if args.cleanup and os.path.exists(dst):
logging.info("removing dir {}".format(dst))
shutil.rmtree(dst)
logging.info("copying sha scenario from {} to {}".format(src,dst))
logging.info("source: {}".format(src))
logging.info("destination: {}".format(dst))
shutil.copytree(src=src, dst=dst)
if args.spatial:
src = pathRegistry.GetPath("Source_External_Spatial_Dir")
dst = pathRegistry.GetPath("External_Spatial_Dir")
if args.cleanup and os.path.exists(dst):
logging.info("removing dir {}".format(dst))
shutil.rmtree(dst)
logging.info("copying external spatial data to local working directory")
logging.info("source: {}".format(src))
logging.info("destination: {}".format(dst))
shutil.copytree(src=src, dst=dst,
ignore = ignore_patterns("*.sr.lock")) #ignore annoying arc lock files
if args.cleanup:
for r in subRegionConfig.GetRegions():
subRegionDir = pathRegistry.GetPath("SubRegionDir", region_path=r["PathName"])
logging.info("dropping working dir {}".format(subRegionDir))
if os.path.exists(subRegionDir):
shutil.rmtree(subRegionDir)
except Exception as ex:
logging.exception("error")
sys.exit(1)
logging.info("all setup tasks finished")
if __name__ == "__main__":
main()