diff --git a/.gitignore b/.gitignore index 3a0620b..5b8b63f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ venv/ -WorkFiles/ \ No newline at end of file +WorkFiles/ +**/__pycache__ \ No newline at end of file diff --git a/Main.py b/Main.py index bbd0492..8355005 100644 --- a/Main.py +++ b/Main.py @@ -1,35 +1,14 @@ -__author__ = "Andre Barbe" -__project__ = "Auto-GTAP" -__created__ = "2018-3-9" - -# Import methods -# Methods from other sources import os import subprocess -import shutil import distutils.dir_util -# Methods defined in this program -from CleanWorkFiles import CleanWorkFiles -from CopyInputFiles import CopyInputFiles -from CreateSTI import CreateSTI -from SimulationCMF import SimulationCMF -from CreateMAP import CreateMAP -from ImportCSV_SL4 import ImportCSV_SL4 -from ModifyDatabase import ModifyDatabase -from ExportDictionary import ExportDictionary -from CreateOutput import CreateOutput -from CreateConfig import CreateConfig -from ModifyHAR import ModifyHAR -from MoveDatabaseFiles import MoveDatabaseFiles -from SplitCommodities import SplitCommodities -from AggregateModelData import AggregateModelData -from GTAPAdjustCMF import GTAPAdjustCMF +# noinspection PyPep8Naming +import AutoGTAP as ag # Call Methods # Load config files that will control program -config = CreateConfig("config-gtap.yaml") +config = ag.CreateConfig("config-gtap.yaml") # Delete working files directory -CleanWorkFiles() +ag.CleanWorkFiles() # For each simulation, perform the different subparts (data aggregation, splitting, # experiment simulation, etc) that make up that simulation for simulation_name in config.simulation_list: @@ -42,7 +21,7 @@ part_work_folder = config.yaml_file["simulations"][simulation_name]["subparts"][part_num]["work_folder"] # Copy input files for this part to the appropriate work directory - CopyInputFiles(simulation_name, part_input_folder, part_work_folder).copy() + ag.CopyInputFiles(simulation_name, part_input_folder, part_work_folder).copy() for additional_input_folder in config.part_additional_input_folders(simulation_name, part_num): # need to use this copy method to overwrite files/folders (or copy files to folders taht already exist) @@ -55,7 +34,7 @@ prev_part_work_folder = config.yaml_file["simulations"][simulation_name]["subparts"][prev_part_num][ "work_folder"] prev_part_type = config.yaml_file["simulations"][simulation_name]["subparts"][prev_part_num]["type"] - MoveDatabaseFiles(simulation_name, prev_part_type, part_type, prev_part_work_folder, part_work_folder) + ag.MoveDatabaseFiles(simulation_name, prev_part_type, part_type, prev_part_work_folder, part_work_folder) # Run the actual work for this part, depending on which type of part it is if part_type == "GTPAg2": @@ -63,23 +42,24 @@ "agg_scheme_file"] data_subfolder = config.yaml_file["simulations"][simulation_name]["subparts"][part_num][ "data_subfolder"] - AggregateModelData(simulation_name, part_work_folder, agg_scheme_file, data_subfolder) + ag.AggregateModelData(simulation_name, part_work_folder, agg_scheme_file, data_subfolder) if part_type == "MSplitCom-Exe": - SplitCommodities(simulation_name) + ag.SplitCommodities(simulation_name) if part_type == "modify_har": # Modify_HAR directly modifies a HAR file # This module should be rarely used - ModifyHAR("WorkFiles\\" + simulation_name, "olddefault", "default", - config.yaml_file["parameter_modifications"][ - config.sim_property(simulation_name, "parameter_modifications")]) + ag.ModifyHAR("WorkFiles\\" + simulation_name, "olddefault", "default", + config.yaml_file["parameter_modifications"][ + config.sim_property(simulation_name, "parameter_modifications")]) if part_type == "GTPVEW-V6" or part_type == "Shocks-V6": model_file_name = config.yaml_file["simulations"][simulation_name]["subparts"][part_num]["model_file_name"] cmf_file_name = config.yaml_file["simulations"][simulation_name]["subparts"][part_num]["cmf_file_name"] - # Change working directory to WorkFiles so all output (and logs) will go there when gemsim or sltoht is called + # Change working directory to WorkFiles so all output (and logs) + # will go there when gemsim or sltoht is called os.chdir("WorkFiles\\{0}\\{1}".format(simulation_name, part_work_folder)) # Create GSS and GST files for shocks and model gemsim subprocess.call("tablo -sti {0}.sti".format(model_file_name)) @@ -91,9 +71,10 @@ if part_type == "GTAP-Adjust": part_shock = config.yaml_file["simulations"][simulation_name]["subparts"][part_num]["shock"] - GTAPAdjustCMF(simulation_name, part_solution_method, part_work_folder, part_shock) + ag.GTAPAdjustCMF(simulation_name, part_solution_method, part_work_folder, part_shock) - # Change working directory to WorkFiles so all output (and logs) will go there when gemsim or sltoht is called + # Change working directory to WorkFiles so all output (and logs) + # will go there when gemsim or sltoht is called os.chdir("WorkFiles\\{0}\\{1}".format(simulation_name, part_work_folder)) subprocess.call("adjust.bat") os.chdir("..") @@ -108,16 +89,17 @@ model_file_name = config.yaml_file["simulations"][simulation_name]["subparts"][part_num]["model_file_name"] map = config.yaml_file["simulations"][simulation_name]["subparts"][part_num]["map"] - SimulationCMF(simulation_name, part_solution_method, part_work_folder, part_shock, part_type) - # Change working directory to WorkFiles so all output (and logs) will go there when gemsim or sltoht is called + ag.SimulationCMF(simulation_name, part_solution_method, part_work_folder, part_shock, part_type) + # Change working directory to WorkFiles so all output (and logs) + # will go there when gemsim or sltoht is called os.chdir("WorkFiles\\{0}\\{1}".format(simulation_name, part_work_folder)) # Create GSS and GST files for shocks and model gemsim subprocess.call("tablo -sti {0}.sti".format(model_file_name)) subprocess.call("gemsim -cmf {0}.cmf".format(model_file_name)) # Use SLtoHT export the results of the simulation from sl4 to a CSV file - CreateMAP("sim", simulation_name, map) # Map file determines which variables to export - CreateSTI(model_file_name, simulation_name, "sltoht") # STI file controls running of sltoht + ag.CreateMAP("sim", simulation_name, map) # Map file determines which variables to export + ag.CreateSTI(model_file_name, simulation_name, "sltoht") # STI file controls running of sltoht subprocess.call("sltoht -sti {0}_sltoht.sti".format(model_file_name)) # Change directory back to WorkFiles @@ -128,11 +110,11 @@ # Import simulation results into a single database os.chdir("WorkFiles") # load the various CSV files created by the experiment simulation into a database -databaseSL4 = ImportCSV_SL4(config.csv_paths()).create() +databaseSL4 = ag.ImportCSV_SL4(config.csv_paths()).create() # Modify the database to make it more readable -databaseMod = ModifyDatabase(databaseSL4).create() +databaseMod = ag.ModifyDatabase(databaseSL4).create() # Export the database to a results csv file -ExportDictionary("Results.csv", databaseMod) +ag.ExportDictionary("Results.csv", databaseMod) # Copy results.csv to the OutputFiles directory, and rename it with a timestamp os.chdir("..") -CreateOutput() +ag.CreateOutput() diff --git a/OutputFiles/2018-06-27 14-26 Results.csv b/OutputFiles/2018-06-27 14-26 Results.csv new file mode 100644 index 0000000..ff1fd49 --- /dev/null +++ b/OutputFiles/2018-06-27 14-26 Results.csv @@ -0,0 +1,5 @@ +simulation name,commodities shocked,Commodity Description,variable,variable property,variable description,value +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",Percent Change,No Variable Description Available,10.000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",PreLevel,No Variable Description Available,1.0000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",PostLevel,No Variable Description Available,1.1000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",Absolute Change,No Variable Description Available,0.10000002 diff --git a/OutputFiles/2018-06-28 10-09 Results.csv b/OutputFiles/2018-06-28 10-09 Results.csv new file mode 100644 index 0000000..ff1fd49 --- /dev/null +++ b/OutputFiles/2018-06-28 10-09 Results.csv @@ -0,0 +1,5 @@ +simulation name,commodities shocked,Commodity Description,variable,variable property,variable description,value +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",Percent Change,No Variable Description Available,10.000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",PreLevel,No Variable Description Available,1.0000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",PostLevel,No Variable Description Available,1.1000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",Absolute Change,No Variable Description Available,0.10000002 diff --git a/OutputFiles/2018-06-28 10-12 Results.csv b/OutputFiles/2018-06-28 10-12 Results.csv new file mode 100644 index 0000000..ff1fd49 --- /dev/null +++ b/OutputFiles/2018-06-28 10-12 Results.csv @@ -0,0 +1,5 @@ +simulation name,commodities shocked,Commodity Description,variable,variable property,variable description,value +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",Percent Change,No Variable Description Available,10.000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",PreLevel,No Variable Description Available,1.0000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",PostLevel,No Variable Description Available,1.1000000 +GTAP-V6//gtap2//gtap.csv,GTAP-V6//gtap2//gtap.csv,Unknown Sector,"pxwreg(""EU_28"")",Absolute Change,No Variable Description Available,0.10000002 diff --git a/AggregateModelData.py b/src/AutoGTAP/AggregateModelData/AggregateModelData.py similarity index 100% rename from AggregateModelData.py rename to src/AutoGTAP/AggregateModelData/AggregateModelData.py diff --git a/src/AutoGTAP/AggregateModelData/README.md b/src/AutoGTAP/AggregateModelData/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/AggregateModelData/__init__.py b/src/AutoGTAP/AggregateModelData/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CleanWorkFiles.py b/src/AutoGTAP/CleanWorkFiles/CleanWorkFiles.py similarity index 100% rename from CleanWorkFiles.py rename to src/AutoGTAP/CleanWorkFiles/CleanWorkFiles.py diff --git a/src/AutoGTAP/CleanWorkFiles/README.md b/src/AutoGTAP/CleanWorkFiles/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/CleanWorkFiles/__init__.py b/src/AutoGTAP/CleanWorkFiles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CopyInputFiles.py b/src/AutoGTAP/CopyInputFiles/CopyInputFiles.py similarity index 94% rename from CopyInputFiles.py rename to src/AutoGTAP/CopyInputFiles/CopyInputFiles.py index f8764d4..002e0c6 100644 --- a/CopyInputFiles.py +++ b/src/AutoGTAP/CopyInputFiles/CopyInputFiles.py @@ -3,7 +3,6 @@ __created__ = "2018-3-12" import shutil -from CreateConfig import CreateConfig class CopyInputFiles(object): diff --git a/src/AutoGTAP/CopyInputFiles/README.md b/src/AutoGTAP/CopyInputFiles/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/CopyInputFiles/__init__.py b/src/AutoGTAP/CopyInputFiles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CreateConfig.py b/src/AutoGTAP/CreateConfig/CreateConfig.py similarity index 100% rename from CreateConfig.py rename to src/AutoGTAP/CreateConfig/CreateConfig.py diff --git a/src/AutoGTAP/CreateConfig/README.md b/src/AutoGTAP/CreateConfig/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/CreateConfig/__init__.py b/src/AutoGTAP/CreateConfig/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CreateMAP.py b/src/AutoGTAP/CreateMAP/CreateMAP.py similarity index 100% rename from CreateMAP.py rename to src/AutoGTAP/CreateMAP/CreateMAP.py diff --git a/src/AutoGTAP/CreateMAP/README.md b/src/AutoGTAP/CreateMAP/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/CreateMAP/__init__.py b/src/AutoGTAP/CreateMAP/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CreateOutput.py b/src/AutoGTAP/CreateOutput/CreateOutput.py similarity index 100% rename from CreateOutput.py rename to src/AutoGTAP/CreateOutput/CreateOutput.py diff --git a/src/AutoGTAP/CreateOutput/__init__.py b/src/AutoGTAP/CreateOutput/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/CreateSTI.py b/src/AutoGTAP/CreateSTI/CreateSTI.py similarity index 100% rename from CreateSTI.py rename to src/AutoGTAP/CreateSTI/CreateSTI.py diff --git a/src/AutoGTAP/CreateSTI/README.md b/src/AutoGTAP/CreateSTI/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/CreateSTI/__init__.py b/src/AutoGTAP/CreateSTI/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ExportDictionary.py b/src/AutoGTAP/ExportDictionary/ExportDictionary.py similarity index 100% rename from ExportDictionary.py rename to src/AutoGTAP/ExportDictionary/ExportDictionary.py diff --git a/src/AutoGTAP/ExportDictionary/README.md b/src/AutoGTAP/ExportDictionary/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/ExportDictionary/__init__.py b/src/AutoGTAP/ExportDictionary/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/GTAPAdjustCMF.py b/src/AutoGTAP/GTAPAdjustCMF/GTAPAdjustCMF.py similarity index 99% rename from GTAPAdjustCMF.py rename to src/AutoGTAP/GTAPAdjustCMF/GTAPAdjustCMF.py index 21efc75..f0b9e23 100644 --- a/GTAPAdjustCMF.py +++ b/src/AutoGTAP/GTAPAdjustCMF/GTAPAdjustCMF.py @@ -2,7 +2,7 @@ __project__ = "Auto-GTAP" __created__ = "2018-5-4" -from Shocks import Shocks +from AutoGTAP.Shocks import Shocks class GTAPAdjustCMF(object): diff --git a/src/AutoGTAP/GTAPAdjustCMF/README.md b/src/AutoGTAP/GTAPAdjustCMF/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/GTAPAdjustCMF/__init__.py b/src/AutoGTAP/GTAPAdjustCMF/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ImportCSV_SL4.py b/src/AutoGTAP/ImportCSV_SL4/ImportCSV_SL4.py similarity index 100% rename from ImportCSV_SL4.py rename to src/AutoGTAP/ImportCSV_SL4/ImportCSV_SL4.py diff --git a/src/AutoGTAP/ImportCSV_SL4/__init__.py b/src/AutoGTAP/ImportCSV_SL4/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ModifyDatabase.py b/src/AutoGTAP/ModifyDatabase/ModifyDatabase.py similarity index 100% rename from ModifyDatabase.py rename to src/AutoGTAP/ModifyDatabase/ModifyDatabase.py diff --git a/src/AutoGTAP/ModifyDatabase/README.md b/src/AutoGTAP/ModifyDatabase/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/ModifyDatabase/__init__.py b/src/AutoGTAP/ModifyDatabase/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ModifyHAR.py b/src/AutoGTAP/ModifyHAR/ModifyHAR.py similarity index 100% rename from ModifyHAR.py rename to src/AutoGTAP/ModifyHAR/ModifyHAR.py diff --git a/src/AutoGTAP/ModifyHAR/README.md b/src/AutoGTAP/ModifyHAR/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/ModifyHAR/__init__.py b/src/AutoGTAP/ModifyHAR/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MoveDatabaseFiles.py b/src/AutoGTAP/MoveDatabaseFiles/MoveDatabaseFiles.py similarity index 100% rename from MoveDatabaseFiles.py rename to src/AutoGTAP/MoveDatabaseFiles/MoveDatabaseFiles.py diff --git a/src/AutoGTAP/MoveDatabaseFiles/README.md b/src/AutoGTAP/MoveDatabaseFiles/README.md new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/MoveDatabaseFiles/__init__.py b/src/AutoGTAP/MoveDatabaseFiles/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Shocks.py b/src/AutoGTAP/Shocks.py similarity index 100% rename from Shocks.py rename to src/AutoGTAP/Shocks.py diff --git a/src/AutoGTAP/SimulationCMF/README.md b/src/AutoGTAP/SimulationCMF/README.md new file mode 100644 index 0000000..e69de29 diff --git a/SimulationCMF.py b/src/AutoGTAP/SimulationCMF/SimulationCMF.py similarity index 99% rename from SimulationCMF.py rename to src/AutoGTAP/SimulationCMF/SimulationCMF.py index 0f54bb1..cf4fa29 100644 --- a/SimulationCMF.py +++ b/src/AutoGTAP/SimulationCMF/SimulationCMF.py @@ -2,7 +2,7 @@ __project__ = "Auto-GTAP" __created__ = "2018-3-13" -from Shocks import Shocks +from AutoGTAP.Shocks import Shocks class SimulationCMF(object): diff --git a/src/AutoGTAP/SimulationCMF/__init__.py b/src/AutoGTAP/SimulationCMF/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/SimulationShocksShale.py b/src/AutoGTAP/SimulationShocksShale.py similarity index 100% rename from SimulationShocksShale.py rename to src/AutoGTAP/SimulationShocksShale.py diff --git a/src/AutoGTAP/SplitCommodities/README.md b/src/AutoGTAP/SplitCommodities/README.md new file mode 100644 index 0000000..e69de29 diff --git a/SplitCommodities.py b/src/AutoGTAP/SplitCommodities/SplitCommodities.py similarity index 100% rename from SplitCommodities.py rename to src/AutoGTAP/SplitCommodities/SplitCommodities.py diff --git a/src/AutoGTAP/SplitCommodities/__init__.py b/src/AutoGTAP/SplitCommodities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/AutoGTAP/__init__.py b/src/AutoGTAP/__init__.py new file mode 100644 index 0000000..4c02c2c --- /dev/null +++ b/src/AutoGTAP/__init__.py @@ -0,0 +1,15 @@ +from AutoGTAP.AggregateModelData.AggregateModelData import AggregateModelData +from AutoGTAP.CleanWorkFiles.CleanWorkFiles import CleanWorkFiles +from AutoGTAP.CopyInputFiles.CopyInputFiles import CopyInputFiles +from AutoGTAP.CreateConfig.CreateConfig import CreateConfig +from AutoGTAP.ExportDictionary.ExportDictionary import ExportDictionary +from AutoGTAP.CreateMAP.CreateMAP import CreateMAP +from AutoGTAP.CreateOutput.CreateOutput import CreateOutput +from AutoGTAP.CreateSTI.CreateSTI import CreateSTI +from AutoGTAP.GTAPAdjustCMF.GTAPAdjustCMF import GTAPAdjustCMF +from AutoGTAP.ImportCSV_SL4.ImportCSV_SL4 import ImportCSV_SL4 +from AutoGTAP.ModifyDatabase.ModifyDatabase import ModifyDatabase +from AutoGTAP.ModifyHAR.ModifyHAR import ModifyHAR +from AutoGTAP.MoveDatabaseFiles.MoveDatabaseFiles import MoveDatabaseFiles +from AutoGTAP.SimulationCMF.SimulationCMF import SimulationCMF +from AutoGTAP.SplitCommodities.SplitCommodities import SplitCommodities \ No newline at end of file