From 9fce116db65ad8737a90e1c5567c4f4347af86ec Mon Sep 17 00:00:00 2001 From: "David P. Chassin" Date: Fri, 5 Nov 2021 15:26:03 -0700 Subject: [PATCH] Develop add insights (#1028) * Add insights utility * Finalize gridlabd usage insights utility * Update insights.py * Delete test_IEEE_13_pole.glm * Update requirements.txt * Update requirements.txt * Update Dockerfile Co-authored-by: Alyona Teyber --- docker/Dockerfile | 2 +- .../powerflow/autotest/test_IEEE_13_pole.glm | 684 ------------------ python_extras/Makefile.mk | 1 + python_extras/insights.py | 125 ++++ python_extras/requirements.txt | 1 + 5 files changed, 128 insertions(+), 685 deletions(-) delete mode 100644 module/powerflow/autotest/test_IEEE_13_pole.glm create mode 100644 python_extras/insights.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 40f03124a..f5e2b0087 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM slacgismo/gridlabd_dockerhub_base:210520 +FROM slacgismo/gridlabd_dockerhub_base:211105 ENV container docker diff --git a/module/powerflow/autotest/test_IEEE_13_pole.glm b/module/powerflow/autotest/test_IEEE_13_pole.glm deleted file mode 100644 index ab28ff119..000000000 --- a/module/powerflow/autotest/test_IEEE_13_pole.glm +++ /dev/null @@ -1,684 +0,0 @@ -// $Id: IEEE13-pole -// SLAC National Accelerator Laboratory - -#define OUTPUT=${FILENAME $modelname}_output.csv - -#ifexist "../${OUTPUT}" -#define DIR=.. -#endif - -#set iteration_limit=100 -module tape -{ - csv_header_type NAME; -} -module reliability{ - report_event_log false; -} -module climate; -module powerflow { - solver_method NR; - line_capacitance true; - } - -clock { - timezone "EST+5EDT"; - starttime "2020-07-01 00:00:00"; - stoptime "2020-07-02 00:00:00"; -} - -object fault_check -{ - check_mode ONCHANGE; - output_filename "fault_check.txt"; - reliability_mode TRUE; -} -object climate { - name weather; - object player { - property wind_speed; - file "${DIR:-.}/test_IEEE_13_pole_wind.player"; - }; -} - -object multi_recorder -{ - file ${OUTPUT}; - property weather:wind_speed, p633:resisting_moment, p633:total_moment, p633:status, p633:tilt_angle, p633:pole_stress; - interval -1; -} -object pole_configuration -{ - name "WOOD-SOUTHERN-PINE-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (34.8205/3.14); - top_diameter (21/3.14); - fiber_strength 8000 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-DOUGLAS-FIR-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (34.8205/3.14); - top_diameter (21/3.14); - fiber_strength 8000 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-RED-PINE-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (36.7949/3.14); - top_diameter (21/3.14); - fiber_strength 6600 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-JACK-PINE-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (36.7949/3.14); - top_diameter (21/3.14); - fiber_strength 6600 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-LODGEPOLE-PINE-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (36.7949/3.14); - top_diameter (21/3.14); - fiber_strength 6600 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-PONDEROSA-PINE-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (38.2756/3.14); - top_diameter (21/3.14); - fiber_strength 6000 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-WESTERN-RED-CEDAR-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (38.2756/3.14); - top_diameter (21/3.14); - fiber_strength 6000 psi; - repair_time 1 h; -} - -object pole_configuration -{ - name "WOOD-NORTHERN-WHITE-CEDAR-45/4"; - pole_type WOOD; - pole_length 45 ft; - pole_depth 6.5 ft; - ground_diameter (43.7051/3.14); - top_diameter (21/3.14); - fiber_strength 4000 psi; - repair_time 1 h; -} -// Phase Conductor for 601: 556,500 26/7 ACSR -object overhead_line_conductor { - name olc6010; - geometric_mean_radius 0.031300; - diameter 0.927 in; - resistance 0.185900; -} - -// Phase Conductor for 602: 4/0 6/1 ACSR -object overhead_line_conductor { - name olc6020; - geometric_mean_radius 0.00814; - diameter 0.56 in; - resistance 0.592000; -} - -// Phase Conductor for 603, 604, 605: 1/0 ACSR -object overhead_line_conductor { - name olc6030; - geometric_mean_radius 0.004460; - diameter 0.4 in; - resistance 1.120000; -} - - -// Phase Conductor for 606: 250,000 AA,CN -object underground_line_conductor { - name ulc6060; - outer_diameter 1.290000; - conductor_gmr 0.017100; - conductor_diameter 0.567000; - conductor_resistance 0.410000; - neutral_gmr 0.0020800; - neutral_resistance 14.87200; - neutral_diameter 0.0640837; - neutral_strands 13.000000; - insulation_relative_permitivitty 2.3; - shield_gmr 0.000000; - shield_resistance 0.000000; -} - -// Phase Conductor for 607: 1/0 AA,TS N: 1/0 Cu -object underground_line_conductor { - name ulc6070; - outer_diameter 1.060000; - conductor_gmr 0.011100; - conductor_diameter 0.368000; - conductor_resistance 0.970000; - neutral_gmr 0.011100; - neutral_resistance 0.970000; // Unsure whether this is correct - neutral_diameter 0.0640837; - neutral_strands 6.000000; - insulation_relative_permitivitty 2.3; - shield_gmr 0.000000; - shield_resistance 0.000000; -} - -// Overhead line configurations -object line_spacing { - name ls500601; - distance_AB 2.5; - distance_AC 4.5; - distance_BC 7.0; - distance_BN 5.656854; - distance_AN 4.272002; - distance_CN 5.0; - distance_AE 28.0; - distance_BE 28.0; - distance_CE 28.0; - distance_NE 24.0; -} - -// Overhead line configurations -object line_spacing { - name ls500602; - distance_AC 2.5; - distance_AB 4.5; - distance_BC 7.0; - distance_CN 5.656854; - distance_AN 4.272002; - distance_BN 5.0; - distance_AE 28.0; - distance_BE 28.0; - distance_CE 28.0; - distance_NE 24.0; -} - -object line_spacing { - name ls505603; - distance_BC 7.0; - distance_CN 5.656854; - distance_BN 5.0; - distance_BE 28.0; - distance_CE 28.0; - distance_NE 24.0; -} - -object line_spacing { - name ls505604; - distance_AC 7.0; - distance_AN 5.656854; - distance_CN 5.0; - distance_AE 28.0; - distance_CE 28.0; - distance_NE 24.0; -} - -object line_spacing { - name ls510; - distance_CN 5.0; - distance_CE 28.0; - distance_NE 24.0; -} - -object line_configuration { - name lc601; - conductor_A olc6010; - conductor_B olc6010; - conductor_C olc6010; - conductor_N olc6020; - spacing ls500601; -} - -object line_configuration { - name lc602; - conductor_A olc6020; - conductor_B olc6020; - conductor_C olc6020; - conductor_N olc6020; - spacing ls500602; -} - -object line_configuration { - name lc603; - conductor_B olc6030; - conductor_C olc6030; - conductor_N olc6030; - spacing ls505603; -} - -object line_configuration { - name lc604; - conductor_A olc6030; - conductor_C olc6030; - conductor_N olc6030; - spacing ls505604; -} - -object line_configuration { - name lc605; - conductor_C olc6030; - conductor_N olc6030; - spacing ls510; -} - -//Underground line configuration -object line_spacing { - name ls515; - distance_AB 0.500000; - distance_BC 0.500000; - distance_AC 1.000000; -} - -object line_spacing { - name ls520; - distance_AN 0.083333; -} - -object line_configuration { - name lc606; - conductor_A ulc6060; - conductor_B ulc6060; - conductor_C ulc6060; - spacing ls515; -} - -object line_configuration { - name lc607; - conductor_A ulc6070; - conductor_N ulc6070; - spacing ls520; -} - -// Define line objects -object overhead_line { - phases "BCN"; - name line_632-645; - from n632; - to l645; - length 500; - configuration lc603; -} - -object overhead_line { - phases "BCN"; - name line_645-646; - from l645; - to l646; - length 300; - configuration lc603; -} - -object overhead_line { //630632 { - phases "ABCN"; - name line_630-632; - from n630; - to n632; - length 2000; - configuration lc601; -} - -//Split line for distributed load -object overhead_line { //6326321 { - phases "ABCN"; - name line_632-6321; - from n632; - to l6321; - length 500; - configuration lc601; -} - -object overhead_line { //6321671 { - phases "ABCN"; - name line_6321-671; - from l6321; - to l671; - length 1500; - configuration lc601; -} -//End split line - -object overhead_line { //671680 { - phases "ABCN"; - name line_671-680; - from l671; - to n680; - length 1000; - configuration lc601; -} - -object overhead_line { //671684 { - phases "ACN"; - name line_671-684; - from l671; - to n684; - length 300; - configuration lc604; -} - - object overhead_line { //684611 { - phases "CN"; - name line_684-611; - from n684; - to l611; - length 300; - configuration lc605; -} - -object underground_line { //684652 { - phases "AN"; - name line_684-652; - from n684; - to l652; - length 800; - configuration lc607; -} - -object underground_line { //692675 { - phases "ABC"; - name line_692-675; - from l692; - to l675; - length 500; - configuration lc606; -} - -object overhead_line { //632633 { - phases "ABCN"; - name line_632-633; - from n632; - to n633; - length 500; - configuration lc602; -} - -// Create node objects -object node { //633 { - name n633; - phases "ABCN"; - object pole { - name p633; - configuration WOOD-SOUTHERN-PINE-45/4; - weather weather; - }; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - nominal_voltage 2401.7771; -} - -object node { //630 { - name n630; - phases "ABCN"; - voltage_A 2401.7771+0j; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - nominal_voltage 2401.7771; -} - -object node { //632 { - name n632; - phases "ABCN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - nominal_voltage 2401.7771; -} - -object node { //650 { - name n650; - phases "ABCN"; - bustype SWING; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - nominal_voltage 2401.7771; -} - -object node { //680 { - name n680; - phases "ABCN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - nominal_voltage 2401.7771; -} - - -object node { //684 { - name n684; - phases "ACN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - nominal_voltage 2401.7771; -} - - - -// Create load objects - -object load { //634 { - name l634; - phases "ABCN"; - voltage_A 480.000+0j; - voltage_B -240.000-415.6922j; - voltage_C -240.000+415.6922j; - constant_power_A 160000+110000j; - constant_power_B 120000+90000j; - constant_power_C 120000+90000j; - nominal_voltage 480.000; -} - -object load { //645 { - name l645; - phases "BCN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_power_B 170000+125000j; - nominal_voltage 2401.7771; -} - -object load { //646 { - name l646; - phases "BCD"; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_impedance_B 56.5993+32.4831j; - nominal_voltage 2401.7771; -} - - -object load { //652 { - name l652; - phases "AN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_impedance_A 31.0501+20.8618j; - nominal_voltage 2401.7771; -} - -object load { //671 { - name l671; - phases "ABCD"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_power_A 385000+220000j; - constant_power_B 385000+220000j; - constant_power_C 385000+220000j; - nominal_voltage 2401.7771; -} - -object load { //675 { - name l675; - phases "ABC"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_power_A 485000+190000j; - constant_power_B 68000+60000j; - constant_power_C 290000+212000j; - constant_impedance_A 0.00-28.8427j; //Shunt Capacitors - constant_impedance_B 0.00-28.8427j; - constant_impedance_C 0.00-28.8427j; - nominal_voltage 2401.7771; -} - -object load { //692 { - name l692; - phases "ABCD"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_current_A 0+0j; - constant_current_B 0+0j; - constant_current_C -17.2414+51.8677j; - nominal_voltage 2401.7771; -} - -object load { //611 { - name l611; - phases "CN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_current_C -6.5443+77.9524j; - constant_impedance_C 0.00-57.6854j; //Shunt Capacitor - nominal_voltage 2401.7771; -} - -// distributed load between node 632 and 671 -// 2/3 of load 1/4 of length down line: Kersting p.56 -object load { //6711 { - name l6711; - parent l671; - phases "ABC"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_power_A 5666.6667+3333.3333j; - constant_power_B 22000+12666.6667j; - constant_power_C 39000+22666.6667j; - nominal_voltage 2401.7771; -} - -object load { //6321 { - name l6321; - phases "ABCN"; - voltage_A 2401.7771; - voltage_B -1200.8886-2080.000j; - voltage_C -1200.8886+2080.000j; - constant_power_A 11333.333+6666.6667j; - constant_power_B 44000+25333.3333j; - constant_power_C 78000+45333.3333j; - nominal_voltage 2401.7771; -} - - - -// Switch -object switch { - phases "ABCN"; - name switch_671-692; - from l671; - to l692; - status CLOSED; -} - -// Transformer -object transformer_configuration { - name tc400; - connect_type WYE_WYE; - install_type PADMOUNT; - power_rating 500; - primary_voltage 4160; - secondary_voltage 480; - resistance 0.011; - reactance 0.02; -} - -object transformer { - phases "ABCN"; - name transformer_633-634; - from n633; - to l634; - configuration tc400; -} - - -// Regulator -object regulator_configuration { - name regconfig6506321; - connect_type 1; - band_center 122.000; - band_width 2.0; - time_delay 30.0; - raise_taps 16; - lower_taps 16; - current_transducer_ratio 700; - power_transducer_ratio 20; - compensator_r_setting_A 3.0; - compensator_r_setting_B 3.0; - compensator_r_setting_C 3.0; - compensator_x_setting_A 9.0; - compensator_x_setting_B 9.0; - compensator_x_setting_C 9.0; - CT_phase "ABC"; - PT_phase "ABC"; - regulation 0.10; - Control MANUAL; - Type A; - tap_pos_A 10; - tap_pos_B 8; - tap_pos_C 11; -} - -object regulator { - name fregn650n630; - phases "ABC"; - from n650; - to n630; - configuration regconfig6506321; -} - -#ifdef DIR -#on_exit 0 diff ${OUTPUT} ../${OUTPUT} -#endif diff --git a/python_extras/Makefile.mk b/python_extras/Makefile.mk index b6873a89c..282af331d 100644 --- a/python_extras/Makefile.mk +++ b/python_extras/Makefile.mk @@ -6,6 +6,7 @@ dist_pkgdata_DATA += python_extras/example/tstat_commit.py dist_pkgdata_DATA += python_extras/example/tstat_init.py dist_pkgdata_DATA += python_extras/gridlabd-editor.png dist_pkgdata_DATA += python_extras/gridlabd-editor.py +dist_pkgdata_DATA += python_extras/insights.py dist_pkgdata_DATA += python_extras/metar2glm.py dist_pkgdata_DATA += python_extras/nsrdb_weather.py dist_pkgdata_DATA += python_extras/ucar_weather.py diff --git a/python_extras/insights.py b/python_extras/insights.py new file mode 100644 index 000000000..b03cdac8b --- /dev/null +++ b/python_extras/insights.py @@ -0,0 +1,125 @@ +"""Gather data on gridlabd usage + +SYNTAX + + $ gridlabd insights CATEGORY [OPTIONS ...] + +DESCRIPTION + +The `insights` utility retrieves gridlabd usage data from the AWS servers. + +The following options are available. + + -d|--debug Enable debugging (raises exception instead of printing errors) + + -h|--help|help Display this help + + -m|--month=MONTH Specify the month number (1-12) + + -o|--output=CSVFILE Specify the output CSV filename + + -p|--prefix=PATH Specify the log prefix pathname + + -y|--year=YEAR Specify the year number (4 digit year) + +CAVEAT + +Currently, only the category `requests` is supported. This returns the number of users +who run gridlabd online each day of the month. + +SEE ALSO + +* [[/Subcommand/Aws]] +""" +import sys +import boto3 +import pandas +import datetime +import matplotlib.pyplot as plot + +today = datetime.datetime.now().date() +bucket = "logs.gridlabd.us" +path = "version" +debug = False +output = "/dev/stdout" + +def error(msg,code=None): + print(f"ERROR [insights]: {msg}",file=sys.stderr) + if debug: + raise Exception(msg) + if code != None: + exit(code) + +def syntax(code=None): + if code == None: + print(__doc__) + exit(1) + else: + print("Syntax: gridlabd insights CATEGORY [OPTIONS ...]") + if type(code) == int: + exit(code) + +def get_requests(year=None,month=None): + + if not year: + year = today.year + if not month: + month = today.month + prefix = f"{path}/{year}-{month}" + + s3 = boto3.client('s3') + objects = s3.list_objects(Bucket=bucket,Prefix=prefix) + dt = [] + ip = [] + contents = objects.get("Contents") + if contents: + for obj in contents: + data = s3.get_object(Bucket=bucket, Key=obj.get("Key")) + records = data["Body"].read().decode("utf-8").split("\n") + for record in records: + field = record.split() + if len(field) > 10 and field[10] == "/version.gridlabd.us": + dt.append(datetime.datetime.strptime(field[2].strip(" []"),"%d/%b/%Y:%H:%M:%S").date()) + ip.append(field[4]) + + df = pandas.DataFrame(ip,index=dt,columns=["requests"]) + df.index.name = "date" + return df.groupby("date").count() + +if __name__ == "__main__": + year = None + month = None + call = None + + if len(sys.argv) == 1: + syntax(1) + for arg in sys.argv[1:]: + args = arg.split("=") + if type(args) is list and len(args) > 1: + tag = args[0] + value = args[1] + else: + tag = arg + value = None + if tag in ["-y","--year"]: + year = value + elif tag in ["-m","--month"]: + month = value + elif tag in ["-d","--debug"]: + debug = True + elif tag in ["-o","--output"]: + output = value + elif tag in ["-p","--prefix"]: + path = value + elif tag in ["-h","--help","help"]: + syntax() + elif "get_"+tag in globals().keys(): + call = globals()["get_"+tag] + else: + error(f"{arg} is invalid",1) + + if not call: + error(f"insight category not specified",2) + else: + result = call(year,month) + result.to_csv(output) diff --git a/python_extras/requirements.txt b/python_extras/requirements.txt index 6fe019b58..c11cc17ed 100644 --- a/python_extras/requirements.txt +++ b/python_extras/requirements.txt @@ -1 +1,2 @@ metar>=1.8.0 +boto3>=1.19.11