Testing 20240616

.gitignore

@@ -41,8 +41,8 @@ auto_rx/dfm09mod
 auto_rx/dft_detect
 auto_rx/fsk_demod
 auto_rx/imet1rs_dft
+auto_rx/iq_dec
 auto_rx/lms6Xmod
-auto_rx/lms6mod
 auto_rx/m10mod
 auto_rx/m20mod
 auto_rx/mk2a1680mod
@@ -52,14 +52,14 @@ auto_rx/imet54mod
 auto_rx/mXXmod
 auto_rx/mp3h1mod
 auto_rx/mts01mod
+auto_rx/weathex301d
 
 m10
 meisei100mod
 mk2a_lms1680
 demod/dfm09ecc
 demod/mod/dfm09mod
 demod/mod/lms6Xmod
-demod/mod/lms6mod
 demod/mod/meisei100mod
 demod/mod/rs41mod
 demod/mod/rs92mod
@@ -81,3 +81,4 @@ rs_module/rs41mod
 rs_module/rs92mod
 scan/reset_usb
 scan/rs_detect
+weathex/weathex301d

Makefile

@@ -13,6 +13,7 @@ SUBDIRS := \
 	mk2a \
 	scan \
 	utils \
+	weathex \
 
 all: $(SUBDIRS)
 

auto_rx/auto_rx.py

@@ -8,6 +8,16 @@
 #   Refer github page for instructions on setup and usage.
 #   https://github.com/projecthorus/radiosonde_auto_rx/
 #
+
+# exit status codes:
+#
+# 0 - normal termination (ctrl-c)
+# 1 - critical error, needs human attention to fix
+# 2 - exit because continous running timeout reached
+# 3 - exception occurred, can rerun after resetting SDR
+# 4 - some of the threads failed to join, SDR reset and restart required
+#     this is mostly caused by hung external utilities
+
 import argparse
 import datetime
 import logging
@@ -28,7 +38,6 @@
 from autorx.decode import SondeDecoder, VALID_SONDE_TYPES, DRIFTY_SONDE_TYPES
 from autorx.logger import TelemetryLogger
 from autorx.email_notification import EmailNotification
-from autorx.habitat import HabitatUploader
 from autorx.aprs import APRSUploader
 from autorx.ozimux import OziUploader
 from autorx.sondehub import SondehubUploader
@@ -44,6 +53,7 @@
     start_flask,
     stop_flask,
     flask_emit_event,
+    flask_running,
     WebHandler,
     WebExporter,
 )
@@ -170,6 +180,7 @@ def start_scanner():
             ppm=autorx.sdr_list[_device_idx]["ppm"],
             bias=autorx.sdr_list[_device_idx]["bias"],
             save_detection_audio=config["save_detection_audio"],
+            wideband_sondes=config["wideband_sondes"],
             temporary_block_list=temporary_block_list,
             temporary_block_time=config["temporary_block_time"],
         )
@@ -232,7 +243,7 @@ def start_decoder(freq, sonde_type, continuous=False):
             _exp_sonde_type = sonde_type
 
         if continuous:
-            _timeout = 0
+            _timeout = 3600*6 # 6 hours before a 'continuous' decoder gets restarted automatically.
         else:
             _timeout = config["rx_timeout"]
 
@@ -261,7 +272,8 @@ def start_decoder(freq, sonde_type, continuous=False):
             rs92_ephemeris=rs92_ephemeris,
             rs41_drift_tweak=config["rs41_drift_tweak"],
             experimental_decoder=config["experimental_decoders"][_exp_sonde_type],
-            save_raw_hex=config["save_raw_hex"]
+            save_raw_hex=config["save_raw_hex"],
+            wideband_sondes=config["wideband_sondes"]
         )
         autorx.sdr_list[_device_idx]["task"] = autorx.task_list[freq]["task"]
 
@@ -322,7 +334,7 @@ def handle_scan_results():
                     if (type(_key) == int) or (type(_key) == float):
                         # Extract the currently decoded sonde type from the currently running decoder.
                         _decoding_sonde_type = autorx.task_list[_key]["task"].sonde_type
-                        
+
                         # Remove any inverted decoder information for the comparison.
                         if _decoding_sonde_type.startswith("-"):
                             _decoding_sonde_type = _decoding_sonde_type[1:]
@@ -432,7 +444,8 @@ def clean_task_list():
 
             else:
                 # Shutdown the SDR, if required for the particular SDR type.
-                shutdown_sdr(config["sdr_type"], _task_sdr)
+                if _key != 'SCAN':
+                    shutdown_sdr(config["sdr_type"], _task_sdr, sdr_hostname=config["sdr_hostname"], frequency=_key)
                 # Release its associated SDR.
                 autorx.sdr_list[_task_sdr]["in_use"] = False
                 autorx.sdr_list[_task_sdr]["task"] = None
@@ -492,6 +505,12 @@ def stop_all():
     for _task in autorx.task_list.keys():
         try:
             autorx.task_list[_task]["task"].stop()
+
+            # Release the SDR channel if necessary
+            _task_sdr = autorx.task_list[_task]["device_idx"]
+            if _task != 'SCAN':
+                shutdown_sdr(config["sdr_type"], _task_sdr, sdr_hostname=config["sdr_hostname"], frequency=_task)
+
         except Exception as e:
             logging.error("Error stopping task - %s" % str(e))
 
@@ -640,6 +659,7 @@ def telemetry_filter(telemetry):
         or ("LMS" in telemetry["type"])
         or ("IMET" in telemetry["type"])
         or ("MTS01" in telemetry["type"])
+        or ("WXR" in telemetry["type"])
     ):
         return "OK"
     else:
@@ -745,9 +765,6 @@ def main():
     )
     args = parser.parse_args()
 
-    # Copy out timeout value, and convert to seconds,
-    _timeout = args.timeout * 60
-
     # Copy out RS92 ephemeris value, if provided.
     if args.ephemeris != "None":
         rs92_ephemeris = args.ephemeris
@@ -770,7 +787,7 @@ def main():
     autorx.logging_path = logging_path
 
     # Configure logging
-    _log_suffix = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S_system.log")
+    _log_suffix = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S_system.log")
     _log_path = os.path.join(logging_path, _log_suffix)
 
     system_log_enabled = False
@@ -806,6 +823,16 @@ def main():
     logging.getLogger("engineio").setLevel(logging.ERROR)
     logging.getLogger("geventwebsocket").setLevel(logging.ERROR)
 
+    # Copy out timeout value, and convert to seconds.
+    if args.timeout > 0:
+        logging.info(f"Will shut down automatically after {args.timeout} minutes.")
+    _timeout = args.timeout * 60
+
+    # Check all the RS utilities exist.
+    logging.debug("Checking if required binaries exist")
+    if not check_rs_utils(config):
+        sys.exit(1)
+
     # Attempt to read in config file
     logging.info("Reading configuration file...")
     _temp_cfg = read_auto_rx_config(args.config)
@@ -816,6 +843,7 @@ def main():
         config = _temp_cfg
         autorx.sdr_list = config["sdr_settings"]
 
+
     # Apply any logging changes based on configuration file settings.
     if config["save_system_log"]:
         # Enable system logging.
@@ -844,9 +872,6 @@ def main():
     web_handler = WebHandler()
     logging.getLogger().addHandler(web_handler)
 
-    # Check all the RS utilities exist.
-    if not check_rs_utils():
-        sys.exit(1)
 
     # If a sonde type has been provided, insert an entry into the scan results,
     # and immediately start a decoder. This also sets the decoder time to 0, which
@@ -874,7 +899,10 @@ def main():
     # Start our exporter options
     # Telemetry Logger
     if config["per_sonde_log"]:
-        _logger = TelemetryLogger(log_directory=logging_path)
+        _logger = TelemetryLogger(
+            log_directory=logging_path,
+            save_cal_data=config["save_cal_data"]
+        )
         exporter_objects.append(_logger)
         exporter_functions.append(_logger.add)
 
@@ -897,6 +925,7 @@ def main():
             ),
             launch_notifications=config["email_launch_notifications"],
             landing_notifications=config["email_landing_notifications"],
+            encrypted_sonde_notifications=config["email_encrypted_sonde_notifications"],
             landing_range_threshold=config["email_landing_range_threshold"],
             landing_altitude_threshold=config["email_landing_altitude_threshold"],
         )
@@ -905,30 +934,6 @@ def main():
         exporter_objects.append(_email_notification)
         exporter_functions.append(_email_notification.add)
 
-    # Habitat Uploader - DEPRECATED - Sondehub DB now in use (>1.5.0)
-    # if config["habitat_enabled"]:
-
-    #     if config["habitat_upload_listener_position"] is False:
-    #         _habitat_station_position = None
-    #     else:
-    #         _habitat_station_position = (
-    #             config["station_lat"],
-    #             config["station_lon"],
-    #             config["station_alt"],
-    #         )
-
-    #     _habitat = HabitatUploader(
-    #         user_callsign=config["habitat_uploader_callsign"],
-    #         user_antenna=config["habitat_uploader_antenna"],
-    #         station_position=_habitat_station_position,
-    #         synchronous_upload_time=config["habitat_upload_rate"],
-    #         callsign_validity_threshold=config["payload_id_valid"],
-    #         url=config["habitat_url"],
-    #     )
-
-    #     exporter_objects.append(_habitat)
-    #     exporter_functions.append(_habitat.add)
-
     # APRS Uploader
     if config["aprs_enabled"]:
 
@@ -1073,7 +1078,7 @@ def main():
             logging.info("Shutdown time reached. Closing.")
             stop_flask(host=config["web_host"], port=config["web_port"])
             stop_all()
-            break
+            sys.exit(2)
 
 
 if __name__ == "__main__":
@@ -1084,9 +1089,13 @@ def main():
         # Upon CTRL+C, shutdown all threads and exit.
         stop_flask(host=config["web_host"], port=config["web_port"])
         stop_all()
+        sys.exit(0)
     except Exception as e:
         # Upon exceptions, attempt to shutdown threads and exit.
         traceback.print_exc()
         print("Main Loop Error - %s" % str(e))
-        stop_flask(host=config["web_host"], port=config["web_port"])
+        if flask_running():
+            stop_flask(host=config["web_host"], port=config["web_port"])
         stop_all()
+        sys.exit(3)
+

auto_rx/auto_rx.service

@@ -3,7 +3,11 @@ Description=auto_rx
 After=syslog.target
 
 [Service]
-ExecStart=/usr/bin/python3 /home/pi/radiosonde_auto_rx/auto_rx/auto_rx.py -t 0
+# For running outside of a python virtual environment
+# ExecStart=/usr/bin/python3 /home/pi/radiosonde_auto_rx/auto_rx/auto_rx.py -t 0 
+
+# For running within a venv, located at /home/pi/radiosonde_auto_rx/auto_rx/venv/
+ExecStart=/home/pi/radiosonde_auto_rx/auto_rx/venv/bin/python3 /home/pi/radiosonde_auto_rx/auto_rx/auto_rx.py -t 0
 Restart=always
 RestartSec=120
 WorkingDirectory=/home/pi/radiosonde_auto_rx/auto_rx/

auto_rx/auto_rx.sh

@@ -6,16 +6,12 @@
 # NOTE: If running this from crontab, make sure to set the appropriate PATH env-vars,
 # else utilities like rtl_power and rtl_fm won't be found.
 #
-#	WARNING - THIS IS DEPRECATED - PLEASE USE THE SYSTEMD SERVICE
+#	WARNING - THIS IS DEPRECATED - PLEASE USE THE SYSTEMD SERVICE OR DOCKER IMAGE
+#   See: https://github.com/projecthorus/radiosonde_auto_rx/wiki#451-option-1---operation-as-a-systemd-service-recommended
+#   Or: https://github.com/projecthorus/radiosonde_auto_rx/wiki/Docker
 #
 
 # change into appropriate directory
 cd $(dirname $0)
 
-# Clean up old files
-rm log_power*.csv
-
-# Start auto_rx process with a 3 hour timeout.
-# auto_rx will exit after this time.
-
-python3 auto_rx.py -t 180
+python3 auto_rx.py -t 180

auto_rx/autorx/__init__.py

@@ -12,7 +12,7 @@
 # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
 # PATCH - Small changes, or minor feature additions.
 
-__version__ = "1.6.1-beta2"
+__version__ = "1.7.3"
 
 
 # Global Variables

auto_rx/autorx/aprs.py

@@ -214,7 +214,7 @@ def generate_station_object(
     _datum = "!w%s%s!" % (_lat_prec, _lon_prec)
 
     # Generate timestamp using current UTC time
-    _aprs_timestamp = datetime.datetime.utcnow().strftime("%H%M%S")
+    _aprs_timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%H%M%S")
 
     # Add version string to position comment, if requested.
     _aprs_comment = comment
@@ -759,13 +759,19 @@ def close(self):
 
         # Wait for all threads to close.
         if self.upload_thread is not None:
-            self.upload_thread.join()
+            self.upload_thread.join(60)
+            if self.upload_thread.is_alive():
+                self.log_error("aprs upload thread failed to join")
 
         if self.timer_thread is not None:
-            self.timer_thread.join()
+            self.timer_thread.join(60)
+            if self.timer_thread.is_alive():
+                self.log_error("aprs timer thread failed to join")
 
         if self.input_thread is not None:
-            self.input_thread.join()
+            self.input_thread.join(60)
+            if self.input_thread.is_alive():
+                self.log_error("aprs input thread failed to join")
 
     def log_debug(self, line):
         """ Helper function to log a debug message with a descriptive heading. 
@@ -801,10 +807,10 @@ def log_warning(self, line):
     # ['frame', 'id', 'datetime', 'lat', 'lon', 'alt', 'temp', 'type', 'freq', 'freq_float', 'datetime_dt']
     test_telem = [
         # These types of DFM serial IDs are deprecated
-        # {'id':'DFM06-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()},
-        # {'id':'DFM09-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()},
-        # {'id':'DFM15-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()},
-        # {'id':'DFM17-12345678', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.utcnow()},
+        # {'id':'DFM06-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)},
+        # {'id':'DFM09-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)},
+        # {'id':'DFM15-123456', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)},
+        # {'id':'DFM17-12345678', 'frame':10, 'lat':-10.0, 'lon':10.0, 'alt':10000, 'temp':1.0, 'type':'DFM', 'freq':'401.520 MHz', 'freq_float':401.52, 'heading':0.0, 'vel_h':5.1, 'vel_v':-5.0, 'datetime_dt':datetime.datetime.now(datetime.timezone.utc)},
         {
             "id": "DFM-19123456",
             "frame": 10,
@@ -821,7 +827,7 @@ def log_warning(self, line):
             "heading": 0.0,
             "vel_h": 5.1,
             "vel_v": -5.0,
-            "datetime_dt": datetime.datetime.utcnow(),
+            "datetime_dt": datetime.datetime.now(datetime.timezone.utc),
         },
         {
             "id": "DFM-123456",
@@ -839,7 +845,7 @@ def log_warning(self, line):
             "heading": 0.0,
             "vel_h": 5.1,
             "vel_v": -5.0,
-            "datetime_dt": datetime.datetime.utcnow(),
+            "datetime_dt": datetime.datetime.now(datetime.timezone.utc),
         },
         {
             "id": "N1234567",
@@ -857,7 +863,7 @@ def log_warning(self, line):
             "heading": 0.0,
             "vel_h": 5.1,
             "vel_v": -5.0,
-            "datetime_dt": datetime.datetime.utcnow(),
+            "datetime_dt": datetime.datetime.now(datetime.timezone.utc),
         },
         {
             "id": "M1234567",
@@ -875,7 +881,7 @@ def log_warning(self, line):
             "heading": 0.0,
             "vel_h": 5.1,
             "vel_v": -5.0,
-            "datetime_dt": datetime.datetime.utcnow(),
+            "datetime_dt": datetime.datetime.now(datetime.timezone.utc),
         },
     ]