Merge pull request #473 from projecthorus/testing

Version 1.5.2 Release

Dockerfile

@@ -39,6 +39,7 @@ RUN case $(uname -m) in \
   apt-get update && \
   apt-get upgrade -y && \
   apt-get install -y \
+  libatomic1 \
   rng-tools \
   rtl-sdr \
   sox \

README.md

@@ -9,7 +9,7 @@ and uploading of
 [Radiosonde](https://en.wikipedia.org/wiki/Radiosonde) telemetry to
 multiple services, including:
 
-* The [Habitat High-Altitude Balloon Tracker](https://tracker.sondehub.org)
+* The [SondeHub Radiosonde Tracker](https://tracker.sondehub.org) - a tracking website specifically designed for tracking radiosondes!
 
 * APRS-IS (for display on sites such as [radiosondy.info](https://radiosondy.info) and [aprs.fi](https://aprs.fi)
 
@@ -64,7 +64,7 @@ instructions](https://github.com/projecthorus/radiosonde_auto_rx/wiki).
 ### Licensing Information
 All software within this repository is licensed under the GNU General Public License v3. Refer this repositories LICENSE file for the full license text.
 
-Radiosonde telemetry data captured via this software and uploaded into the [Habitat](https://habitat.habhub.org/) and [Sondehub](https://sondehub.org/) Database systems is licensed under [Creative Commons BY-SA v2.0](https://creativecommons.org/licenses/by-sa/2.0/). 
+Radiosonde telemetry data captured via this software and uploaded into the [Sondehub](https://sondehub.org/) Database system is licensed under [Creative Commons BY-SA v2.0](https://creativecommons.org/licenses/by-sa/2.0/). 
 Telemetry data uploaded into the APRS-IS network is generally considered to be released into the public domain. 
 
 By uploading data into these systems (by enabling the relevant uploaders within the `station.cfg` file) you as the user agree for your data to be made available under these licenses. Note that uploading to Sondehub and Habitat is enabled by default.

auto_rx/auto_rx.py

@@ -36,7 +36,7 @@
     rtlsdr_test,
     position_info,
     check_rs_utils,
-    check_autorx_version,
+    version_startup_check,
 )
 from autorx.config import read_auto_rx_config
 from autorx.web import (
@@ -94,7 +94,7 @@
 
 
 def allocate_sdr(check_only=False, task_description=""):
-    """ Allocate an un-used SDR for a task.
+    """Allocate an un-used SDR for a task.
 
     Args:
         check_only (bool) : If True, don't set the free SDR as in-use. Used to check if there are any free SDRs.
@@ -124,7 +124,7 @@ def allocate_sdr(check_only=False, task_description=""):
 
 
 def start_scanner():
-    """ Start a scanner thread on the first available SDR """
+    """Start a scanner thread on the first available SDR"""
     global config, RS_PATH, temporary_block_list
 
     if "SCAN" in autorx.task_list:
@@ -151,9 +151,9 @@ def start_scanner():
             min_freq=config["min_freq"],
             max_freq=config["max_freq"],
             search_step=config["search_step"],
-            whitelist=config["whitelist"],
-            greylist=config["greylist"],
-            blacklist=config["blacklist"],
+            only_scan=config["only_scan"],
+            always_scan=config["always_scan"],
+            never_scan=config["never_scan"],
             snr_threshold=config["snr_threshold"],
             min_distance=config["min_distance"],
             quantization=config["quantization"],
@@ -180,7 +180,7 @@ def start_scanner():
 
 
 def stop_scanner():
-    """ Stop a currently running scan thread, and release the SDR it was using. """
+    """Stop a currently running scan thread, and release the SDR it was using."""
 
     if "SCAN" not in autorx.task_list:
         # No scanner thread running!
@@ -199,7 +199,7 @@ def stop_scanner():
 
 
 def start_decoder(freq, sonde_type):
-    """ Attempt to start a decoder thread for a given sonde.
+    """Attempt to start a decoder thread for a given sonde.
 
     Args:
         freq (float): Radiosonde frequency in Hz.
@@ -255,7 +255,7 @@ def start_decoder(freq, sonde_type):
 
 
 def handle_scan_results():
-    """ Read in Scan results via the scan results Queue.
+    """Read in Scan results via the scan results Queue.
 
     Depending on how many SDRs are available, two things can happen:
     - If there is a free SDR, allocate it to a decoder.
@@ -293,7 +293,7 @@ def handle_scan_results():
                             if abs(_key - _freq) < config["decoder_spacing_limit"]:
                                 # At this point, we can be pretty sure that there is another decoder already decoding this particular sonde ID.
                                 # Without actually starting another decoder and matching IDs, we can't be 100% sure, but it's a good chance.
-                                logging.error(
+                                logging.warning(
                                     "Task Manager - Detected %s sonde on %.3f MHz, but this is within %d kHz of an already running decoder. (This limit can be set using the 'decoder_spacing_limit' advanced config option.)"
                                     % (
                                         _type,
@@ -314,7 +314,7 @@ def handle_scan_results():
                     if temporary_block_list[_freq] > (
                         time.time() - config["temporary_block_time"] * 60
                     ):
-                        logging.error(
+                        logging.warning(
                             "Task Manager - Attempted to start a decoder on a temporarily blocked frequency (%.3f MHz)"
                             % (_freq / 1e6)
                         )
@@ -343,7 +343,7 @@ def handle_scan_results():
 
                 # Break if we don't support this sonde type.
                 if _check_type not in VALID_SONDE_TYPES:
-                    logging.error(
+                    logging.warning(
                         "Task Manager - Unsupported sonde type: %s" % _check_type
                     )
                     # TODO - Potentially add the frequency of the unsupported sonde to the temporary block list?
@@ -368,7 +368,7 @@ def handle_scan_results():
 
 
 def clean_task_list():
-    """ Check the task list to see if any tasks have stopped running. If so, release the associated SDR """
+    """Check the task list to see if any tasks have stopped running. If so, release the associated SDR"""
 
     for _key in autorx.task_list.copy().keys():
         # Attempt to get the state of the task
@@ -444,7 +444,7 @@ def clean_task_list():
 
 
 def stop_all():
-    """ Shut-down all decoders, scanners, and exporters. """
+    """Shut-down all decoders, scanners, and exporters."""
     global exporter_objects
     logging.info("Starting shutdown of all threads.")
     for _task in autorx.task_list.keys():
@@ -464,7 +464,7 @@ def stop_all():
 
 
 def telemetry_filter(telemetry):
-    """ Filter incoming radiosonde telemetry based on various factors,
+    """Filter incoming radiosonde telemetry based on various factors,
         - Invalid Position
         - Invalid Altitude
         - Abnormal range from receiver.
@@ -545,16 +545,18 @@ def telemetry_filter(telemetry):
             return "TempBlock"
 
     # DateTime Check
-    _delta_time = (datetime.datetime.now(datetime.timezone.utc) - parse(telemetry['datetime'])).total_seconds()
+    _delta_time = (
+        datetime.datetime.now(datetime.timezone.utc) - parse(telemetry["datetime"])
+    ).total_seconds()
     logging.debug("Delta time: %d" % _delta_time)
 
-    if abs(_delta_time) > (3600*config["sonde_time_threshold"]):
+    if abs(_delta_time) > (3600 * config["sonde_time_threshold"]):
         logging.warning(
-            "Sonde reported time too far from current UTC time. Either sonde time or system time is invalid. (Threshold: %d hours)" % config["sonde_time_threshold"]
+            "Sonde reported time too far from current UTC time. Either sonde time or system time is invalid. (Threshold: %d hours)"
+            % config["sonde_time_threshold"]
         )
         return False
 
-
     # Payload Serial Number Checks
     _serial = telemetry["id"]
     # Run a Regex to match known Vaisala RS92/RS41 serial numbers (YWWDxxxx)
@@ -565,17 +567,19 @@ def telemetry_filter(telemetry):
     # ~2025-2030, so have expanded the regex to match (and also support some older RS92s)
     vaisala_callsign_valid = re.match(r"[E-Z][0-5][\d][1-7]\d{4}", _serial)
 
-    # Regex to check DFM callsigns are valid.
-    # DFM serial numbers have at least 6 numbers (newer sondes have 8)
-    dfm_callsign_valid = re.match(r"DFM-\d{6}", _serial)
+    # Just make sure we're not getting the 'xxxxxxxx' unknown serial from the DFM decoder.
+    if "DFM" in telemetry["type"]:
+        dfm_callsign_valid = "x" not in _serial.split("-")[1]
+    else:
+        dfm_callsign_valid = False
 
     # Check Meisei sonde callsigns for validity.
     # meisei_ims returns a callsign of IMS100-xxxxxx until it receives the serial number, so we filter based on the x's being present or not.
     if "MEISEI" in telemetry["type"]:
         meisei_callsign_valid = "x" not in _serial.split("-")[1]
     else:
         meisei_callsign_valid = False
-    
+
     if "MRZ" in telemetry["type"]:
         mrz_callsign_valid = "x" not in _serial.split("-")[1]
     else:
@@ -598,7 +602,7 @@ def telemetry_filter(telemetry):
         # Add in a note about DFM sondes and their oddness...
         if "DFM" in telemetry["id"]:
             _id_msg += " Note: DFM sondes may take a while to get an ID."
-        
+
         if "MRZ" in telemetry["id"]:
             _id_msg += " Note: MRZ sondes may take a while to get an ID."
 
@@ -607,7 +611,7 @@ def telemetry_filter(telemetry):
 
 
 def station_position_update(position):
-    """ Handle a callback from GPSDAdaptor object, and update each exporter object. """
+    """Handle a callback from GPSDAdaptor object, and update each exporter object."""
     global exporter_objects
     # Quick sanity check of the incoming data
     if "valid" not in position:
@@ -627,7 +631,7 @@ def station_position_update(position):
 
 
 def email_error(message="foo"):
-    """ Helper function to email an error message, if the email exporter is available """
+    """Helper function to email an error message, if the email exporter is available"""
     global email_exporter
 
     if email_exporter and config["email_error_notifications"]:
@@ -640,7 +644,7 @@ def email_error(message="foo"):
 
 
 def main():
-    """ Main Loop """
+    """Main Loop"""
     global config, exporter_objects, exporter_functions, logging_level, rs92_ephemeris, gpsd_adaptor, email_exporter
 
     # Command line arguments.
@@ -662,7 +666,7 @@ def main():
         "--frequency",
         type=float,
         default=0.0,
-        help="Sonde Frequency Override (MHz). This overrides the scan whitelist with the supplied frequency.",
+        help="Sonde Frequency Override (MHz). This overrides the only_scan list with the supplied frequency.",
     )
     parser.add_argument(
         "-m",
@@ -773,7 +777,9 @@ def main():
     # allows it to run indefinitely.
     if args.type != None:
         if args.type in VALID_SONDE_TYPES:
-            logging.warning("Overriding RX timeout for manually specified radiosonde type. Decoders will not automatically stop!")
+            logging.warning(
+                "Overriding RX timeout for manually specified radiosonde type. Decoders will not automatically stop!"
+            )
             config["rx_timeout"] = 0
             autorx.scan_results.put([[args.frequency * 1e6, args.type]])
         else:
@@ -784,10 +790,10 @@ def main():
     # This needs to occur AFTER logging is setup, else logging breaks horribly for some reason.
     start_flask(host=config["web_host"], port=config["web_port"])
 
-    # If we have been supplied a frequency via the command line, override the whitelist settings
+    # If we have been supplied a frequency via the command line, override the only_scan list settings
     # to only include the supplied frequency.
     if args.frequency != 0.0:
-        config["whitelist"] = [args.frequency]
+        config["only_scan"] = [args.frequency]
 
     # Start our exporter options
     # Telemetry Logger
@@ -822,29 +828,29 @@ def main():
         exporter_objects.append(_email_notification)
         exporter_functions.append(_email_notification.add)
 
-    # Habitat Uploader
-    if config["habitat_enabled"]:
-
-        if config["habitat_upload_listener_position"] is False:
-            _habitat_station_position = None
-        else:
-            _habitat_station_position = (
-                config["station_lat"],
-                config["station_lon"],
-                config["station_alt"],
-            )
-
-        _habitat = HabitatUploader(
-            user_callsign=config["habitat_uploader_callsign"],
-            user_antenna=config["habitat_uploader_antenna"],
-            station_position=_habitat_station_position,
-            synchronous_upload_time=config["habitat_upload_rate"],
-            callsign_validity_threshold=config["payload_id_valid"],
-            url=config["habitat_url"],
-        )
-
-        exporter_objects.append(_habitat)
-        exporter_functions.append(_habitat.add)
+    # Habitat Uploader - DEPRECATED - Sondehub DB now in use (>1.5.0)
+    # if config["habitat_enabled"]:
+
+    #     if config["habitat_upload_listener_position"] is False:
+    #         _habitat_station_position = None
+    #     else:
+    #         _habitat_station_position = (
+    #             config["station_lat"],
+    #             config["station_lon"],
+    #             config["station_alt"],
+    #         )
+
+    #     _habitat = HabitatUploader(
+    #         user_callsign=config["habitat_uploader_callsign"],
+    #         user_antenna=config["habitat_uploader_antenna"],
+    #         station_position=_habitat_station_position,
+    #         synchronous_upload_time=config["habitat_upload_rate"],
+    #         callsign_validity_threshold=config["payload_id_valid"],
+    #         url=config["habitat_url"],
+    #     )
+
+    #     exporter_objects.append(_habitat)
+    #     exporter_functions.append(_habitat.add)
 
     # APRS Uploader
     if config["aprs_enabled"]:
@@ -925,6 +931,7 @@ def main():
         exporter_objects.append(_rotator)
         exporter_functions.append(_rotator.add)
 
+    # Sondehub v2 Database
     if config["sondehub_enabled"]:
         if config["habitat_upload_listener_position"] is False:
             _sondehub_station_position = None
@@ -934,7 +941,7 @@ def main():
                 config["station_lon"],
                 config["station_alt"],
             )
-        
+
         _sondehub = SondehubUploader(
             user_callsign=config["habitat_uploader_callsign"],
             user_position=_sondehub_station_position,
@@ -958,7 +965,7 @@ def main():
             callback=station_position_update,
         )
 
-    check_autorx_version()
+    version_startup_check()
 
     # Note the start time.
     _start_time = time.time()

auto_rx/autorx/__init__.py

@@ -17,7 +17,7 @@
 # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
 # PATCH - Small changes, or minor feature additions.
 
-__version__ = "1.5.1"
+__version__ = "1.5.2"
 
 
 # Global Variables