ENH: Exponential backoff decorator (fix #449)

CHANGELOG.md

@@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- ENH: Exponential backoff decorator (fix #449) [#588](https://github.com/RocketPy-Team/RocketPy/pull/588)
 - ENH: Add new stability margin properties to Flight class [#572](https://github.com/RocketPy-Team/RocketPy/pull/572)
 - ENH: adds `Function.remove_outliers` method [#554](https://github.com/RocketPy-Team/RocketPy/pull/554)
 

rocketpy/environment/environment.py

@@ -3,7 +3,7 @@
 import re
 import warnings
 from collections import namedtuple
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 import numpy as np
 import numpy.ma as ma
@@ -13,6 +13,7 @@
 from ..mathutils.function import Function, funcify_method
 from ..plots.environment_plots import _EnvironmentPlots
 from ..prints.environment_prints import _EnvironmentPrints
+from ..tools import exponential_backoff
 
 try:
     import netCDF4
@@ -680,18 +681,9 @@
 
         #     self.elevation = elev
 
-        elif self.latitude != None and self.longitude != None:
-            try:
-                print("Fetching elevation from open-elevation.com...")
-                request_url = "https://api.open-elevation.com/api/v1/lookup?locations={:f},{:f}".format(
-                    self.latitude, self.longitude
-                )
-                response = requests.get(request_url)
-                results = response.json()["results"]
-                self.elevation = results[0]["elevation"]
-                print("Elevation received:", self.elevation)
-            except:
-                raise RuntimeError("Unable to reach Open-Elevation API servers.")
+        elif self.latitude is not None and self.longitude is not None:
+            self.elevation = self.__fetch_open_elevation()
+            print("Elevation received: ", self.elevation)
         else:
             raise ValueError(
                 "Latitude and longitude must be set to use"
@@ -1303,26 +1295,8 @@
                     "v_wind": "vgrdprs",
                 }
                 # Attempt to get latest forecast
-                time_attempt = datetime.utcnow()
-                success = False
-                attempt_count = 0
-                while not success and attempt_count < 10:
-                    time_attempt -= timedelta(hours=6 * attempt_count)
-                    file = "https://nomads.ncep.noaa.gov/dods/gens_bc/gens{:04d}{:02d}{:02d}/gep_all_{:02d}z".format(
-                        time_attempt.year,
-                        time_attempt.month,
-                        time_attempt.day,
-                        6 * (time_attempt.hour // 6),
-                    )
-                    try:
-                        self.process_ensemble(file, dictionary)
-                        success = True
-                    except OSError:
-                        attempt_count += 1
-                if not success:
-                    raise RuntimeError(
-                        "Unable to load latest weather data for GEFS through " + file
-                    )
+                self.__fetch_gefs_ensemble(dictionary)
+
             elif file == "CMC":
                 # Define dictionary
                 dictionary = {
@@ -1338,27 +1312,7 @@
                     "u_wind": "ugrdprs",
                     "v_wind": "vgrdprs",
                 }
-                # Attempt to get latest forecast
-                time_attempt = datetime.utcnow()
-                success = False
-                attempt_count = 0
-                while not success and attempt_count < 10:
-                    time_attempt -= timedelta(hours=12 * attempt_count)
-                    file = "https://nomads.ncep.noaa.gov/dods/cmcens/cmcens{:04d}{:02d}{:02d}/cmcens_all_{:02d}z".format(
-                        time_attempt.year,
-                        time_attempt.month,
-                        time_attempt.day,
-                        12 * (time_attempt.hour // 12),
-                    )
-                    try:
-                        self.process_ensemble(file, dictionary)
-                        success = True
-                    except OSError:
-                        attempt_count += 1
-                if not success:
-                    raise RuntimeError(
-                        "Unable to load latest weather data for CMC through " + file
-                    )
+                self.__fetch_cmc_ensemble(dictionary)
             # Process other forecasts or reanalysis
             else:
                 # Check if default dictionary was requested
@@ -1650,20 +1604,7 @@
             model.
         """
 
-        # Process the model string
-        model = model.lower()
-        if model[-1] == "u":  # case iconEu
-            model = "".join([model[:4], model[4].upper(), model[4 + 1 :]])
-        # Load data from Windy.com: json file
-        url = f"https://node.windy.com/forecast/meteogram/{model}/{self.latitude}/{self.longitude}/?step=undefined"
-        try:
-            response = requests.get(url).json()
-        except:
-            if model == "iconEu":
-                raise ValueError(
-                    "Could not get a valid response for Icon-EU from Windy. Check if the latitude and longitude coordinates set are inside Europe.",
-                )
-            raise
+        response = self.__fetch_atmospheric_data_from_windy(model)
 
         # Determine time index from model
         time_array = np.array(response["data"]["hours"])
@@ -1824,18 +1765,7 @@
         None
         """
         # Request Wyoming Sounding from file url
-        response = requests.get(file)
-        if response.status_code != 200:
-            raise ImportError("Unable to load " + file + ".")
-        if len(re.findall("Can't get .+ Observations at", response.text)):
-            raise ValueError(
-                re.findall("Can't get .+ Observations at .+", response.text)[0]
-                + " Check station number and date."
-            )
-        if response.text == "Invalid OUTPUT: specified\n":
-            raise ValueError(
-                "Invalid OUTPUT: specified. Make sure the output is Text: List."
-            )
+        response = self.__fetch_wyoming_sounding(file)
 
         # Process Wyoming Sounding by finding data table and station info
         response_split_text = re.split("(<.{0,1}PRE>)", response.text)
@@ -1961,9 +1891,7 @@
         None
         """
         # Request NOAA Ruc Sounding from file url
-        response = requests.get(file)
-        if response.status_code != 200 or len(response.text) < 10:
-            raise ImportError("Unable to load " + file + ".")
+        response = self.__fetch_noaaruc_sounding(file)
 
         # Split response into lines
         lines = response.text.split("\n")
@@ -3538,6 +3466,110 @@
                 f"The reference system {datum} for Earth geometry " "is not recognized."
             )
 
+    # Auxiliary functions - Fetching Data from 3rd party APIs
+
+    @exponential_backoff(max_attempts=3, base_delay=1, max_delay=60)
+    def __fetch_open_elevation(self):
+        print("Fetching elevation from open-elevation.com...")
+        request_url = (
+            "https://api.open-elevation.com/api/v1/lookup?locations"
+            f"={self.latitude},{self.longitude}"
+        )
+        try:
+            response = requests.get(request_url)
+        except Exception as e:
+            raise RuntimeError("Unable to reach Open-Elevation API servers.")
+        results = response.json()["results"]
+        return results[0]["elevation"]
+
+    @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60)
+    def __fetch_atmospheric_data_from_windy(self, model):
+        model = model.lower()
+        if model[-1] == "u":  # case iconEu
+            model = "".join([model[:4], model[4].upper(), model[4 + 1 :]])
+        url = (
+            f"https://node.windy.com/forecast/meteogram/{model}/"
+            f"{self.latitude}/{self.longitude}/?step=undefined"
+        )
+        try:
+            response = requests.get(url).json()
+        except Exception as e:
+            if model == "iconEu":
+                raise ValueError(
+                    "Could not get a valid response for Icon-EU from Windy. "
+                    "Check if the coordinates are set inside Europe."
+                )
+        return response
+
+    @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60)
+    def __fetch_wyoming_sounding(self, file):
+        response = requests.get(file)
+        if response.status_code != 200:
+            raise ImportError(f"Unable to load {file}.")
+        if len(re.findall("Can't get .+ Observations at", response.text)):
+            raise ValueError(
+                re.findall("Can't get .+ Observations at .+", response.text)[0]
+                + " Check station number and date."
+            )
+        if response.text == "Invalid OUTPUT: specified\n":
+            raise ValueError(
+                "Invalid OUTPUT: specified. Make sure the output is Text: List."
+            )
+        return response
+
+    @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60)
+    def __fetch_noaaruc_sounding(self, file):
+        response = requests.get(file)
+        if response.status_code != 200 or len(response.text) < 10:
+            raise ImportError("Unable to load " + file + ".")
+
+    @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60)
+    def __fetch_gefs_ensemble(self, dictionary):
+        time_attempt = datetime.now(tz=timezone.utc)
+        success = False
+        attempt_count = 0
+        while not success and attempt_count < 10:
+            time_attempt -= timedelta(hours=6 * attempt_count)
+            file = (
+                f"https://nomads.ncep.noaa.gov/dods/gens_bc/gens"
+                f"{time_attempt.year:04d}{time_attempt.month:02d}"
+                f"{time_attempt.day:02d}/"
+                f"gep_all_{6 * (time_attempt.hour // 6):02d}z"
+            )
+            try:
+                self.process_ensemble(file, dictionary)
+                success = True
+            except OSError:
+                attempt_count += 1
+        if not success:
+            raise RuntimeError(
+                "Unable to load latest weather data for GEFS through " + file
+            )
+
+    @exponential_backoff(max_attempts=5, base_delay=2, max_delay=60)
+    def __fetch_cmc_ensemble(self, dictionary):
+        # Attempt to get latest forecast
+        time_attempt = datetime.now(tz=timezone.utc)
+        success = False
+        attempt_count = 0
+        while not success and attempt_count < 10:
+            time_attempt -= timedelta(hours=12 * attempt_count)
+            file = (
+                f"https://nomads.ncep.noaa.gov/dods/cmcens/"
+                f"cmcens{time_attempt.year:04d}{time_attempt.month:02d}"
+                f"{time_attempt.day:02d}/"
+                f"cmcens_all_{12 * (time_attempt.hour // 12):02d}z"
+            )
+            try:
+                self.process_ensemble(file, dictionary)
+                success = True
+            except OSError:
+                attempt_count += 1
+        if not success:
+            raise RuntimeError(
+                "Unable to load latest weather data for CMC through " + file
+            )
+
     # Auxiliary functions - Geodesic Coordinates
 
     @staticmethod

rocketpy/tools.py

@@ -1,6 +1,8 @@
+import functools
 import importlib
 import importlib.metadata
 import re
+import time
 from bisect import bisect_left
 
 import numpy as np
@@ -382,6 +384,25 @@
     return True
 
 
+def exponential_backoff(max_attempts, base_delay=1, max_delay=60):
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            delay = base_delay
+            for i in range(max_attempts):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    if i == max_attempts - 1:
+                        raise e from None
+                    delay = min(delay * 2, max_delay)
+                    time.sleep(delay)
+
+        return wrapper
+
+    return decorator
+
+
 def parallel_axis_theorem_from_com(com_inertia_moment, mass, distance):
     """Calculates the moment of inertia of a object relative to a new axis using
     the parallel axis theorem. The new axis is parallel to and at a distance

tests/fixtures/environment/environment_fixtures.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 
 import pytest
+
 from rocketpy import Environment, EnvironmentAnalysis
 
 
@@ -54,8 +55,8 @@ def env_analysis():
     EnvironmentAnalysis
     """
     env_analysis = EnvironmentAnalysis(
-        start_date=datetime.datetime(2019, 10, 23),
-        end_date=datetime.datetime(2021, 10, 23),
+        start_date=datetime(2019, 10, 23),
+        end_date=datetime(2021, 10, 23),
         latitude=39.3897,
         longitude=-8.28896388889,
         start_hour=6,

tests/test_environment.py

@@ -1,5 +1,4 @@
 import datetime
-import time
 from unittest.mock import patch
 
 import pytest
@@ -64,13 +63,8 @@ def test_wyoming_sounding_atmosphere(mock_show, example_plain_env):
     #        "file" option, instead of receiving the URL as a string.
     URL = "http://weather.uwyo.edu/cgi-bin/sounding?region=samer&TYPE=TEXT%3ALIST&YEAR=2019&MONTH=02&FROM=0500&TO=0512&STNM=83779"
     # give it at least 5 times to try to download the file
-    for i in range(5):
-        try:
-            example_plain_env.set_atmospheric_model(type="wyoming_sounding", file=URL)
-            break
-        except:
-            time.sleep(1)  # wait 1 second before trying again
-            pass
+    example_plain_env.set_atmospheric_model(type="wyoming_sounding", file=URL)
+
     assert example_plain_env.all_info() == None
     assert abs(example_plain_env.pressure(0) - 93600.0) < 1e-8
     assert (

tests/unit/test_environment.py

@@ -1,10 +1,10 @@
+import json
 import os
 
 import numpy as np
 import numpy.ma as ma
 import pytest
 import pytz
-import json
 
 from rocketpy import Environment