Skip to content

Commit

Permalink
Bugfix for cases where the external site cannot be accessed
Browse files Browse the repository at this point in the history
  • Loading branch information
@joergschultzelutter
joergschultzelutter committed Jun 11, 2021
1 parent 06aac24 commit b7e91c4
Showing 1 changed file with 171 additions and 147 deletions.
318 changes: 171 additions & 147 deletions src/radiosonde_modules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# Radiosonde landing prediction
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# it under the terms of the GNU General Public License as published byz
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
Expand Down Expand Up @@ -174,7 +174,12 @@ def get_kml_data_from_habhub(
# Send the payload to the site. If all goes well, Habhub responds
# with a HTTP200 and provides us with a UUID
url = "http://predict.habhub.org/ajax.php?action=submitForm"
resp = requests.post(url=url, data=hubhab_payload)

try:
resp = requests.post(url=url, data=hubhab_payload)
except:
logger.info(f"Cannot connect to {url}")
resp = None
if resp:
if resp.status_code == 200:
json_content = resp.json()
Expand Down Expand Up @@ -342,6 +347,9 @@ def get_radiosonde_landing_prediction(aprsfi_callsign: str, aprsdotfi_api_key: s
success = False
else:
success = False
logger.info("Found on aprs.fi but does not contain a comment")
else:
logger.info("Not found on aprs.fi")
return success, landing_latitude, landing_longitude, landing_timestamp, landing_url


Expand Down Expand Up @@ -520,117 +528,82 @@ def get_radiosondy_data(sonde_id: str):
dyn_url = f"https://radiosondy.info/dyn/get_sondeinfo.php?sondenumber={sonde_id}"

# Get the main URL
page = d.get(url=main_url, headers=headers)

if page.last_response.status_code == 200:
success = True
# In case the response's URL indicates that the request got redirected to the archived data
if "sonde_archive.php" in page.last_response.url:
logger.info("Parsing static Radiosondy data")
soup = BeautifulSoup(page.last_response.response.text, "html.parser")
# Archived probe; we have proper tables and can parse them. Page has STATIC content
# Parse Table "Status Changes"
table = soup.find("table", attrs={"id": "Table2"})
if table:
# get first row
rows = table.find("tr", attrs={"class": "bg_1"})
# There was at least one minimal change
if rows:
# There was at least one minimal change for this radiosonde
cols = rows.find_all("td")
if cols and len(cols) == 9:
launch_site = cols[0].string
probe_type = cols[1].string
probe_aux = cols[2].string
probe_freq = cols[3].string
probe_status = cols[4].string
probe_finder = cols[5].string
landing_point = cols[6].string
landing_description = cols[7].string
changes_made = cols[8].string

regex_string = r"^(-?\d*[.]\d*),\s*(-?\d*[.]\d*)$"
matches = re.search(
pattern=regex_string,
string=landing_point,
flags=re.IGNORECASE,
)
if matches:
try:
landing_point_latitude = float(matches[1])
landing_point_longitude = float(matches[2])
except ValueError:
landing_point_longitude = landing_point_longitude = 0.0
else:
# This branch gets executed in case the probe's status has never changed since its inception
# With the exception of the APRS data, the data that we want / need is stored as regular
# text. We use the text's icons in order to identify the content
html_response_dict = parse_radiosondy_html_content(
html_raw_content=page.last_response.response.text
)
sonde_number = html_response_dict["sonde_number"]
launch_site = html_response_dict["launch_site"]
probe_type = html_response_dict["probe_type"]
probe_freq = html_response_dict["probe_freq"]
probe_aux = html_response_dict["probe_aux"]
probe_status = html_response_dict["probe_status"]
max_speed = html_response_dict["max_speed"]
max_speed_height = html_response_dict["max_speed_height"]
avg_speed_kmh = html_response_dict["avg_speed_kmh"]
max_altitude = html_response_dict["max_altitude"]
avg_ascent_speed = html_response_dict["avg_ascent_speed"]
avg_descent_speed = html_response_dict["avg_descent_speed"]

# pparse APRS data
table = soup.find("table", attrs={"id": "Table1"})
if table:
# get first row
rows = table.find("tr", attrs={"class": "bg_1"})
if rows:
cols = rows.find_all("td")
if cols and len(cols) == 9:
receiver = cols[0].string
sonde_number = cols[1].string
datetime_utc = cols[2].string
latitude = cols[3].string
longitude = cols[4].string
course_deg = cols[5].string
speed_kmh = cols[6].string
altitude_m = cols[7].string
aprs_comment = cols[8].string
else:
# Probe is either planned or still in process. We have DYNAMIC content and need to get this from a different URL
logger.info("parsing dynamic URL")
page = d.get(url=dyn_url, headers=headers)
if page.last_response.status_code == 200:

# With the exception of the APRS data, the data that we want / need is stored as regular
# text. We use the text's icons in order to identify the content
html_response_dict = parse_radiosondy_html_content(
html_raw_content=page.last_response.response.text
)
sonde_number = html_response_dict["sonde_number"]
launch_site = html_response_dict["launch_site"]
probe_type = html_response_dict["probe_type"]
probe_freq = html_response_dict["probe_freq"]
probe_aux = html_response_dict["probe_aux"]
probe_status = html_response_dict["probe_status"]
max_speed = html_response_dict["max_speed"]
max_speed_height = html_response_dict["max_speed_height"]
avg_speed_kmh = html_response_dict["avg_speed_kmh"]
max_altitude = html_response_dict["max_altitude"]
avg_ascent_speed = html_response_dict["avg_ascent_speed"]
avg_descent_speed = html_response_dict["avg_descent_speed"]

try:
page = d.get(url=main_url, headers=headers)
except:
logger.info(f"Cannot access {main_url}")
page = None

if page:
if page.last_response.status_code == 200:
success = True
# In case the response's URL indicates that the request got redirected to the archived data
if "sonde_archive.php" in page.last_response.url:
logger.info("Parsing static Radiosondy data")
soup = BeautifulSoup(page.last_response.response.text, "html.parser")
# Parse the APRS data
# Archived probe; we have proper tables and can parse them. Page has STATIC content
# Parse Table "Status Changes"
table = soup.find("table", attrs={"id": "Table2"})
if table:
# get first row
rows = table.find("tr", attrs={"class": "bg_1"})
# There was at least one minimal change
if rows:
# There was at least one minimal change for this radiosonde
cols = rows.find_all("td")
if cols and len(cols) == 9:
launch_site = cols[0].string
probe_type = cols[1].string
probe_aux = cols[2].string
probe_freq = cols[3].string
probe_status = cols[4].string
probe_finder = cols[5].string
landing_point = cols[6].string
landing_description = cols[7].string
changes_made = cols[8].string

regex_string = r"^(-?\d*[.]\d*),\s*(-?\d*[.]\d*)$"
matches = re.search(
pattern=regex_string,
string=landing_point,
flags=re.IGNORECASE,
)
if matches:
try:
landing_point_latitude = float(matches[1])
landing_point_longitude = float(matches[2])
except ValueError:
landing_point_longitude = (
landing_point_longitude
) = 0.0
else:
# This branch gets executed in case the probe's status has never changed since its inception
# With the exception of the APRS data, the data that we want / need is stored as regular
# text. We use the text's icons in order to identify the content
html_response_dict = parse_radiosondy_html_content(
html_raw_content=page.last_response.response.text
)
sonde_number = html_response_dict["sonde_number"]
launch_site = html_response_dict["launch_site"]
probe_type = html_response_dict["probe_type"]
probe_freq = html_response_dict["probe_freq"]
probe_aux = html_response_dict["probe_aux"]
probe_status = html_response_dict["probe_status"]
max_speed = html_response_dict["max_speed"]
max_speed_height = html_response_dict["max_speed_height"]
avg_speed_kmh = html_response_dict["avg_speed_kmh"]
max_altitude = html_response_dict["max_altitude"]
avg_ascent_speed = html_response_dict["avg_ascent_speed"]
avg_descent_speed = html_response_dict["avg_descent_speed"]

# pparse APRS data
table = soup.find("table", attrs={"id": "Table1"})
if table:
# get first row
rows = table.find("tr", attrs={"class": "bg_1"})
if rows:
cols = rows.find_all("td")
if cols and len(cols) == 13:
if cols and len(cols) == 9:
receiver = cols[0].string
sonde_number = cols[1].string
datetime_utc = cols[2].string
Expand All @@ -639,48 +612,99 @@ def get_radiosondy_data(sonde_id: str):
course_deg = cols[5].string
speed_kmh = cols[6].string
altitude_m = cols[7].string
climbing = cols[8].string
temperature = cols[9].string
pressure = cols[10].string
humidity = cols[11].string
aux_o3 = cols[12].string

# Remove the additional content such as units of measure etc.
# Yes, this is quick and dirty
climbing = remove_trailing_content(
source_string=climbing, trailing_content=" m/s"
)
altitude_m = remove_trailing_content(
source_string=altitude_m, trailing_content=" m"
)
aux_o3 = remove_trailing_content(
source_string=aux_o3, trailing_content=" mPa"
)
course_deg = remove_trailing_content(
source_string=course_deg, trailing_content=" °"
)
humidity = remove_trailing_content(
source_string=humidity, trailing_content=" %"
)
pressure = remove_trailing_content(
source_string=pressure, trailing_content=" hPa"
)
speed_kmh = remove_trailing_content(
source_string=speed_kmh, trailing_content=" km/h"
)
temperature = remove_trailing_content(
source_string=temperature, trailing_content=" °C"
)
latitude = remove_trailing_content(
source_string=latitude, trailing_content=" φ"
)
longitude = remove_trailing_content(
source_string=longitude, trailing_content=" λ"
)

aprs_comment = cols[8].string
else:
# We were unable to access the dynamic PHP data - return an error to the user
success = False
# Probe is either planned or still in process. We have DYNAMIC content and need to get this from a different URL
logger.info("parsing dynamic URL")
try:
page = d.get(url=dyn_url, headers=headers)
except:
page = None
success = False
if page:
if page.last_response.status_code == 200:

# With the exception of the APRS data, the data that we want / need is stored as regular
# text. We use the text's icons in order to identify the content
html_response_dict = parse_radiosondy_html_content(
html_raw_content=page.last_response.response.text
)
sonde_number = html_response_dict["sonde_number"]
launch_site = html_response_dict["launch_site"]
probe_type = html_response_dict["probe_type"]
probe_freq = html_response_dict["probe_freq"]
probe_aux = html_response_dict["probe_aux"]
probe_status = html_response_dict["probe_status"]
max_speed = html_response_dict["max_speed"]
max_speed_height = html_response_dict["max_speed_height"]
avg_speed_kmh = html_response_dict["avg_speed_kmh"]
max_altitude = html_response_dict["max_altitude"]
avg_ascent_speed = html_response_dict["avg_ascent_speed"]
avg_descent_speed = html_response_dict["avg_descent_speed"]

soup = BeautifulSoup(
page.last_response.response.text, "html.parser"
)
# Parse the APRS data
table = soup.find("table", attrs={"id": "Table1"})
if table:
# get first row
rows = table.find("tr", attrs={"class": "bg_1"})
if rows:
cols = rows.find_all("td")
if cols and len(cols) == 13:
receiver = cols[0].string
sonde_number = cols[1].string
datetime_utc = cols[2].string
latitude = cols[3].string
longitude = cols[4].string
course_deg = cols[5].string
speed_kmh = cols[6].string
altitude_m = cols[7].string
climbing = cols[8].string
temperature = cols[9].string
pressure = cols[10].string
humidity = cols[11].string
aux_o3 = cols[12].string

# Remove the additional content such as units of measure etc.
# Yes, this is quick and dirty
climbing = remove_trailing_content(
source_string=climbing, trailing_content=" m/s"
)
altitude_m = remove_trailing_content(
source_string=altitude_m, trailing_content=" m"
)
aux_o3 = remove_trailing_content(
source_string=aux_o3, trailing_content=" mPa"
)
course_deg = remove_trailing_content(
source_string=course_deg, trailing_content=" °"
)
humidity = remove_trailing_content(
source_string=humidity, trailing_content=" %"
)
pressure = remove_trailing_content(
source_string=pressure, trailing_content=" hPa"
)
speed_kmh = remove_trailing_content(
source_string=speed_kmh,
trailing_content=" km/h",
)
temperature = remove_trailing_content(
source_string=temperature,
trailing_content=" °C",
)
latitude = remove_trailing_content(
source_string=latitude, trailing_content=" φ"
)
longitude = remove_trailing_content(
source_string=longitude, trailing_content=" λ"
)
else:
# We were unable to access the dynamic PHP data - return an error to the user
success = False

radiosondy_response = {
"launch_site": launch_site,
"probe_type": probe_type,
Expand Down

0 comments on commit b7e91c4

Please sign in to comment.