PIPELINE-1860 Added Belize normalization pipeline

packages/libs/utils/utils/convert.py

@@ -2,6 +2,10 @@ def to_float(value):
     return float(value) if value is not None else None
 
 
+def to_string(value):
+    return f"{value}".strip() if value is not None else None
+
+
 def dms_to_float(dms_value: str):
     """
     Convert Degrees, Minutes, Seconds coordinate to float value

packages/pipe-vms-ingestion/common/transforms/calculate_implied_course.py

@@ -0,0 +1,23 @@
+import math
+
+
+def calculate_implied_course(lat1, lon1, lat2, lon2):
+    if None in [lat1, lon1, lat2, lon2]:
+        return None
+    # Calculates forward azimuth - course along a great circle
+    # from a to b
+    alat = math.radians(lat1)
+    alon = math.radians(lon1)
+    blat = math.radians(lat2)
+    blon = math.radians(lon2)
+    dlon = blon - alon
+    return (
+        360
+        + math.degrees(
+            math.atan2(
+                math.sin(dlon) * math.cos(blat),
+                math.cos(alat) * math.sin(blat)
+                - math.sin(alat) * math.cos(blat) * math.cos(dlon),
+            )
+        )
+    ) % 360

packages/pipe-vms-ingestion/common/transforms/calculate_implied_speed.py

@@ -0,0 +1,11 @@
+from haversine import Unit, haversine
+
+
+def calculate_implied_speed_kt(ts1, lat1, lon1, ts2, lat2, lon2):
+    if None in [ts1, lat1, lon1, ts2, lat2, lon2]:
+        return None
+    prev = (lat1, lon1)
+    current = (lat2, lon2)
+    distance_kt = abs(haversine(current, prev, unit=Unit.NAUTICAL_MILES))
+    timedelta_h = abs((ts2 - ts1).total_seconds() / (60 * 60))
+    return 0 if timedelta_h == 0 else (distance_kt / timedelta_h)

packages/pipe-vms-ingestion/tests/vms_ingestion/common/transforms/data/implied_speed_course.py

@@ -0,0 +1,269 @@
+import datetime
+
+MESSAGES_WITH_IMPLIED_SPEED_AND_COURSE = [
+    # From Belize id 34 (mmsi: 312891000)
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 0, 1, 22, tzinfo=datetime.timezone.utc),
+        16.252783,
+        -16.671667,
+        174.72,
+        4.68,
+        None,
+        None,
+        None,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 1, 1, 22, tzinfo=datetime.timezone.utc),
+        16.172967,
+        -16.672,
+        180.23,
+        4.77,
+        datetime.datetime(2023, 12, 27, 0, 1, 22, tzinfo=datetime.timezone.utc),
+        16.252783,
+        -16.671667,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 2, 1, 22, tzinfo=datetime.timezone.utc),
+        16.092283,
+        -16.68325,
+        187.63,
+        4.86,
+        datetime.datetime(2023, 12, 27, 1, 1, 22, tzinfo=datetime.timezone.utc),
+        16.172967,
+        -16.672,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 3, 1, 22, tzinfo=datetime.timezone.utc),
+        16.078967,
+        -16.7444,
+        257.24,
+        3.62,
+        datetime.datetime(2023, 12, 27, 2, 1, 22, tzinfo=datetime.timezone.utc),
+        16.092283,
+        -16.68325,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 4, 1, 22, tzinfo=datetime.timezone.utc),
+        16.094583,
+        -16.682083,
+        75.37,
+        3.72,
+        datetime.datetime(2023, 12, 27, 3, 1, 22, tzinfo=datetime.timezone.utc),
+        16.078967,
+        -16.7444,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 5, 1, 22, tzinfo=datetime.timezone.utc),
+        16.1258,
+        -16.6983,
+        333.48,
+        2.09,
+        datetime.datetime(2023, 12, 27, 4, 1, 22, tzinfo=datetime.timezone.utc),
+        16.094583,
+        -16.682083,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 6, 1, 22, tzinfo=datetime.timezone.utc),
+        16.067667,
+        -16.705017,
+        186.34,
+        3.5,
+        datetime.datetime(2023, 12, 27, 5, 1, 22, tzinfo=datetime.timezone.utc),
+        16.1258,
+        -16.6983,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 7, 1, 22, tzinfo=datetime.timezone.utc),
+        16.068,
+        -16.78825,
+        270.25,
+        4.81,
+        datetime.datetime(2023, 12, 27, 6, 1, 22, tzinfo=datetime.timezone.utc),
+        16.067667,
+        -16.705017,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 8, 1, 22, tzinfo=datetime.timezone.utc),
+        16.119367,
+        -16.827733,
+        323.56,
+        3.82,
+        datetime.datetime(2023, 12, 27, 7, 1, 22, tzinfo=datetime.timezone.utc),
+        16.068,
+        -16.78825,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 9, 1, 22, tzinfo=datetime.timezone.utc),
+        16.105217,
+        -16.796133,
+        114.99,
+        2.01,
+        datetime.datetime(2023, 12, 27, 8, 1, 22, tzinfo=datetime.timezone.utc),
+        16.119367,
+        -16.827733,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 10, 1, 22, tzinfo=datetime.timezone.utc),
+        16.1781,
+        -16.793033,
+        2.34,
+        4.36,
+        datetime.datetime(2023, 12, 27, 9, 1, 22, tzinfo=datetime.timezone.utc),
+        16.105217,
+        -16.796133,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 11, 1, 22, tzinfo=datetime.timezone.utc),
+        16.131883,
+        -16.8035,
+        192.27,
+        2.83,
+        datetime.datetime(2023, 12, 27, 10, 1, 22, tzinfo=datetime.timezone.utc),
+        16.1781,
+        -16.793033,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 12, 1, 22, tzinfo=datetime.timezone.utc),
+        16.162917,
+        -16.8503,
+        304.63,
+        3.28,
+        datetime.datetime(2023, 12, 27, 11, 1, 22, tzinfo=datetime.timezone.utc),
+        16.131883,
+        -16.8035,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 13, 1, 22, tzinfo=datetime.timezone.utc),
+        16.093383,
+        -16.731867,
+        121.42,
+        8.0,
+        datetime.datetime(2023, 12, 27, 12, 1, 22, tzinfo=datetime.timezone.utc),
+        16.162917,
+        -16.8503,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 14, 1, 22, tzinfo=datetime.timezone.utc),
+        16.119217,
+        -16.735867,
+        351.54,
+        1.56,
+        datetime.datetime(2023, 12, 27, 13, 1, 22, tzinfo=datetime.timezone.utc),
+        16.093383,
+        -16.731867,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 15, 1, 22, tzinfo=datetime.timezone.utc),
+        16.150217,
+        -16.749867,
+        336.55,
+        2.02,
+        datetime.datetime(2023, 12, 27, 14, 1, 22, tzinfo=datetime.timezone.utc),
+        16.119217,
+        -16.735867,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 16, 1, 22, tzinfo=datetime.timezone.utc),
+        16.162617,
+        -16.744583,
+        22.26,
+        0.8,
+        datetime.datetime(2023, 12, 27, 15, 1, 22, tzinfo=datetime.timezone.utc),
+        16.150217,
+        -16.749867,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 17, 1, 22, tzinfo=datetime.timezone.utc),
+        16.084217,
+        -16.744283,
+        179.79,
+        4.68,
+        datetime.datetime(2023, 12, 27, 16, 1, 22, tzinfo=datetime.timezone.utc),
+        16.162617,
+        -16.744583,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 18, 1, 22, tzinfo=datetime.timezone.utc),
+        16.1046,
+        -16.721633,
+        46.87,
+        1.79,
+        datetime.datetime(2023, 12, 27, 17, 1, 22, tzinfo=datetime.timezone.utc),
+        16.084217,
+        -16.744283,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 19, 1, 22, tzinfo=datetime.timezone.utc),
+        16.181833,
+        -16.736117,
+        349.79,
+        4.69,
+        datetime.datetime(2023, 12, 27, 18, 1, 22, tzinfo=datetime.timezone.utc),
+        16.1046,
+        -16.721633,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 20, 1, 22, tzinfo=datetime.timezone.utc),
+        16.253017,
+        -16.697233,
+        27.67,
+        4.81,
+        datetime.datetime(2023, 12, 27, 19, 1, 22, tzinfo=datetime.timezone.utc),
+        16.181833,
+        -16.736117,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 21, 1, 22, tzinfo=datetime.timezone.utc),
+        16.3347,
+        -16.67925,
+        11.93,
+        4.99,
+        datetime.datetime(2023, 12, 27, 20, 1, 22, tzinfo=datetime.timezone.utc),
+        16.253017,
+        -16.697233,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 22, 1, 22, tzinfo=datetime.timezone.utc),
+        16.274333,
+        -16.672617,
+        173.98,
+        3.63,
+        datetime.datetime(2023, 12, 27, 21, 1, 22, tzinfo=datetime.timezone.utc),
+        16.3347,
+        -16.67925,
+    ),
+    (
+        "312891000",
+        datetime.datetime(2023, 12, 27, 23, 1, 22, tzinfo=datetime.timezone.utc),
+        16.194967,
+        -16.677633,
+        183.47,
+        4.75,
+        datetime.datetime(2023, 12, 27, 22, 1, 22, tzinfo=datetime.timezone.utc),
+        16.274333,
+        -16.672617,
+    ),
+]

packages/pipe-vms-ingestion/tests/vms_ingestion/common/transforms/test_calculate_implied_course.py

@@ -0,0 +1,32 @@
+import pytest
+from common.transforms.calculate_implied_course import calculate_implied_course
+from tests.vms_ingestion.common.transforms.data.implied_speed_course import (
+    MESSAGES_WITH_IMPLIED_SPEED_AND_COURSE,
+)
+
+
+class TestCalculateImpliedCourse:
+
+    @pytest.mark.parametrize(
+        "id,timestamp,lat,lon,course,speed,prev_timestamp,prev_lat,prev_lon",
+        MESSAGES_WITH_IMPLIED_SPEED_AND_COURSE,
+    )
+    def test_calculate_implied_course(
+        self, id, timestamp, lat, lon, course, speed, prev_timestamp, prev_lat, prev_lon
+    ):
+        implied_course = calculate_implied_course(prev_lat, prev_lon, lat, lon)
+
+        course_is_not_defined = (
+            implied_course is None
+            and prev_timestamp is None
+            and prev_lat is None
+            and prev_lon is None
+        )
+        # when course is defined
+        # calculated course matches approximatelly (1% tolerance) the expected course
+        # near the 360º calculated course matches approximatelly (1% tolerance) the expected course
+        assert (
+            course_is_not_defined
+            or implied_course == pytest.approx(course, abs=360 * 0.01)
+            or abs(360 - implied_course) == pytest.approx(course, abs=360 * 0.01)
+        )

packages/pipe-vms-ingestion/tests/vms_ingestion/common/transforms/test_calculate_implied_speed.py

@@ -0,0 +1,27 @@
+import pytest
+from common.transforms.calculate_implied_speed import calculate_implied_speed_kt
+from tests.vms_ingestion.common.transforms.data.implied_speed_course import (
+    MESSAGES_WITH_IMPLIED_SPEED_AND_COURSE,
+)
+
+
+class TestCalculateImpliedSpeed:
+
+    @pytest.mark.parametrize(
+        "id,timestamp,lat,lon,course,speed,prev_timestamp,prev_lat,prev_lon",
+        MESSAGES_WITH_IMPLIED_SPEED_AND_COURSE,
+    )
+    def test_calculate_implied_speed_kt(
+        self, id, timestamp, lat, lon, course, speed, prev_timestamp, prev_lat, prev_lon
+    ):
+        implied_speed = calculate_implied_speed_kt(
+            prev_timestamp, prev_lat, prev_lon, timestamp, lat, lon
+        )
+
+        speed_is_not_defined = (
+            implied_speed is None
+            and prev_timestamp is None
+            and prev_lat is None
+            and prev_lon is None
+        )
+        assert speed_is_not_defined or implied_speed == pytest.approx(speed, abs=0.1)