Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gtc 3025/filter alerts by forest #170

Merged
merged 9 commits into from
Nov 7, 2024
2 changes: 1 addition & 1 deletion app/routes/titiler/algorithms/alerts.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class Alerts(BaseAlgorithm):
title: str = "Deforestation Alerts"
description: str = "Decode and visualize alerts"

conf_colors: OrderedDict = None
conf_colors: Optional[OrderedDict] = None

record_start_date: str = "2014-12-31"

Expand Down
41 changes: 41 additions & 0 deletions app/routes/titiler/algorithms/dist_alerts.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
from collections import OrderedDict
from typing import Optional

from pydantic import ConfigDict
from rio_tiler.models import ImageData

from app.models.enumerators.titiler import AlertConfidence

Expand All @@ -9,6 +13,7 @@ class DISTAlerts(Alerts):
title: str = "Land Disturbance (DIST) Alerts"
description: str = "Decode and visualize DIST alerts"

model_config = ConfigDict(arbitrary_types_allowed=True)
conf_colors: OrderedDict = OrderedDict(
{
AlertConfidence.low: AlertConfig(
Expand All @@ -21,3 +26,39 @@ class DISTAlerts(Alerts):
)

record_start_date: str = "2020-12-31"

tree_cover_density_mask: Optional[int] = None
tree_cover_density_data: Optional[ImageData] = None

tree_cover_height_mask: Optional[int] = None
tree_cover_height_data: Optional[ImageData] = None

tree_cover_loss_mask: Optional[int] = None
tree_cover_loss_data: Optional[ImageData] = None

def create_mask(self):
mask = super().create_mask()

if self.tree_cover_density_mask:
mask *= (
self.tree_cover_density_data.array[0, :, :]
>= self.tree_cover_density_mask
)

if self.tree_cover_height_mask:
mask *= (
self.tree_cover_height_data.array[0, :, :]
>= self.tree_cover_height_mask
)

if self.tree_cover_loss_mask:
# Tree cover loss before 2020 can't be used to filter out pixels as not forest
# if they had tree cover loss. Instead we use tree cover height taken that year
# is used as source of truth.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the informative comment, Solomon. 💪

mask *= (
(self.tree_cover_loss_data.array[0, :, :] >= self.tree_cover_loss_mask)
| (self.tree_cover_loss_data.array[0, :, :] == 0)
| (self.tree_cover_loss_data.array[0, :, :] <= 2020)
)

return mask
47 changes: 43 additions & 4 deletions app/routes/titiler/umd_glad_dist_alerts.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from dateutil.relativedelta import relativedelta
from fastapi import APIRouter, Depends, Query, Response
from rio_tiler.io import COGReader
from titiler.core.resources.enums import ImageType
from titiler.core.utils import render_image

Expand Down Expand Up @@ -49,24 +50,62 @@ async def glad_dist_alerts_raster_tile(
AlertConfidence.low,
description="Show alerts that are at least of this confidence level",
),
tree_cover_density: Optional[int] = Query(
None,
ge=0,
le=100,
description="Alerts in pixels with tree cover density (in percent) below this threshold won't be displayed. `umd_tree_cover_density_2010` is used for this masking.",
),
tree_cover_height: Optional[int] = Query(
None,
description="Alerts in pixels with tree cover height (in meters) below this threshold won't be displayed. `umd_tree_cover_height_2020` dataset in the API is used for this masking.",
),
tree_cover_loss_cutoff: bool = Query(
False,
ge=2021,
description="""This filter is to be used on conjunction with `tree_cover_density` and `tree_cover_height` filters to detect only alerts in forests. """
"""Alerts for pixels that have had tree cover loss this year or earlier (to 2021) won't be displayed.""",
),
) -> Response:
"""UMD GLAD DIST alerts raster tiles."""

tile_x, tile_y, zoom = xyz
bands = ["default", "intensity"]
folder: str = f"s3://{DATA_LAKE_BUCKET}/{dataset}/{version}/raster/epsg-4326/cog"
with AlertsReader(input=folder) as reader:
tile_x, tile_y, zoom = xyz

# NOTE: the bands in the output `image_data` array will be in the order of
# the input `bands` list
image_data = reader.tile(tile_x, tile_y, zoom, bands=bands)

processed_image = DISTAlerts(
dist_alert = DISTAlerts(
start_date=start_date,
end_date=end_date,
render_type=render_type,
alert_confidence=alert_confidence,
)(image_data)
tree_cover_density_mask=tree_cover_density,
tree_cover_height_mask=tree_cover_height,
tree_cover_loss_mask=tree_cover_loss_cutoff,
)

if tree_cover_density:
with COGReader(
f"s3://{DATA_LAKE_BUCKET}/umd_tree_cover_density_2010/v1.6/raster/epsg-4326/cog/default.tif"
) as reader:
dist_alert.tree_cover_density_data = reader.tile(tile_x, tile_y, zoom)

if tree_cover_height:
with COGReader(
f"s3://{DATA_LAKE_BUCKET}/umd_tree_cover_height_2020/v2022/raster/epsg-4326/cog/default.tif"
) as reader:
dist_alert.tree_cover_height_data = reader.tile(tile_x, tile_y, zoom)

if tree_cover_loss_cutoff:
with COGReader(
f"s3://{DATA_LAKE_BUCKET}/umd_tree_cover_loss/v1.10.1/raster/epsg-4326/cog/default.tif"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Put in a TODO here to fix the version name (to v1.11) once we go from staging to production?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How often will we need to change the code? Should this be a config variable somewhere?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TCL is updated once a year I believe, so not that frequent but still good idea to not hardcode it here.

Copy link
Member Author

@solomon-negusse solomon-negusse Nov 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@gtempus: I hit what's potentially a terraform bug setting an env variable that's a json string with the dataset versions - same variable that works fine with the docker-compose file. So, for now I have moved out the dataset versions into the global config file where we'll have to update it (693e0c2) as interim solution and opened an issue in terraform aws provider that I hope will get sorted out.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a good interim solution, @solomon-negusse. Thanks!
It's much better than having to dig into the codebase. 😄
Bummer about the terraform bug. 😦

) as reader:
dist_alert.tree_cover_loss_data = reader.tile(tile_x, tile_y, zoom)

processed_image = dist_alert(image_data)

content, media_type = render_image(
processed_image,
Expand Down
75 changes: 75 additions & 0 deletions tests/titiler/test_alerts_algo.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from rio_tiler.models import ImageData

from app.models.enumerators.titiler import IntegratedAlertConfidence, RenderType
from app.routes.titiler.algorithms.dist_alerts import DISTAlerts
from app.routes.titiler.algorithms.integrated_alerts import IntegratedAlerts
from tests.conftest import DATE_CONF_TIF, INTENSITY_TIF

Expand All @@ -24,6 +25,50 @@ def get_tile_data():
return ImageData(data)


def get_tcl_data():
"""Tree Cover Loss test data."""
with rasterio.open(DATE_CONF_TIF) as date_conf_file:
date_conf = date_conf_file.read(1)

data = np.zeros_like(date_conf)

data[122, 109] = 2019
data[120, 109] = 2022
data[154, 71] = 2023

return ImageData(data)


def get_tch_data():
"""Tree Cover Height test data."""

with rasterio.open(DATE_CONF_TIF) as date_conf_file:
date_conf = date_conf_file.read(1)

data = np.zeros_like(date_conf)

data[122, 109] = 2
data[120, 109] = 5
data[154, 71] = 4

return ImageData(data)


def get_tcd_data():
"""Tree Cover Density test data."""

with rasterio.open(DATE_CONF_TIF) as date_conf_file:
date_conf = date_conf_file.read(1)

data = np.zeros_like(date_conf)

data[122, 109] = 40
data[120, 109] = 30
data[154, 71] = 20

return ImageData(data)


def test_integrated_alerts_defaults():
"""Test default values of the Alerts class."""
alerts = IntegratedAlerts()
Expand Down Expand Up @@ -104,3 +149,33 @@ def test_encoded_rgba():

# test high confidence in alpha channel
assert rgba.array[3, 154, 71] == 8


def test_forest_mask():
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for continuing to improve the test coverage, @solomon-negusse ! ❤️

"""Test that only alerts that match forest criteria are shown."""

alerts_data = get_tile_data()
tcl = get_tcl_data()
tch = get_tch_data()
tcd = get_tcd_data()

alerts = DISTAlerts(
tree_cover_density_mask=30,
tree_cover_height_mask=3,
tree_cover_loss_mask=2021,
render_type=RenderType.true_color,
)

alerts.tree_cover_density_data = tcd
alerts.tree_cover_height_data = tch
alerts.tree_cover_loss_data = tcl

rgba = alerts(alerts_data)

np.testing.assert_array_equal(rgba.array[:, 122, 109], np.array([220, 102, 153, 0]))

np.testing.assert_array_equal(rgba.array[:, 154, 71], np.array([220, 102, 153, 0]))

np.testing.assert_array_equal(
rgba.array[:, 120, 109], np.array([220, 102, 153, 255])
)
Loading