Skip to content

Commit

Permalink
PERF: maintain root when signing ItemCollections (#30)
Browse files Browse the repository at this point in the history
Losing the root collection was causing `to_dict()` on signed `ItemCollections` to be extremely slow, like stac-utils/pystac#546. By preserving the root catalog stac-utils/pystac-client#72, it's much faster.
  • Loading branch information
gjoseph92 authored Oct 29, 2021
1 parent ab740f1 commit d7fd87e
Show file tree
Hide file tree
Showing 6 changed files with 72 additions and 18 deletions.
11 changes: 5 additions & 6 deletions planetary_computer/sas.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,12 +179,11 @@ def sign_item_collection(item_collection: ItemCollection) -> ItemCollection:
a "msft:expiry" property is added to the Item properties indicating the
earliest expiry time for any assets that were signed.
"""
return ItemCollection.from_dict(
{
"type": "FeatureCollection",
"features": [sign(item).to_dict() for item in item_collection],
}
)
new = item_collection.clone()
for item in new:
for key in item.assets:
_sign_asset_in_place(item.assets[key])
return new


@sign.register(ItemSearch)
Expand Down
29 changes: 29 additions & 0 deletions tests/data-files/catalog.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"type": "Catalog",
"id": "microsoft-pc-demo",
"stac_version": "1.0.0",
"description": "Microsoft Planetary Computer STAC API",
"links": [
{
"rel": "root",
"href": "./catalog.json",
"type": "application/json"
},
{
"rel": "item",
"href": "./sample-item.json",
"type": "application/json"
},
{
"rel": "item",
"href": "./sample-tabular-item.json",
"type": "application/json"
},
{
"rel": "item",
"href": "./sample-zarr-item.json",
"type": "application/json"
}
],
"stac_extensions": []
}
8 changes: 7 additions & 1 deletion tests/data-files/sample-item.json
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,13 @@
"roles": ["logo"]
}
},
"links": [],
"links": [
{
"rel": "root",
"href": "./catalog.json",
"type": "application/json"
}
],
"stac_extensions":[
"eo",
"projection"
Expand Down
8 changes: 7 additions & 1 deletion tests/data-files/sample-tabular-item.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,13 @@
"datetime": "2020-06-01T00:00:00Z"
},
"geometry": null,
"links": [],
"links": [
{
"rel": "root",
"href": "./catalog.json",
"type": "application/json"
}
],
"assets": {
"data": {
"href": "abfs://cpdata/raw/fia/plot.parquet",
Expand Down
8 changes: 7 additions & 1 deletion tests/data-files/sample-zarr-item.json
Original file line number Diff line number Diff line change
Expand Up @@ -588,7 +588,13 @@
]
]
},
"links": [],
"links": [
{
"rel": "root",
"href": "./catalog.json",
"type": "application/json"
}
],
"assets": {
"zarr-https": {
"href": "https://ai4edataeuwest.blob.core.windows.net/gridmet/gridmet.zarr",
Expand Down
26 changes: 17 additions & 9 deletions tests/test_signing.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
import os
import json
from typing import Any, Dict
import unittest
from urllib.parse import parse_qs, urlparse
from pathlib import Path
Expand Down Expand Up @@ -35,31 +33,37 @@
HERE = Path(__file__).parent


def get_sample_item_dict() -> Dict[str, Any]:
file_path = HERE.joinpath("data-files/sample-item.json").absolute()
with open(file_path) as json_file:
return json.load(json_file)
def resolve(item: Item) -> Item:
item.resolve_links()
return item


def get_sample_item() -> Item:
return Item.from_dict(get_sample_item_dict())
file_path = os.fspath(HERE.joinpath("data-files/sample-item.json"))
return resolve(Item.from_file(file_path))


def get_sample_zarr_item() -> Item:
file_path = os.fspath(HERE.joinpath("data-files/sample-zarr-item.json"))
return Item.from_file(file_path)
return resolve(Item.from_file(file_path))


def get_sample_tabular_item() -> Item:
file_path = os.fspath(HERE.joinpath("data-files/sample-tabular-item.json"))
return Item.from_file(file_path)
return resolve(Item.from_file(file_path))


def get_sample_item_collection() -> ItemCollection:
return ItemCollection([get_sample_item()])


class TestSigning(unittest.TestCase):
def assertRootResolved(self, item: Item) -> None:
root_link = item.get_root_link()
self.assertIsNotNone(root_link)
assert root_link # for type checker
self.assertTrue(root_link.is_resolved())

def assertSigned(self, url: str) -> None:
# Ensure the signed item has an "se" URL parameter added to it,
# which indicates it has been signed
Expand Down Expand Up @@ -96,6 +100,7 @@ def test_signed_assets(self) -> None:
signed_item = pc.sign(get_sample_item())
self.verify_signed_urls_in_item(signed_item)
self.verify_asset_owner(signed_item)
self.assertRootResolved(signed_item)

def test_read_signed_asset(self) -> None:
signed_href = pc.sign(SENTINEL_THUMBNAIL)
Expand All @@ -107,6 +112,7 @@ def test_signed_item_collection(self) -> None:
self.assertEqual(len(list(signed_item_collection)), 1)
for signed_item in signed_item_collection:
self.verify_signed_urls_in_item(signed_item)
self.assertRootResolved(signed_item)

def test_search_and_sign(self) -> None:
# Filter out a resource warning coming from within the pystac-client search
Expand Down Expand Up @@ -158,13 +164,15 @@ def test_sign_zarr_item(self) -> None:
"credential",
result.assets["zarr-abfs"].extra_fields["xarray:storage_options"],
)
self.assertRootResolved(item)

def test_sign_tabular_item(self) -> None:
item = get_sample_tabular_item()
result = pc.sign(item)
self.assertIn(
"credential", result.assets["data"].extra_fields["table:storage_options"]
)
self.assertRootResolved(item)


class TestUtils(unittest.TestCase):
Expand Down

0 comments on commit d7fd87e

Please sign in to comment.