Skip to content

Commit

Permalink
Issue #134 drop 0.4.0 support in job result/asset handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@soxofaan
soxofaan committed Mar 7, 2023
1 parent d7e8dbf commit c8c80c2
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 100 deletions.
10 changes: 4 additions & 6 deletions openeo/rest/job.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,12 +380,10 @@ def get_assets(self) -> List[ResultAsset]:
"""
# TODO: add arguments to filter on metadata, e.g. to only get assets of type "image/tiff"
metadata = self.get_metadata()
if "assets" in metadata:
# API 1.0 style: dictionary mapping filenames to metadata dict (with at least a "href" field)
assets = metadata["assets"]
else:
# Best effort translation of on old style to "assets" style (#134)
assets = {a["href"].split("/")[-1]: a for a in metadata["links"]}
# API 1.0 style: dictionary mapping filenames to metadata dict (with at least a "href" field)
assets = metadata.get("assets", {})
if not assets:
logger.warning("No assets found in job result metadata.")
return [
ResultAsset(job=self._job, name=name, href=asset["href"], metadata=asset)
for name, asset in assets.items()
Expand Down
118 changes: 48 additions & 70 deletions tests/rest/test_job.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,6 @@
TIFF_CONTENT = b'T1f7D6t6l0l' * 1000


@pytest.fixture
def session040(requests_mock):
# TODO #134 eliminate 0.4.0 support
requests_mock.get(API_URL + "/", json={
"api_version": "0.4.0",
"endpoints": [{"path": "/credentials/basic", "methods": ["GET"]}]
})
session = openeo.connect(API_URL)
return session


@pytest.fixture
def con100(requests_mock):
Expand Down Expand Up @@ -59,16 +49,27 @@ def test_execute_batch(con100, requests_mock, tmpdir):
requests_mock.get(API_URL + "/collections/SENTINEL2", json={"foo": "bar"})
requests_mock.post(API_URL + "/jobs", status_code=201, headers={"OpenEO-Identifier": "f00ba5"})
requests_mock.post(API_URL + "/jobs/f00ba5/results", status_code=202)
requests_mock.get(API_URL + "/jobs/f00ba5", [
{'json': {"status": "submitted"}},
{'json': {"status": "queued"}},
{'json': {"status": "running", "progress": 15}},
{'json': {"status": "running", "progress": 80}},
{'json': {"status": "finished", "progress": 100}},
])
requests_mock.get(API_URL + "/jobs/f00ba5/results", json={
"links": [{"href": API_URL + "/jobs/f00ba5/files/output.tiff"}]
})
requests_mock.get(
API_URL + "/jobs/f00ba5",
[
{"json": {"status": "submitted"}},
{"json": {"status": "queued"}},
{"json": {"status": "running", "progress": 15}},
{"json": {"status": "running", "progress": 80}},
{"json": {"status": "finished", "progress": 100}},
],
)
requests_mock.get(
API_URL + "/jobs/f00ba5/results",
json={
"assets": {
"output.tiff": {
"href": API_URL + "/jobs/f00ba5/files/output.tiff",
"type": "image/tiff; application=geotiff",
},
}
},
)
requests_mock.get(API_URL + "/jobs/f00ba5/files/output.tiff", text="tiffdata")
requests_mock.get(API_URL + "/jobs/f00ba5/logs", json={'logs': []})

Expand Down Expand Up @@ -174,16 +175,27 @@ def test_execute_batch_with_soft_errors(con100, requests_mock, tmpdir, error_res
requests_mock.get(API_URL + "/collections/SENTINEL2", json={"foo": "bar"})
requests_mock.post(API_URL + "/jobs", status_code=201, headers={"OpenEO-Identifier": "f00ba5"})
requests_mock.post(API_URL + "/jobs/f00ba5/results", status_code=202)
requests_mock.get(API_URL + "/jobs/f00ba5", [
{'json': {"status": "queued"}},
{'json': {"status": "running", "progress": 15}},
error_response,
{'json': {"status": "running", "progress": 80}},
{'json': {"status": "finished", "progress": 100}},
])
requests_mock.get(API_URL + "/jobs/f00ba5/results", json={
"links": [{"href": API_URL + "/jobs/f00ba5/files/output.tiff"}]
})
requests_mock.get(
API_URL + "/jobs/f00ba5",
[
{"json": {"status": "queued"}},
{"json": {"status": "running", "progress": 15}},
error_response,
{"json": {"status": "running", "progress": 80}},
{"json": {"status": "finished", "progress": 100}},
],
)
requests_mock.get(
API_URL + "/jobs/f00ba5/results",
json={
"assets": {
"output.tiff": {
"href": API_URL + "/jobs/f00ba5/files/output.tiff",
"type": "image/tiff; application=geotiff",
},
}
},
)
requests_mock.get(API_URL + "/jobs/f00ba5/files/output.tiff", text="tiffdata")
requests_mock.get(API_URL + "/jobs/f00ba5/logs", json={'logs': []})

Expand Down Expand Up @@ -263,7 +275,7 @@ def test_execute_batch_with_excessive_soft_errors(con100, requests_mock, tmpdir,
]


def test_get_job_logs(session040, requests_mock):
def test_get_job_logs(con100, requests_mock):
requests_mock.get(API_URL + "/jobs/f00ba5/logs", json={
'logs': [{
'id': "123abc",
Expand All @@ -272,13 +284,12 @@ def test_get_job_logs(session040, requests_mock):
}]
})

log_entries = session040.job('f00ba5').logs(offset="123abc")
log_entries = con100.job("f00ba5").logs(offset="123abc")

assert log_entries[0].message == "error processing batch job"


# TODO: do we keep testing on sesssion040, or should we switch to con100?
def test_get_job_logs_returns_debug_loglevel_by_default(session040, requests_mock):
def test_get_job_logs_returns_debug_loglevel_by_default(con100, requests_mock):
requests_mock.get(
API_URL + "/jobs/f00ba5/logs",
json={
Expand Down Expand Up @@ -307,7 +318,7 @@ def test_get_job_logs_returns_debug_loglevel_by_default(session040, requests_moc
},
)

log_entries = session040.job("f00ba5").logs()
log_entries = con100.job("f00ba5").logs()

assert len(log_entries) == 4
assert log_entries[0].level == "error"
Expand Down Expand Up @@ -335,7 +346,7 @@ def test_get_job_logs_returns_debug_loglevel_by_default(session040, requests_moc
],
)
def test_get_job_logs_keeps_loglevel_that_is_higher_or_equal(
session040, requests_mock, log_level, exp_num_messages
con100, requests_mock, log_level, exp_num_messages
):
requests_mock.get(
API_URL + "/jobs/f00ba5/logs",
Expand Down Expand Up @@ -365,7 +376,7 @@ def test_get_job_logs_keeps_loglevel_that_is_higher_or_equal(
},
)

log_entries = session040.job("f00ba5").logs(log_level=log_level)
log_entries = con100.job("f00ba5").logs(log_level=log_level)
assert len(log_entries) == exp_num_messages


Expand All @@ -384,19 +395,6 @@ def check_request(request):
con100.create_job({"foo1": {"process_id": "foo"}}, title="Foo", description="just testing")


def test_download_result_040(session040, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results", json={"links": [
{"href": API_URL + "/dl/jjr1.tiff"},
]})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
job = BatchJob("jj", connection=session040)
assert job.list_results() == {'links': [{'href': 'https://oeo.test/dl/jjr1.tiff'}]}
target = tmp_path / "result.tiff"
res = job.download_result(target)
assert res == target
with target.open("rb") as f:
assert f.read() == TIFF_CONTENT


@pytest.fixture
def job_with_1_asset(con100, requests_mock, tmp_path) -> BatchJob:
Expand Down Expand Up @@ -496,26 +494,6 @@ def test_get_results_multiple_download_single_by_wrong_name(job_with_2_assets: B
job.get_results().download_file(target, name="foobar.tiff")


def test_download_results_040(session040, requests_mock, tmp_path):
requests_mock.get(API_URL + "/jobs/jj/results", json={"links": [
{"href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff"},
{"href": API_URL + "/dl/jjr2.tiff", "type": "image/tiff"},
]})
requests_mock.get(API_URL + "/dl/jjr1.tiff", content=TIFF_CONTENT)
requests_mock.get(API_URL + "/dl/jjr2.tiff", content=TIFF_CONTENT)
job = BatchJob("jj", connection=session040)
target = tmp_path / "folder"
target.mkdir()
downloads = job.download_results(target)
assert downloads == {
target / "jjr1.tiff": {"href": API_URL + "/dl/jjr1.tiff", "type": "image/tiff"},
target / "jjr2.tiff": {"href": API_URL + "/dl/jjr2.tiff", "type": "image/tiff"},
}
assert set(p.name for p in target.iterdir()) == {"jjr1.tiff", "jjr2.tiff"}
with (target / "jjr1.tiff").open("rb") as f:
assert f.read() == TIFF_CONTENT
with (target / "jjr2.tiff").open("rb") as f:
assert f.read() == TIFF_CONTENT


def test_download_results(job_with_2_assets: BatchJob, tmp_path):
Expand Down
24 changes: 0 additions & 24 deletions tests/rest/test_job_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,6 @@
API_URL = "https://oeo.net"


@pytest.fixture
def session040(requests_mock):
# TODO #134 eliminate 0.4.0 support
requests_mock.get(API_URL + "/", json={"api_version": "0.4.0"})
session = openeo.connect(API_URL)
return session


@pytest.fixture
def con100(requests_mock):
Expand All @@ -28,14 +21,6 @@ def test_load_json(con100, requests_mock):
assert json == {'2018-08-03T00:00:00Z': [[], [], [None, None]]}


def test_load_json_040(session040, requests_mock):
requests_mock.get(API_URL + '/jobs/1/results', json={'links': [{'href': API_URL + '/jobs/1/results/out'}]})
requests_mock.get(API_URL + '/jobs/1/results/out', json={'2018-08-03T00:00:00Z': [[], [], [None, None]]})

json = session040.job('1').get_result().load_json()

assert json == {'2018-08-03T00:00:00Z': [[], [], [None, None]]}


def test_load_bytes(con100, requests_mock):
requests_mock.get(API_URL + '/jobs/1/results', json={'assets': {'out': {'href': API_URL + '/jobs/1/results/out'}}})
Expand All @@ -44,12 +29,3 @@ def test_load_bytes(con100, requests_mock):
content = con100.job('1').get_result().load_bytes()

assert content == b'\x01\x02\x03\x04\x05'


def test_load_bytes_040(session040, requests_mock):
requests_mock.get(API_URL + '/jobs/1/results', json={'links': [{'href': API_URL + '/jobs/1/results/out'}]})
requests_mock.get(API_URL + '/jobs/1/results/out', content=b'\x01\x02\x03\x04\x05')

content = session040.job('1').get_result().load_bytes()

assert content == b'\x01\x02\x03\x04\x05'

0 comments on commit c8c80c2

Please sign in to comment.