Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replace send_job with create_job #283

Merged
merged 1 commit into from
Mar 17, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Drop `ImageCollection` from `DataCube`'s class hierarchy.
- This practically removes very old (pre-0.4.0) methods like `date_range_filter` and `bbox_filter` from `DataCube`.
([#100](https://github.com/Open-EO/openeo-python-client/issues/100), [#278](https://github.com/Open-EO/openeo-python-client/issues/278))
- Deprecate `DataCube.send_job` in favor of `DataCube.create_job` for better consistency (internally and with other libraries) ([#276](https://github.com/Open-EO/openeo-python-client/issues/276))

### Removed

Expand Down
2 changes: 1 addition & 1 deletion docs/cookbook/sampling.rst
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Combining all of this, results in the following sample code::
temporal_extent=["2020-05-01","2020-06-01"]
)
s2_bands = s2_bands.filter_spatial("https://artifactory.vgt.vito.be/testdata-public/parcels/test_10.geojson")
job = s2_bands.send_job(title="Sentinel2", description="Sentinel-2 L2A bands",out_format="netCDF",sample_by_feature=True)
job = s2_bands.create_job(title="Sentinel2", description="Sentinel-2 L2A bands",out_format="netCDF",sample_by_feature=True)

Sampling only works for batch jobs, because it results in multiple output files, which can not be conveniently transferred
in a synchronous call.
Expand Down
2 changes: 1 addition & 1 deletion examples/R_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@

# Test Job

job = pg.send_job()
job = pg.create_job()
print(job.job_id)
print(job.start_job())
print (job.describe_job())
Expand Down
2 changes: 1 addition & 1 deletion examples/eodc_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
composite = timeseries.min_time()
logging.debug(timeseries.to_json(indent=None))

job = timeseries.send_job()
job = timeseries.create_job()
logging.debug("{}".format(job.job_id))

status = job.queue()
Expand Down
2 changes: 1 addition & 1 deletion examples/eurac_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
print(datacube.to_json())

# Submit your process graph as new batch job to back-end
job = datacube.send_job()
job = datacube.create_job()

# Launch processing of submitted batch job
if job.job_id:
Expand Down
2 changes: 1 addition & 1 deletion examples/gee_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@


# Send Job to backend
job = datacube.send_job()
job = datacube.create_job()
print(job.describe_job())

# Wait for job to finish and download
Expand Down
2 changes: 1 addition & 1 deletion examples/gee_uc1_pol.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
print(datacube.to_json())

# Send Job to backend
job = datacube.send_job()
job = datacube.create_job()

res = job.start_and_wait().download_results()
for key, val in res.items():
Expand Down
2 changes: 1 addition & 1 deletion examples/gee_uc1_temp.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
print(datacube.to_json())

# Send Job to backend
job = datacube.send_job()
job = datacube.create_job()
job.start_and_wait().download_results()
#print(job.job_id)
#print(job.start_job())
Expand Down
2 changes: 1 addition & 1 deletion examples/mundialis_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@

# datacube.download("/tmp/testfile.tiff", format="GeoTIFF")

job = datacube.send_job()
job = datacube.create_job()
if job:
print(job.job_id)
print(job.start_job())
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/EODC_Forum_2019/EODC.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"job.start_job()\n",
"job"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/EODC_Forum_2019/EURAC.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"job.start_job()\n",
"job"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/EODC_Forum_2019/GEE.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -734,7 +734,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"job.start_job()\n",
"job"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/EODC_Forum_2019/RBackend.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"job.start_job()\n",
"job"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/EODC_Forum_2019/VITO.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -763,7 +763,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"job.start_job()\n",
"job"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/PoC_EODC.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = timeseries.send_job()\n",
"job = timeseries.create_job()\n",
"\n",
"job"
]
Expand Down
4 changes: 2 additions & 2 deletions examples/notebooks/Sentinel2_NDVI_GEE.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -2117,15 +2117,15 @@
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mJobFailedException\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-10-b5e05ed255c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Sending the job to the backend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatacube\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_and_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mOUTPUT_FILE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mjob\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-10-b5e05ed255c3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Sending the job to the backend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdatacube\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_job\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_and_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload_results\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mOUTPUT_FILE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mjob\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/data/REPO/openeo-python-client/openeo/rest/job.py\u001b[0m in \u001b[0;36mstart_and_wait\u001b[0;34m(self, print, max_poll_interval, connection_retry_interval)\u001b[0m\n\u001b[1;32m 222\u001b[0m raise JobFailedException(\"Batch job {i} didn't finish properly. Status: {s} (after {t}).\".format(\n\u001b[1;32m 223\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjob_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0melapsed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m ), job=self)\n\u001b[0m\u001b[1;32m 225\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mJobFailedException\u001b[0m: Batch job XfUBRlYFgKe3SBvA didn't finish properly. Status: error (after 0:00:11.604513)."
]
}
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"job.start_and_wait().download_results(OUTPUT_FILE)\n",
"job"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/SimpleCompositor.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1455,7 +1455,7 @@
}
],
"source": [
"composite_job = composite.save_result(format='gtiff').send_job()\n",
"composite_job = composite.save_result(format='gtiff').create_job()\n",
"composite_job.start_and_wait().get_results().download_file(\"./composite.tiff\")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/UC1_GEE_Pol.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"results = job.start_and_wait().download_results()\n",
"results"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/UC1_GEE_Temp.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@
],
"source": [
"# Sending the job to the backend\n",
"job = datacube.send_job()\n",
"job = datacube.create_job()\n",
"results = job.start_and_wait().download_results()\n",
"results"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/openEO_Python_Sentinel2_EURAC.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@
"outputs": [],
"source": [
"# submit your process graph as new batch job to back-end\n",
"job = datacube.send_job()"
"job = datacube.create_job()"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/notebooks/openeo-terrascope-webinar.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1201,7 +1201,7 @@
"metadata": {},
"outputs": [],
"source": [
"job = masked_timeseries.send_job()"
"job = masked_timeseries.create_job()"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion examples/vito_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

datacube.download("/tmp/testfile.tiff")

job = datacube.send_job()
job = datacube.create_job()
if job:
print(job.job_id)
print(job.run_synchronous("/tmp/testfile"))
Expand Down
9 changes: 5 additions & 4 deletions openeo/imagecollection.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@

from openeo.rest.job import RESTJob
from openeo.rest.service import Service
from openeo.util import get_temporal_extent, first_not_none, dict_no_none

from openeo.util import get_temporal_extent, first_not_none, dict_no_none, legacy_alias

if hasattr(typing, 'TYPE_CHECKING') and typing.TYPE_CHECKING:
# Imports for type checking only (circular import issue at runtime). `hasattr` is Python 3.5 workaround #210
Expand Down Expand Up @@ -539,7 +538,7 @@ def execute_batch(
This method is mostly recommended if the batch job is expected to run in a reasonable amount of time.

For very long running jobs, you probably do not want to keep the client running. In that case, using
:func:`~openeo.imagecollection.ImageCollection.send_job` might be more appropriate.
:func:`~openeo.imagecollection.ImageCollection.create_job` might be more appropriate.

:param job_options: A dictionary containing (custom) job options
:param outputfile: The path of a file to which a result can be written
Expand All @@ -549,7 +548,7 @@ def execute_batch(
"""
pass

def send_job(self, out_format:str=None, job_options:Dict=None, **format_options) -> RESTJob:
def create_job(self, out_format:str=None, job_options:Dict=None, **format_options) -> RESTJob:
"""
Sends a job to the backend and returns a RESTJob instance. The job will still need to be started and managed explicitly.
The :func:`~openeo.imagecollection.ImageCollection.execute_batch` method allows you to run batch jobs without managing it.
Expand All @@ -561,6 +560,8 @@ def send_job(self, out_format:str=None, job_options:Dict=None, **format_options)
"""
pass

send_job = legacy_alias(create_job, name="send_job")

def pipe(self, func: Callable, *args, **kwargs):
"""
Pipe the image collection through a function and return the result.
Expand Down
1 change: 0 additions & 1 deletion openeo/rest/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -1048,7 +1048,6 @@ def create_job(
:return: job_id: String Job id of the new created job
"""
# TODO move all this (RESTJob factory) logic to RESTJob?
# TODO: unify Connection.create_job vs DataCube.send_job. #276
req = self._build_request_with_process_graph(
process_graph=process_graph,
**dict_no_none(title=title, description=description, plan=plan, budget=budget)
Expand Down
7 changes: 4 additions & 3 deletions openeo/rest/datacube.py
Original file line number Diff line number Diff line change
Expand Up @@ -1543,13 +1543,13 @@ def execute_batch(
:param format_options: String Parameters for the job result format

"""
job = self.send_job(out_format, job_options=job_options, **format_options)
job = self.create_job(out_format, job_options=job_options, **format_options)
return job.run_synchronous(
outputfile=outputfile,
print=print, max_poll_interval=max_poll_interval, connection_retry_interval=connection_retry_interval
)

def send_job(
def create_job(
self, out_format=None, title: str = None, description: str = None, plan: str = None, budget=None,
job_options=None, **format_options
) -> RESTJob:
Expand All @@ -1563,7 +1563,6 @@ def send_job(
:return: status: Job resulting job.
"""
# TODO: add option to also automatically start the job?
# TODO: unify Connection.create_job vs DataCube.send_job. #276
img = self
if out_format:
# add `save_result` node
Expand All @@ -1573,6 +1572,8 @@ def send_job(
title=title, description=description, plan=plan, budget=budget, additional=job_options
)

send_job = legacy_alias(create_job, name="send_job")

def save_user_defined_process(
self,
user_defined_process_id: str,
Expand Down
6 changes: 4 additions & 2 deletions openeo/rest/imagecollectionclient.py
Original file line number Diff line number Diff line change
Expand Up @@ -1003,14 +1003,14 @@ def execute_batch(
:param format_options: String Parameters for the job result format

"""
job = self.send_job(out_format, job_options=job_options, **format_options)
job = self.create_job(out_format, job_options=job_options, **format_options)
return job.run_synchronous(
# TODO #135 support multi file result sets too
outputfile=outputfile,
print=print, max_poll_interval=max_poll_interval, connection_retry_interval=connection_retry_interval
)

def send_job(
def create_job(
self, out_format=None, title: str = None, description: str = None, plan: str = None, budget=None,
job_options=None, **format_options
) -> RESTJob:
Expand All @@ -1033,6 +1033,8 @@ def send_job(
title=title, description=description, plan=plan, budget=budget, additional=job_options
)

send_job = legacy_alias(create_job, name="send_job")

def execute(self) -> Dict:
"""Executes the process graph of the imagery. """
newbuilder = self.builder.shallow_copy()
Expand Down
6 changes: 6 additions & 0 deletions openeo/rest/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,18 +44,21 @@ def _repr_html_(self):
def describe_job(self) -> dict:
""" Get all job information."""
# GET /jobs/{job_id}
# TODO: rename to just `describe`? #280
return self.connection.get("/jobs/{}".format(self.job_id), expected_status=200).json()

def update_job(self, process_graph=None, output_format=None,
output_parameters=None, title=None, description=None,
plan=None, budget=None, additional=None):
""" Update a job."""
# PATCH /jobs/{job_id}
# TODO: rename to just `update`? #280
raise NotImplementedError

def delete_job(self):
""" Delete a job."""
# DELETE /jobs/{job_id}
# TODO: rename to just `delete`? #280
self.connection.delete("/jobs/{}".format(self.job_id), expected_status=204)

def estimate_job(self):
Expand All @@ -68,11 +71,14 @@ def estimate_job(self):
def start_job(self):
""" Start / queue a job for processing."""
# POST /jobs/{job_id}/results
# TODO: rename to just `start`? #280
# TODO: return self, to allow chained calls
self.connection.post("/jobs/{}/results".format(self.job_id), expected_status=202)

def stop_job(self):
""" Stop / cancel job processing."""
# DELETE /jobs/{job_id}/results
# TODO: rename to just `stop`? #280
self.connection.delete("/jobs/{}/results".format(self.job_id), expected_status=204)

@deprecated("Use :py:meth:`~RESTJOB.get_results` instead.", version="0.4.10")
Expand Down
5 changes: 2 additions & 3 deletions openeo/rest/mlmodel.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,16 @@ def execute_batch(
:param outputfile: The path of a file to which a result can be written
:param out_format: (optional) Format of the job result.
:param format_options: String Parameters for the job result format

"""
# TODO: check/warn about final `save_ml_model` node?
job = self.send_job(additional=job_options)
job = self.create_job(additional=job_options)
return job.run_synchronous(
# TODO #135 support multi file result sets too
outputfile=outputfile,
print=print, max_poll_interval=max_poll_interval, connection_retry_interval=connection_retry_interval
)

def send_job(self, **kwargs) -> RESTJob:
def create_job(self, **kwargs) -> RESTJob:
"""
Sends a job to the backend and returns a ClientJob instance.

Expand Down
6 changes: 4 additions & 2 deletions openeo/rest/vectorcube.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,14 +122,14 @@ def execute_batch(
:param format_options: String Parameters for the job result format

"""
job = self.send_job(out_format, job_options=job_options, **format_options)
job = self.create_job(out_format, job_options=job_options, **format_options)
return job.run_synchronous(
# TODO #135 support multi file result sets too
outputfile=outputfile,
print=print, max_poll_interval=max_poll_interval, connection_retry_interval=connection_retry_interval
)

def send_job(self, out_format=None, job_options=None, **format_options) -> RESTJob:
def create_job(self, out_format=None, job_options=None, **format_options) -> RESTJob:
"""
Sends a job to the backend and returns a ClientJob instance.

Expand All @@ -143,3 +143,5 @@ def send_job(self, out_format=None, job_options=None, **format_options) -> RESTJ
# add `save_result` node
shp = shp.save_result(format=out_format, options=format_options)
return self._connection.create_job(process_graph=shp.flat_graph(), additional=job_options)

send_job = legacy_alias(create_job, name="send_job")
Loading