EP-3555 restructure pubic processes.py vs internal helper modules

docs/processes.rst

@@ -311,7 +311,7 @@ which can be used directly:
 
 .. code-block:: python
 
-    from openeo.processes.processes import absolute, max
+    from openeo.processes import absolute, max
 
     cube.apply(absolute)
     cube.reduce_dimension(max, dimension="t")
@@ -327,7 +327,7 @@ or normal Python functions:
 
 .. code-block:: python
 
-    from openeo.processes.processes import array_element
+    from openeo.processes import array_element
 
     def my_bandmath(data):
         band1 = array_element(data, index=1)
@@ -339,14 +339,14 @@ or normal Python functions:
 
 
 The argument that is passed to these functions is
-an instance of :py:class:`openeo.processes.processes.ProcessBuilder`.
+an instance of :py:class:`openeo.processes.ProcessBuilder`.
 This is a helper object with predefined methods for all standard processes,
 allowing to use an object oriented coding style to define the callback.
 For example:
 
 .. code-block:: python
 
-    from openeo.processes.processes import ProcessBuilder
+    from openeo.processes import ProcessBuilder
 
     def avg(data: ProcessBuilder):
         return data.mean()
@@ -368,7 +368,7 @@ The following examples result in the same callback:
 
 .. code-block:: python
 
-    from openeo.processes.processes import ProcessBuilder, mean, cos, add
+    from openeo.processes import ProcessBuilder, mean, cos, add
 
     # Chained methods
     cube.reduce_dimension(
@@ -396,7 +396,7 @@ Specifying callbacks through Python functions (or lambdas)
 looks intuitive and straightforward, but it should be noted
 that not everything is allowed in these functions.
 You should just limit yourself to calling
-:py:mod:`openeo.process.processes` functions, :py:class:`openeo.processes.processes.ProcessBuilder` methods and basic math operators.
+:py:mod:`openeo.process.processes` functions, :py:class:`openeo.processes.ProcessBuilder` methods and basic math operators.
 Don't call functions from other libraries like numpy or scipy.
 Don't use Python control flow statements like ``if/else`` constructs
 or ``for`` loops.

openeo/processes/generator.py → openeo/internal/processes/generator.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Union, List, Iterator
 
-from openeo.processes.parse import Process, parse_all_from_dir
+from openeo.internal.processes.parse import Process, parse_all_from_dir
 
 
 class PythonRenderer:
@@ -148,7 +148,7 @@ def __pow__(self, other) -> 'ProcessBuilder':
 
 def main():
     # Usage example (from project root, assuming the `openeo-process` repo is checked out as well):
-    #     python openeo/processes//generator.py  ../openeo-processes  --output openeo/processes/processes.py
+    #     python openeo/internal/processes/generator.py  ../openeo-processes  --output openeo/processes.py
     arg_parser = argparse.ArgumentParser()
     arg_parser.add_argument("dir", help="""Directory that holds openEO process definitions in JSON format""")
     arg_parser.add_argument("--output", help="Path to output 'processes.py' file")

openeo/processes/processes.py → openeo/processes.py

@@ -2,7 +2,7 @@
 # This file is automatically generated.
 # Do not edit directly.
 
-from openeo.processes.builder import ProcessBuilderBase, UNSET
+from openeo.internal.processes.builder import ProcessBuilderBase, UNSET
 
 
 class ProcessBuilder(ProcessBuilderBase):

openeo/rest/datacube.py

@@ -18,19 +18,19 @@
 import shapely.geometry
 import shapely.geometry.base
 from deprecated import deprecated
-from openeo.processes.processes import ProcessBuilder
 from shapely.geometry import Polygon, MultiPolygon, mapping
 
 import openeo
+import openeo.processes
 from openeo.imagecollection import ImageCollection, CollectionMetadata
 from openeo.internal.graph_building import PGNode, ReduceNode
+from openeo.metadata import Band
+from openeo.processes import ProcessBuilder
 from openeo.rest import BandMathException, OperatorException, OpenEoClientException
 from openeo.rest.job import RESTJob
 from openeo.rest.udp import RESTUserDefinedProcess
 from openeo.util import get_temporal_extent, dict_no_none
 from openeo.vectorcube import VectorCube
-import openeo.processes.processes
-from openeo.metadata import Band
 import numpy
 from builtins import staticmethod
 
@@ -520,8 +520,8 @@ def get_parameter_names(process: typing.Callable) -> List[str]:
             pg = process
         elif isinstance(process, str):
             # Assume given reducer is a simple predefined reduce process_id
-            if process in openeo.processes.processes.__dict__:
-                process_params = get_parameter_names(openeo.processes.processes.__dict__[process])
+            if process in openeo.processes.__dict__:
+                process_params = get_parameter_names(openeo.processes.__dict__[process])
             else:
                 # Best effort guess
                 process_params = parent_parameters

tests/processes/test_generator.py → tests/internal/processes/test_generator.py

@@ -1,7 +1,7 @@
 from textwrap import dedent
 
-from openeo.processes.generator import PythonRenderer
-from openeo.processes.parse import Process
+from openeo.internal.processes.generator import PythonRenderer
+from openeo.internal.processes.parse import Process
 
 
 def test_render_basic():

tests/processes/test_parse.py → tests/internal/processes/test_parse.py

@@ -1,4 +1,4 @@
-from openeo.processes.parse import Parameter, Schema, Returns, Process
+from openeo.internal.processes.parse import Parameter, Schema, Returns, Process
 
 
 def test_schema():

tests/rest/datacube/test_datacube100.py

@@ -280,7 +280,7 @@ def between(min, max) -> PGNode:
 
 
 def test_load_collection_properties_process_builder_function(con100):
-    from openeo.processes.processes import between, eq
+    from openeo.processes import between, eq
     im = con100.load_collection(
         "S2",
         spatial_extent={"west": 16.1, "east": 16.6, "north": 48.6, "south": 47.2},

tests/rest/datacube/test_processbuilder.py

@@ -1,5 +1,5 @@
 from openeo.internal.graph_building import PGNode
-from openeo.processes.processes import ProcessBuilder
+from openeo.processes import ProcessBuilder
 
 from ... import load_json_resource
 
@@ -24,7 +24,7 @@ def test_apply_callback_absolute_lambda_method(con100):
 
 def test_apply_callback_absolute_function(con100):
     im = con100.load_collection("S2")
-    from openeo.processes.processes import absolute
+    from openeo.processes import absolute
     result = im.apply(absolute)
     assert result.graph == load_json_resource('data/1.0.0/apply_absolute.json')
 
@@ -46,14 +46,14 @@ def test_apply_callback_chain_lambda_method(con100):
 
 def test_apply_callback_chain_lambda_functions(con100):
     im = con100.load_collection("S2")
-    from openeo.processes.processes import absolute, cos, add
+    from openeo.processes import absolute, cos, add
     result = im.apply(lambda data: add(cos(absolute(data)), 1.23))
     assert result.graph == load_json_resource('data/1.0.0/apply_chain.json')
 
 
 def test_apply_callback_chain_lambda_mixed_and_operator(con100):
     im = con100.load_collection("S2")
-    from openeo.processes.processes import cos
+    from openeo.processes import cos
     result = im.apply(lambda data: cos(data.absolute()) + 1.23)
     assert result.graph == load_json_resource('data/1.0.0/apply_chain.json')
 
@@ -68,7 +68,7 @@ def transform(x: ProcessBuilder) -> ProcessBuilder:
 
 
 def test_apply_callback_chain_custom_function_functions(con100):
-    from openeo.processes.processes import absolute, cos, add
+    from openeo.processes import absolute, cos, add
 
     def transform(x: ProcessBuilder) -> ProcessBuilder:
         return add(cos(absolute(x)), y=1.23)
@@ -79,7 +79,7 @@ def transform(x: ProcessBuilder) -> ProcessBuilder:
 
 
 def test_apply_callback_chain_custom_function_mixed_and_operator(con100):
-    from openeo.processes.processes import cos
+    from openeo.processes import cos
 
     def transform(x: ProcessBuilder) -> ProcessBuilder:
         return cos(x.absolute()) + 1.23
@@ -133,7 +133,7 @@ def test_apply_neighborhood_trim_pgnode(con100):
 
 
 def test_apply_neighborhood_trim_callable(con100):
-    from openeo.processes.processes import trim_cube
+    from openeo.processes import trim_cube
     im = con100.load_collection("S2")
     result = im.apply_neighborhood(
         process=trim_cube,
@@ -187,7 +187,7 @@ def callback(data: ProcessBuilder):
 def test_apply_neighborhood_complex_callback(con100):
     collection = con100.load_collection("S2")
 
-    from openeo.processes.processes import max
+    from openeo.processes import max
     neighbors = collection.apply_neighborhood(process=lambda data: max(data).absolute(), size=[
         {'dimension': 'x', 'value': 128, 'unit': 'px'},
         {'dimension': 'y', 'value': 128, 'unit': 'px'}
@@ -230,7 +230,7 @@ def test_apply_dimension_max_pgnode(con100):
 
 def test_apply_dimension_max_callable(con100):
     im = con100.load_collection("S2")
-    from openeo.processes.processes import max
+    from openeo.processes import max
     res = im.apply_dimension(process=max, dimension="bands")
     assert res.graph == load_json_resource('data/1.0.0/apply_dimension_max.json')
 
@@ -242,7 +242,7 @@ def test_apply_dimension_max_lambda(con100):
 
 
 def test_apply_dimension_bandmath_lambda(con100):
-    from openeo.processes.processes import array_element
+    from openeo.processes import array_element
     im = con100.load_collection("S2")
     res = im.apply_dimension(
         process=lambda d: array_element(d, index=1) + array_element(d, index=2),
@@ -265,7 +265,7 @@ def test_reduce_dimension_max_pgnode(con100):
 
 def test_reduce_dimension_max_callable(con100):
     im = con100.load_collection("S2")
-    from openeo.processes.processes import max
+    from openeo.processes import max
     res = im.reduce_dimension(reducer=max, dimension="bands")
     assert res.graph == load_json_resource('data/1.0.0/reduce_dimension_max.json')
 
@@ -277,7 +277,7 @@ def test_reduce_dimension_max_lambda(con100):
 
 
 def test_reduce_dimension_bandmath_lambda(con100):
-    from openeo.processes.processes import array_element
+    from openeo.processes import array_element
     im = con100.load_collection("S2")
     res = im.reduce_dimension(
         reducer=lambda data: array_element(data, index=1) + array_element(data, index=2),
@@ -306,7 +306,7 @@ def test_merge_cubes_add_pgnode(con100):
 def test_merge_cubes_add_callable(con100):
     im1 = con100.load_collection("S2")
     im2 = con100.load_collection("MASK")
-    from openeo.processes.processes import add
+    from openeo.processes import add
     res = im1.merge_cubes(other=im2, overlap_resolver=add)
     assert res.graph == load_json_resource('data/1.0.0/merge_cubes_add.json')
 
@@ -338,7 +338,7 @@ def test_merge_cubes_max_pgnode(con100):
 def test_merge_cubes_max_callable(con100):
     im1 = con100.load_collection("S2")
     im2 = con100.load_collection("MASK")
-    from openeo.processes.processes import max
+    from openeo.processes import max
     res = im1.merge_cubes(other=im2, overlap_resolver=max)
     assert res.graph == load_json_resource('data/1.0.0/merge_cubes_max.json')