Merge pull request #710 from MetaCell/feature/CH-87

Feature/ch 87

tools/deployment-cli-tools/ch_cli_tools/skaffold.py

@@ -3,7 +3,7 @@
 import json
 import time
 
-from os.path import join, dirname, relpath, basename
+from os.path import join, relpath, basename, exists, abspath
 from cloudharness_model import ApplicationTestConfig, HarnessMainConfig
 
 from cloudharness_utils.constants import APPS_PATH, DEPLOYMENT_CONFIGURATION_PATH, \
@@ -51,7 +51,7 @@ def build_artifact(image_name, context_path, requirements=None, dockerfile_path=
                                          in requirements]
         return artifact_spec
 
-    
+
     base_images = set()
 
     def process_build_dockerfile(dockerfile_path, root_path, global_context=False, requirements=None, app_name=None):
@@ -78,21 +78,21 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r
 
         for dockerfile_path in base_dockerfiles:
             process_build_dockerfile(dockerfile_path, root_path, global_context=True)
-            
-    
+
+
     release_config = skaffold_conf['deploy']['helm']['releases'][0]
     release_config['name'] = helm_values.namespace
     release_config['namespace'] = helm_values.namespace
     release_config['artifactOverrides'][KEY_APPS] = {}
-    
+
     static_images = set()
     for root_path in root_paths:
         static_dockerfiles = find_dockerfiles_paths(
             join(root_path, STATIC_IMAGES_PATH))
 
         for dockerfile_path in static_dockerfiles:
             process_build_dockerfile(dockerfile_path, root_path)
-           
+
 
     for root_path in root_paths:
         apps_path = join(root_path, APPS_PATH)
@@ -138,22 +138,45 @@ def process_build_dockerfile(dockerfile_path, root_path, global_context=False, r
                         }
                 }
 
-            flask_main = find_file_paths(context_path, '__main__.py')
-
-            if flask_main:
+            mains_candidates = find_file_paths(context_path, '__main__.py')
+
+            def identify_unicorn_based_main(candidates):
+                import re
+                gunicorn_pattern = re.compile(r"gunicorn")
+                for candidate in candidates:
+                    dockerfile_path = f"{candidate}/.."
+                    while not exists(f"{dockerfile_path}/Dockerfile") and abspath(dockerfile_path) != abspath(root_path):
+                        dockerfile_path += "/.."
+                    dockerfile = f"{dockerfile_path}/Dockerfile"
+                    if not exists(dockerfile):
+                        continue
+                    with open(dockerfile, 'r') as file:
+                        if re.search(gunicorn_pattern, file.read()):
+                            return candidate
+                    requirements = f"{candidate}/../requirements.txt"
+                    if not exists(requirements):
+                        continue
+                    with open(requirements, 'r') as file:
+                        if re.search(gunicorn_pattern, file.read()):
+                            return candidate
+                return None
+
+            task_main_file = identify_unicorn_based_main(mains_candidates)
+
+            if task_main_file:
                 release_config['overrides']['apps'][app_key] = \
                     {
                         'harness': {
                             'deployment': {
                                 'command': ['python'],
-                                'args': [f'/usr/src/app/{os.path.basename(flask_main[0])}/__main__.py']
+                                'args': [f'/usr/src/app/{os.path.basename(task_main_file)}/__main__.py']
                             }
                         }
                 }
-            
+
             test_config: ApplicationTestConfig = helm_values.apps[app_key].harness.test
             if test_config.unit.enabled and test_config.unit.commands:
-                    
+
                     skaffold_conf['test'].append(dict(
                         image=get_image_tag(app_name),
                         custom=[dict(command="docker run $IMAGE " + cmd) for cmd in test_config.unit.commands]
@@ -209,7 +232,7 @@ def get_image_tag(name):
                                                                                                        "/usr/src/app"),
                     }
                 })
-                
+
 
     if not os.path.exists(os.path.dirname(vscode_launch_path)):
         os.makedirs(os.path.dirname(vscode_launch_path))

tools/deployment-cli-tools/tests/resources_buggy/.dockerignore

@@ -0,0 +1,22 @@
+**/node_modules
+.tox
+docs
+applications
+/infrastructure
+/blueprint
+test
+/tools/deployment-cli-tools
+.github
+.git
+.vscode
+/deployment
+skaffold.yaml
+*.egg-info
+__pycache__
+.hypothesis
+.coverage
+.pytest_cache
+/application-templates
+/deployment-configuration
+/cloud-harness
+.openapi-generator

tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values-prod.yaml

@@ -0,0 +1,7 @@
+kafka:
+  resources:
+    limits:
+      cpu: overridden-prod
+    requests:
+      cpu: 50m
+      memory: 100Mi

tools/deployment-cli-tools/tests/resources_buggy/applications/events/deploy/values.yaml

@@ -0,0 +1,8 @@
+kafka:
+  resources:
+    limits:
+      cpu: 600m
+      memory: overridden
+    requests:
+      cpu: 50m
+      memory: 100Mi

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/.dockerignore

@@ -0,0 +1 @@
+*.ignored

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/Dockerfile

@@ -0,0 +1,4 @@
+ARG CLOUDHARNESS_FLASK
+FROM $CLOUDHARNESS_FLASK
+
+RUN pip install gunicorn

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-dev.yaml

@@ -0,0 +1,11 @@
+harness:
+  name: "I'm useless"
+  subdomain: mysubdomain
+  dependencies:
+    soft:
+      - legacy
+    build:
+      - cloudharness-flask
+      - my-common
+a: b
+dev: true

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-test.yaml

@@ -0,0 +1,2 @@
+a: test
+test: true

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withmongo.yaml

@@ -0,0 +1,4 @@
+harness:
+  database:
+    auto: true
+    type: mongo

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values-withpostgres.yaml

@@ -0,0 +1,2 @@
+harness:
+  database: {auto: true, type: postgres}

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/deploy/values.yaml

@@ -0,0 +1,14 @@
+harness:
+  name: "I'm useless"
+  subdomain: mysubdomain
+  dependencies:
+    soft:
+      - legacy
+    build:
+      - cloudharness-flask
+  test:
+    unit:
+      commands:
+      - tox
+      - echo "hello"
+a: b

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/myapp_code/__main__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+from cloudharness.utils.server import init_flask, main
+
+
+app = init_flask(title="Cloudharness sample application", webapp=True)
+
+if __name__ == '__main__':
+    main()

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp/venv/matplotlib/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from something import something_else
+
+
+def fake_content():
+    ...

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/.dockerignore

@@ -0,0 +1 @@
+*.ignored

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/Dockerfile

@@ -0,0 +1,4 @@
+ARG CLOUDHARNESS_FLASK
+FROM $CLOUDHARNESS_FLASK
+
+RUN pip install -r requirements.txt

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-dev.yaml

@@ -0,0 +1,11 @@
+harness:
+  name: "I'm useless"
+  subdomain: mysubdomain
+  dependencies:
+    soft:
+      - legacy
+    build:
+      - cloudharness-flask
+      - my-common
+a: b
+dev: true

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-test.yaml

@@ -0,0 +1,2 @@
+a: test
+test: true

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withmongo.yaml

@@ -0,0 +1,4 @@
+harness:
+  database:
+    auto: true
+    type: mongo

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values-withpostgres.yaml

@@ -0,0 +1,2 @@
+harness:
+  database: {auto: true, type: postgres}

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/deploy/values.yaml

@@ -0,0 +1,14 @@
+harness:
+  name: "I'm useless"
+  subdomain: mysubdomain
+  dependencies:
+    soft:
+      - legacy
+    build:
+      - cloudharness-flask
+  test:
+    unit:
+      commands:
+      - tox
+      - echo "hello"
+a: b

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/myapp_code/__main__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+from cloudharness.utils.server import init_flask, main
+
+
+app = init_flask(title="Cloudharness sample application", webapp=True)
+
+if __name__ == '__main__':
+    main()

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/requirements.txt

@@ -0,0 +1 @@
+gunicorn

tools/deployment-cli-tools/tests/resources_buggy/applications/myapp2/venv/matplotlib/__main__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+from something import something_else
+
+
+def fake_content():
+    ...

tools/deployment-cli-tools/tests/test_skaffold.py

@@ -7,6 +7,7 @@
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 RESOURCES = os.path.join(HERE, 'resources')
+RESOURCES_BUGGY = os.path.join(HERE, 'resources_buggy')
 OUT = '/tmp/deployment'
 CLOUDHARNESS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
 
@@ -70,7 +71,7 @@ def test_create_skaffold_configuration():
         a for a in sk['build']['artifacts'] if a['image'] == 'reg/cloudharness/cloudharness-flask')
 
 
-    assert os.path.samefile(cloudharness_flask_artifact['context'], 
+    assert os.path.samefile(cloudharness_flask_artifact['context'],
        join(CLOUDHARNESS_ROOT, 'infrastructure/common-images/cloudharness-flask')
     )
 
@@ -102,3 +103,75 @@ def test_create_skaffold_configuration():
 
     shutil.rmtree(OUT)
     shutil.rmtree(BUILD_DIR)
+
+
+def test_create_skaffold_configuration_with_conflicting_dependencies(tmp_path):
+    values = create_helm_chart(
+        [CLOUDHARNESS_ROOT, RESOURCES_BUGGY],
+        output_path=OUT,
+        include=['myapp'],
+        exclude=['events'],
+        domain="my.local",
+        namespace='test',
+        env='dev',
+        local=False,
+        tag=1,
+        registry='reg'
+    )
+    root_paths = preprocess_build_overrides(
+        root_paths=[CLOUDHARNESS_ROOT, RESOURCES_BUGGY],
+        helm_values=values,
+        merge_build_path=str(tmp_path)
+    )
+
+    sk = create_skaffold_configuration(
+        root_paths=root_paths,
+        helm_values=values,
+        output_path=OUT
+    )
+
+    releases = sk['deploy']['helm']['releases']
+    assert len(releases) == 1  # Ensure we only found 1 deployment (for myapp)
+
+    release = releases[0]
+    assert 'myapp' in release['overrides']['apps']
+    assert 'matplotlib' not in release['overrides']['apps']
+
+    myapp_config = release['overrides']['apps']['myapp']
+    assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py'
+
+
+def test_create_skaffold_configuration_with_conflicting_dependencies_requirements_file(tmp_path):
+    values = create_helm_chart(
+        [CLOUDHARNESS_ROOT, RESOURCES_BUGGY],
+        output_path=OUT,
+        include=['myapp2'],
+        exclude=['events'],
+        domain="my.local",
+        namespace='test',
+        env='dev',
+        local=False,
+        tag=1,
+        registry='reg'
+    )
+    root_paths = preprocess_build_overrides(
+        root_paths=[CLOUDHARNESS_ROOT, RESOURCES_BUGGY],
+        helm_values=values,
+        merge_build_path=str(tmp_path)
+    )
+
+    sk = create_skaffold_configuration(
+        root_paths=root_paths,
+        helm_values=values,
+        output_path=OUT
+    )
+
+    releases = sk['deploy']['helm']['releases']
+    assert len(releases) == 1  # Ensure we only found 1 deployment (for myapp)
+
+    release = releases[0]
+    assert 'myapp2' in release['overrides']['apps']
+    assert 'matplotlib' not in release['overrides']['apps']
+
+    myapp_config = release['overrides']['apps']['myapp2']
+    assert myapp_config['harness']['deployment']['args'][0] == '/usr/src/app/myapp_code/__main__.py'