Merge pull request #719 from mendix/develop

Release 2024-03-21

.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "develop"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "develop"

.github/workflows/build.yml

@@ -33,7 +33,7 @@ jobs:
           echo "::set-output name=current_version_tag::${CURRENT_VERSION_TAG}"
       - name: Auto-generate future version
         id: autogenerate-version
-        uses: paulhatch/semantic-version@v5.3.0
+        uses: paulhatch/semantic-version@v5.4.0
         with:
           tag_prefix: "v"
           major_pattern: "(major)"
@@ -46,7 +46,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.10.6
           cache: pip
@@ -61,7 +61,8 @@ jobs:
       - name: Build
         run: VERSION=${{ needs.get-version.outputs.version_tag }} make build
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: dist
           path: dist
+          overwrite: true

.github/workflows/deploy.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist

.github/workflows/test.yml

@@ -40,7 +40,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - id: set-matrix
       name: Set up test matrix
       run: |
@@ -59,13 +59,13 @@ jobs:
       TEST_MEMORY: 1G
       TEST_PROCESSES: 1
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/download-artifact@v3
+    - uses: actions/checkout@v4
+    - uses: actions/download-artifact@v4
       with:
         name: dist
         path: dist
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.10.6
         cache: pip

.github/workflows/veracodescan.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: build
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist
           path: dist

Makefile

@@ -5,8 +5,8 @@ TEST_FILES ?= tests/integration/test_*.py
 VERSION ?= $(shell git tag --list --sort=-version:refname "v*" | head -n 1)
 COMMIT ?= $(shell git rev-parse --short HEAD)
 
-PIP_TOOLS_VERSION ?= 7.3.0
-PIP_VERSION ?= 23.3.1
+PIP_TOOLS_VERSION ?= 7.4.0
+PIP_VERSION ?= 24.0
 PYTHON_PLATFORM ?= manylinux2014_x86_64
 PYTHON_VERSION ?= 310
 
@@ -104,5 +104,5 @@ format:
 
 .PHONY: lint
 lint:
-	ruff check buildpack lib/m2ee/* tests/*/ --show-source
+	ruff check buildpack lib/m2ee/* tests/*/ --output-format=full
 	pylint --disable=W,R,C buildpack lib/m2ee/* tests/*/

buildpack/core/runtime.py

@@ -12,6 +12,7 @@
 from buildpack import util
 from lib.m2ee import M2EE as m2ee_class
 from lib.m2ee.version import MXVersion
+from lib.m2ee.util import strtobool
 
 from . import security
 
@@ -173,6 +174,14 @@ def _activate_license():
   <entry key="license_key" value="{{LICENSE_KEY}}"/>
 </map>"""
 
+    if(strtobool(os.environ.get("MXRUNTIME_License.UseLicenseServer","false"))):
+        prefs_template = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+        <!DOCTYPE map SYSTEM "http://java.sun.com/dtd/preferences.dtd">
+        <map MAP_XML_VERSION="1.0">
+          <entry key="id2" value="{{LICENSE_ID}}"/>
+          <entry key="license_key2" value="{{LICENSE_KEY}}"/>
+        </map>"""
+
     license_key = os.environ.get(
         "FORCED_LICENSE_KEY", os.environ.get("LICENSE_KEY", None)
     )

buildpack/telemetry/datadog.py

@@ -12,6 +12,7 @@
 import json
 import logging
 import os
+import shutil
 import socket
 import subprocess
 from collections import OrderedDict
@@ -30,13 +31,21 @@
 ROOT_DIR = os.path.abspath(".local")
 SIDECAR_ROOT_DIR = os.path.join(ROOT_DIR, NAMESPACE)
 AGENT_USER_CHECKS_DIR = os.path.abspath("/home/vcap/app/datadog_integrations")
-
+DOT_DATADOG_PATH = os.path.abspath("/home/vcap/app/.datadog")
 STATSD_PORT = 8125
 LOGS_PORT = 9032
 
 
 def _get_agent_dir(root=ROOT_DIR):
-    return os.path.join(root, NAMESPACE, "lib")
+    # DD-CF-buildpack assumes the root DATADOG_DIR to be
+    # /home/vcap/app/.datadog
+    # So once we've downloaded and installed the binaries
+    # return the path with ".datadog/"
+    path_with_lib = os.path.join(root, NAMESPACE, "lib")
+    if os.path.exists(path_with_lib):
+        return os.path.join(root, NAMESPACE, "lib")
+    else:
+        return DOT_DATADOG_PATH
 
 
 def _get_user_checks_dir():
@@ -389,6 +398,19 @@ def _set_up_environment(model_version, runtime_version):
     e["DD_ENABLE_CHECKS"] = str(bool(_is_checks_enabled())).lower()
     e["LOGS_CONFIG"] = json.dumps(_get_logging_config())
 
+    # Append the ruby binary path
+    ruby_path = os.path.join(ROOT_DIR, "ruby", "bin")
+    e["PATH"] += os.pathsep + ruby_path
+
+    # Add an empty .global_env file as the DD buildpack expects a .global_env
+    # This is in parity with how the DD buildpack sets up the environment before
+    # installing the DD agents. Sourced from bin/supply scripts
+    # https://github.com/DataDog/datadog-cloudfoundry-buildpack/blob/master/bin/supply#L31
+    # Keeping it empty as we're already adding the RUBY path as part of process env
+    global_env_file = os.path.join(_get_agent_dir(), ".global_env")
+    with open(global_env_file, mode='a'):
+        pass
+
     return e
 
 
@@ -492,6 +514,16 @@ def update_config(
 
 
 def _download(buildpack_dir, build_path, cache_dir):
+    # DD for cflinuxfs4 requires ruby to be available, so installing
+    # the ruby dependency first before we proceed
+    # https://github.com/DataDog/datadog-cloudfoundry-buildpack/pull/174
+    util.resolve_dependency(
+        "ruby",
+        os.path.join(build_path, "ruby"),
+        buildpack_dir=buildpack_dir,
+        cache_dir=cache_dir,
+    )
+
     util.resolve_dependency(
         f"{NAMESPACE}.buildpack",
         os.path.join(build_path, NAMESPACE),
@@ -525,13 +557,48 @@ def _patch_run_datadog_script(script_dir):
         lines = file_handler.readlines()
         file_handler.seek(0)
         for line in lines:
-            if "stop_datadog &" in line:
+            if "monit_datadog &" in line:
+                file_handler.write(f"# {line}")
+            else:
+                file_handler.write(line)
+        file_handler.truncate()
+
+
+def _patch_create_logs_config_script(script_dir):
+    # Seems like there is a race condition between the
+    # File.exists check and Dir.mkdir call, as a result we get an exception.
+    # Commenting out the Dir.mkdir call, as it is already created as part of
+    # run-datadog.sh script
+    script = os.path.join(script_dir, "scripts", "create_logs_config.rb")
+    pattern = 'Dir.mkdir(logs_config_dir)'
+    with open(script, "r+") as file_handler:
+        lines = file_handler.readlines()
+        file_handler.seek(0)
+        for line in lines:
+            if pattern in line:
                 file_handler.write(f"# {line}")
             else:
                 file_handler.write(line)
         file_handler.truncate()
 
 
+def _prep_datadog_dir(datadog_install_path):
+    # Tries to replicate what the datadog-cloudfoundry-buildpack
+    # supply script does before running the datadog startup script.
+    # The DD buildpack assumes /home/vcap/app/.datadog to be the default
+    # directory for all configs and binaries though it provides DATADOG_DIR
+    # as an environment variable. So here we try to move all files under lib/
+    # to /home/vcap/app/.datadog/
+    shutil.copytree(
+        datadog_install_path,
+        DOT_DATADOG_PATH,
+        copy_function=shutil.move,
+        dirs_exist_ok=True,
+    )
+    # finally cleanup up the lib/ dir
+    shutil.rmtree(datadog_install_path)
+
+
 def stage(buildpack_path, build_path, cache_path):
     if not is_enabled():
         return
@@ -545,6 +612,9 @@ def stage(buildpack_path, build_path, cache_path):
     logging.debug("Patching run-datadog.sh...")
     _patch_run_datadog_script(_get_agent_dir(build_path))
 
+    logging.debug("Patching create_logs_config.rb ...")
+    _patch_create_logs_config_script(_get_agent_dir(build_path))
+
 
 def run(model_version, runtime_version):
     if not is_enabled():
@@ -557,8 +627,11 @@ def run(model_version, runtime_version):
         )
         return
 
+    logging.debug("Prep DATADOG_DIR ... %s", _get_agent_dir())
+    _prep_datadog_dir(_get_agent_dir())
+
     logging.debug("Setting Datadog Agent script permissions if required...")
-    util.set_executable(f"{_get_agent_dir()}/*.sh")
+    util.set_executable(f"{_get_agent_dir()}/scripts/*.sh")
 
     # Start the run script "borrowed" from the official DD buildpack
     # and include settings as environment variables

dependencies.yml

@@ -20,10 +20,10 @@ dependencies:
     buildpack:
       alias: cf-datadog-sidecar
       artifact: datadog/datadog-cloudfoundry-buildpack-{{ version }}.zip
-      version: 4.33.0
+      version: 4.37.0
     trace-agent:
       artifact: datadog/dd-java-agent-{{ version }}.jar
-      version: 0.101.0
+      version: 1.27.0
   dynatrace:
     agent:
       artifact: "{{ url }}/e/{{ environment }}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={{ token }}"
@@ -92,6 +92,11 @@ dependencies:
     fs: cflinuxfs4
     cpe: cpe:2.3:a:f5:nginx:{{ version }}:*:*:*:*:*:*:*
     version: 1.22.1
+  ruby:
+    artifact: ruby/ruby_{{ version }}_linux_x64_{{ fs }}_{{ commit }}.tgz
+    commit: 5fed98f8
+    fs: cflinuxfs4
+    version: 3.2.2
   telegraf:
     agent:
       artifact: telegraf/telegraf-{{ version }}_linux_amd64.tar.gz

pyproject.toml

@@ -10,7 +10,7 @@ exclude = [
     "tests/unit/test_databroker.py"
 ]
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 # Unlike Flake8, default to a complexity level of 10.
 max-complexity = 10
 

requirements-dev.in

@@ -1,11 +1,11 @@
 click==8.1.7
 idna==3.6
-pytest==7.4.3
-pytest-timer==0.0.11
+pytest==8.0.1
+pytest-timer==1.0.0
 pytest-timeout==2.2.0
-pylint==3.0.2
-pyopenssl==23.3.0
+pylint==3.0.3
+pyopenssl==24.0.0
 randomname==0.2.1
 requests-mock==1.11.0
-ruff==0.1.6
+ruff==0.2.2
 parameterized==0.9.0

requirements.in

@@ -1,11 +1,11 @@
 backoff==2.2.1
-certifi==2023.11.17
-cryptography==41.0.7
-distro==1.8.0
+certifi==2024.2.2
+cryptography==42.0.4
+distro==1.9.0
 httplib2==0.22.0
-jinja2==3.1.2
+jinja2==3.1.3
 omegaconf==2.3.0
 psycopg2-binary==2.9.9
 pyyaml==6.0.1
 requests==2.31.0
-urllib3==2.1.0
+urllib3==2.2.1

requirements.txt

@@ -8,23 +8,23 @@ antlr4-python3-runtime==4.9.3
     # via omegaconf
 backoff==2.2.1
     # via -r requirements.in
-certifi==2023.11.17
+certifi==2024.2.2
     # via
     #   -r requirements.in
     #   requests
 cffi==1.14.4
     # via cryptography
 charset-normalizer==2.0.3
     # via requests
-cryptography==41.0.7
+cryptography==42.0.4
     # via -r requirements.in
-distro==1.8.0
+distro==1.9.0
     # via -r requirements.in
 httplib2==0.22.0
     # via -r requirements.in
 idna==2.8
     # via requests
-jinja2==3.1.2
+jinja2==3.1.3
     # via -r requirements.in
 markupsafe==2.0.1
     # via jinja2
@@ -42,7 +42,7 @@ pyyaml==6.0.1
     #   omegaconf
 requests==2.31.0
     # via -r requirements.in
-urllib3==2.1.0
+urllib3==2.2.1
     # via
     #   -r requirements.in
     #   requests

tests/integration/test_datadog.py

@@ -59,8 +59,8 @@ def _test_logsubscriber_active(self):
         logsubscribers = json.loads(logsubscribers_json.text)
         self.assertTrue("DatadogSubscriber" in logsubscribers["feedback"])
 
-    def test_datadog_mx7(self):
-        self._test_datadog("BuildpackTestApp-mx-7-16.mda")
-
     def test_datadog_mx8(self):
         self._test_datadog("Mendix8.1.1.58432_StarterApp.mda")
+
+    def test_datadog_mx9(self):
+        self._test_datadog("BuildpackTestApp-mx9-7.mda")