diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..b3c537517 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + target-branch: "develop" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + target-branch: "develop" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82e4493f0..8c1517731 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: echo "::set-output name=current_version_tag::${CURRENT_VERSION_TAG}" - name: Auto-generate future version id: autogenerate-version - uses: paulhatch/semantic-version@v5.3.0 + uses: paulhatch/semantic-version@v5.4.0 with: tag_prefix: "v" major_pattern: "(major)" @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.10.6 cache: pip @@ -61,7 +61,8 @@ jobs: - name: Build run: VERSION=${{ needs.get-version.outputs.version_tag }} make build - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist + overwrite: true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 99522ce90..ffc23a4a8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: dist path: dist diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6ea8ed31..62f14a3b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: set-matrix name: Set up test matrix run: | @@ -59,13 +59,13 @@ jobs: TEST_MEMORY: 1G TEST_PROCESSES: 1 steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: dist path: dist - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.10.6 cache: pip diff --git a/.github/workflows/veracodescan.yml b/.github/workflows/veracodescan.yml index ad54e905c..83e253ed3 100644 --- a/.github/workflows/veracodescan.yml +++ b/.github/workflows/veracodescan.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: dist path: dist diff --git a/Makefile b/Makefile index 64a2ed87b..dfba7c789 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,8 @@ TEST_FILES ?= tests/integration/test_*.py VERSION ?= $(shell git tag --list --sort=-version:refname "v*" | head -n 1) COMMIT ?= $(shell git rev-parse --short HEAD) -PIP_TOOLS_VERSION ?= 7.3.0 -PIP_VERSION ?= 23.3.1 +PIP_TOOLS_VERSION ?= 7.4.0 +PIP_VERSION ?= 24.0 PYTHON_PLATFORM ?= manylinux2014_x86_64 PYTHON_VERSION ?= 310 @@ -104,5 +104,5 @@ format: .PHONY: lint lint: - ruff check buildpack lib/m2ee/* tests/*/ --show-source + ruff check buildpack lib/m2ee/* tests/*/ --output-format=full pylint --disable=W,R,C buildpack lib/m2ee/* tests/*/ diff --git a/buildpack/core/runtime.py b/buildpack/core/runtime.py index 99f61949b..a1caf9b9b 100644 --- a/buildpack/core/runtime.py +++ b/buildpack/core/runtime.py @@ -12,6 +12,7 @@ from buildpack import util from lib.m2ee import M2EE as m2ee_class from lib.m2ee.version import MXVersion +from lib.m2ee.util import strtobool from . import security @@ -173,6 +174,14 @@ def _activate_license(): """ + if(strtobool(os.environ.get("MXRUNTIME_License.UseLicenseServer","false"))): + prefs_template = """ + + + + + """ + license_key = os.environ.get( "FORCED_LICENSE_KEY", os.environ.get("LICENSE_KEY", None) ) diff --git a/buildpack/telemetry/datadog.py b/buildpack/telemetry/datadog.py index f08761ded..026e3cdf6 100644 --- a/buildpack/telemetry/datadog.py +++ b/buildpack/telemetry/datadog.py @@ -12,6 +12,7 @@ import json import logging import os +import shutil import socket import subprocess from collections import OrderedDict @@ -30,13 +31,21 @@ ROOT_DIR = os.path.abspath(".local") SIDECAR_ROOT_DIR = os.path.join(ROOT_DIR, NAMESPACE) AGENT_USER_CHECKS_DIR = os.path.abspath("/home/vcap/app/datadog_integrations") - +DOT_DATADOG_PATH = os.path.abspath("/home/vcap/app/.datadog") STATSD_PORT = 8125 LOGS_PORT = 9032 def _get_agent_dir(root=ROOT_DIR): - return os.path.join(root, NAMESPACE, "lib") + # DD-CF-buildpack assumes the root DATADOG_DIR to be + # /home/vcap/app/.datadog + # So once we've downloaded and installed the binaries + # return the path with ".datadog/" + path_with_lib = os.path.join(root, NAMESPACE, "lib") + if os.path.exists(path_with_lib): + return os.path.join(root, NAMESPACE, "lib") + else: + return DOT_DATADOG_PATH def _get_user_checks_dir(): @@ -389,6 +398,19 @@ def _set_up_environment(model_version, runtime_version): e["DD_ENABLE_CHECKS"] = str(bool(_is_checks_enabled())).lower() e["LOGS_CONFIG"] = json.dumps(_get_logging_config()) + # Append the ruby binary path + ruby_path = os.path.join(ROOT_DIR, "ruby", "bin") + e["PATH"] += os.pathsep + ruby_path + + # Add an empty .global_env file as the DD buildpack expects a .global_env + # This is in parity with how the DD buildpack sets up the environment before + # installing the DD agents. Sourced from bin/supply scripts + # https://github.com/DataDog/datadog-cloudfoundry-buildpack/blob/master/bin/supply#L31 + # Keeping it empty as we're already adding the RUBY path as part of process env + global_env_file = os.path.join(_get_agent_dir(), ".global_env") + with open(global_env_file, mode='a'): + pass + return e @@ -492,6 +514,16 @@ def update_config( def _download(buildpack_dir, build_path, cache_dir): + # DD for cflinuxfs4 requires ruby to be available, so installing + # the ruby dependency first before we proceed + # https://github.com/DataDog/datadog-cloudfoundry-buildpack/pull/174 + util.resolve_dependency( + "ruby", + os.path.join(build_path, "ruby"), + buildpack_dir=buildpack_dir, + cache_dir=cache_dir, + ) + util.resolve_dependency( f"{NAMESPACE}.buildpack", os.path.join(build_path, NAMESPACE), @@ -525,13 +557,48 @@ def _patch_run_datadog_script(script_dir): lines = file_handler.readlines() file_handler.seek(0) for line in lines: - if "stop_datadog &" in line: + if "monit_datadog &" in line: + file_handler.write(f"# {line}") + else: + file_handler.write(line) + file_handler.truncate() + + +def _patch_create_logs_config_script(script_dir): + # Seems like there is a race condition between the + # File.exists check and Dir.mkdir call, as a result we get an exception. + # Commenting out the Dir.mkdir call, as it is already created as part of + # run-datadog.sh script + script = os.path.join(script_dir, "scripts", "create_logs_config.rb") + pattern = 'Dir.mkdir(logs_config_dir)' + with open(script, "r+") as file_handler: + lines = file_handler.readlines() + file_handler.seek(0) + for line in lines: + if pattern in line: file_handler.write(f"# {line}") else: file_handler.write(line) file_handler.truncate() +def _prep_datadog_dir(datadog_install_path): + # Tries to replicate what the datadog-cloudfoundry-buildpack + # supply script does before running the datadog startup script. + # The DD buildpack assumes /home/vcap/app/.datadog to be the default + # directory for all configs and binaries though it provides DATADOG_DIR + # as an environment variable. So here we try to move all files under lib/ + # to /home/vcap/app/.datadog/ + shutil.copytree( + datadog_install_path, + DOT_DATADOG_PATH, + copy_function=shutil.move, + dirs_exist_ok=True, + ) + # finally cleanup up the lib/ dir + shutil.rmtree(datadog_install_path) + + def stage(buildpack_path, build_path, cache_path): if not is_enabled(): return @@ -545,6 +612,9 @@ def stage(buildpack_path, build_path, cache_path): logging.debug("Patching run-datadog.sh...") _patch_run_datadog_script(_get_agent_dir(build_path)) + logging.debug("Patching create_logs_config.rb ...") + _patch_create_logs_config_script(_get_agent_dir(build_path)) + def run(model_version, runtime_version): if not is_enabled(): @@ -557,8 +627,11 @@ def run(model_version, runtime_version): ) return + logging.debug("Prep DATADOG_DIR ... %s", _get_agent_dir()) + _prep_datadog_dir(_get_agent_dir()) + logging.debug("Setting Datadog Agent script permissions if required...") - util.set_executable(f"{_get_agent_dir()}/*.sh") + util.set_executable(f"{_get_agent_dir()}/scripts/*.sh") # Start the run script "borrowed" from the official DD buildpack # and include settings as environment variables diff --git a/dependencies.yml b/dependencies.yml index 11eebdadc..37cffd6d6 100644 --- a/dependencies.yml +++ b/dependencies.yml @@ -20,10 +20,10 @@ dependencies: buildpack: alias: cf-datadog-sidecar artifact: datadog/datadog-cloudfoundry-buildpack-{{ version }}.zip - version: 4.33.0 + version: 4.37.0 trace-agent: artifact: datadog/dd-java-agent-{{ version }}.jar - version: 0.101.0 + version: 1.27.0 dynatrace: agent: artifact: "{{ url }}/e/{{ environment }}/api/v1/deployment/installer/agent/unix/paas/latest?include=java&bitness=64&Api-Token={{ token }}" @@ -92,6 +92,11 @@ dependencies: fs: cflinuxfs4 cpe: cpe:2.3:a:f5:nginx:{{ version }}:*:*:*:*:*:*:* version: 1.22.1 + ruby: + artifact: ruby/ruby_{{ version }}_linux_x64_{{ fs }}_{{ commit }}.tgz + commit: 5fed98f8 + fs: cflinuxfs4 + version: 3.2.2 telegraf: agent: artifact: telegraf/telegraf-{{ version }}_linux_amd64.tar.gz diff --git a/pyproject.toml b/pyproject.toml index e73e06c23..a9a12f25e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ exclude = [ "tests/unit/test_databroker.py" ] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] # Unlike Flake8, default to a complexity level of 10. max-complexity = 10 diff --git a/requirements-dev.in b/requirements-dev.in index dbd7bb33a..d76475d0c 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,11 +1,11 @@ click==8.1.7 idna==3.6 -pytest==7.4.3 -pytest-timer==0.0.11 +pytest==8.0.1 +pytest-timer==1.0.0 pytest-timeout==2.2.0 -pylint==3.0.2 -pyopenssl==23.3.0 +pylint==3.0.3 +pyopenssl==24.0.0 randomname==0.2.1 requests-mock==1.11.0 -ruff==0.1.6 +ruff==0.2.2 parameterized==0.9.0 diff --git a/requirements.in b/requirements.in index 230527d90..3236c03bd 100644 --- a/requirements.in +++ b/requirements.in @@ -1,11 +1,11 @@ backoff==2.2.1 -certifi==2023.11.17 -cryptography==41.0.7 -distro==1.8.0 +certifi==2024.2.2 +cryptography==42.0.4 +distro==1.9.0 httplib2==0.22.0 -jinja2==3.1.2 +jinja2==3.1.3 omegaconf==2.3.0 psycopg2-binary==2.9.9 pyyaml==6.0.1 requests==2.31.0 -urllib3==2.1.0 \ No newline at end of file +urllib3==2.2.1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8c9b7c45e..6d152a1b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ antlr4-python3-runtime==4.9.3 # via omegaconf backoff==2.2.1 # via -r requirements.in -certifi==2023.11.17 +certifi==2024.2.2 # via # -r requirements.in # requests @@ -16,15 +16,15 @@ cffi==1.14.4 # via cryptography charset-normalizer==2.0.3 # via requests -cryptography==41.0.7 +cryptography==42.0.4 # via -r requirements.in -distro==1.8.0 +distro==1.9.0 # via -r requirements.in httplib2==0.22.0 # via -r requirements.in idna==2.8 # via requests -jinja2==3.1.2 +jinja2==3.1.3 # via -r requirements.in markupsafe==2.0.1 # via jinja2 @@ -42,7 +42,7 @@ pyyaml==6.0.1 # omegaconf requests==2.31.0 # via -r requirements.in -urllib3==2.1.0 +urllib3==2.2.1 # via # -r requirements.in # requests diff --git a/tests/integration/test_datadog.py b/tests/integration/test_datadog.py index 739142dbe..a78daeee9 100644 --- a/tests/integration/test_datadog.py +++ b/tests/integration/test_datadog.py @@ -59,8 +59,8 @@ def _test_logsubscriber_active(self): logsubscribers = json.loads(logsubscribers_json.text) self.assertTrue("DatadogSubscriber" in logsubscribers["feedback"]) - def test_datadog_mx7(self): - self._test_datadog("BuildpackTestApp-mx-7-16.mda") - def test_datadog_mx8(self): self._test_datadog("Mendix8.1.1.58432_StarterApp.mda") + + def test_datadog_mx9(self): + self._test_datadog("BuildpackTestApp-mx9-7.mda")