update python to 3.12.5

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### 🛑 Breaking changes 🛑
+
+- (Splunk) Update Python to 3.12.5 in the Smart Agent bundle for Linux and Windows. Check [What’s New In Python 3.12](https://docs.python.org/3/whatsnew/3.12.html) for details. ([#5298](https://github.com/signalfx/splunk-otel-collector/pull/5298))
+
 ## v0.108.1
 
 This Splunk OpenTelemetry Collector release includes changes from the [opentelemetry-collector v0.108.1](https://github.com/open-telemetry/opentelemetry-collector/releases/tag/v0.108.1) and the [opentelemetry-collector-contrib v0.108.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.108.0) releases where appropriate.

internal/signalfx-agent/bundle/Dockerfile

@@ -15,15 +15,15 @@
 ARG ARCH=amd64
 ARG JDK_VERSION=11.0.24_8
 ARG MAVEN_VERSION=3.6.0
-ARG PYTHON_VERSION=3.11.9
-ARG PIP_VERSION=21.0.1
+ARG PYTHON_VERSION=3.12.5
+ARG PIP_VERSION=24.2
 ARG COLLECTD_VERSION=5.8.0-sfx0
 ARG COLLECTD_COMMIT=4d3327b14cf4359029613baf4f90c4952702105e
 ARG DOCKER_REPO=docker.io
 
 
 ######## Base image for subsequent stages ########
-FROM ${DOCKER_REPO}/ubuntu:22.04 as base
+FROM ${DOCKER_REPO}/ubuntu:22.04 AS base
 
 ARG ARCH
 ENV DEBIAN_FRONTEND noninteractive
@@ -33,7 +33,7 @@ RUN sed -i -e '/^deb-src/d' /etc/apt/sources.list && \
 
 
 ######## Java monitor dependencies and monitor jar compilation ########
-FROM base as java
+FROM base AS java
 
 ARG JDK_VERSION
 ARG MAVEN_VERSION
@@ -82,7 +82,7 @@ RUN rm -rf /tmp/apache-maven*
 
 
 ######## Python bundle builder image ########
-FROM base as python
+FROM base AS python
 
 ARG PYTHON_VERSION
 
@@ -169,13 +169,13 @@ RUN wget -nv -O /tmp/Python-${PYTHON_VERSION}.tgz https://www.python.org/ftp/pyt
     make -j`nproc` && make install libinstall && \
     ldconfig
 
-RUN rm -rf /usr/local/lib/python3.11/test /usr/local/lib/python3.11/unittest
+RUN rm -rf /usr/local/lib/python3.12/test /usr/local/lib/python3.12/unittest
 
 RUN rm -rf /tmp/Python*
 
 
 ######## Collectd builder image ########
-FROM python as collectd
+FROM python AS collectd
 
 ARG COLLECTD_VERSION
 ARG COLLECTD_COMMIT
@@ -216,7 +216,7 @@ COPY --from=java /opt/root/jdk/ /opt/root/jdk/
 WORKDIR /usr/src/collectd
 
 ARG extra_cflags="-O2"
-ENV CFLAGS "-Wno-error -fPIC $extra_cflags"
+ENV CFLAGS "-Wno-error -Wno-format-truncation -Wno-stringop-truncation -fPIC $extra_cflags"
 ENV CXXFLAGS $CFLAGS
 ENV JAVA_HOME /opt/root/jdk
 
@@ -267,9 +267,9 @@ RUN DISABLE_XENCPU=""; if [ "$ARCH" = "ppc64le" ]; then DISABLE_XENCPU="--disabl
         --disable-silent-rules \
         --disable-static \
         --with-java=${JAVA_HOME} \
-        LIBPYTHON_LDFLAGS="$(python3.11-config --ldflags) -lpython3.11" \
-        LIBPYTHON_CPPFLAGS="$(python3.11-config --includes)/cpython" \
-        LIBPYTHON_LIBS="$(python3.11-config --libs) -lpython3.11"
+        LIBPYTHON_LDFLAGS="$(python3.12-config --ldflags) -lpython3.12" \
+        LIBPYTHON_CPPFLAGS="$(python3.12-config --includes)/cpython" \
+        LIBPYTHON_LIBS="$(python3.12-config --libs) -lpython3.12"
 
 # Compile all of collectd first, including plugins
 RUN make -j`nproc` &&\
@@ -284,22 +284,21 @@ RUN patchelf --add-needed libm-2.31.so /opt/deps/libvarnishapi.so.2.0.0
 
 
 ######## Python Plugin Image ########
-FROM collectd as python-plugins
+FROM collectd AS python-plugins
 
 ARG PIP_VERSION
 
 RUN /usr/local/bin/python3 -m pip install --upgrade pip==$PIP_VERSION
 
 # Mirror the same dir structure that exists in the original source
 COPY scripts/get-collectd-plugins.py /opt/scripts/
-COPY scripts/requirements.txt /opt/
+COPY scripts/get-collectd-plugins-requirements.txt /opt/scripts/
+COPY scripts/pyyaml-requirements.txt /opt/scripts/
 COPY collectd-plugins.yaml /opt/
 
 # workaround for https://github.com/yaml/pyyaml/issues/724
-RUN /usr/local/bin/python3 -m pip install 'wheel==0.40.0'
-RUN /usr/local/bin/python3 -m pip install 'Cython<3.0' 'PyYaml~=5.0' --no-build-isolation
-
-RUN /usr/local/bin/python3 -m pip install -r /opt/requirements.txt
+RUN /usr/local/bin/python3 -m pip install -r /opt/scripts/pyyaml-requirements.txt
+RUN /usr/local/bin/python3 -m pip install -r /opt/scripts/get-collectd-plugins-requirements.txt --no-build-isolation
 
 RUN mkdir -p /opt/collectd-python &&\
     /usr/local/bin/python3 -u /opt/scripts/get-collectd-plugins.py /opt/collectd-python
@@ -319,19 +318,19 @@ RUN /usr/local/bin/python3 -m pip list
 # Remove pip to avoid usage in python monitor and in exec'd container
 RUN /usr/local/bin/python3 -m pip uninstall pip -y
 # Delete all bundled wheels
-RUN find /usr/local/lib/python3.11 -name "*.whl" -delete
+RUN find /usr/local/lib/python3.12 -name "*.whl" -delete
 
 # Delete all compiled python to save space
-RUN find /usr/local/lib/python3.11 -name "*.pyc" -o -name "*.pyo" | xargs rm
+RUN find /usr/local/lib/python3.12 -name "*.pyc" -o -name "*.pyo" | xargs rm
 # We don't support compiling extension modules so don't need this directory
-RUN rm -rf /usr/local/lib/python3.11/config-*-linux-gnu
+RUN rm -rf /usr/local/lib/python3.12/config-*-linux-gnu
 
-RUN find /usr/local/lib/python3.11 -type d -name __pycache__ | xargs rm -rf {} \;
-RUN find /usr/local/lib/python3.11 -wholename "*test*.key" -delete -or -wholename "*test*.pem" -delete
+RUN find /usr/local/lib/python3.12 -type d -name __pycache__ | xargs rm -rf {} \;
+RUN find /usr/local/lib/python3.12 -wholename "*test*.key" -delete -or -wholename "*test*.pem" -delete
 
 
 ######## Extra packages that don't make sense to pull down in any other stage ########
-FROM base as extra-packages
+FROM base AS extra-packages
 
 RUN apt-get update && apt-get install -qq -y \
         host \
@@ -373,9 +372,9 @@ RUN mkdir -p /opt/root/bin &&\
 COPY --from=collectd /usr/local/bin/patchelf /usr/bin/
 
 # Gather Python dependencies
-COPY --from=python-plugins /usr/local/lib/python3.11 /opt/root/lib/python3.11
-COPY --from=python-plugins /usr/local/lib/libpython3.11.so.1.0 /opt/root/lib
-COPY --from=python-plugins /usr/local/bin/python3.11 /opt/root/bin/python
+COPY --from=python-plugins /usr/local/lib/python3.12 /opt/root/lib/python3.12
+COPY --from=python-plugins /usr/local/lib/libpython3.12.so.1.0 /opt/root/lib
+COPY --from=python-plugins /usr/local/bin/python3.12 /opt/root/bin/python
 
 # Gather compiled collectd plugin libraries
 COPY --from=collectd /usr/sbin/collectd /opt/root/bin/collectd
@@ -392,7 +391,7 @@ RUN patch-rpath /opt/root
 # This build stage is meant as the final target when running the agent in a
 # container environment (e.g. directly with Docker or on K8s).  The stages
 # below this are special-purpose.
-FROM scratch as final-image
+FROM scratch AS final-image
 
 COPY --from=collectd /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 ENV SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt

internal/signalfx-agent/bundle/scripts/build.sh

@@ -38,7 +38,7 @@ CACHE_FROM_OPTS=""
 CACHE_TO_OPTS=""
 BUILDER=""
 
-ALL_STAGES=$( grep '^FROM .* as .*' ${SCRIPT_DIR}/../Dockerfile | sed -e 's/.*as //' )
+ALL_STAGES=$( grep '^FROM .* AS .*' ${SCRIPT_DIR}/../Dockerfile | sed -e 's/.* AS //' )
 
 export DOCKER_BUILDKIT=1
 

internal/signalfx-agent/bundle/scripts/get-collectd-plugins-requirements.txt

@@ -0,0 +1 @@
+PyYaml~=5.0

internal/signalfx-agent/bundle/scripts/pyyaml-requirements.txt

@@ -0,0 +1,3 @@
+Cython<3.0
+setuptools==74.0.0
+wheel==0.44.0

internal/signalfx-agent/bundle/scripts/security-requirements.txt

@@ -1,4 +1,6 @@
 certifi>=2024.7.4
+dbus-python==1.3.2; sys_platform == "linux"
+idna>=3.7,<4
 requests>=2.32.0
 urllib3>=1.26.19,<2
-idna>=3.7,<4
+WMI==1.5.1; sys_platform == "win32"

internal/signalfx-agent/bundle/scripts/windows/bundle.ps1

@@ -18,8 +18,8 @@ $repoDir = "$scriptDir\..\..\..\..\.."
 . $scriptDir\common.ps1
 
 $BUILD_DIR="$repoDir\build"
-$PYTHON_VERSION="3.11.9"
-$PIP_VERSION="21.0.1"
+$PYTHON_VERSION="3.12.5"
+$PIP_VERSION="24.2"
 $NUGET_URL="https://aka.ms/nugetclidl"
 $NUGET_EXE="nuget.exe"
 
@@ -32,16 +32,16 @@ function download_collectd([string]$collectdCommit, [string]$outputDir="$BUILD_D
 function get_collectd_plugins ([string]$buildDir=$BUILD_DIR) {
     mkdir "$buildDir\collectd-python" -ErrorAction Ignore
     $collectdPlugins = Resolve-Path "$buildDir\collectd-python"
-    $requirements = Resolve-Path "$scriptDir\..\requirements.txt"
+    $script_requirements = Resolve-Path "$scriptDir\..\get-collectd-plugins-requirements.txt"
+    $pyyaml_requirements = Resolve-Path "$scriptDir\..\pyyaml-requirements.txt"
     $security_requirements = Resolve-Path "$scriptDir\..\security-requirements.txt"
     $script = Resolve-Path "$scriptDir\..\get-collectd-plugins.py"
     $python = Resolve-Path "$buildDir\python\python.exe"
 
     # workaround for https://github.com/yaml/pyyaml/issues/724
-    & $python -m pip install 'wheel==0.40.0'
-    & $python -m pip install 'Cython<3.0' 'PyYaml~=5.0' --no-build-isolation
-
-    & $python -m pip install -qq -r $requirements
+    & $python -m pip install -qq -r $pyyaml_requirements
+    if ($lastexitcode -ne 0){ throw }
+    & $python -m pip install -qq -r $script_requirements --no-build-isolation
     if ($lastexitcode -ne 0){ throw }
 
     & $python $script $collectdPlugins