Adopts the orca testing into the releaser pipeline CI/CD.

Significant fixes and some reorganization. The gateway and collectd tests will reappear in separate PRs in order to keep cognitive load down on this one. Signed-off-by: James Hughes <jameshughes@splunk.com>
signalfx · Feb 4, 2025 · 584780e · 584780e
1 parent 9eb4afa
commit 584780e
Show file tree

Hide file tree

Showing 15 changed files with 299 additions and 246 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,3 @@
+# See https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings#example
+*.cmd text eol=crlf
+*.ps1 text eol=crlf
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,21 +1,35 @@
 default:
-  image: '${DOCKER_CICD_REPO}/ci-container/debian-bookworm:3.5.0'
+  image: "${DOCKER_CICD_REPO}/ci-container/debian-bookworm:3.5.0"
+  id_tokens:
+    CI_JOB_JWT:
+      aud:
+        - "$CICD_VAULT_ADDR"
+        - https://secrets.splunkit.io
 
 variables:
   WIN_2019_BASE_IMAGE: mcr.microsoft.com/windows/servercore:ltsc2019
   WIN_2022_BASE_IMAGE: mcr.microsoft.com/windows/servercore:ltsc2022
+  SPLUNK_OTELCOL_DOWNLOAD_BASE:
+    value: 'https://github.com/signalfx/splunk-otel-collector/releases/download'
+    description: 'URL to download artifacts from'
+  BUILD_DIR:
+    value: '${CI_PROJECT_DIR}/build'
+    description: 'in this CI, where builds are stored'
+  PYPI_READER_ROLE: "artifactory:v2/cloud/role/pypi-reader-role"
 
 stages:
   - update-deps
   - sast-oss-scan
   - build
+  - code_analysis
   - sign-binaries
   - package
   - sign-packages
   - release
   - docker-manifest-release
   - xray-scan
   - github-release
+  - Deploy and Verify
 
 include:
   - project: 'prodsec/scp-scanning/gitlab-checkmarx'
@@ -412,6 +426,108 @@ agent-bundle-windows:
     paths:
       - dist/agent-bundle_windows_amd64.zip
 
+.auth-setup:
+  variables:
+    TEST_FOLDER: $BUILD_DIR/$CI_JOB_ID
+  before_script: |
+      creds-helper init
+      creds-helper pypi "$PYPI_READER_ROLE"
+      eval "$(creds-helper artifactory --eval "$PYPI_READER_ROLE" )"
+      pip install --upgrade pip
+      pip install --upgrade splunk-orca
+      splunk_orca --help
+      cd ~
+      echo ${ORCA_TAR_GZ} | base64 -d > orca_cred.tar.gz
+      tar -xzvf ~/orca_cred.tar.gz
+      echo "area = otel-collector" >> ~/.orca/orca.conf
+      cd -
+
+build-all-platforms:
+  image: "${DOCKER_CICD_REPO}/ci-container/python-3.11-bullseye:1.19.0"
+  stage: "build"
+  script: |
+    make distribute-ta -e
+  artifacts:
+    paths:
+      - "$BUILD_DIR/out/distribution/Splunk_TA_otel*.tgz"
+      - "$BUILD_DIR/packaging-scripts"
+      - "$BUILD_DIR/Makefile"
+  parallel:
+    matrix:
+      - PLATFORM: ["all"]
+        ARCH: ["amd64"]
+
+test-happypath-ta:
+  image: "${DOCKER_CICD_REPO}/ci-container/python-3.11-bullseye:1.19.0"
+  extends: .auth-setup
+  stage: Deploy and Verify
+  dependencies:
+    - "build-all-platforms"
+  parallel:
+    matrix:
+      - UF_VERSION: ["9.0.2", "8.2.7"]
+        SPLUNK_PLATFORM: ["x64_centos_7"]
+        ORCA_CLOUD: ["aws"]
+        ORCA_OPTION: [""]
+        PLATFORM: ["linux"]
+        ARCH: ["amd64"]
+      - UF_VERSION: ["9.0.2", "8.2.7"]
+        SPLUNK_PLATFORM: ["x64_windows_2019"]
+        ORCA_CLOUD: ["aws"]
+        ORCA_OPTION: [""]
+        PLATFORM: ["windows"]
+        ARCH: ["amd64"]
+  script: |
+    make orca-test-ta -e
+  after_script:
+    # remove orca
+    - "cd ${CI_PROJECT_DIR}"
+    - deployment_id=$(grep "orca_deployment_id" $TEST_FOLDER/orca_deployment.json | awk -F ':' '{print $2}' | awk -F '"' '{print $2}')
+    - orca_container=$(splunk_orca --cloud ${ORCA_CLOUD} show containers --deployment-id "${deployment_id}" | grep "Container Name" | awk -F " " '{print $8}')
+    - splunk_orca --cloud "${ORCA_CLOUD}" copy from "${orca_container}" ${ORCA_OPTION} --source /opt/container_artifact/ansible.log --destination "$BUILD_DIR/"
+    - splunk_orca --cloud ${ORCA_CLOUD} destroy "${deployment_id}"
+  artifacts:
+    when: always
+    paths:
+      - "$BUILD_DIR/out/**/*"
+      - "$BUILD_DIR/$CI_JOB_ID/**/*"
+      - "$BUILD_DIR/container_details.txt"
+      - "$BUILD_DIR/orca_deployment.json"
+      - "$BUILD_DIR/ansible*.log"
+      - "$BUILD_DIR/cpu_util.json"
+      - "$BUILD_DIR/orca-stderr.log"
+      - "$BUILD_DIR/orca-stdout.log"
+AppInspect_local:
+  image: "${DOCKER_CICD_REPO}/ci-container/python-3.11-bullseye:1.19.0"
+  dependencies:
+    - "build-all-platforms"
+  stage: code_analysis
+  # From https://cd.splunkdev.com/splcore/dynamic-data-self-storage-app/-/blob/develop/.gitlab-ci.yml#L193
+  # AppInspect regex bugs out on python 3.11
+  image: docker.repo.splunkdev.net/ci-cd/ci-container/python-3.9:1.26.0
+  before_script:
+    - pip install splunk-add-on-ucc-framework
+    - pip install splunk-packaging-toolkit
+    - pip install splunk-appinspect
+  script:
+    - 'which "$BUILD_DIR/out/distribution/Splunk_TA_otel.tgz" && exit 1'
+    - splunk-appinspect inspect --mode precert --included-tags cloud --output-file "$BUILD_DIR/out/appinspect_report_local.json" "$BUILD_DIR/out/distribution/Splunk_TA_otel.tgz"
+  artifacts:
+    paths:
+      - "$BUILD_DIR/out/appinspect_report_local.json"
+      - "$BUILD_DIR/out/distribution/**/*"
+    expire_in: 21 day
+    when: always
+  allow_failure: false
+
+spell check:
+  image: "${DOCKER_CICD_REPO}/ci-container/node-18-alpine:1.9.0"
+  stage: code_analysis
+  script: |
+    npm install -g cspell@latest
+    cd $CI_PROJECT_DIR/packaging/technical-addon/Splunk_TA_otel
+    cspell --config $CI_PROJECT_DIR/packaging/technical-addon/packaging-scripts/cspell/cspell.json --color "**/*.md" --exclude "**/agent-bundle/**"
+
 .instrumentation-deb-rpm:
   extends: .trigger-filter
   stage: package

diff --git a/packaging/technical-addon/CONTRIBUTING.md b/packaging/technical-addon/CONTRIBUTING.md
@@ -0,0 +1,32 @@
+# Orca
+(For internal splunkers only)
+To use orca,
+0. Set up a venv `mkdir -p ~/.venvs; python3 -m venv ~/.venvs/orca`
+1. Activate the venv `source ~/.venvs/orca/bin/activate`
+2. Ensure you're on vpn and have authenticated with artifactory (ex `okta-artifactory-login`)
+3. Install orca `pip install --upgrade orca`
+
+# Testing
+Currently, we use orca, a wrapper for splunk-ansible.
+
+To auth into splunk orca, run `splunk_orca config auth`.  It's usually best to install orca to a virtual env (assuming you've run `okta-artifactory-login`)
+```
+python3.9 -m venv ../.orca-latest && source ../.orca-latest/bin/activate && pip install --upgrade splunk_orca
+```
+
+To test locally, grab an access token and run something like
+
+```
+make install-tools && PLATFORM=all make distribute-ta && OLLY_ACCESS_TOKEN="<REDACTED>" UF_VERSION=9.0.2 SPLUNK_PLATFORM=x64_windows_2022 PLATFORM=windows ARCH=amd64 ORCA_OPTION="" ORCA_CLOUD="aws" SPLUNK_CONFIG='$SPLUNK_OTEL_TA_HOME/configs/ta-agent-config.yaml' BUILD_DIR=$(pwd)/build make orca-test-ta
+```
+
+If you want to test windows, you may need to run something like 
+```
+ rm -rf build && PLATFORM=all make distribute-ta && OLLY_ACCESS_TOKEN="REDACTED" UF_VERSION=9.0.2 SPLUNK_PLATFORM=x64_windows_2022 PLATFORM=windows ARCH=amd64 ORCA_OPTION="" ORCA_CLOUD="aws" make -e orca-test-ta
+```
+
+When debugging orca itself, you can directly invoke provide a local path to your TA and any ansible as such:
+```
+splunk_orca -vvv --cloud aws --printer sdd-json --deployment-file /home/jameshughes/workspace/otel-github/splunk-otel-collector/build/orca_deployment.json --ansible-log ansible-local.log create --prefix happypath --env SPLUNK_CONNECTION_TIMEOUT=600 --splunk-version 9.0.2 --platform x64_windows_2022 --local-apps /home/jameshughes/workspace/otel-github/splunk-otel-collector/build/ci-cd/Splunk_TA_otel.tgz
+```
+
diff --git a/packaging/technical-addon/Makefile b/packaging/technical-addon/Makefile
@@ -1,6 +1,6 @@
 # Used for building
 SOURCE_DIR?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-BUILD_DIR?=./build
+BUILD_DIR?=$(realpath .)/build
 OTEL_COLLECTOR_VERSION?=0.111.0
 SPLUNK_OTELCOL_DOWNLOAD_BASE?=https://github.com/signalfx/splunk-otel-collector/releases/download
 PLATFORM?=linux
@@ -87,8 +87,8 @@ package-ta: env-guard-ta
 .PHONY: distribute-ta
 distribute-ta: generate-technical-addon download-release package-ta
 
-.PHONY: verify-ta
-verify-ta: env-guard-verify
+.PHONY: orca-test-ta
+orca-test-ta: env-guard-verify
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	BUILD_DIR="$(BUILD_DIR)" \
 	ARCH="$(ARCH)" \
@@ -98,7 +98,7 @@ verify-ta: env-guard-verify
 	UF_VERSION="$(UF_VERSION)" \
 	SPLUNK_PLATFORM="$(SPLUNK_PLATFORM)" \
 	OLLY_ACCESS_TOKEN="$(OLLY_ACCESS_TOKEN)" \
-	$(SOURCE_DIR)/packaging-scripts/deploy-and-verify.sh
+	$(SOURCE_DIR)/packaging-scripts/cicd-tests/happypath-test.sh
 
 .PHONY: smoketest-ta
 smoketest-ta:
@@ -116,7 +116,6 @@ discovery-test-ta:
 	PLATFORM="$(PLATFORM)" \
 	$(SOURCE_DIR)/packaging-scripts/cicd-tests/discovery/discovery-docker-test.sh
 
-
 .PHONY: update-ta-deps
 update-ta-deps:
 	SOURCE_DIR="$(SOURCE_DIR)" \

diff --git a/packaging/technical-addon/README.md b/packaging/technical-addon/README.md
@@ -36,6 +36,8 @@ operating system (ex windows, linux).
 This can be accomplished by removing either `Splunk_TA_Otel/linux_x86_64` or
 `Splunk_TA_Otel/windows_x86_64`, respectively.
 
+Further, they may remove the agent bundle downloaded to the `bin/` folder in these platform specific directories if they don't need smart agent support.
+
 ## Maintaining configuration between upgrades
 As with all TAs, any changes made to `configs` will be overwritten.
 Customers should copy any relevant custom configuration from `configs/` or `defaults/`

diff --git a/packaging/technical-addon/Splunk_TA_otel/linux_x86_64/bin/Splunk_TA_otel.sh b/packaging/technical-addon/Splunk_TA_otel/linux_x86_64/bin/Splunk_TA_otel.sh
@@ -521,7 +521,7 @@ extract_bundle() {
     BUNDLE_PARENT="$(realpath --canonicalize-missing "$SPLUNK_BUNDLE_DIR/../")"
     splunk_TA_otel_log_msg "DEBUG" "will cd to $BUNDLE_PARENT"
     tar -C "$BUNDLE_PARENT" -xzf "$splunk_TA_otel_script_directory/$SMART_AGENT_BUNDLE" || splunk_TA_otel_log_error "failed to extract $splunk_TA_otel_script_directory/$SMART_AGENT_BUNDLE to $SPLUNK_BUNDLE_DIR"
-    splunk_TA_otel_log_msg "INFO" "agent bundle extracted to $SPLUNK_BUNDLE_DIR"
+    splunk_TA_otel_log_msg "INFO" "Done extracting agent bundle to $SPLUNK_BUNDLE_DIR"
 
     # Patch everything in agent-bundle/bin to use our provided .so
     root_path="$(readlink -f "$splunk_bundle_dir_value")"

diff --git a/packaging/technical-addon/Splunk_TA_otel/windows_x86_64/bin/Splunk_TA_otel.cmd b/packaging/technical-addon/Splunk_TA_otel/windows_x86_64/bin/Splunk_TA_otel.cmd
@@ -6,8 +6,9 @@ set "splunk_otel_common_ps_name=Splunk_TA_otel_utils.ps1"
 
 set "SPLUNK_OTEL_TA_HOME="
 set "SPLUNK_OTEL_TA_PLATFORM_HOME="
-
 set "SPLUNK_OTEL_FLAGS="
+
+
 :: BEGIN AUTOGENERATED CODE
 set "configd_name=configd"
 set "configd_value="
@@ -61,25 +62,25 @@ if "%1%"=="" goto splunk_TA_otel_run_agent
 if "%1%"=="--scheme" goto splunk_TA_otel_scheme
 if "%1%"=="--validate-arguments" goto splunk_TA_otel_validate_arg
 :: exit if no matching handler for argument
-exit /b
+exit /b 1
 
 :: main entry hooks
 
 :splunk_TA_otel_scheme
 setlocal
 echo "display scheme called"
 endlocal
-exit /B
+exit /b 0
 
 :splunk_TA_otel_validate_arg
 setlocal
 echo "validate args called"
 endlocal
-exit /B 0
+exit /b 0
 
 
 :splunk_TA_otel_run_agent
-setlocal
+setlocal enabledelayedexpansion
 :: READING CONFIGURATION FROM STDIN
 call :splunk_TA_otel_read_configs
 
@@ -88,8 +89,8 @@ call :splunk_TA_otel_log_msg "INFO" "Logging TA notices to %splunk_TA_otel_log_f
 :: By default, otel will register itself as a windows service. In context of the TA, we want splunk to manage our lifecycle, so turn this off.
 set "NO_WINDOWS_SERVICE=1"
 
-if "%SPLUNK_ACCESS_TOKEN%"=="" (
-    call :splunk_TA_otel_log_msg "INFO" "Grabbing SPLUNK_ACCESS_TOKEN from file %splunk_access_token_file%"
+if "%SPLUNK_ACCESS_TOKEN%" == "" (
+    call :splunk_TA_otel_log_msg "INFO" "Grabbing SPLUNK_ACCESS_TOKEN from file %splunk_access_token_file_value%"
     call :get_access_token
 ) else (
     call :splunk_TA_otel_log_msg "INFO" "Environment variable SPLUNK_ACCESS_TOKEN already set."
@@ -115,13 +116,11 @@ if "%splunk_bundle_dir_value%" == "" (
 if "%splunk_config_value%" == "" (
     call :splunk_TA_otel_log_msg "DEBUG" "Param %splunk_config_name% not set"
 ) else (
-    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --config=%splunk_config_value%"
     set "SPLUNK_CONFIG=%splunk_config_value%"
 )
 if "%splunk_config_dir_value%" == "" (
     call :splunk_TA_otel_log_msg "DEBUG" "Param %splunk_config_dir_name% not set"
 ) else (
-    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --config-dir=%splunk_config_dir_value%"
     set "SPLUNK_CONFIG_DIR=%splunk_config_dir_value%"
 )
 if "%splunk_collectd_dir_value%" == "" (
@@ -184,28 +183,18 @@ if "%splunk_access_token_file_value%" == "" (
 ) else (
     set "SPLUNK_ACCESS_TOKEN_FILE=%splunk_access_token_file_value%"
 )
-if "%configd_value%" == "true" (
-    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --configd"
-) else (
-    call :splunk_TA_otel_log_msg "DEBUG" "Optional flag %configd_name% not set"
-)
-if "%discovery_value%" == "true" (
-    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --discovery"
-) else (
-    call :splunk_TA_otel_log_msg "DEBUG" "Optional flag %discovery_name% not set"
-)
-if "%discovery_properties_value%" != "" (
-    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% --discovery-properties=%discovery_properties_value%"
-) else (
+if "%discovery_properties_value%" == "" (
     call :splunk_TA_otel_log_msg "DEBUG" "Optional flag %discovery_properties_name% not set"
+) else (
+    set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% /discovery-properties='%discovery_properties_value%'"
 )
 :: END AUTOGENERATED CODE
 
 :: Extract agent bundle
 call :extract_bundle
 
 set "command_line=%splunk_TA_otel_app_directory%%splunk_otel_process_name%"
-start /B "" "%command_line%" "%SPLUNK_OTEL_FLAGS%" > "%splunk_otel_log_file_value%" 2>&1
+start /B "" "%command_line%" %SPLUNK_OTEL_FLAGS% > "%splunk_otel_log_file_value%" 2>&1
 set "splunk_otel_common_ps=%splunk_TA_otel_app_directory%%splunk_otel_common_ps_name%"
 start /B "" /I /WAIT powershell "& '%splunk_otel_common_ps%' '%splunk_otel_process_name%' '%splunk_TA_otel_log_file%'"
 
@@ -228,18 +217,16 @@ for /f "delims=" %%a in ('powershell -noninteractive -noprofile -command "get-da
 if not "%log_type%%SPLUNK_OTEL_TA_DEBUG%" == "DEBUG" (
     echo "%log_date%" "%log_type%" "%log_msg%" >> "%splunk_TA_otel_log_file%"
 )
-
+echo off
 endlocal
-exit /B 0
+exit /b 0
 
 :splunk_TA_otel_read_configs
 echo "INFO grabbing config from stdin..."
 
-
-
-for /F "tokens=1,2 delims==" %%I in ('powershell -noninteractive -noprofile -command "$input | Select-String -Pattern '.*?(%configd_name%|%discovery_name%|%discovery_properties_name%|%gomemlimit_name%|%splunk_access_token_file_name%|%splunk_api_url_name%|%splunk_ballast_size_mib_name%|%splunk_bundle_dir_name%|%splunk_collectd_dir_name%|%splunk_config_name%|%splunk_config_dir_name%|%splunk_config_yaml_name%|%splunk_debug_config_server_name%|%splunk_gateway_url_name%|%splunk_hec_url_name%|%splunk_ingest_url_name%|%splunk_listen_interface_name%|%splunk_memory_limit_mib_name%|%splunk_memory_total_mib_name%|%splunk_otel_log_file_name%|%splunk_realm_name%).*?>(.*?)<' | ForEach-Object { $_.Matches.Groups[1].Value + '=' + $_.Matches.Groups[2].Value }"') do (
-    if "%%I"=="%configd_name%" set "configd_value=%%J"
-    if "%%I"=="%discovery_name%" set "discovery_value=%%J"
+for /F "tokens=1,2 delims==" %%I in ('powershell -noninteractive -noprofile -command "$input | Select-String -Pattern '.*?(%configd_name%|%discovery_name%|%discovery_properties_name%|%gomemlimit_name%|%splunk_api_url_name%|%splunk_bundle_dir_name%|%splunk_config_name%|%splunk_config_dir_name%|%splunk_collectd_dir_name%|%splunk_debug_config_server_name%|%splunk_config_yaml_name%|%splunk_gateway_url_name%|%splunk_hec_url_name%|%splunk_listen_interface_name%|%splunk_memory_limit_mib_name%|%splunk_memory_total_mib_name%|%splunk_otel_log_file_name%|%splunk_ingest_url_name%|%splunk_realm_name%|%splunk_access_token_file_name%|session_key).*?>(.*?)<' | ForEach-Object { $_.Matches.Groups[1].Value + '=' + $_.Matches.Groups[2].Value }"') do (
+    if "%%I"=="%configd_name%" set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% /configd"
+    if "%%I"=="%discovery_name%" set "SPLUNK_OTEL_FLAGS=%SPLUNK_OTEL_FLAGS% /discovery"
     if "%%I"=="%discovery_properties_name%" set "discovery_properties_value=%%J"
     if "%%I"=="%gomemlimit_name%" set "gomemlimit_value=%%J"
     if "%%I"=="%splunk_api_url_name%" set "splunk_api_url_value=%%J"
@@ -366,7 +353,6 @@ for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%splu
 for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%splunk_otel_log_file_value%' -replace '\$SPLUNK_HOME', '%SPLUNK_HOME%'"') do (
     set "splunk_otel_log_file_value=%%i"
 )
-
 :: expand params in discovery_properties_value
 for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%discovery_properties_value%' -replace '\$SPLUNK_OTEL_TA_PLATFORM_HOME', '%SPLUNK_OTEL_TA_PLATFORM_HOME%'"') do (
     set "discovery_properties_value=%%i"
@@ -377,15 +363,22 @@ for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%disc
 for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "'%discovery_properties_value%' -replace '\$SPLUNK_HOME', '%SPLUNK_HOME%'"') do (
     set "discovery_properties_value=%%i"
 )
+
+
 :: END AUTOGENERATED CODE
+exit /b 0
 
 :get_access_token
 set /p SPLUNK_ACCESS_TOKEN=<"%splunk_access_token_file_value%"
-exit /B 0
+exit /b 0
 
 :extract_bundle
+setlocal
 call :splunk_TA_otel_log_msg "INFO" "Extract agent bundle from '%splunk_TA_otel_app_directory%\agent-bundle_windows_amd64.zip' to %splunk_bundle_dir%"
 for /f "delims=" %%i in ('powershell -noninteractive -noprofile -command "if (-not (Test-Path '%splunk_bundle_dir%')) {Expand-Archive -LiteralPath '%splunk_TA_otel_app_directory%\agent-bundle_windows_amd64.zip' -Destination '%splunk_bundle_dir%\..\' -Force }"') do (
     call :splunk_TA_otel_log_msg "DEBUG" "result from extract: %%i"
 )
-exit /B 0
+call :splunk_TA_otel_log_msg "INFO" "Done extracting agent bundle from '%splunk_TA_otel_app_directory%\agent-bundle_windows_amd64.zip' to %splunk_bundle_dir%"
+endlocal
+exit /b 0
+