Update main

# Conflicts: # apps/services/e2_metric_connector_manager.cpp # apps/services/e2_metric_connector_manager.h
srsran · Oct 9, 2024 · 45da8c1 · 45da8c1
2 parents e73b461 + 356a9df
commit 45da8c1
Show file tree

Hide file tree

Showing 171 changed files with 2,744 additions and 1,203 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -85,7 +85,6 @@ trigger builder:
     - if: $ON_MR
       changes:
         paths:
-          - .gitlab/ci/src_cache.yml
           - .gitlab/ci/builders.yml
           - .gitlab/ci/builders/**/*
           - docker/scripts/**/*
@@ -115,36 +114,76 @@ trigger docker:
   variables:
     PY_DIR: docker/metrics_server
 
-helm:
+matlab nightly:
   stage: ci
   rules:
-    - if: $CI_DESCRIPTION =~ /Weekly/
+    - if: $CI_DESCRIPTION =~ /Nightly Build Unit Tests/
       variables:
-        CI_DESCRIPTION: Synchronization with srsGNB
+        ON_SCHEDULE: "true"
+        CI_DESCRIPTION: Nightly
+        SRSRAN_COMMIT: $CI_COMMIT_SHA
         NOTIFY_SLACK: "true"
+      when: delayed
+      start_in: 3 hours
   inherit:
-    variables: false        
+    variables: false
   trigger:
-    project: softwareradiosystems/ci/srsgnb_k8s
-    branch: main
-  needs:
-    - job: trigger docker
-      artifacts: false
+    project: softwareradiosystems/srsgnb_matlab
+    branch: master
 
-enterprise:
+enterprise nightly:
   stage: ci
   rules:
     - if: $CI_DESCRIPTION =~ /Nightly Build Unit Tests/
       variables:
+        ON_SCHEDULE: "true"
         CI_DESCRIPTION: Nightly
         SRSRAN_COMMIT: $CI_COMMIT_SHA
         NOTIFY_SLACK: "true"
+      when: delayed
+      start_in: 3 hours
   inherit:
     variables: false
   trigger:
     project: softwareradiosystems/srsran_5g_enterprise
     branch: main
 
+matlab weekly:
+  stage: ci
+  rules:
+    - if: $CI_DESCRIPTION =~ /Weekly/
+      variables:
+        ON_SCHEDULE: "true"
+        CI_DESCRIPTION: Nightly
+        SRSRAN_COMMIT: $CI_COMMIT_SHA
+        NOTIFY_SLACK: "true"
+      when: delayed
+      start_in: 20 hours
+  inherit:
+    variables: false
+  trigger:
+    project: softwareradiosystems/srsgnb_matlab
+    branch: master
+
+helm weekly:
+  stage: ci
+  rules:
+    - if: $CI_DESCRIPTION =~ /Weekly/
+      variables:
+        ON_SCHEDULE: "true"
+        CI_DESCRIPTION: Synchronization with srsGNB
+        NOTIFY_SLACK: "true"
+      when: delayed
+      start_in: 20 hours
+  inherit:
+    variables: false
+  trigger:
+    project: softwareradiosystems/ci/srsgnb_k8s
+    branch: main
+  needs:
+    - job: trigger docker
+      artifacts: false
+
 ################################################################################
 ## Static
 ################################################################################
@@ -184,7 +223,7 @@ full-code-format:
   image: ${CR_REGISTRY_URI}/srsgnb/codechecker:${DOCKER_BUILDER_VERSION}
   stage: static
   tags:
-    - ${AMD64_TAG}
+    - amd64
   needs:
     - job: builder version
       optional: false
@@ -298,7 +337,7 @@ clangsa:
     KUBERNETES_MEMORY_REQUEST: 12Gi
     KUBERNETES_MEMORY_LIMIT: 12Gi
   tags:
-    - ${AMD64_TAG}
+    - amd64
   interruptible: false
   timeout: 2 hours
   script:
@@ -400,10 +439,10 @@ unit coverage:
     - |
       export metric_prefix=""
       if [[ $ON_MR == "true" ]]; then metric_prefix="_pr"; fi
-      echo "coverage${metric_prefix},pipeline=${CI_PIPELINE_ID},os=${OS} value=${coverage_value}" >> build_time_metrics.txt
+      echo "coverage${metric_prefix},pipeline=${CI_PIPELINE_ID},os=${OS} value=${coverage_value}" >> build/build_time_metrics.txt
     - |
       influx write --host $INFLUXDB_URL --token $INFLUXDB_TOKEN --org $INFLUXDB_ORG \
-        --bucket ci --file build_time_metrics.txt
+        --bucket ci --file build/build_time_metrics.txt
   coverage: /^\s*Line coverage:\s*\d+.\d+\%/
   artifacts: &unit_coverage_artifacts
     paths:

diff --git a/.gitlab/ci-shared/build.yml b/.gitlab/ci-shared/build.yml
@@ -0,0 +1,102 @@
+variables:
+  INFRASTRUCTURE_TAG:
+    description: Computer architecture and supported instruction sets
+    options:
+      - amd64
+      - amd64-avx2
+      - amd64-avx2-avx512
+      - arm64
+      - on-prem-amd64
+      - on-prem-arm64
+      - aws-spot-amd64
+      - aws-spot-arm64
+    value: "amd64-avx2"
+  OS:
+    description: Operating system
+    options:
+      - "ubuntu-24.10"
+      - "ubuntu-24.04"
+      - "ubuntu-23.10"
+      - "ubuntu-22.04"
+      - "debian-12"
+      - "debian-11"
+      - "archlinux-latest"
+      - "rhel-8"
+    value: "ubuntu-24.04"
+  COMPILER:
+    description: Compiler to use
+    options:
+      - "gcc"
+      - "clang"
+    value: "gcc"
+  BUILD_ARGS:
+    description: It will be passed to cmake
+    value: ""
+  MAKE_ARGS:
+    description: It will be passed to make
+    value: ""
+  UHD_VERSION:
+    description: must be one version supported in the specified OS
+    value: ""
+  DPDK_VERSION:
+    description: must be one version supported in the specified OS
+    value: ""
+
+.build_and_unit_template:
+  image: ${CR_REGISTRY_URI}/srsgnb/builder-${OS}:${DOCKER_BUILDER_VERSION}
+  variables:
+    # TEST
+    TEST_EXECUTION_TIMEOUT: 0
+    # CI
+    KUBERNETES_CPU_REQUEST: 6
+    KUBERNETES_CPU_LIMIT: 6
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 12Gi
+  tags:
+    - ${INFRASTRUCTURE_TAG}
+  before_script:
+    - |
+      BUILD_CMD="${BUILD_ARGS}"
+      if [ -n "${DPDK_VERSION}" ]; then
+        BUILD_CMD="-d ${DPDK_VERSION} ${BUILD_CMD}"
+        export LD_LIBRARY_PATH=/opt/dpdk/${DPDK_VERSION}/lib/x86_64-linux-gnu/:/opt/dpdk/${DPDK_VERSION}/lib/aarch64-linux-gnu/:${LD_LIBRARY_PATH}
+      fi
+      if [ -n "${UHD_VERSION}" ]; then
+        BUILD_CMD="-u ${UHD_VERSION} ${BUILD_CMD}"
+        export LD_LIBRARY_PATH=/opt/uhd/${UHD_VERSION}/lib:${LD_LIBRARY_PATH}
+      fi
+      if [ -n "${COMPILER}" ]; then
+        BUILD_CMD="-c ${COMPILER} ${BUILD_CMD}"
+      fi
+    - |
+      build_srsran() {
+        target="$1"
+        shift
+        set -x
+        builder.sh -m "-j${KUBERNETES_CPU_REQUEST} ${target}" ${BUILD_CMD} "$@" ${SRSRANDIR}
+        { set +x; } 2>/dev/null
+      }
+    - |
+      build_plugin() {
+        target="$1"
+        shift      
+        set -x
+        builder.sh -m "-j${KUBERNETES_CPU_REQUEST} ${target}" ${BUILD_CMD} "$@" ${CI_PROJECT_DIR}
+        { set +x; } 2>/dev/null
+      }
+    - |
+      launch_tests_srsran() {
+        cd ${SRSRANDIR}/build
+        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+        set -x
+        ctest -j${KUBERNETES_CPU_REQUEST} --schedule-random --output-on-failure --output-junit xunit.xml "$@"
+        { set +x; } 2>/dev/null
+      }
+    - |
+      launch_tests_plugin() {
+        cd ${CI_PROJECT_DIR}/build
+        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
+        set -x
+        ctest -j${KUBERNETES_CPU_REQUEST} --schedule-random --output-on-failure --output-junit xunit.xml "$@"
+        { set +x; } 2>/dev/null
+      }
diff --git a/.gitlab/ci-shared/e2e.yml b/.gitlab/ci-shared/e2e.yml
@@ -0,0 +1,161 @@
+variables:
+  TESTBED:
+    description: Retina Testbed Description
+    options:
+      - "zmq"
+      - "zmq_uesim"
+      - "zmq_deb"
+      - "zmq_srsue"
+      - "zmq_cudu"
+      - "rf_b200"
+      - "rf_hp"
+      - "android_b200"
+      - "android_x300"
+      - "viavi"
+      - "none"
+    value: "zmq"
+  MARKERS:
+    description: Selected pytest marker (label / tag). Tests with that mark will run.
+    value: ""
+  KEYWORDS:
+    description: Select tests by keyword expressions. This will run tests which contain names that match the given string expression (case-insensitive), which can include Python operators that use filenames, class names and function names as variables
+    value: ""
+  PYTEST_ARGS:
+    description: Extra pytest args for the e2e suite to run
+    value: ""
+  RETINA_PARAM_ARGS:
+    description: Extra retina args for the e2e suite to run
+    value: ""
+  RETINA_LAUNCHER_ARGS:
+    description: Extra launcher args
+    value: ""
+  E2E_LOG_LEVEL:
+    description: Log level to set for software under test in the e2e tests
+    options:
+      - debug
+      - info
+      - warning
+      - error
+    value: "info"
+
+.load retina variables:
+  stage: ci
+  script:
+    - cat ${SRSRANDIR}/.gitlab/ci/e2e/.env
+  artifacts:
+    reports:
+      dotenv: ${SRSRANDIR}/.gitlab/ci/e2e/.env
+
+.prepare_test:
+  variables:
+    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
+    KUBECONFIG_VAR_NAME_EXTRA: "RETINA_NAMESPACE_KUBECONFIG_EXTRA"
+  before_script: &setup_kube_config
+    - |
+      export FORCE_COLOR=1
+      eval K_PATH="\$$KUBECONFIG_VAR_NAME"
+      export KUBECONFIG=$K_PATH
+
+      eval K_PATH_EXTRA="\$$KUBECONFIG_VAR_NAME_EXTRA"
+      export KUBECONFIG_EXTRA=$K_PATH_EXTRA
+
+.e2e-run-template:
+  resource_group: e2e-${GROUP}
+  timeout: 3 hours
+  image:
+    name: ${RETINA_REGISTRY_PREFIX}/launcher:${RETINA_VERSION}
+    entrypoint: ["/bin/sh", "-c"]
+  variables:
+    ARTIFACT_COMPRESSION_LEVEL: "slowest"
+    KUBERNETES_CPU_REQUEST: 2
+    KUBERNETES_CPU_LIMIT: 2
+    KUBERNETES_MEMORY_REQUEST: 2Gi
+    KUBERNETES_MEMORY_LIMIT: 2Gi
+    KUBERNETES_EPHEMERAL_STORAGE_REQUEST: "20G"
+    KUBERNETES_EPHEMERAL_STORAGE_LIMIT: "20G"
+    KUBECONFIG_VAR_NAME: "RETINA_NAMESPACE_KUBECONFIG"
+    KUBECONFIG_VAR_NAME_EXTRA: "RETINA_NAMESPACE_KUBECONFIG_EXTRA"
+    GROUP: zmq
+  tags:
+    - "${RETINA_TAG}"
+  extends:
+    - .prepare_test
+  artifacts:
+    paths:
+      - ${SRSRANDIR}/tests/e2e/log
+    when: always
+    expire_in: 1 week
+    reports:
+      junit: ${SRSRANDIR}/tests/e2e/out.xml
+  script:
+    - |
+      # Print E2E parameters
+      echo "GROUP=${GROUP}"
+      echo "TESTBED=${TESTBED}"
+      echo "MARKERS=${MARKERS}"
+      echo "KEYWORDS=${KEYWORDS}"
+      echo "PYTEST_ARGS=${PYTEST_ARGS}"
+      echo "RETINA_PARAM_ARGS=${RETINA_PARAM_ARGS}"
+      echo "RETINA_LAUNCHER_ARGS=${RETINA_LAUNCHER_ARGS}"
+      echo "E2E_LOG_LEVEL=${E2E_LOG_LEVEL}"
+    - cd ${SRSRANDIR}
+    # Clean LFS files
+    - |
+      while read -r line; do
+        git rm --cached "$line" >/dev/null
+      done < <(git lfs ls-files | sed -r 's/^.{13}//')
+      echo "srsGNB sources+build size is" $(du -hs .)
+    # Remove any existing retina resource for this group
+    - retina-delete-orchestration-network --user-name ci_${GROUP}
+    # Add extra config env variables to the .env file
+    - |
+      echo "" >> .gitlab/ci/e2e/.env
+      cat $RETINA_CONFIG_ENV >> .gitlab/ci/e2e/.env
+    # Modify request to shared the complete folder with the gnb container
+    - |
+      yq -i '(.[] | select(.type == "gnb") | .shared_files) += [{"local_path": "../../", "remote_path": env(SRSRANDIR), "is_executable": false}]' ${SRSRANDIR}/.gitlab/ci/e2e/retina_request_${TESTBED}.yml
+    # Set username for retina
+    - |
+      cd tests/e2e
+      export LOGNAME=ci_${GROUP}
+    # Run Retina
+    - |
+      E2E_CMD="retina-launcher ${RETINA_LAUNCHER_ARGS} --retina-request=${SRSRANDIR}/.gitlab/ci/e2e/retina_request_${TESTBED}.yml ${PYTEST_ARGS} -k '${KEYWORDS}' -m '${MARKERS}' --register-parameter ue.all.log_level=$E2E_LOG_LEVEL gnb.all.log_level=$E2E_LOG_LEVEL ${RETINA_PARAM_ARGS}"
+      echo "${E2E_CMD}"
+      eval $E2E_CMD
+  after_script:
+    # Remove any existing retina resource for this group
+    - *setup_kube_config
+    - retina-delete-orchestration-network --user-name ci_${GROUP}
+    # Push test metrics
+    - |
+      find . -iname "test_metrics.csv" -exec \
+        influx write --host $INFLUXDB_URL --token $INFLUXDB_TOKEN --org $INFLUXDB_ORG \
+        --bucket ci --file {} \;
+    # Artifact size
+    - echo -e "\e[0Ksection_start:`date +%s`:e2e_folder_section[collapsed=true]\r\e[0KLog folder's tree"
+    - |
+      print_tree() {
+        local dir="$1"
+        local prefix="$2"
+
+        # List directories first
+        find "$dir" -mindepth 1 -maxdepth 1 -type d | while read -r subdir; do
+          local size=$(du -sh "$subdir" | awk '{print $1}')
+          echo "${prefix}├── $(basename "$subdir") [$size]"
+          print_tree "$subdir" "$prefix│   "
+        done
+
+        # List files afterwards
+        find "$dir" -mindepth 1 -maxdepth 1 -type f | while read -r file; do
+          local size=$(du -sh "$file" | awk '{print $1}')
+          echo "${prefix}├── $(basename "$file") [$size]"
+        done
+      }
+      print_tree "${SRSRANDIR}/tests/e2e/log/" ""
+    - echo -e "\e[0Ksection_end:`date +%s`:e2e_folder_section\r\e[0K"
+    - |
+      subpath=$(echo "$SRSRANDIR" | sed "s|^$CI_PROJECT_DIR||")
+      echo "*******************************************************************************************************************************"
+      echo "Test report ---> https://softwareradiosystems.gitlab.io/-/$CI_PROJECT_NAME/-/jobs/$CI_JOB_ID/artifacts${subpath}/tests/e2e/log/report.html"
+      echo "*******************************************************************************************************************************"