From a07b13f9ba508e9e85a5966f68f43e73ae2ebe8e Mon Sep 17 00:00:00 2001 From: sirknightj Date: Thu, 6 Feb 2025 22:10:13 -0800 Subject: [PATCH] Adding sample checks & invalid dts fix --- .github/workflows/ci.yml | 18 +- .github/workflows/raspberry-pi.yaml | 8 +- .github/workflows/samples.yml | 290 +++++++++++++++++++ samples/kvs_gstreamer_audio_video_sample.cpp | 2 +- samples/kvs_gstreamer_multistream_sample.cpp | 11 +- 5 files changed, 314 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/samples.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4f0a62f..3abc1edb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: role-duration-seconds: 10800 - name: Run tests run: | - cd build + cd build ./tst/producerTest mac-os-build-gcc: @@ -73,10 +73,10 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} role-duration-seconds: 10800 - name: Run tests - run: | + run: | cd build ./tst/producerTest - + linux-gcc-code-coverage: runs-on: ubuntu-20.04 env: @@ -88,7 +88,7 @@ jobs: - name: Clone repository uses: actions/checkout@v3 - name: Install dependencies - run: | + run: | sudo apt clean && sudo apt update sudo apt install -y libunwind-dev sudo apt-get install -y libssl-dev libcurl4-openssl-dev liblog4cplus-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-plugins-base-apps gstreamer1.0-plugins-bad gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-tools @@ -115,7 +115,7 @@ jobs: cd build for test_file in $(find CMakeFiles/KinesisVideoProducer.dir gstkvssink.dir KinesisVideoProducerJNI.dir -name '*.gcno'); do gcov $test_file; done bash <(curl -s https://codecov.io/bash) - + address-sanitizer: runs-on: ubuntu-20.04 permissions: @@ -189,7 +189,7 @@ jobs: timeout --signal=SIGABRT 60m ./tst/producerTest # memory-sanitizer: - # runs-on: ubuntu-20.04 + # runs-on: ubuntu-20.04 # permissions: # id-token: write # contents: read @@ -237,9 +237,9 @@ jobs: # mkdir build && cd build # cmake .. -DBUILD_TEST=TRUE -DTHREAD_SANITIZER=TRUE -DBUILD_GSTREAMER_PLUGIN=TRUE -DBUILD_JNI=TRUE # make - # ulimit -c unlimited -S + # ulimit -c unlimited -S # timeout --signal=SIGABRT 20m ./tst/producerTest - + ubuntu-gcc: runs-on: ubuntu-20.04 env: @@ -312,7 +312,7 @@ jobs: - name: Run tests run: | $env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\producer\open-source\local\lib;D:\producer\open-source\local\bin' - & "D:\producer\build\tst\producerTest.exe" + & "D:\producer\build\tst\producerTest.exe" arm64-cross-compilation: runs-on: ubuntu-20.04 diff --git a/.github/workflows/raspberry-pi.yaml b/.github/workflows/raspberry-pi.yaml index 6d3c934d..b3aec002 100644 --- a/.github/workflows/raspberry-pi.yaml +++ b/.github/workflows/raspberry-pi.yaml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 timeout-minutes: 30 permissions: id-token: write @@ -70,7 +70,7 @@ jobs: make -j$(nproc) export GST_PLUGIN_PATH=$(pwd) - + set +e # Disable exit on error for the timeout command timeout --preserve-status --signal=SIGINT --kill-after=15s 30s \ gst-launch-1.0 -v videotestsrc is-live=true \ @@ -81,10 +81,10 @@ jobs: ! kvssink stream-name="cpp-producer-rpi-${{ matrix.os }}" EXIT_CODE=$? set -e # Re-enable exit on error - + # 0: Process exited after interrupt with code 0 # 1: Process exited with error code 1 - # 137: Process killed by SIGKILL (if the --kill-after timeout is reached) + # 137: Process killed by SIGKILL (if the --kill-after timeout is reached) echo "Command exited with code: $EXIT_CODE" if [ $EXIT_CODE -ne 0 ]; then echo "Command did not exit gracefully after interrupt." diff --git a/.github/workflows/samples.yml b/.github/workflows/samples.yml new file mode 100644 index 00000000..9b3ca262 --- /dev/null +++ b/.github/workflows/samples.yml @@ -0,0 +1,290 @@ +name: Producer CPP Sample Checks + +on: + push: + branches: + - develop + - master + pull_request: + branches: + - develop + - master + +jobs: + sample-checks: + name: ${{ matrix.runner.id }} - ${{ matrix.sample.name }} + + strategy: + matrix: + sample: + - name: kvs_gstreamer_audio_video_sample + args: -f sample.mp4 + - name: kvs_gstreamer_file_uploader_sample + args: sample.mp4 0 audio-video + - name: kvs_gstreamer_sample + args: sample.mp4 + - name: kvssink_gstreamer_sample + args: sample.mp4 + runner: + - id: macos-13 + image: macos-13 + + - id: ubuntu-22.04 + image: ubuntu-latest + docker: public.ecr.aws/ubuntu/ubuntu:22.04_stable + + - id: ubuntu-20.04 + image: ubuntu-latest + docker: public.ecr.aws/ubuntu/ubuntu:20.04_stable + + - id: windows-2022 + image: windows-2022 + + fail-fast: false + + runs-on: ${{ matrix.runner.image }} + container: ${{ matrix.runner.docker || '' }} + timeout-minutes: 30 + + env: + AWS_KVS_LOG_LEVEL: 2 + KVS_DEBUG_DUMP_DATA_FILE_DIR: ${{ github.workspace }}/build/debug_output + DEBIAN_FRONTEND: noninteractive + GST_PLUGIN_PATH: ${{ github.workspace }}/build + + permissions: + id-token: write + contents: read + + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew install gstreamer log4cplus mkvtoolnix + + - name: Install dependencies (Linux) + if: runner.os == 'Linux' + run: | + apt-get update + apt-get install -y git cmake build-essential pkg-config libssl-dev libcurl4-openssl-dev \ + liblog4cplus-dev libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \ + gstreamer1.0-plugins-base-apps gstreamer1.0-plugins-bad gstreamer1.0-plugins-good \ + gstreamer1.0-plugins-ugly gstreamer1.0-tools curl mkvtoolnix + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + run: | + choco install nasm strawberryperl pkgconfiglite mkvtoolnix + choco install gstreamer --version=1.22.8 + choco install gstreamer-devel --version=1.22.8 + + - name: Build samples (Linux & Mac) + if: runner.os == 'Linux' || runner.os == 'macOS' + run: | + mkdir build && cd build + mkdir -p $KVS_DEBUG_DUMP_DATA_FILE_DIR + cmake .. -DBUILD_GSTREAMER_PLUGIN=ON -DBUILD_DEPENDENCIES=OFF + make -j$(nproc) + + - name: Build samples (Windows) + if: runner.os == 'Windows' + run: | + $env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\producer\open-source\local\lib;D:\producer\open-source\local\bin' + mkdir D:\producer + Move-Item -Path "D:\a\amazon-kinesis-video-streams-producer-sdk-cpp\amazon-kinesis-video-streams-producer-sdk-cpp\*" -Destination "D:\producer" + cd D:\producer + git config --system core.longpaths true + dir + .github\build_windows.bat + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }} + aws-region: ${{ secrets.AWS_REGION }} + role-duration-seconds: 10800 + + - name: Run ${{ matrix.sample.name }} (Linux & Mac) + if: runner.os == 'Linux' || runner.os == 'macOS' + working-directory: ./build + run: | + curl -fsSL -o sample.mp4 https://awsj-iot-handson.s3-ap-northeast-1.amazonaws.com/kvs-workshop/sample.mp4 + ./${{ matrix.sample.name }} demo-stream-producer-cpp-${{ matrix.runner.id }}-ci-${{ matrix.sample.name }} ${{ matrix.sample.args }} + + - name: Run ${{ matrix.sample.name }} (Windows) + if: runner.os == 'Windows' + env: + GST_PLUGIN_PATH: D:\producer\build + KVS_DEBUG_DUMP_DATA_FILE_DIR: D:\producer\debug_output + working-directory: D:\producer\build + run: | + # Equivalent to set -x + Set-PSDebug -Trace 1 + + $env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\producer\open-source\local\lib;D:\producer\open-source\local\bin;D:\gstreamer\1.0\msvc_x86_64\bin' + + mkdir D:\producer\debug_output + + Invoke-WebRequest -Uri https://awsj-iot-handson.s3-ap-northeast-1.amazonaws.com/kvs-workshop/sample.mp4 -OutFile sample.mp4 + dir + $exePath = Join-Path $PWD ${{ matrix.sample.name }}.exe + & $exePath demo-stream-producer-cpp-${{ matrix.runner.id }}-ci-${{ matrix.sample.name }} ${{ matrix.sample.args }} + + - name: Verify MKV dump (Mac & Linux) + if: runner.os == 'Linux' || runner.os == 'macOS' + working-directory: ./build/debug_output + run: | + shopt -s nullglob # Ensure globbing works correctly and avoids errors when no files are found + + ls -tlrh + mkvfiles=(*.mkv) + if [ ${#mkvfiles[@]} -eq 0 ]; then + echo "No MKV files found in debug_output" + exit 1 + fi + + for file in "${mkvfiles[@]}"; do + echo "Verifying $file with mkvinfo (verbose and hexdump):" + mkvinfo -v -X "$file" + done + shell: bash + + - name: Verify MKV dump (Windows) + if: runner.os == 'Windows' + working-directory: D:\producer\build + run: | + $env:Path += ";C:\Program Files\MKVToolNix" + dir D:\producer\debug_output + $mkvFiles = Get-ChildItem -Path "D:\producer\debug_output" -Filter *.mkv + if ($mkvFiles.Count -eq 0) { + Write-Error "No MKV files found in D:\producer\build\debug_output" + exit 1 + } + # Run mkvinfo on each MKV file + foreach ($file in $mkvFiles) { + Write-Output "Verifying $($file.FullName) with mkvinfo (verbose and hexdump):" + mkvinfo.exe -v -X "$($file.FullName)" + } + + multistream-sample: + name: Multistream sample on Mac + runs-on: macos-13 + timeout-minutes: 30 + + env: + AWS_KVS_LOG_LEVEL: 2 + KVS_DEBUG_DUMP_DATA_FILE_DIR: ${{ github.workspace }}/build/debug_output + GST_PLUGIN_PATH: ${{ github.workspace }}/build + DEBIAN_FRONTEND: noninteractive + + permissions: + id-token: write + contents: read + + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + brew install gstreamer log4cplus mkvtoolnix coreutils + brew install --cask docker + + - name: Build samples + run: | + mkdir build && cd build + mkdir -p $KVS_DEBUG_DUMP_DATA_FILE_DIR + cmake .. -DBUILD_GSTREAMER_PLUGIN=ON -DBUILD_DEPENDENCIES=OFF + make -j$(nproc) + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }} + aws-region: ${{ secrets.AWS_REGION }} + role-duration-seconds: 10800 + + - name: Run multistream sample + working-directory: ./build + run: | + set -x + + open -a /Applications/Docker.app --args --unattended --accept-license + echo "We are waiting for Docker to be up and running. It can take over 2 minutes..." + while ! /Applications/Docker.app/Contents/Resources/bin/docker info &>/dev/null; do sleep 1; done + + sudo ln -s ~/.docker/run/docker.sock /var/run/docker.sock + + docker run -d --rm -it -e RTSP_PROTOCOLS=tcp -p 8554:8554 bluenviron/mediamtx:latest + docker run -d --rm -it -e RTSP_PROTOCOLS=tcp -p 8555:8554 bluenviron/mediamtx:latest + + ( + ffmpeg -re -f lavfi -i "testsrc=size=640x480:rate=10" -vcodec libx264 -x264-params keyint=25 -f rtsp rtsp://localhost:8554/mystream + ) & + ( + ffmpeg -re -f lavfi -i "testsrc=size=640x480:rate=10" -vcodec libx264 -x264-params keyint=25 -f rtsp rtsp://localhost:8555/mystream + ) & + + echo "rtsp://0.0.0.0:8554/mystream" > rtsp-urls.txt + echo "rtsp://0.0.0.0:8555/mystream" >> rtsp-urls.txt + + sleep 10 + gst-discoverer-1.0 rtsp://0.0.0.0:8554/mystream + gst-discoverer-1.0 rtsp://0.0.0.0:8555/mystream + + set +e # Disable exit on error for the timeout command + gtimeout --preserve-status --signal=SIGINT --kill-after=15s 30s \ + ./kvs_gstreamer_multistream_sample demo-stream-producer-cpp-macos-13-ci-kvs_gstreamer_multistream_sample rtsp-urls.txt + EXIT_CODE=$? + set -e # Re-enable exit on error + + # 130 (128 + 2): Process killed by SIGINT + # 137: Process killed by SIGKILL (if the --kill-after timeout is reached) + echo "Command exited with code: $EXIT_CODE" + if [ $EXIT_CODE -ne 130 ]; then + echo "Command did not exit gracefully after interrupt." + exit 1 + fi + + shell: bash + + - name: Verify MKV dump + working-directory: ./build/debug_output + run: | + shopt -s nullglob # Ensure globbing works correctly and avoids errors when no files are found + + ls -tlrh + mkvfiles=(*.mkv) + if [ ${#mkvfiles[@]} -eq 0 ]; then + echo "No MKV files found in debug_output" + exit 1 + fi + + found_0=0 + found_1=0 + + for file in "${mkvfiles[@]}"; do + if [[ "$file" == demo-stream-producer-cpp-macos-13-ci-kvs_gstreamer_multistream_sample_0* ]]; then + found_0=1 + elif [[ "$file" == demo-stream-producer-cpp-macos-13-ci-kvs_gstreamer_multistream_sample_1* ]]; then + found_1=1 + fi + done + + if [ $found_0 -eq 0 ] || [ $found_1 -eq 0 ]; then + echo "Expected at least one file starting with each prefix:" + echo " - demo-stream-producer-cpp-macos-13-ci-kvs_gstreamer_multistream_sample_0" + echo " - demo-stream-producer-cpp-macos-13-ci-kvs_gstreamer_multistream_sample_1" + exit 1 + fi + + for file in "${mkvfiles[@]}"; do + echo "Verifying $file with mkvinfo (verbose and hexdump):" + mkvinfo -v -X "$file" + done + shell: bash diff --git a/samples/kvs_gstreamer_audio_video_sample.cpp b/samples/kvs_gstreamer_audio_video_sample.cpp index 38ea3837..4d6b9613 100644 --- a/samples/kvs_gstreamer_audio_video_sample.cpp +++ b/samples/kvs_gstreamer_audio_video_sample.cpp @@ -1011,7 +1011,7 @@ int main(int argc, char *argv[]) { if (argc < 2) { LOG_ERROR( - "Usage: AWS_ACCESS_KEY_ID=SAMPLEKEY AWS_SECRET_ACCESS_KEY=SAMPLESECRET ./kinesis_video_gstreamer_audio_video_sample_app my-stream-name /path/to/file" + "Usage: AWS_ACCESS_KEY_ID=SAMPLEKEY AWS_SECRET_ACCESS_KEY=SAMPLESECRET ./kinesis_video_gstreamer_audio_video_sample_app my-stream-name -f /path/to/file" "AWS_ACCESS_KEY_ID=SAMPLEKEY AWS_SECRET_ACCESS_KEY=SAMPLESECRET ./kinesis_video_gstreamer_audio_video_sample_app my-stream-name"); return 1; } diff --git a/samples/kvs_gstreamer_multistream_sample.cpp b/samples/kvs_gstreamer_multistream_sample.cpp index fe2b9e22..476aced0 100644 --- a/samples/kvs_gstreamer_multistream_sample.cpp +++ b/samples/kvs_gstreamer_multistream_sample.cpp @@ -53,6 +53,7 @@ LOGGER_TAG("com.amazonaws.kinesis.video.gstreamer"); #define DEFAULT_BUFFER_SIZE (1 * 1024 * 1024) #define DEFAULT_STORAGE_SIZE (128 * 1024 * 1024) #define DEFAULT_ROTATION_TIME_SECONDS 3600 +#define DEFAULT_FRAME_DURATION_MS 2 namespace com { namespace amazonaws { namespace kinesis { namespace video { @@ -180,6 +181,7 @@ typedef struct _CustomData { // Pts of first frame map first_pts_map; map producer_start_time_map; + map last_dts_map; } CustomData; void create_kinesis_video_frame(Frame *frame, const nanoseconds &pts, const nanoseconds &dts, FRAME_FLAGS flags, @@ -259,6 +261,12 @@ static GstFlowReturn on_new_sample(GstElement *sink, CustomData *data) { buffer->pts += data->producer_start_time_map[stream_handle_key] - data->first_pts_map[stream_handle_key]; + if (!GST_BUFFER_DTS_IS_VALID(buffer)) { + buffer->dts = data->last_dts_map[stream_handle_key] + DEFAULT_FRAME_DURATION_MS * HUNDREDS_OF_NANOS_IN_A_MILLISECOND * DEFAULT_TIME_UNIT_IN_NANOS; + } + + data->last_dts_map[stream_handle_key] = buffer->dts; + if (false == put_frame(data->kinesis_video_stream_handles[stream_handle_key], data->frame_data_map[stream_handle_key], buffer_size, std::chrono::nanoseconds(buffer->pts), std::chrono::nanoseconds(buffer->dts), kinesis_video_flags)) { GST_WARNING("Dropped frame"); @@ -405,7 +413,8 @@ int gstreamer_init(int argc, char *argv[]) { data.frame_data_map = map(); data.frame_data_size_map = map(); data.first_pts_map = map(); - data.producer_start_time_map = map();; + data.producer_start_time_map = map(); + data.last_dts_map = map();; /* init GStreamer */ gst_init(&argc, &argv);