Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/docker update #59

Merged
merged 17 commits into from
Nov 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
.idea/
.github/
.git/
8 changes: 8 additions & 0 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ jobs:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Python 3.11
uses: actions/setup-python@v3
with:
python-version: 3.11

- name: Run pre-commit checks
uses: pre-commit/action@v3.0.1

- name: Prepare
id: prepare
run: |
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
scripts/gstreamer
scripts/gstreamer
.idea/
23 changes: 23 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-added-large-files
- id: check-executables-have-shebangs
- id: check-merge-conflict
- id: check-shebang-scripts-are-executable

- repo: https://github.com/AleksaC/hadolint-py
rev: v2.12.1b3
hooks:
- id: hadolint
args: [--ignore, DL3008]

- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.10.0.1
hooks:
- id: shellcheck
26 changes: 16 additions & 10 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,15 @@ ARG LIBCAMERA_VERSION
ARG RPICAM_ENABLED
ARG GST_OMX_ENABLED

COPY ./scripts/build_gst.sh /build_gst.sh
COPY --link ./scripts/build_gst.sh /build_gst.sh
ENV CCACHE_DIR=/ccache
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
--mount=type=cache,target=/root/.cache,sharing=locked \
--mount=type=cache,target=/ccache,sharing=locked \
./build_gst.sh \
&& rm -f /build_gst.sh
&& rm -f /build_gst.sh \
&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*


# Stage 3: Final Image
Expand All @@ -50,9 +53,11 @@ ARG RPICAM_ENABLED
RUN <<-EOF
set -e
RCFILE_PATH="/etc/blueosrc"
echo "alias cat='batcat --paging=never'" >> "$RCFILE_PATH"
echo "alias ls=exa" >> "$RCFILE_PATH"
echo "cd ~" >> "$RCFILE_PATH"
{
echo "alias cat='batcat --paging=never'"
echo "alias ls=exa"
echo "cd ~"
} >> "$RCFILE_PATH"
echo "source $RCFILE_PATH" >> "/etc/bash.bashrc"
EOF

Expand Down Expand Up @@ -133,23 +138,24 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
libyaml-0-2 \
libx264-164 \
libx265-199 \
libxml2
libxml2 \
&& apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Install some tools
COPY ./scripts/install_viu.sh /install_viu.sh
COPY --link ./scripts/install_viu.sh /install_viu.sh
RUN ./install_viu.sh && rm /install_viu.sh

COPY ./scripts/install_gping.sh /install_gping.sh
COPY --link ./scripts/install_gping.sh /install_gping.sh
RUN ./install_gping.sh && rm /install_gping.sh

COPY ./scripts/install_simple_http_server.sh /install_simple_http_server.sh
COPY --link ./scripts/install_simple_http_server.sh /install_simple_http_server.sh
RUN ./install_simple_http_server.sh && rm /install_simple_http_server.sh

# Install Pre-built GStreamer
COPY --from=gstreamer /artifacts/. /.

# Update links for the installed libraries and check if GStreamer is setup correctly
COPY ./scripts/inspect_gst_plugins.sh /inspect_gst_plugins.sh
COPY --link ./scripts/inspect_gst_plugins.sh /inspect_gst_plugins.sh
RUN ldconfig \
&& /inspect_gst_plugins.sh \
&& mkdir -p /home/pi/tools \
Expand Down
70 changes: 35 additions & 35 deletions scripts/build_gst.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,35 +66,35 @@ GST_MESON_OPTIONS_DEFAULT=(
-D tools=enabled
-D webrtc=enabled
)
GST_MESON_OPTIONS=${GST_MESON_OPTIONS:-${GST_MESON_OPTIONS_DEFAULT[@]}}
GST_MESON_OPTIONS=("${GST_MESON_OPTIONS[@]:-${GST_MESON_OPTIONS_DEFAULT[@]}}")
# If enabled, add OMX build configurations to the GST_MESON_OPTIONS array
# Note: GStreamer >= 1.24.0 doesn't support it, and won't recognize the `omx` property
if [ $GST_OMX_ENABLED == true ]; then
if [ "$GST_OMX_ENABLED" == true ]; then
GST_MESON_OPTIONS+=(
-D omx=enabled
)
if [[ $ARM == true ]]; then
if [[ "$ARM" == true ]]; then
# To build omx for the "rpi" target, we need to provide the raspberrypi
# IL headers:
USERLAND_PATH=/tmp/userland
GST_MESON_OPTIONS+=(
-D gst-omx:target=rpi
-D gst-omx:header_path=$USERLAND_PATH/interface/vmcs_host/khronos/IL
-D gst-omx:header_path="$USERLAND_PATH"/interface/vmcs_host/khronos/IL
)
else
GST_MESON_OPTIONS+=(
-D gst-omx:target=generic
)
fi
fi
fi
if [ $LIBCAMERA_ENABLED == true ]; then
if [ "$LIBCAMERA_ENABLED" == true ]; then
GST_MESON_OPTIONS+=(
-D custom_subprojects=libcamera
-D libcamera:cam=disabled
-D libcamera:cpp_std=c++17
-D libcamera:documentation=disabled
-D libcamera:gstreamer=enabled
-D libcamera:ipas=ipu3,rkisp1,rpi/vc4
-D libcamera:ipas="ipu3,rkisp1,rpi/vc4"
-D libcamera:lc-compliance=disabled
-D libcamera:pipelines=auto
-D libcamera:pycamera=disabled
Expand All @@ -105,11 +105,11 @@ if [ $LIBCAMERA_ENABLED == true ]; then
-D libcamera:v4l2=true
)
fi
if [ $RPICAM_ENABLED == true ]; then
if [ "$RPICAM_ENABLED" == true ]; then
GST_MESON_OPTIONS+=(
-D gst-plugins-good:rpicamsrc=enabled
-D gst-plugins-good:rpi-header-dir=$GST_INSTALL_DIR/opt/vc/include
-D gst-plugins-good:rpi-lib-dir=$GST_INSTALL_DIR/opt/vc/lib
-D gst-plugins-good:rpi-header-dir="$GST_INSTALL_DIR"/opt/vc/include
-D gst-plugins-good:rpi-lib-dir="$GST_INSTALL_DIR"/opt/vc/lib
-D gst-plugins-good:replaygain=disabled
)
fi
Expand All @@ -132,7 +132,7 @@ GST_BUILD_TOOLS_DEFAULT=(
pkg-config
python-gi-dev
)
GST_BUILD_TOOLS=${GST_BUILD_TOOLS:-${GST_BUILD_TOOLS_DEFAULT[@]}}
GST_BUILD_TOOLS=("${GST_BUILD_TOOLS:-${GST_BUILD_TOOLS_DEFAULT[@]}}")

# Although to build GStreamer essentially we need only a few libraries, here we
# are actively providing several libraries which would otherwise be compiled
Expand Down Expand Up @@ -177,8 +177,8 @@ GST_BUILD_LIBS_DEFAULT=(
libudev-dev
openssl
)
GST_BUILD_LIBS=${GST_BUILD_LIBS:-${GST_BUILD_LIBS_DEFAULT[@]}}
if [ $LIBCAMERA_ENABLED == true ]; then
GST_BUILD_LIBS=("${GST_BUILD_LIBS:-${GST_BUILD_LIBS_DEFAULT[@]}}")
if [ "$LIBCAMERA_ENABLED" == true ]; then
GST_BUILD_LIBS+=(
libboost-dev
libgnutls28-dev
Expand All @@ -191,7 +191,7 @@ GST_PIP_DEPENDENCIES=(
"markdown==3.3.7"
"meson==1.3.2"
)
if [ $LIBCAMERA_ENABLED == true ]; then
if [ "$LIBCAMERA_ENABLED" == true ]; then
GST_PIP_DEPENDENCIES+=(
"jinja2==3.1.2"
"ply==3.11"
Expand Down Expand Up @@ -224,40 +224,40 @@ EOF

# Install all dependencies

apt update
apt install --assume-yes --no-install-recommends --mark-auto \
${GST_BUILD_TOOLS[@]} ${GST_BUILD_LIBS[@]}
pip3 install ${GST_PIP_DEPENDENCIES[@]}
apt-get update
apt-get install --assume-yes --no-install-recommends --mark-auto \
"${GST_BUILD_TOOLS[@]}" "${GST_BUILD_LIBS[@]}"
python3 -m pip install --no-cache-dir "${GST_PIP_DEPENDENCIES[@]}"

# Download and install IL headers if needed:
if [ -n "$USERLAND_PATH" ]; then
git clone https://github.com/raspberrypi/userland.git $USERLAND_PATH
cd $USERLAND_PATH
git checkout 54fd97ae4066a10b6b02089bc769ceed328737e0
git clone https://github.com/raspberrypi/userland.git "$USERLAND_PATH" --branch 54fd97ae4066a10b6b02089bc769ceed328737e0 \
--single-branch --depth 1
cd "$USERLAND_PATH"

sed -i "s/sudo//g" buildme # remove any sudo call
./buildme $GST_INSTALL_DIR
./buildme "$GST_INSTALL_DIR"

# Let linux aware of userland libs, needed in runtime
mkdir -p $GST_INSTALL_DIR/etc/ld.so.conf.d/
echo "/opt/vc/lib" > $GST_INSTALL_DIR/etc/ld.so.conf.d/userland.conf
mkdir -p "$GST_INSTALL_DIR"/etc/ld.so.conf.d/
echo "/opt/vc/lib" > "$GST_INSTALL_DIR"/etc/ld.so.conf.d/userland.conf

cd $OLDPWD
cd "$OLDPWD"
fi

# Setup ccache
update-ccache-symlinks
echo 'export PATH="/usr/lib/ccache:$PATH"'
echo "export PATH='/usr/lib/ccache:$PATH'"

# Download, build, and pre-install Gstreamer

GSTREAMER_GIT_DIR=/tmp/gstreamer

git clone --branch $GST_VERSION --single-branch --depth=1 \
$GST_GIT_URL $GSTREAMER_GIT_DIR
cd $GSTREAMER_GIT_DIR
git clone --branch "$GST_VERSION" --single-branch --depth=1 \
"$GST_GIT_URL" "$GSTREAMER_GIT_DIR"
cd "$GSTREAMER_GIT_DIR"

if [ $LIBCAMERA_ENABLED == true ]; then
if [ "$LIBCAMERA_ENABLED" == true ]; then
cat << EOF > subprojects/libcamera.wrap
[wrap-git]
directory=libcamera
Expand All @@ -267,9 +267,9 @@ EOF
fi

GST_BUILD_DIR=builddir
meson setup ${GST_MESON_OPTIONS[@]} $GST_BUILD_DIR
meson setup "$GST_BUILD_DIR" "${GST_MESON_OPTIONS[@]}"
Copy link
Member

@joaoantoniocardoso joaoantoniocardoso Nov 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this work? The help says meson setup [args] [builddir] [sourcedir]

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The CI is green, maybe we should add tests to ensure the gst installation ?

Copy link
Member

@joaoantoniocardoso joaoantoniocardoso Nov 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From the logs, meson receives those parameters correctly, so that works.

Regarding tests, we do have some tests to ensure (some) plugins are installed with no missing dynamic libraries in the inspect_gst_plugins.sh script.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's true, forgot about those

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ha... The documents I read got the the parameters in the other order... I didnt' double check with the version used so maybe I am wrong on this.
It was building thus

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@joaoantoniocardoso meson docs is a bit confusing about this but on current version the syntax is this one.
The documentation example also state this on hints : https://mesonbuild.com/Running-Meson.html#configuring-the-build-directory
So it should be good


DESTDIR=$GST_INSTALL_DIR ninja install -C $GST_BUILD_DIR
DESTDIR="$GST_INSTALL_DIR" ninja install -C "$GST_BUILD_DIR"

# Pre-install RTSP helpers
GST_RTSP_HELPERS=(
Expand All @@ -278,7 +278,7 @@ GST_RTSP_HELPERS=(
test-netclock
test-netclock-client
)
for file in ${GST_RTSP_HELPERS[@]}; do
install -Dm755 $GST_BUILD_DIR/subprojects/gst-rtsp-server/examples/$file \
$GST_INSTALL_DIR/usr/local/bin/$file
for file in "${GST_RTSP_HELPERS[@]}"; do
install -Dm755 "$GST_BUILD_DIR"/subprojects/gst-rtsp-server/examples/"$file" \
"$GST_INSTALL_DIR"/usr/local/bin/"$file"
done
14 changes: 7 additions & 7 deletions scripts/inspect_gst_plugins.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,21 +53,21 @@ PLUGINS=(
x265enc
)

if [ $RPICAM_ENABLED == true ] && [ -f /dev/vchiq ]; then
if [ "$RPICAM_ENABLED" == true ] && [ -f /dev/vchiq ]; then
# This test needs to be run in a Raspberry Pi hardware to work.
PLUGINS+=(
rpicamsrc
)
fi

if [ $GST_OMX_ENABLED == true ]; then
if [ "$GST_OMX_ENABLED" == true ]; then
PLUGINS+=(
omxh264enc
omx
)
fi

if [ $LIBCAMERA_ENABLED == true ]; then
if [ "$LIBCAMERA_ENABLED" == true ]; then
PLUGINS+=(
libcamera
libcamerasrc
Expand All @@ -79,15 +79,15 @@ clear_cache
# Here we are individually checking for each plugin because gst-inspect-1.0 only returns the error
# code for the last item when a list is passed.
errors=0
for plugin in ${PLUGINS[@]}; do \
for plugin in "${PLUGINS[@]}"; do \
# Check if gst-inspect can find the plugin
filename=$(gst-inspect-1.0 $plugin | grep Filename | awk '{print $2}')
filename=$(gst-inspect-1.0 "$plugin" | grep Filename | awk '{print $2}')
if [ -z "$filename" ]; then
let "errors++"
(( errors++ ))
# If found, check for possible missing links
elif ldd -r "$filename" 2>&1 | grep -qF "undefined symbol\|not found\|???"; then
echo "Error: $filename has at least one undefined symbol."
let "errors++"
(( errors++ ))
fi
done
if [ $errors -gt 0 ]; then
Expand Down
4 changes: 2 additions & 2 deletions scripts/install_gping.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ if [[ "$(uname -m)" == "x86_64"* ]]; then
fi

wget $REMOTE_BINARY_URL
tar xzf *.tar.gz
tar xzf ./*.tar.gz

cp gping /usr/bin/gping

# Go to original folder
cd -
rm -rf "$TEMPORAY_PATH"
rm -rf "$TEMPORAY_PATH"
2 changes: 1 addition & 1 deletion scripts/install_viu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ if [[ "$(uname -m)" == "x86_64"* ]]; then
fi

wget $REMOTE_BINARY_URL -O /usr/bin/viu
chmod +x /usr/bin/viu
chmod +x /usr/bin/viu
Loading