Skip to content

Commit

Permalink
Update to debian bookworm and python 3.12
Browse files Browse the repository at this point in the history
  • Loading branch information
@ostefano
ostefano committed Jul 15, 2024
1 parent 1c89ebc commit b06bac9
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 92 deletions.
155 changes: 91 additions & 64 deletions core/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,42 @@
ARG DOCKER_HUB_PROXY=""

FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build

FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as php-base
ENV DEBIAN_FRONTEND noninteractive

# Uncomment when building in corporate environments
# COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem
# COPY ./rootca.crt /usr/lib/ssl/cert.pem

RUN apt-get update; apt-get install -y --no-install-recommends \
lsb-release \
ca-certificates \
curl
RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb
RUN dpkg -i /tmp/debsuryorg-archive-keyring.deb
RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list
RUN apt-get update


FROM php-base as composer-build
ENV DEBIAN_FRONTEND noninteractive
ENV COMPOSER_ALLOW_SUPERUSER 1
ARG CORE_TAG
ARG CORE_COMMIT

RUN apt-get update; apt-get install -y --no-install-recommends \
ca-certificates \
php \
php-apcu \
php-curl \
php-xml \
php-intl \
php-bcmath \
php-mbstring \
php-mysql \
php-redis \
php-gd \
php-fpm \
php-zip \
RUN apt-get install -y --no-install-recommends \
php7.4 \
php7.4-apcu \
php7.4-curl \
php7.4-xml \
php7.4-intl \
php7.4-bcmath \
php7.4-mbstring \
php7.4-mysql \
php7.4-redis \
php7.4-gd \
php7.4-fpm \
php7.4-zip \
unzip \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

Expand All @@ -29,41 +46,53 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
RUN composer config --no-interaction allow-plugins.composer/installers true
RUN composer install
RUN composer require --with-all-dependencies --no-interaction \
supervisorphp/supervisor:^4.0 \
guzzlehttp/guzzle \
lstrojny/fxmlrpc \
php-http/message \
php-http/message-factory \
# docker image specific dependencies
elasticsearch/elasticsearch:^8.7.0 \
jakub-onderka/openid-connect-php:^1.0.0 \
aws/aws-sdk-php

FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
supervisorphp/supervisor:^4.0 \
guzzlehttp/guzzle \
lstrojny/fxmlrpc \
php-http/message \
php-http/message-factory \
# docker image specific dependencies
elasticsearch/elasticsearch:^8.7.0 \
jakub-onderka/openid-connect-php:^1.0.0 \
aws/aws-sdk-php


FROM php-base as php-build
ENV DEBIAN_FRONTEND noninteractive
ENV TZ Etc/UTC

RUN apt-get update; apt-get install -y --no-install-recommends \
RUN apt-get install -y --no-install-recommends \
gcc \
g++ \
make \
php7.4 \
php7.4-dev \
php7.4-xml \
libfuzzy-dev \
ca-certificates \
php \
php-dev \
php-xml \
php-pear \
librdkafka-dev \
libsimdjson-dev \
libzstd-dev \
git \
php-pear \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson && pecl install zstd
RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
cd php-ext-brotli && phpize && ./configure && make && make install
RUN apt-cache search pecl

RUN update-alternatives --set php /usr/bin/php7.4
RUN update-alternatives --set php-config /usr/bin/php-config7.4
RUN update-alternatives --set phpize /usr/bin/phpize7.4

RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib
RUN pecl channel-update pecl.php.net && \
pecl install ssdeep && \
pecl install rdkafka && \
pecl install simdjson && \
pecl install zstd
RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
cd php-ext-brotli && phpize && ./configure && make && make install

FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build

FROM php-base as python-build
ENV DEBIAN_FRONTEND noninteractive
ARG CORE_TAG
ARG CORE_COMMIT
Expand All @@ -77,8 +106,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
ARG PYPI_CYBOX_VERSION
ARG PYPI_PYMISP_VERSION

RUN apt-get update; apt-get install -y --no-install-recommends \
python3-pip \
RUN apt-get install -y --no-install-recommends \
git \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

Expand All @@ -89,7 +117,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
else
git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP
fi

cd /var/www/MISP || exit; git submodule update --init --recursive .
EOF

Expand Down Expand Up @@ -124,7 +151,7 @@ EOF
fi
done;

pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt

# Remove files we do not care for
rm -r /var/www/MISP/PyMISP
Expand All @@ -134,13 +161,14 @@ EOF
find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} +
EOF

FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"

FROM php-base
ENV DEBIAN_FRONTEND noninteractive
ARG CORE_TAG
ARG CORE_COMMIT
ARG PHP_VER

RUN apt-get update; apt-get install -y --no-install-recommends \
RUN apt-get install -y --no-install-recommends \
gettext \
procps \
sudo \
Expand All @@ -150,42 +178,41 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
openssl \
gpg \
gpg-agent \
ssdeep \
libfuzzy2 \
mariadb-client \
rsync \
# Python Requirements
python3 \
python3-setuptools \
python3-pip \
# PHP Requirements
php \
php-apcu \
php-curl \
php-xml \
php-intl \
php-bcmath \
php-mbstring \
php-mysql \
php-redis \
php-gd \
php-fpm \
php-zip \
php-ldap \
php7.4 \
php7.4-apcu \
php7.4-curl \
php7.4-xml \
php7.4-intl \
php7.4-bcmath \
php7.4-mbstring \
php7.4-mysql \
php7.4-redis \
php7.4-gd \
php7.4-fpm \
php7.4-zip \
php7.4-ldap \
libmagic1 \
libldap-common \
librdkafka1 \
libbrotli1 \
libsimdjson5 \
libsimdjson14 \
libzstd1 \
ssdeep \
libfuzzy2 \
# Unsure we need these
zip unzip \
# Require for advanced an unattended configuration
curl jq \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

RUN update-alternatives --set php /usr/bin/php7.4

# Install python modules
COPY --from=python-build /wheels /wheels
RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels
RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels

# PHP: install prebuilt libraries, then install the app's PHP deps
COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"]
Expand All @@ -208,7 +235,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
EOF

# nginx
RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs
RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs

# Make a copy of the file and configuration stores, so we can sync from it

Expand Down
66 changes: 38 additions & 28 deletions modules/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
ARG DOCKER_HUB_PROXY=""

FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build
ENV DEBIAN_FRONTEND noninteractive
ARG MODULES_TAG
ARG MODULES_COMMIT
Expand All @@ -9,72 +9,82 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
git \
python3-dev \
python3-pip \
python3-wheel \
build-essential \
pkg-config \
libpoppler-cpp-dev \
libfuzzy-dev \
libffi-dev \
libxml2-dev \
libxslt-dev \
libssl-dev \
ninja-build \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

RUN mkdir /wheels
WORKDIR /srv

RUN <<-EOF
if [ ! -z ${MODULES_COMMIT} ]; then
git clone https://github.com/MISP/misp-modules.git /srv/misp-modules && cd /srv/misp-modules && git checkout ${MODULES_COMMIT}
else
git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules
fi

cd /srv/misp-modules
sed -i "s/numpy.*/numpy<2/" REQUIREMENTS
echo "pyeti" >> REQUIREMENTS
echo "greynoise" >> REQUIREMENTS
echo "Google-Search-API" >> REQUIREMENTS
pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/
rm -rf /srv/misp-modules
EOF

WORKDIR /srv/misp-modules
RUN pip install pipenv
RUN sed -i "s/python_version.*/python_version = \"3.12\"/" Pipfile
RUN sed -i "/\[packages\]/a numpy = \"<2.0.0\"" Pipfile
RUN sed -i "/\[packages\]/a pysafebrowsing = \"*\"" Pipfile
RUN sed -i "/\[packages\]/a sigmf = \"*\"" Pipfile
RUN sed -i "/\[packages\]/a matplotlib = \"*\"" Pipfile
RUN pipenv lock
RUN pipenv requirements > requirements.txt
RUN pip wheel -r requirements.txt --no-cache-dir -w /wheels/

WORKDIR /srv/
RUN rm -rf /srv/misp-modules

RUN <<-EOF
git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup
cd /srv/faup
if [ ! -z ${LIBFAUP_COMMIT} ]; then
git checkout ${LIBFAUP_COMMIT}
fi

cd /srv/faup/build
cmake -G "Ninja" ../
ninja
ninja install
cd /srv/faup/src/lib/bindings/python
pip3 wheel --no-cache-dir --no-dependencies -w /wheels/ .
rm -rf /srv/faup
EOF

WORKDIR /srv/faup/build
RUN cmake -G "Ninja" ../
RUN ninja
RUN ninja install
WORKDIR /srv/faup/src/lib/bindings/python
RUN pip wheel --no-cache-dir --no-dependencies -w /wheels/ .

WORKDIR /srv/
RUN rm -rf /srv/faup

FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye"

FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm"
ENV DEBIAN_FRONTEND noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
libglib2.0-0 \
libzbar0 \
libxrender1 \
libxext6 \
libpoppler-cpp0v5 \
libgl1 \
libfuzzy2 \
libffi8 \
libxext6 \
libxml2 \
libxslt1.1 \
libzbar0 \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

COPY --from=python-build /wheels /wheels
COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/
RUN pip3 install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels
RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels

# Since we compile faup ourselves and lua is not required anymore, we can load our own library
# and skip the pre-compiled blob to improve compatibility with other architectures like ARM
RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
/usr/local/lib/python3.11/site-packages/pyfaup/__init__.py
/usr/local/lib/python3.12/site-packages/pyfaup/__init__.py

# Disable (all) warnings raised when using 'future'
RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \
Expand Down

0 comments on commit b06bac9

Please sign in to comment.