Skip to content

Commit

Permalink
[Build] Support j2 template for debian sources (sonic-net#12557)
Browse files Browse the repository at this point in the history
Why I did it
Unify the Debian mirror sources
Make easy to upgrade to the next Debian release, not source url code change required.
Support to customize the Debian mirror sources during the build
Relative issue: sonic-net#12523
  • Loading branch information
xumia committed Apr 20, 2023
1 parent c45c109 commit de05de5
Show file tree
Hide file tree
Showing 20 changed files with 72 additions and 191 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,7 @@ files/build/tmp
dockers/**/buildinfo
platform/**/buildinfo
sonic-slave*/**/buildinfo

# Debian mirror Sources
sources.list.*
!sources.list*.j2
14 changes: 11 additions & 3 deletions Makefile.work
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC)
# Generate the version control build info
$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
scripts/generate_buildinfo_config.sh)

# Generate the slave Dockerfile, and prepare build info for it
Expand Down Expand Up @@ -280,13 +281,20 @@ SONIC_BUILD_INSTRUCTION := make \
SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
SLAVE_DIR=$(SLAVE_DIR) \
MIRROR_URLS=$(MIRROR_URLS) \
MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \
MIRROR_SNAPSHOT=$(MIRROR_SNAPSHOT) \
$(SONIC_OVERRIDE_BUILD_VARS)

.PHONY: sonic-slave-build sonic-slave-bash init reset

.DEFAULT_GOAL := all

%::
export MIRROR_URLS
export MIRROR_SECURITY_URLS
export SONIC_VERSION_CONTROL_COMPONENTS

%:: | sonic-build-hooks
ifeq ($(MULTIARCH_QEMU_ENVIRON), y)
@$(DOCKER_MULTIARCH_CHECK)
ifneq ($(BLDENV), )
Expand All @@ -296,8 +304,6 @@ endif
endif
@$(OVERLAY_MODULE_CHECK)

@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
{ [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \
$(DOCKER_BASE_PULL) && \
Expand All @@ -322,6 +328,8 @@ endif
sonic-build-hooks:
@pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd
@cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo
@[ "$(MULTIARCH_QEMU_ENVIRON)" == y ] && scripts/build_mirror_config.sh $(SLAVE_DIR) amd64 $(BLDENV)
@scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)

sonic-slave-base-build : sonic-build-hooks
@$(OVERLAY_MODULE_CHECK)
Expand Down
1 change: 1 addition & 0 deletions build_debian.sh
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount
[ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/

## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates
scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO
sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list
sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/

Expand Down
8 changes: 1 addition & 7 deletions dockers/docker-base-buster/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive

# Configure data sources for apt/dpkg
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
{% if CONFIGURED_ARCH == "armhf" %}
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
{% elif CONFIGURED_ARCH == "arm64" %}
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
{% else %}
COPY ["sources.list", "/etc/apt/sources.list"]
{% endif %}
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]

Expand Down
13 changes: 0 additions & 13 deletions dockers/docker-base-buster/sources.list

This file was deleted.

11 changes: 0 additions & 11 deletions dockers/docker-base-buster/sources.list.arm64

This file was deleted.

11 changes: 0 additions & 11 deletions dockers/docker-base-buster/sources.list.armhf

This file was deleted.

8 changes: 1 addition & 7 deletions dockers/docker-base-stretch/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive

# Configure data sources for apt/dpkg
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
{% if CONFIGURED_ARCH == "armhf" %}
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
{% elif CONFIGURED_ARCH == "arm64" %}
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
{% else %}
COPY ["sources.list", "/etc/apt/sources.list"]
{% endif %}
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]

Expand Down
11 changes: 0 additions & 11 deletions dockers/docker-base-stretch/sources.list

This file was deleted.

11 changes: 0 additions & 11 deletions dockers/docker-base-stretch/sources.list.arm64

This file was deleted.

11 changes: 0 additions & 11 deletions dockers/docker-base-stretch/sources.list.armhf

This file was deleted.

13 changes: 0 additions & 13 deletions files/apt/sources.list.amd64

This file was deleted.

13 changes: 0 additions & 13 deletions files/apt/sources.list.arm64

This file was deleted.

13 changes: 0 additions & 13 deletions files/apt/sources.list.armhf

This file was deleted.

20 changes: 20 additions & 0 deletions files/apt/sources.list.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# The configuration is generated by template
# Please add additional sources in /etc/apt/sources.list.d

{% for mirror_url in MIRROR_URLS.split(',') %}
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free
{% endfor %}
{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %}
{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %}
{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %}
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free
{% else %}
deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free
{% endif %}
{% endfor %}
26 changes: 26 additions & 0 deletions scripts/build_mirror_config.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/bash

# Generate the sources.list.<arch> in the config path
CONFIG_PATH=$1
export ARCHITECTURE=$2
export DISTRIBUTION=$3

# The default mirror urls
DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/
DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/

# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead
if [ "$ARCHITECTURE" == "armhf" ]; then
DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/
DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/
fi

[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS
[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS

TEMPLATE=files/apt/sources.list.j2
[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2
[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2
[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2

MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE
4 changes: 4 additions & 0 deletions scripts/prepare_docker_buildinfo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ if [ -z "$DISTRO" ]; then
[ -z "$DISTRO" ] && DISTRO=jessie
fi

if [[ "$IMAGENAME" == docker-base-* ]]; then
scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO
fi

# add script for reproducible build. using sha256 instead of tag for docker base image.
scripts/docker_version_control.sh $@

Expand Down
2 changes: 2 additions & 0 deletions slave.mk
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ export TRUSTED_GPG_URLS
export SONIC_VERSION_CONTROL_COMPONENTS
DEFAULT_CONTAINER_REGISTRY := $(SONIC_DEFAULT_CONTAINER_REGISTRY)
export DEFAULT_CONTAINER_REGISTRY
export MIRROR_URLS
export MIRROR_SECURITY_URLS

ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y)
ENABLE_PFCWD_ON_START = y
Expand Down
34 changes: 1 addition & 33 deletions sonic-slave-buster/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -12,39 +12,7 @@ FROM {{ prefix }}debian:buster
MAINTAINER gulv@microsoft.com

COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"]

RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list

{%- if CONFIGURED_ARCH == "armhf" %}
RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
echo "deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
{%- elif CONFIGURED_ARCH == "arm64" %}
RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \
echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \
echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \
echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list
{%- endif %}
COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]

## Make apt-get non-interactive
ENV DEBIAN_FRONTEND=noninteractive
Expand Down
Loading

0 comments on commit de05de5

Please sign in to comment.