From 8395de69d38f2019383bbd181097ccd842668b43 Mon Sep 17 00:00:00 2001 From: xumia <59720581+xumia@users.noreply.github.com> Date: Fri, 30 Dec 2022 09:47:33 +0800 Subject: [PATCH] [Build] Support j2 template for debian sources (#12557) (#13185) Why I did it Unify the Debian mirror sources Make easy to upgrade to the next Debian release, not source url code change required. Support to customize the Debian mirror sources during the build Relative issue: #12523 How I did it How to verify it --- .gitignore | 3 ++ Makefile.work | 8 +++++ build_debian.sh | 1 + dockers/docker-base-bullseye/Dockerfile.j2 | 8 +---- dockers/docker-base-bullseye/sources.list | 14 -------- .../docker-base-bullseye/sources.list.arm64 | 9 ----- .../docker-base-bullseye/sources.list.armhf | 9 ----- dockers/docker-base-buster/Dockerfile.j2 | 8 +---- dockers/docker-base-buster/sources.list | 14 -------- dockers/docker-base-buster/sources.list.arm64 | 12 ------- dockers/docker-base-buster/sources.list.armhf | 12 ------- dockers/docker-base-stretch/Dockerfile.j2 | 8 +---- dockers/docker-base-stretch/sources.list | 11 ------ .../docker-base-stretch/sources.list.arm64 | 11 ------ .../docker-base-stretch/sources.list.armhf | 11 ------ files/apt/sources.list.amd64 | 13 ------- files/apt/sources.list.arm64 | 13 ------- files/apt/sources.list.armhf | 18 ---------- files/apt/sources.list.j2 | 20 +++++++++++ scripts/build_mirror_config.sh | 26 ++++++++++++++ scripts/prepare_docker_buildinfo.sh | 4 +++ slave.mk | 2 ++ sonic-slave-bullseye/Dockerfile.j2 | 35 +------------------ sonic-slave-buster/Dockerfile.j2 | 34 +----------------- sonic-slave-stretch/Dockerfile.j2 | 35 +------------------ 25 files changed, 70 insertions(+), 269 deletions(-) delete mode 100644 dockers/docker-base-bullseye/sources.list delete mode 100644 dockers/docker-base-bullseye/sources.list.arm64 delete mode 100644 dockers/docker-base-bullseye/sources.list.armhf delete mode 100644 dockers/docker-base-buster/sources.list delete mode 100644 dockers/docker-base-buster/sources.list.arm64 delete mode 100644 dockers/docker-base-buster/sources.list.armhf delete mode 100644 dockers/docker-base-stretch/sources.list delete mode 100644 dockers/docker-base-stretch/sources.list.arm64 delete mode 100644 dockers/docker-base-stretch/sources.list.armhf delete mode 100644 files/apt/sources.list.amd64 delete mode 100644 files/apt/sources.list.arm64 delete mode 100644 files/apt/sources.list.armhf create mode 100644 files/apt/sources.list.j2 create mode 100755 scripts/build_mirror_config.sh diff --git a/.gitignore b/.gitignore index 5f6f41b1a09c..60c328d62289 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,6 @@ htmlcov/ .vscode/ .idea/ +# Debian mirror Sources +sources.list.* +!sources.list*.j2 diff --git a/Makefile.work b/Makefile.work index 1d599a9d1935..b5373067eb23 100644 --- a/Makefile.work +++ b/Makefile.work @@ -387,6 +387,8 @@ SONIC_BUILD_INSTRUCTION := make \ ENABLE_FIPS_FEATURE=$(ENABLE_FIPS_FEATURE) \ ENABLE_FIPS=$(ENABLE_FIPS) \ SONIC_SLAVE_DOCKER_DRIVER=$(SONIC_SLAVE_DOCKER_DRIVER) \ + MIRROR_URLS=$(MIRROR_URLS) \ + MIRROR_SECURITY_URLS=$(MIRROR_SECURITY_URLS) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset @@ -396,6 +398,10 @@ SONIC_BUILD_INSTRUCTION := make \ %:: ifeq ($(MULTIARCH_QEMU_ENVIRON), y) @$(DOCKER_MULTIARCH_CHECK) + +export MIRROR_URLS +export MIRROR_SECURITY_URLS + ifneq ($(BLDENV), ) @$(DOCKER_SERVICE_MULTIARCH_CHECK) @$(DOCKER_SERVICE_DOCKERFS_CHECK) @@ -405,6 +411,7 @@ endif @pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo + @scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV) @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { [ $(ENABLE_DOCKER_BASE_PULL) == y ] && { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Pulling...; } && \ $(DOCKER_BASE_PULL) && \ @@ -432,6 +439,7 @@ docker-cleanup: sonic-build-hooks: @pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_DIR)/buildinfo + @scripts/build_mirror_config.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV) sonic-slave-base-build : sonic-build-hooks ifeq ($(MULTIARCH_QEMU_ENVIRON), y) diff --git a/build_debian.sh b/build_debian.sh index c7331c737d84..f1e17e4bcb23 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -108,6 +108,7 @@ sudo LANG=C chroot $FILESYSTEM_ROOT mount [ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/ ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates +scripts/build_mirror_config.sh files/apt $CONFIGURED_ARCH $IMAGE_DISTRO sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ diff --git a/dockers/docker-base-bullseye/Dockerfile.j2 b/dockers/docker-base-bullseye/Dockerfile.j2 index f47d56468102..464b64a453d8 100644 --- a/dockers/docker-base-bullseye/Dockerfile.j2 +++ b/dockers/docker-base-bullseye/Dockerfile.j2 @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] -{% if CONFIGURED_ARCH == "armhf" %} -COPY ["sources.list.armhf", "/etc/apt/sources.list"] -{% elif CONFIGURED_ARCH == "arm64" %} -COPY ["sources.list.arm64", "/etc/apt/sources.list"] -{% else %} -COPY ["sources.list", "/etc/apt/sources.list"] -{% endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] diff --git a/dockers/docker-base-bullseye/sources.list b/dockers/docker-base-bullseye/sources.list deleted file mode 100644 index c45ef0811f10..000000000000 --- a/dockers/docker-base-bullseye/sources.list +++ /dev/null @@ -1,14 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian bullseye-updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free - -# Debian mirror supports multiple versions for a package -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/dockers/docker-base-bullseye/sources.list.arm64 b/dockers/docker-base-bullseye/sources.list.arm64 deleted file mode 100644 index 5a0652aa3c7e..000000000000 --- a/dockers/docker-base-bullseye/sources.list.arm64 +++ /dev/null @@ -1,9 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=arm64] http://deb.debian.org/debian bullseye main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian bullseye-updates main contrib non-free -deb [arch=arm64] http://security.debian.org bullseye-security main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian bullseye-backports main -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free diff --git a/dockers/docker-base-bullseye/sources.list.armhf b/dockers/docker-base-bullseye/sources.list.armhf deleted file mode 100644 index 435f4692de9f..000000000000 --- a/dockers/docker-base-bullseye/sources.list.armhf +++ /dev/null @@ -1,9 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian bullseye-updates main contrib non-free -deb [arch=armhf] http://security.debian.org bullseye-security main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian bullseye-backports main -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index 52c784e9a7fe..77886ede4408 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] -{% if CONFIGURED_ARCH == "armhf" %} -COPY ["sources.list.armhf", "/etc/apt/sources.list"] -{% elif CONFIGURED_ARCH == "arm64" %} -COPY ["sources.list.arm64", "/etc/apt/sources.list"] -{% else %} -COPY ["sources.list", "/etc/apt/sources.list"] -{% endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] diff --git a/dockers/docker-base-buster/sources.list b/dockers/docker-base-buster/sources.list deleted file mode 100644 index 473c9eb22e76..000000000000 --- a/dockers/docker-base-buster/sources.list +++ /dev/null @@ -1,14 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free - -# Debian mirror supports multiple versions for a package -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.arm64 b/dockers/docker-base-buster/sources.list.arm64 deleted file mode 100644 index 249efc17b6fd..000000000000 --- a/dockers/docker-base-buster/sources.list.arm64 +++ /dev/null @@ -1,12 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free -deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free -deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free -deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian/ buster-backports main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.armhf b/dockers/docker-base-buster/sources.list.armhf deleted file mode 100644 index ff6d5787b212..000000000000 --- a/dockers/docker-base-buster/sources.list.armhf +++ /dev/null @@ -1,12 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free -deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free -deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free -deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian/ buster-backports main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-stretch/Dockerfile.j2 b/dockers/docker-base-stretch/Dockerfile.j2 index 5603dc502792..9e11c4ea35f0 100644 --- a/dockers/docker-base-stretch/Dockerfile.j2 +++ b/dockers/docker-base-stretch/Dockerfile.j2 @@ -27,13 +27,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Configure data sources for apt/dpkg COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"] -{% if CONFIGURED_ARCH == "armhf" %} -COPY ["sources.list.armhf", "/etc/apt/sources.list"] -{% elif CONFIGURED_ARCH == "arm64" %} -COPY ["sources.list.arm64", "/etc/apt/sources.list"] -{% else %} -COPY ["sources.list", "/etc/apt/sources.list"] -{% endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"] COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"] diff --git a/dockers/docker-base-stretch/sources.list b/dockers/docker-base-stretch/sources.list deleted file mode 100644 index 0c29b339bb87..000000000000 --- a/dockers/docker-base-stretch/sources.list +++ /dev/null @@ -1,11 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch-backports main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.arm64 b/dockers/docker-base-stretch/sources.list.arm64 deleted file mode 100644 index 520c46519919..000000000000 --- a/dockers/docker-base-stretch/sources.list.arm64 +++ /dev/null @@ -1,11 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free -deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free -deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free -deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free -deb [arch=arm64] http://deb.debian.org/debian/ stretch-backports main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.armhf b/dockers/docker-base-stretch/sources.list.armhf deleted file mode 100644 index 58077f310424..000000000000 --- a/dockers/docker-base-stretch/sources.list.armhf +++ /dev/null @@ -1,11 +0,0 @@ -## Debian mirror for ARM repo - -# ARM repo -deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free -deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free -deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free -deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian/ stretch-backports main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/files/apt/sources.list.amd64 b/files/apt/sources.list.amd64 deleted file mode 100644 index fb0a1e994932..000000000000 --- a/files/apt/sources.list.amd64 +++ /dev/null @@ -1,13 +0,0 @@ -## Debian mirror on Microsoft Azure -## Ref: http://debian-archive.trafficmanager.net/ - -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb-src [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb-src [arch=amd64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/files/apt/sources.list.arm64 b/files/apt/sources.list.arm64 deleted file mode 100644 index 75bebe1a91d1..000000000000 --- a/files/apt/sources.list.arm64 +++ /dev/null @@ -1,13 +0,0 @@ -## Debian mirror for ARM -## Not the repo mirror site can change in future, and needs to be updated to be in sync - -deb [arch=arm64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb-src [arch=arm64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -deb [arch=arm64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb-src [arch=arm64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -deb [arch=arm64] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb-src [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb-src [arch=arm64] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/files/apt/sources.list.armhf b/files/apt/sources.list.armhf deleted file mode 100644 index 4f9d4d0a9020..000000000000 --- a/files/apt/sources.list.armhf +++ /dev/null @@ -1,18 +0,0 @@ -## Debian mirror for ARM -## Not the repo mirror site can change in future, and needs to be updated to be in sync - -deb [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free -deb-src [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free -deb [arch=armhf] http://security.debian.org bullseye-security main contrib non-free -deb-src [arch=armhf] http://security.debian.org bullseye-security main contrib non-free -deb [arch=armhf] http://deb.debian.org/debian bullseye-backports main contrib non-free -#deb [arch=armhf] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -#deb-src [arch=armhf] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free -#deb [arch=armhf] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -#deb-src [arch=armhf] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free -#deb [arch=armhf] http://debian-archive.trafficmanager.net/debian/ bullseye-backports main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb-src [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb-src [arch=armhf] http://packages.trafficmanager.net/debian/debian-security/ bullseye-security main contrib non-free -deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye-backports main contrib non-free diff --git a/files/apt/sources.list.j2 b/files/apt/sources.list.j2 new file mode 100644 index 000000000000..57d53b919b03 --- /dev/null +++ b/files/apt/sources.list.j2 @@ -0,0 +1,20 @@ +# The configuration is generated by template +# Please add additional sources in /etc/apt/sources.list.d + +{% for mirror_url in MIRROR_URLS.split(',') %} +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }} main contrib non-free +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-updates main contrib non-free +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-backports main contrib non-free +{% endfor %} +{% for mirror_url in MIRROR_SECURITY_URLS.split(',') %} +{% set dist_separator='/' %}{% if 'packages.trafficmanager.net/debian' in mirror_url %}{% set dist_separator='_' %}{% endif %} +{% if DISTRIBUTION == 'stretch' or DISTRIBUTION == 'buster' %} +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}{{ dist_separator }}updates main contrib non-free +{% else %} +deb [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free +deb-src [arch={{ ARCHITECTURE }}] {{ mirror_url }} {{ DISTRIBUTION }}-security main contrib non-free +{% endif %} +{% endfor %} diff --git a/scripts/build_mirror_config.sh b/scripts/build_mirror_config.sh new file mode 100755 index 000000000000..5f94e701cbde --- /dev/null +++ b/scripts/build_mirror_config.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Generate the sources.list. in the config path +CONFIG_PATH=$1 +export ARCHITECTURE=$2 +export DISTRIBUTION=$3 + +# The default mirror urls +DEFAULT_MIRROR_URLS=http://debian-archive.trafficmanager.net/debian/,http://packages.trafficmanager.net/debian/debian/ +DEFAULT_MIRROR_SECURITY_URLS=http://debian-archive.trafficmanager.net/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ + +# The debian-archive.trafficmanager.net does not support armhf, use debian.org instead +if [ "$ARCHITECTURE" == "armhf" ]; then + DEFAULT_MIRROR_URLS=http://deb.debian.org/debian/,http://packages.trafficmanager.net/debian/debian/ + DEFAULT_MIRROR_SECURITY_URLS=http://deb.debian.org/debian-security/,http://packages.trafficmanager.net/debian/debian-security/ +fi + +[ -z "$MIRROR_URLS" ] && MIRROR_URLS=$DEFAULT_MIRROR_URLS +[ -z "$MIRROR_SECURITY_URLS" ] && MIRROR_SECURITY_URLS=$DEFAULT_MIRROR_SECURITY_URLS + +TEMPLATE=files/apt/sources.list.j2 +[ -f files/apt/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=files/apt/sources.list.$ARCHITECTURE.j2 +[ -f $CONFIG_PATH/sources.list.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.j2 +[ -f $CONFIG_PATH/sources.list.$ARCHITECTURE.j2 ] && TEMPLATE=$CONFIG_PATH/sources.list.$ARCHITECTURE.j2 + +MIRROR_URLS=$MIRROR_URLS MIRROR_SECURITY_URLS=$MIRROR_SECURITY_URLS j2 $TEMPLATE | sed '/^$/N;/^\n$/D' > $CONFIG_PATH/sources.list.$ARCHITECTURE diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index d2b2c57185d1..3c1104eddc53 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -22,6 +22,10 @@ if [ -z "$DISTRO" ]; then [ -z "$DISTRO" ] && DISTRO=jessie fi +if [[ "$IMAGENAME" == docker-base-* ]]; then + scripts/build_mirror_config.sh ${DOCKERFILE_PATH} $ARCH $DISTRO +fi + # add script for reproducible build. using sha256 instead of tag for docker base image. scripts/docker_version_control.sh $@ diff --git a/slave.mk b/slave.mk index b2d2c3bbf446..e3b4e7490e3b 100644 --- a/slave.mk +++ b/slave.mk @@ -132,6 +132,8 @@ export TRUSTED_GPG_URLS export SONIC_VERSION_CONTROL_COMPONENTS DEFAULT_CONTAINER_REGISTRY := $(SONIC_DEFAULT_CONTAINER_REGISTRY) export DEFAULT_CONTAINER_REGISTRY +export MIRROR_URLS +export MIRROR_SECURITY_URLS ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y) ENABLE_PFCWD_ON_START = y diff --git a/sonic-slave-bullseye/Dockerfile.j2 b/sonic-slave-bullseye/Dockerfile.j2 index f172317cb1f3..67409ecca899 100644 --- a/sonic-slave-bullseye/Dockerfile.j2 +++ b/sonic-slave-bullseye/Dockerfile.j2 @@ -14,40 +14,7 @@ FROM {{ prefix }}debian:bullseye MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] - -## TODO: Re-add in any necessary mirror URLs here as they become available -RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian bullseye-backports main" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list - -{%- if CONFIGURED_ARCH == "armhf" %} -RUN echo "deb [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CONFIGURED_ARCH == "arm64" %} -RUN echo "deb [arch=arm64] http://deb.debian.org/debian bullseye main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://security.debian.org bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security bullseye-security main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian bullseye-updates main contrib non-free" >> /etc/apt/sources.list -{%- endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index 9c9378ec0c43..eefa8df1c678 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -12,39 +12,7 @@ FROM {{ prefix }}debian:buster MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] - -RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list - -{%- if CONFIGURED_ARCH == "armhf" %} -RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CONFIGURED_ARCH == "arm64" %} -RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security buster_updates main contrib non-free" >> /etc/apt/sources.list -{%- endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 39554633eb19..6dea980ab9b9 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -10,40 +10,7 @@ FROM {{ prefix }}debian:stretch MAINTAINER gulv@microsoft.com COPY ["no-check-valid-until", "/etc/apt/apt.conf.d/"] - -RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.microsoft.com/debian/9/prod stretch main" >> /etc/apt/sources.list - -{%- if CONFIGURED_ARCH == "armhf" %} -RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list -{%- elif CONFIGURED_ARCH == "arm64" %} -RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian-security stretch_updates main contrib non-free" >> /etc/apt/sources.list -{%- endif %} +COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"] ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive