From 0a18bdba581671018135abe6671c8a71338c4656 Mon Sep 17 00:00:00 2001 From: Bo YU Date: Tue, 26 Dec 2023 19:28:15 +0800 Subject: [PATCH] inew test Signed-off-by: Bo YU --- .github/workflows/main.yml | 2 +- Dockerfile | 32 ++++++------- Makefile | 10 ---- backup/Dockerfile | 46 ++++++++++++++++++ backup/create_image.sh | 96 +++++++++++++++++++++++++++++++++++++ create_image.sh | 21 ++++++-- rootfs/multistrap_nvme.conf | 4 +- rootfs/setup_rootfs.sh | 42 +++++++++------- 8 files changed, 203 insertions(+), 50 deletions(-) delete mode 100644 Makefile create mode 100644 backup/Dockerfile create mode 100755 backup/create_image.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dd64fd6..6abc6b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: - name: Create vf2 rootfs run: | export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 vf2 - KERNEL_VERSION=${{ github.ref_name }} docker-compose up vf2 + KERNEL_VERSION=${{ github.ref_name }} docker-compose up vf2 ls -l image/* - name: Release uses: softprops/action-gh-release@v1 diff --git a/Dockerfile b/Dockerfile index 2021eca..601afb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,36 +11,36 @@ RUN --mount=type=cache,sharing=shared,target=/var/cache \ --mount=type=tmpfs,target=/usr/share/doc \ apt-get update \ && apt-get install -y eatmydata \ - && eatmydata apt-get install -y debootstrap qemu-user-static \ - binfmt-support debian-ports-archive-keyring gdisk kpartx \ + && eatmydata apt-get install -y qemu-user-static \ + binfmt-support gdisk kpartx \ parted \ - autoconf automake autotools-dev bc binfmt-support \ + autoconf automake autotools-dev bc \ build-essential cpio curl \ dosfstools e2fsprogs fdisk flex gawk \ git gperf kmod libexpat-dev \ libgmp-dev libmpc-dev libmpfr-dev libssl-dev \ - libtool mmdebstrap multistrap openssl parted \ + libtool mmdebstrap openssl parted \ patchutils python3 python3-dev python3-distutils \ - python3-setuptools qemu-user-static swig \ + python3-setuptools swig \ systemd-container texinfo zlib1g-dev wget # build rootfs -FROM builder as build_rootfs -WORKDIR /build -COPY rootfs/multistrap_nvme.conf multistrap.conf +#FROM builder as build_rootfs +#WORKDIR /build +#COPY rootfs/multistrap_nvme.conf multistrap.conf -RUN --mount=type=cache,sharing=shared,target=/var/cache \ - --mount=type=cache,sharing=shared,target=/var/lib/apt/lists \ - --mount=type=tmpfs,target=/usr/share/man \ - --mount=type=tmpfs,target=/usr/share/doc \ - eatmydata multistrap -f multistrap.conf +#RUN --mount=type=cache,sharing=shared,target=/var/cache \ +# --mount=type=cache,sharing=shared,target=/var/lib/apt/lists \ +# --mount=type=tmpfs,target=/usr/share/man \ +# --mount=type=tmpfs,target=/usr/share/doc \ +# eatmydata multistrap -f multistrap.conf FROM builder as build_image WORKDIR /builder -COPY --from=build_rootfs /build/rv64-sid/ ./rv64-port/ +#COPY --from=build_rootfs /build/rv64-sid/ ./rv64-port/ COPY create_image.sh build.sh ./ -COPY rootfs/setup_rootfs.sh ./rv64-port/ +COPY rootfs/setup_rootfs.sh ./ -CMD eatmydata /builder/build.sh ${KERNEL_VERSION} +CMD /builder/build.sh ${KERNEL_VERSION} diff --git a/Makefile b/Makefile deleted file mode 100644 index 9b423a2..0000000 --- a/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -unmatched: - sudo rm -rf ./image/* - DOCKER_BUILDKIT=1 docker-compose build vf2 - docker-compose up vf2 - - -clean: - sudo rm -rf ./image/* - DOCKER_BUILDKIT=1 docker-compose build --no-cache - docker-compose up diff --git a/backup/Dockerfile b/backup/Dockerfile new file mode 100644 index 0000000..2021eca --- /dev/null +++ b/backup/Dockerfile @@ -0,0 +1,46 @@ +FROM debian:sid as builder +MAINTAINER Bo YU "tsu.yubo@gmail.com" + +ENV KERNEL_VERSION=${KERNEL_VERSION} + +ARG DEBIAN_FRONTEND=noninteractive + +RUN --mount=type=cache,sharing=shared,target=/var/cache \ + --mount=type=cache,sharing=shared,target=/var/lib/apt/lists \ + --mount=type=tmpfs,target=/usr/share/man \ + --mount=type=tmpfs,target=/usr/share/doc \ + apt-get update \ + && apt-get install -y eatmydata \ + && eatmydata apt-get install -y debootstrap qemu-user-static \ + binfmt-support debian-ports-archive-keyring gdisk kpartx \ + parted \ + autoconf automake autotools-dev bc binfmt-support \ + build-essential cpio curl \ + dosfstools e2fsprogs fdisk flex gawk \ + git gperf kmod libexpat-dev \ + libgmp-dev libmpc-dev libmpfr-dev libssl-dev \ + libtool mmdebstrap multistrap openssl parted \ + patchutils python3 python3-dev python3-distutils \ + python3-setuptools qemu-user-static swig \ + systemd-container texinfo zlib1g-dev wget + +# build rootfs +FROM builder as build_rootfs +WORKDIR /build +COPY rootfs/multistrap_nvme.conf multistrap.conf + +RUN --mount=type=cache,sharing=shared,target=/var/cache \ + --mount=type=cache,sharing=shared,target=/var/lib/apt/lists \ + --mount=type=tmpfs,target=/usr/share/man \ + --mount=type=tmpfs,target=/usr/share/doc \ + eatmydata multistrap -f multistrap.conf + + +FROM builder as build_image +WORKDIR /builder +COPY --from=build_rootfs /build/rv64-sid/ ./rv64-port/ +COPY create_image.sh build.sh ./ +COPY rootfs/setup_rootfs.sh ./rv64-port/ + +CMD eatmydata /builder/build.sh ${KERNEL_VERSION} + diff --git a/backup/create_image.sh b/backup/create_image.sh new file mode 100755 index 0000000..28954b3 --- /dev/null +++ b/backup/create_image.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -ex + +# It woule be good to test it outside of Docker +OUT_DIR="$1" +KERNEL_VERSION="$2" + +ROOTFS_IMG="${OUT_DIR}/vf2-rootfs.img" +SD_DD_OPTS="bs=4k iflag=fullblock oflag=direct conv=fsync status=progress" + +function make_image(){ + + if [ -f ${ROOTFS_IMG} ]; then + echo "deleting nvme rootfs image..." + rm ${ROOTFS_IMG} + fi + + dd if=/dev/zero of="${ROOTFS_IMG}" bs=1M count=4096 + + echo "Creating Blank ${ROOTFS_IMG}" + + sgdisk -g --clear --set-alignment=1 \ + -g --clear --new=1:0:+16M: --new=2:0:+100M: -t 2:EF00 --new=3:0:-1M: --attributes 3:set:2 -d 1 ${ROOTFS_IMG} + + + part_id=$(kpartx -av ${ROOTFS_IMG} | awk '{print $3}' | awk 'NR==1 {print $1}'| awk -F 'p' '{print $2}') + echo "the output is $part_id" + + LOOP_X_P2="loop${part_id}p2" + LOOP_X_P3="loop${part_id}p3" + + echo "formatting loopback device ${LOOP_X_P2}" + + mkfs.vfat "/dev/mapper/${LOOP_X_P2}" + + mkfs.ext4 -m 0 -L root "/dev/mapper/${LOOP_X_P3}" + + # Copy Files, first the rootfs partition + echo "Mounting partitions ${LOOPDEV}" + ROOTFS_POINT=/nvme_rootfs + mkdir -p "${ROOTFS_POINT}" + + mount "/dev/mapper/${LOOP_X_P3}" "${ROOTFS_POINT}" + + #mount ${LOOPDEV} /mnt + #debootstrap --arch=riscv64 --keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring,ca-certificates unstable ${ROOTFS_POINT} https://mirror.iscas.ac.cn/debian-ports + # + copy_rootfs + + kpartx -d ${ROOTFS_IMG} +} + +function get_kernel_deb(){ + vf2_image_repo="https://github.com/yuzibo/vf2-linux" + kernel_deb_url="${vf2_image_repo}/releases/download/${KERNEL_VERSION}/vf2-mainline-kernel-gcc-13.tar.gz" + + wget --no-check-certificate ${kernel_deb_url} + + tar -zxvf vf2-mainline-kernel-gcc-13.tar.gz -C /tmp + +} + +function copy_rootfs(){ + + echo "copy rootfs" + # from docker image has been defined + cp -av /builder/rv64-port/* "${ROOTFS_POINT}" + + # Copy the rootfs + cp -v /usr/bin/qemu-riscv64-static ${ROOTFS_POINT}/usr/bin/ + + # should judge deb if exist + get_kernel_deb + + cp -v /tmp/vf2_kernel_deb/*.deb ${ROOTFS_POINT}/tmp + chroot "${ROOTFS_POINT}" qemu-riscv64-static /bin/sh /setup_rootfs.sh + rm "${ROOTFS_POINT}/setup_rootfs.sh" "${ROOTFS_POINT}/usr/bin/qemu-riscv64-static" + + umount "${ROOTFS_POINT}" + + rm -rf "${ROOTFS_POINT}" + +} + +make_image + +if [ $? -eq 0 ]; then + echo "make_image executed successfully." + # Now compress the image + echo "Compressing the image: ${ROOTFS_IMG}" + cd "${OUT_DIR}" && xz -T0 "${ROOTFS_IMG}" +else + kpartx -d ${ROOTFS_IMG} + exit 1 +fi diff --git a/create_image.sh b/create_image.sh index 02736a8..ffc38fc 100755 --- a/create_image.sh +++ b/create_image.sh @@ -1,4 +1,7 @@ #!/bin/bash +# refer to https://github.com/revyos/mkimg-th1520/blob/main/mkrootfs.sh + +set -ex # It woule be good to test it outside of Docker OUT_DIR="$1" @@ -7,6 +10,10 @@ KERNEL_VERSION="$2" ROOTFS_IMG="${OUT_DIR}/vf2-rootfs.img" SD_DD_OPTS="bs=4k iflag=fullblock oflag=direct conv=fsync status=progress" + +BASE_TOOLS="binutils file tree sudo bash-completion u-boot-menu initramfs-tools openssh-server network-manager dnsmasq-base libpam-systemd ppp wireless-regdb wpasupplicant libengine-pkcs11-openssl iptables systemd-timesyncd vim usbutils libgles2 parted exfatprogs systemd-sysv mesa-vulkan-drivers" +EXTRA_TOOLS="i2c-tools net-tools ethtool xdotool" + function make_image(){ if [ -f ${ROOTFS_IMG} ]; then @@ -44,6 +51,12 @@ function make_image(){ #mount ${LOOPDEV} /mnt #debootstrap --arch=riscv64 --keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring,ca-certificates unstable ${ROOTFS_POINT} https://mirror.iscas.ac.cn/debian-ports # + mmdebstrap -v --architectures=riscv64 \ + --include="ca-certificates debian-archive-keyring locales dosfstools \ + $BASE_TOOLS $EXTRA_TOOLS " \ + sid "${ROOTFS_POINT}" \ + "deb http://deb.debian.org/debian/ sid main" + copy_rootfs kpartx -d ${ROOTFS_IMG} @@ -53,17 +66,15 @@ function get_kernel_deb(){ vf2_image_repo="https://github.com/yuzibo/vf2-linux" kernel_deb_url="${vf2_image_repo}/releases/download/${KERNEL_VERSION}/vf2-mainline-kernel-gcc-13.tar.gz" - wget --no-check-certificate ${kernel_deb_url} + wget --no-verbose --no-check-certificate ${kernel_deb_url} tar -zxvf vf2-mainline-kernel-gcc-13.tar.gz -C /tmp } + function copy_rootfs(){ - echo "copy rootfs" - # from docker image has been defined - cp -av /builder/rv64-port/* "${ROOTFS_POINT}" # Copy the rootfs cp -v /usr/bin/qemu-riscv64-static ${ROOTFS_POINT}/usr/bin/ @@ -72,7 +83,7 @@ function copy_rootfs(){ get_kernel_deb cp -v /tmp/vf2_kernel_deb/*.deb ${ROOTFS_POINT}/tmp - chroot "${ROOTFS_POINT}" qemu-riscv64-static /bin/sh /setup_rootfs.sh + chroot "${ROOTFS_POINT}" qemu-riscv64-static /bin/sh setup_rootfs.sh rm "${ROOTFS_POINT}/setup_rootfs.sh" "${ROOTFS_POINT}/usr/bin/qemu-riscv64-static" umount "${ROOTFS_POINT}" diff --git a/rootfs/multistrap_nvme.conf b/rootfs/multistrap_nvme.conf index 02bfa66..a31d197 100644 --- a/rootfs/multistrap_nvme.conf +++ b/rootfs/multistrap_nvme.conf @@ -13,12 +13,12 @@ packages=apt debconf debian-archive-keyring debianutils diffutils dpkg e2fsprogs packages=zlib1g apt-utils cpio cron debconf-i18n dmidecode dmsetup fdisk ifupdown iproute2 iputils-ping kmod less libapparmor1 libargon2-1 libbpf1 libbsd0 libcap2-bin libcryptsetup12 libdevmapper1.02.1 libedit2 libelf1 libestr0 libfastjson4 libfdisk1 libip4tc2 libjansson4 libjson-c5 libkmod2 liblocale-gettext-perl liblognorm5 libmd0 libmnl0 libncurses6 libncursesw6 libnewt0.52 libnftables1 libnftnl11 libpopt0 libreadline8 libslang2 libssl3 libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl libxtables12 logrotate nano netbase nftables procps readline-common sensible-utils packages=pciutils p11-kit udev vim whiptail xxd packages=util-linux util-linux-extra rsyslog dbus -packages=systemd systemd-sysv libsystemd0 sysvinit-utils init-system-helpers init +packages=systemd systemd-sysv libsystemd0 packages=tasksel tasksel-data packages=net-tools openssh-server network-manager packages=curl gpg openssl ca-certificates sudo psmisc swig tmux packages=lshw lm-sensors -packages=u-boot-menu +packages=u-boot-menu initramfs-tools packages=eatmydata source=http://deb.debian.org/debian/ keyring=debian-archive-keyring diff --git a/rootfs/setup_rootfs.sh b/rootfs/setup_rootfs.sh index 3acdc71..7c9f0c3 100755 --- a/rootfs/setup_rootfs.sh +++ b/rootfs/setup_rootfs.sh @@ -9,10 +9,12 @@ export LC_ALL=C LANGUAGE=C LANG=C /var/lib/dpkg/info/sgml-base.preinst install mkdir -p /etc/sgml -dpkg --configure -a mount proc -t proc /proc +mount -B sys /sys +mount -B run /run +mount -B dev /dev +#mount devpts -t devpts /dev/pts dpkg --configure -a -umount /proc # Needed because we get permissions problems for some reason chmod 0666 /dev/null @@ -40,7 +42,9 @@ cat > /etc/fstab <> /etc/default/u-boot U_BOOT_PARAMETERS="rw console=tty0 console=ttyS0,115200 earlycon rootwait stmmaceth=chain_mode:1 selinux=0" EOF -dpkg-reconfigure ${kernel_image} +cat /boot/extlinux/extlinux.conf + +#dpkg-reconfigure ${kernel_image} + +# update u-boot +#update-initramfs -c -k all + +u-boot-update # boot from sd card -sed -i -e 's|append |append root=/dev/mmcblk1p3 |' /boot/extlinux/extlinux.conf +cat <> /boot/scripts.txt +sed -i -e 's|root=[^ ]*|root=/dev/mmcblk1p3|' /boot/extlinux/extlinux.conf +cd ${lib_dir}/starfive/ +ln -s jh7110-starfive-visionfive-2-v1.2a.dtb jh7110-visionfive-v2.dtb +cd - +EOF # double check cat /boot/extlinux/extlinux.conf @@ -88,8 +97,6 @@ kernel_comp_addr_r=0xb0000000 kernel_comp_size=0x10000000 EOF - - # Set hostname echo vf2 > /etc/hostname @@ -101,8 +108,6 @@ EOF # Add needed modules in initrd #echo "nvme" >> /etc/initramfs-tools/modules - - # # Enable system services # @@ -121,6 +126,11 @@ EOF # apt-get clean +umount /proc +umount /sys +umount /run +umount /dev + # set the time immediately at startup #sed -i 's/^DAEMON_OPTS="/DAEMON_OPTS="-s /' /etc/default/openntpd