Merge pull request #7 from ArendJan/move-to-packe

Move to packer

.github/workflows/build.yaml

@@ -0,0 +1,58 @@
+name: Build and Upload
+
+on:
+    [push]
+permissions:
+    contents: write
+
+jobs:
+    shellcheck:
+      uses: ./.github/workflows/shellcheck.yml
+    shfmt:
+      uses: ./.github/workflows/shfmt.yml
+    build-and-push:
+        # container:
+        #   image: ubuntu:22.04
+        #   options: --privileged
+        runs-on: ubuntu-latest
+        needs: [shellcheck, shfmt]
+        strategy:
+            matrix:
+                image: [
+                        # mirte_orangepizero, 
+                        mirte_orangepizero2,
+                        mirte_orangepi3b,
+                        mirte_rpi4b
+                        ]
+            fail-fast: false
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v2
+
+            - name: setup this repo
+              run: |
+                    sudo apt update
+                    sudo apt install qemu-user-static parted -y
+                    ./packerInstall.sh
+            # - name: setup build files
+            #   run: |
+            #         echo -e "#!/bin/bash\n\nexport INSTALL_DOCS=false\nexport INSTALL_ROS=false\nexport INSTALL_ARDUINO=false\nexport INSTALL_WEB=false\nexport INSTALL_PYTHON=false\nexport INSTALL_JUPYTER=false\nexport EXPIRE_PASSWD=true\nexport INSTALL_NETWORK=true\nexport INSTALL_PROVISIONING=true\nexport INSTALL_VSCODE=true\nexport INSTALL_PAM=true\nexport EXTRA_SCRIPTS=()\nexport PARALLEL=true" > settings.sh
+            # - name: setup repos
+            #   run: |
+            #     cp main_repos.yaml repos.yaml
+            #     yq e -i '.repositories.mirte-ros-packages.version = "fix-intro-rob"' repos.yaml
+            #     yq e -i '.repositories.mirte-telemetrix4arduino.url =  "https://github.com/arendjan/telemetrix4arduino.git"' repos.yaml
+            #     yq e -i '.repositories.mirte-telemetrix4arduino.version = "fix-intro-robotics"' repos.yaml
+            - name: Build
+              run: |
+                    ./packerBuild.sh ${{ matrix.image }}
+            - name: artifacts
+              uses: actions/upload-artifact@v2
+              with:
+                path: build/*.img.xz
+                name: ${{ matrix.image }}
+            - name: Push to release
+              uses: softprops/action-gh-release@v1
+              if: startsWith(github.ref, 'refs/tags/')
+              with:
+                  files: build/*.img.xz

.github/workflows/publish_repos.yaml

@@ -0,0 +1,40 @@
+name: Publish other repos
+
+on:
+    release:
+        types: [published]
+permissions:
+    contents: write
+
+jobs:
+    upload-repos:
+        # container:
+        #   image: ubuntu:22.04
+        #   options: --privileged
+        runs-on: ubuntu-latest
+        strategy:
+            matrix:
+                repo:
+                    [
+                        "mirte-install-scripts",
+                        "mirte-ros-packages",
+                        "mirte-python",
+                        "telemetrix4rpipico",
+                        "mirte-web-interface",
+                        "mirte-frame",
+                        "tmx-pico-aio",
+                        "telemetrix-aio",
+                        "telemetrix4arduino",
+                        "mirte-pcb",
+                        "mirte-documentation",
+                    ]
+            fail-fast: false
+        steps:
+            - run: |
+                  git clone https://github.com/mirte-robot/${{ matrix.repo }}.git
+                  zip -r ${{ matrix.repo }}.zip ${{ matrix.repo }}
+            - name: Push to release
+              uses: softprops/action-gh-release@v1
+              if: startsWith(github.ref, 'refs/tags/')
+              with:
+                  files: ${{ matrix.repo }}.zip

.github/workflows/shellcheck.yml

@@ -0,0 +1,18 @@
+on:
+  # - push
+  - pull_request
+  - workflow_call
+
+name: "Shellcheck"
+permissions: {}
+
+jobs:
+  shellcheck:
+    name: Shellcheck
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+        with:
+            severity: warning

.github/workflows/shfmt.yml

@@ -0,0 +1,17 @@
+name: shfmt
+on:
+  - pull_request
+  # - push
+  - workflow_call
+jobs:
+  sh-checker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run the sh-checker
+        uses: luizm/action-sh-checker@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          sh_checker_shellcheck_disable: true
+          sh_checker_comment: true

.gitignore

@@ -1,5 +1,12 @@
-*.sif
-*.img
-*.zip
-git_local/
+/build
+/git_local
+/log.txt
+/pishrink.sh
+/test
+settings.sh
+logs/
+workdir/
+settings.sh
+shell_workdir/
 repos.yaml
+artifacts/

README.md

@@ -1,110 +1,40 @@
-## Prerequisites
+# Build Mirte image using Packer
 
-You need singularity container >=2.3. You can follow the instructions of the singulairy manal to compile it ([from source](https://sylabs.io/guides/3.0/user-guide/installation.html)), or use a ([package](https://sylabs.io/guides/3.0/user-guide/installation.html#distribution-packages-of-singularity)). For Ubuntu 20.04 this means:
+## Install
+run sudo `./packerInstall.sh` to download required packages.
 
-   ```
-   $ sudo wget -O- http://neuro.debian.net/lists/focal.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list && \
-     sudo apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \
-     sudo apt-get update
+## Build
+Run sudo `./packerBuild.sh` to build the image. Will take some time
+Put your local files in `git_local/` and they will be copied
+edit `settings.sh` to select features and extra scripts and edit `repos.yaml` to select the repositories/branches.
 
-   $ sudo apt install singularity-container
-   ```
+## Build as workflow in Docker
+Install [act](https://github.com/nektos/act) and run `./actBuild.sh`. It will run the workflows and copy the artifacts to `./artifacts/`. This will not use any changes in git_local or repos.yaml.
 
-Note that for other distributions you should follow the steps found on the 'NeuroDebian' site (under [Distribution Packages of Singularity](https://sylabs.io/guides/3.0/user-guide/installation.html#distribution-packages-of-singularity) on the singularity site)
+## Shell to edit
+run ./shell.sh \<img file> and you will get a shell after some time. You can create new shells by using `sudo chroot /tmp/armimg-XXX`. Stop by removing the `/stopshell` file. It will create your fresh image in `shell_workdir`.
 
-## Installing the image builder
-
-1. Clone this repository
-   ```
-   $ git clone https://github.com/mirte-robot/mirte_sd_image_tools.git
-   ```
-2. Install the sigularity image
-   ```
-   $ cd /mirte_sd_image_tools
-   $ sudo ./install.sh
-   ```
-
-## Generating the Mirte SD image
-
-1. Create the sd card image (this will take some time) (NOTE: this will overwrite existing images)
-   ```
-   $ ./run.sh build_sd_image
-   ```
-2. This will generate a mirte_orangepi_sd.img in the current directory
-3. Use an image burning tool (e.g. dd or etcher ([link](https://www.balena.io/etcher/)) to burn it to an SD card
-
-
-## Generating the Mirte SD image for a Raspberry Pi (2,3,4)
-
-1. Create the sd card image (this will take some time) (NOTE: this will overwrite existing images)
-   ```
-   $ sudo ./run.sh build_sd_card raspberry
-   ```
-2. This will generate a mirte_raspberry_sd.img in the current directory
-3. Use an image burning tool (e.g. dd or etcher ([link](https://www.balena.io/etcher/)) to burn it to an SD card
-
-
-## Generating an SD card image using branches of Mirte repos
-
-1. In the repository folder create a file called repos.yaml.
-   ```
-   $ nano repos.yaml
-   ```
-2. This yaml format used is the same as [vcstool](https://github.com/dirk-thomas/vcstool). The example below will create an image with the my_branch branch of the mirte-web-interface repository. For an overview of all the repositories used, have a look at the version in [install_scripts](https://github.com/mirte-robot/mirte_install_scripts/blob/main/repos.yaml).
-```yaml
-repositories:
-  mirte-web-interface:
-    type: git
-    url: https://github.com/mirte-robot/mirte-web-interface
-    version: my_branch
+# /bin/sh Exec format error:
+```sh
+sudo apt remove qemu-user-static -y && sudo apt install qemu-user-static
 ```
 
-3. Create the sd card image (this will take some time) (NOTE: this will overwrite existing images)
-   ```   
-   $ ./run.sh build_sd_image [raspberrypi]
-   ```
-
-## (For Developer) Generating an SD card image using a local repository
-
-1. Make sure you have one or more local repositories in <mirte-sd-image-tools>/git_local/.
-
-```bash
-.
-├── git_local
-│   └── mirte-install-scripts
+# VCS issues:
+When you get 
 ```
-
-2. Now create (or use an existing) repos.yaml, but now use another url by prepeding "/working_dir/" as can be seen in the example below. Please note that this will checkout the branch mentioned, so you need to commit changes to that branch.
-
-```yaml
-repositories:
-  mirte-install-scripts:
-    type: git
-    url: /working_dir/git_local/mirte-install-scripts
-    version: master
+=== ./mirte-arduino-libraries (git) ===
+    arm-image.mirte_orangepizero: Could not clone repository 'https://github.com/arendjan/mirte-arduino-libraries.git': fatal: destination path '.' already exists and is not an empty directory.
 ```
+when building for orange pi Zero (1), you have a qemu version that has some issues, including a ``` qemu: uncaught target signal 11 (segmentation fault) - core dumped``` when using git. Update the qemu installation on your host computer by adding a ppa ( ```sh sudo add-apt-repository ppa:canonical-server/server-backports```) and updating qemu. This should resolve the issues.
 
-3. Create the sd card image (this will take some time) (NOTE: this will overwrite existing images)
-   ```
-   $ sudo ./run.sh build_sd_card [raspberry]
-   ```
+# TODOS:
+- npm prebuilt
 
-## Generating an SD card image using modifications made on a Mirte robot
-TODO
-
-
-## Run created Armbian image on non ARM machine
-1. Make sure you have a Mirte image called mirte_orangepi_sd.img in the repository directory
-
-2. Start an ARM shell
-   ```
-   $ sudo ./run.sh image_shell [orangepi|raspberry]
-   ```
-3. Note that you are logged in as root now. In order to switch to the mirte user please do (which will ask you to reset the default password (mirte_mirte)):
-   ```
-   # su mirte
-   ```
-4. You can now find all Mirte repositories in /usr/local/src/mirte
-   ```
-   $ ls /usr/local/src/mirte
-   ```
+# Editing the latest build
+Install gh or download from releases.
+```
+gh run download #select type
+sudo ./shell.sh ...
+# at end, exit out of the chroot
+# output in shell_workdir/....shrunk_$date.img.xz
+```

actBuild.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+rm -rf /tmp/artifacts || true
+mkdir /tmp/artifacts || true
+act --container-options="-v /dev:/dev -v /proc:/proc --privileged" --artifact-server-path /tmp/artifacts
+mkdir artifacts || true
+cp -r /tmp/artifacts/* ./artifacts
+# somehow the files get an extra __ at the end
+find ./artifacts -name "*.gz__" -exec sh -c 'mv "$1" "${1%.gz__}.gz"' _ {} \;
+find ./artifacts -name "*.gz" -exec sh -c 'gunzip "$1"' _ {} \;

add_partition_local/add_partition.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -ex
+
+image_file=$1
+parent_path=$(
+	cd "$(dirname "${BASH_SOURCE[0]}")"
+	pwd -P
+)
+
+add_partition() {
+	startLocation=$(sfdisk -l -o start -N1 "$image_file" | tail -1)
+	# should be 40960 for zero2, 8192 for zero1
+
+	extraSize="1G"
+
+	dd if=/dev/zero bs=1M count=1024 >>"$image_file"
+	echo "+$extraSize" | sfdisk --move-data -N 1 "$image_file"
+	echo "$startLocation, $extraSize, b" | sfdisk -a "$image_file"
+	sleep 5
+	loop=$(kpartx -av "$image_file")
+	echo $loop
+	loopvar=$(echo $loop | grep -oP 'loop[0-9]*' | head -1)
+	echo $loopvar
+	mkfs.fat /dev/mapper/${loopvar}p2 -n "MIRTE" -i "9EE2A262" # some random id from a previous build, must be the same as in build.pkr.hcl
+	mount_dir=$(mktemp -d)
+	echo $mount_dir
+	mount -v /dev/mapper/${loopvar}p2 $mount_dir
+	cp -r "$parent_path/default_partition_files/." $mount_dir
+	ls $mount_dir
+	sleep 5
+	umount -v /dev/mapper/${loopvar}p2
+	sleep 5
+	kpartx -dv /dev/${loopvar}
+	rm -rf $mount_dir
+}
+
+copy_files() {
+	loop=$(kpartx -av "$image_file")
+	echo $loop
+	loopvar=$(echo $loop | grep -oP 'loop[0-9]*' | head -1)
+	echo $loopvar
+	mount_dir=$(mktemp -d)
+	echo $mount_dir
+	mount -v /dev/mapper/${loopvar}p2 $mount_dir
+	cp -r "$parent_path/default_partition_files/." $mount_dir
+	ls $mount_dir
+	sleep 5
+	umount -v /dev/mapper/${loopvar}p2
+	sleep 5
+	kpartx -dv /dev/${loopvar}
+	rm -rf $mount_dir
+}
+
+if sfdisk -l "$image_file" | grep -q '.img2'; then
+	echo "Already contains extra partition, only copying default files"
+else
+	add_partition
+fi
+
+copy_files
+
+$parent_path/../pishrink.sh "$image_file"
+
+echo "done"

add_partition_local/default_partition_files/machine_config.yaml

@@ -0,0 +1,3 @@
+access_points:
+  - ssid: SSID
+    password: PASSWORD