Feature/multi platform

.github/workflows/product.yaml

@@ -15,23 +15,23 @@ jobs:
   compat:
     uses: ./.github/workflows/reusable-workflow.yaml
     with:
-      build_command: make system-images/compat/compat.tar
+      build_command: make system-images/compat/docker-images/aarch64.tar
       artifact_name: compat.tar
-      artifact_path: system-images/compat/compat.tar
+      artifact_path: system-images/compat/docker-images/aarch64.tar
 
   utils:
     uses: ./.github/workflows/reusable-workflow.yaml
     with:
-      build_command: make system-images/utils/utils.tar
+      build_command: make system-images/utils/docker-images/aarch64.tar
       artifact_name: utils.tar
-      artifact_path: system-images/utils/utils.tar
+      artifact_path: system-images/utils/docker-images/aarch64.tar
 
   binfmt:
     uses: ./.github/workflows/reusable-workflow.yaml
     with:
-      build_command: make system-images/binfmt/binfmt.tar
+      build_command: make system-images/binfmt/docker-images/aarch64.tar
       artifact_name: binfmt.tar
-      artifact_path: system-images/binfmt/binfmt.tar
+      artifact_path: system-images/binfmt/docker-images/aarch64.tar
 
   nc-broadcast:
     uses: ./.github/workflows/reusable-workflow.yaml
@@ -126,12 +126,9 @@ jobs:
         key: cache-raspios
 
     - name: Build image
-      run: "make V=1 NO_KEY=1 eos.img --debug"
-
-    - name: Compress image
-      run: "make gzip"
+      run: "make V=1 ARCH=aarch64 embassyos-raspi.img --debug"
 
     - uses: actions/upload-artifact@v3
       with:
         name: image
-        path: eos.tar.gz
+        path: embassyos-raspi.img

.gitignore

@@ -15,4 +15,5 @@ secrets.db
 /cargo-deps/**/*
 /ENVIRONMENT.txt
 /GIT_HASH.txt
-/eos.tar.gz
+/embassyos-*.tar.gz
+/*.deb

Makefile

@@ -1,14 +1,14 @@
-ARCH = aarch64
-ENVIRONMENT_FILE := $(shell ./check-environment.sh)
-GIT_HASH_FILE := $(shell ./check-git-hash.sh)
+ARCH = $(shell uname -m)
+ENVIRONMENT_FILE = $(shell ./check-environment.sh)
+GIT_HASH_FILE = $(shell ./check-git-hash.sh)
 EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias
 EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui
-EMBASSY_SRC := raspios.img product_key.txt $(EMBASSY_BINS) backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build)
-COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name compat.tar -and -not -name target)
-UTILS_SRC := $(shell find system-images/utils/ -not -name utils.tar)
-BINFMT_SRC := $(shell find system-images/binfmt/ -not -name binfmt.tar)
+EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build)
+COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name *.tar -and -not -name target)
+UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar)
+BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar)
 BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) backend/Cargo.toml backend/Cargo.lock
-FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell find frontend/assets) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
+FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json
 FRONTEND_UI_SRC := $(shell find frontend/projects/ui)
 FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard)
 FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui)
@@ -18,45 +18,81 @@ $(shell sudo true)
 
 .DELETE_ON_ERROR:
 
-.PHONY: all gzip clean format sdk snapshots frontends ui backend
-all: eos.img
+.PHONY: all gzip install clean format sdk snapshots frontends ui backend
 
-gzip: eos.tar.gz
+all: $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
 
-eos.tar.gz: eos.img
-	tar --format=posix -cS -f- eos.img | $(GZIP_BIN) > eos.tar.gz
+gzip: embassyos-raspi.tar.gz
+
+embassyos-raspi.tar.gz: embassyos-raspi.img
+	tar --format=posix -cS -f- embassyos-raspi.img | $(GZIP_BIN) > embassyos-raspi.tar.gz
 
 clean:
-	rm -f eos.img
+	rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip
+	rm -f raspios.img
+	rm -f embassyos-raspi.img
+	rm -f embassyos-raspi.tar.gz
 	rm -f ubuntu.img
 	rm -f product_key.txt
 	rm -f system-images/**/*.tar
-	sudo rm -f $(EMBASSY_BINS)
+	rm -rf system-images/compat/target
+	rm -rf backend/target
+	rm -rf frontend/.angular
 	rm -f frontend/config.json
 	rm -rf frontend/node_modules
 	rm -rf frontend/dist
+	rm -rf libs/target
 	rm -rf patch-db/client/node_modules
 	rm -rf patch-db/client/dist
-	sudo rm -rf cargo-deps
+	rm -rf patch-db/target
+	rm -rf cargo-deps
+	rm ENVIRONMENT.txt
+	rm GIT_HASH.txt
 
 format:
 	cd backend && cargo +nightly fmt
 	cd libs && cargo +nightly fmt
 
-sdk: 
+sdk:
 	cd backend/ && ./install-sdk.sh
 
-eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar system-images/binfmt/binfmt.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE)
-	! test -f eos.img || rm eos.img
-	if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi
-
-system-images/compat/compat.tar: $(COMPAT_SRC)
+embassyos-raspi.img: all raspios.img cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast
+	! test -f embassyos-raspi.img || rm embassyos-raspi.img
+	./build/raspberry-pi/make-image.sh
+
+# For creating os images. DO NOT USE
+install: all
+	mkdir -p $(DESTDIR)/usr/bin
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init $(DESTDIR)/usr/bin/
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd $(DESTDIR)/usr/bin/
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli $(DESTDIR)/usr/bin/
+	cp backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias $(DESTDIR)/usr/bin/
+
+	mkdir -p $(DESTDIR)/usr/lib
+	rm -rf $(DESTDIR)/usr/lib/embassy
+	cp -r build/lib $(DESTDIR)/usr/lib/embassy
+
+	cp ENVIRONMENT.txt $(DESTDIR)/usr/lib/embassy/
+	cp GIT_HASH.txt $(DESTDIR)/usr/lib/embassy/
+
+	mkdir -p $(DESTDIR)/usr/lib/embassy/system-images
+	cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/usr/lib/embassy/system-images/compat.tar
+	cp system-images/utils/docker-images/$(ARCH).tar $(DESTDIR)/usr/lib/embassy/system-images/utils.tar
+	cp system-images/binfmt/docker-images/$(ARCH).tar $(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar
+
+	mkdir -p $(DESTDIR)/var/www/html
+	cp -r frontend/dist/diagnostic-ui $(DESTDIR)/var/www/html/diagnostic
+	cp -r frontend/dist/setup-wizard $(DESTDIR)/var/www/html/setup
+	cp -r frontend/dist/ui $(DESTDIR)/var/www/html/main
+	cp index.html $(DESTDIR)/var/www/html/
+
+system-images/compat/docker-images/aarch64.tar: $(COMPAT_SRC)
 	cd system-images/compat && make
 
-system-images/utils/utils.tar: $(UTILS_SRC)
+system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/x86_64.tar: $(UTILS_SRC)
 	cd system-images/utils && make
 
-system-images/binfmt/binfmt.tar: $(BINFMT_SRC)
+system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/x86_64.tar: $(BINFMT_SRC)
 	cd system-images/binfmt && make
 
 raspios.img:
@@ -75,7 +111,7 @@ snapshots: libs/snapshot-creator/Cargo.toml
 	cd libs/  && ./build-arm-v8-snapshot.sh
 
 $(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/patchdb-ui-seed.json
-	cd backend && ./build-prod.sh
+	cd backend && ARCH=$(ARCH) ./build-prod.sh
 	touch $(EMBASSY_BINS)
 
 frontend/node_modules: frontend/package.json

backend/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Aiden McClelland <me@drbonez.dev>"]
 description = "The core of the Start9 Embassy Operating System"
 documentation = "https://docs.rs/embassy-os"
-edition = "2018"
+edition = "2021"
 keywords = [
   "self-hosted",
   "raspberry-pi",
@@ -42,7 +42,7 @@ path = "src/bin/avahi-alias.rs"
 
 [features]
 avahi = ["avahi-sys"]
-default = ["avahi", "sound", "metal", "js_engine"]
+default = ["avahi", "metal", "sound", "js_engine"]
 dev = []
 metal = []
 sound = []

backend/build-prod.sh

@@ -3,6 +3,10 @@
 set -e
 shopt -s expand_aliases
 
+if [ -z "$ARCH" ]; then
+	ARCH=$(uname -m)
+fi
+
 if [ "$0" != "./build-prod.sh" ]; then
 	>&2 echo "Must be run from backend directory"
 	exit 1
@@ -24,10 +28,10 @@ if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then
 	FLAGS="dev,$FLAGS"
 fi
 if [[ "$FLAGS" = "" ]]; then
-	rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked)"
+	rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked --target=$ARCH-unknown-linux-gnu)"
 else
 	echo "FLAGS=$FLAGS"
-	rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked)"
+	rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)"
 fi
 cd backend
 

backend/embassy-init.service

@@ -7,7 +7,7 @@ Wants=avahi-daemon.service nginx.service tor.service
 [Service]
 Type=oneshot
 Environment=RUST_LOG=embassy_init=debug,embassy=debug,js_engine=debug,patch_db=warn
-ExecStart=/usr/local/bin/embassy-init
+ExecStart=/usr/bin/embassy-init
 RemainAfterExit=true
 StandardOutput=append:/var/log/embassy-init.log
 

backend/embassyd.service

@@ -6,7 +6,7 @@ Requires=embassy-init.service
 [Service]
 Type=simple
 Environment=RUST_LOG=embassyd=debug,embassy=debug,js_engine=debug,patch_db=warn
-ExecStart=/usr/local/bin/embassyd
+ExecStart=/usr/bin/embassyd
 Restart=always
 RestartSec=3
 ManagedOOMPreference=avoid

backend/src/action.rs

@@ -8,17 +8,15 @@ use rpc_toolkit::command;
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
+use crate::config::{Config, ConfigSpec};
 use crate::context::RpcContext;
 use crate::id::ImageId;
+use crate::procedure::docker::DockerContainer;
 use crate::procedure::{PackageProcedure, ProcedureName};
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
 use crate::util::Version;
 use crate::volume::Volumes;
-use crate::{
-    config::{Config, ConfigSpec},
-    procedure::docker::DockerContainer,
-};
 use crate::{Error, ResultExt};
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct Actions(pub BTreeMap<ActionId, Action>);

backend/src/backup/mod.rs

@@ -14,17 +14,18 @@ use tokio::io::AsyncWriteExt;
 use tracing::instrument;
 
 use self::target::PackageBackupInfo;
+use crate::context::RpcContext;
 use crate::dependencies::reconfigure_dependents_with_live_pointers;
 use crate::id::ImageId;
 use crate::install::PKG_ARCHIVE_DIR;
 use crate::net::interface::{InterfaceId, Interfaces};
+use crate::procedure::docker::DockerContainer;
 use crate::procedure::{NoOutput, PackageProcedure, ProcedureName};
 use crate::s9pk::manifest::PackageId;
 use crate::util::serde::IoFormat;
 use crate::util::Version;
 use crate::version::{Current, VersionT};
 use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
-use crate::{context::RpcContext, procedure::docker::DockerContainer};
 use crate::{Error, ErrorKind, ResultExt};
 
 pub mod backup_bulk;

backend/src/backup/restore.rs

@@ -236,7 +236,7 @@ pub async fn recover_full_embassy(
         os_backup.tor_key.public().get_onion_address(),
         os_backup.root_ca_cert,
         async move {
-            let rpc_ctx = RpcContext::init(ctx.config_path.as_ref(), disk_guid).await?;
+            let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid).await?;
             let mut db = rpc_ctx.db.handle();
 
             let ids = backup_guard

backend/src/backup/target/mod.rs

@@ -139,8 +139,10 @@ pub async fn list(
     #[context] ctx: RpcContext,
 ) -> Result<BTreeMap<BackupTargetId, BackupTarget>, Error> {
     let mut sql_handle = ctx.secret_store.acquire().await?;
-    let (disks_res, cifs) =
-        tokio::try_join!(crate::disk::util::list(), cifs::list(&mut sql_handle),)?;
+    let (disks_res, cifs) = tokio::try_join!(
+        crate::disk::util::list(&ctx.os_partitions),
+        cifs::list(&mut sql_handle),
+    )?;
     Ok(disks_res
         .into_iter()
         .flat_map(|mut disk| {

backend/src/bin/embassy-init.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -27,8 +27,11 @@ fn status_fn(_: i32) -> StatusCode {
 }
 
 #[instrument]
-async fn setup_or_init(cfg_path: Option<&str>) -> Result<(), Error> {
-    if tokio::fs::metadata("/embassy-os/disk.guid").await.is_err() {
+async fn setup_or_init(cfg_path: Option<PathBuf>) -> Result<(), Error> {
+    if tokio::fs::metadata("/media/embassy/config/disk.guid")
+        .await
+        .is_err()
+    {
         #[cfg(feature = "avahi")]
         let _mdns = MdnsController::init();
         tokio::fs::write(
@@ -68,7 +71,7 @@ async fn setup_or_init(cfg_path: Option<&str>) -> Result<(), Error> {
         .with_kind(embassy::ErrorKind::Network)?;
     } else {
         let cfg = RpcContextConfig::load(cfg_path).await?;
-        let guid_string = tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+        let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
             .await?;
         let guid = guid_string.trim();
         let requires_reboot = embassy::disk::main::import(
@@ -119,7 +122,7 @@ async fn run_script_if_exists<P: AsRef<Path>>(path: P) {
 }
 
 #[instrument]
-async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
+async fn inner_main(cfg_path: Option<PathBuf>) -> Result<Option<Shutdown>, Error> {
     if tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() {
         tokio::fs::remove_file(STANDBY_MODE_PATH).await?;
         Command::new("sync").invoke(ErrorKind::Filesystem).await?;
@@ -129,10 +132,10 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
 
     embassy::sound::BEP.play().await?;
 
-    run_script_if_exists("/embassy-os/preinit.sh").await;
+    run_script_if_exists("/media/embassy/config/preinit.sh").await;
 
-    let res = if let Err(e) = setup_or_init(cfg_path).await {
-        async {
+    let res = if let Err(e) = setup_or_init(cfg_path.clone()).await {
+        async move {
             tracing::error!("{}", e.source);
             tracing::debug!("{}", e.source);
             embassy::sound::BEETHOVEN.play().await?;
@@ -156,9 +159,12 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
                 .await?;
             let ctx = DiagnosticContext::init(
                 cfg_path,
-                if tokio::fs::metadata("/embassy-os/disk.guid").await.is_ok() {
+                if tokio::fs::metadata("/media/embassy/config/disk.guid")
+                    .await
+                    .is_ok()
+                {
                     Some(Arc::new(
-                        tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
+                        tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy
                             .await?
                             .trim()
                             .to_owned(),
@@ -200,7 +206,7 @@ async fn inner_main(cfg_path: Option<&str>) -> Result<Option<Shutdown>, Error> {
         Ok(None)
     };
 
-    run_script_if_exists("/embassy-os/postinit.sh").await;
+    run_script_if_exists("/media/embassy/config/postinit.sh").await;
 
     res
 }
@@ -217,7 +223,7 @@ fn main() {
 
     EmbassyLogger::init();
 
-    let cfg_path = matches.value_of("config");
+    let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned());
     let res = {
         let rt = tokio::runtime::Builder::new_multi_thread()
             .enable_all()