From dce575bbf52a55a6d042bd3378f6c16c9fd85aa8 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Sat, 10 Sep 2022 13:35:10 -0600 Subject: [PATCH 01/53] wip --- DEBIAN/control | 26 ++++++++++++++++++++++++++ backend/src/backup/target/mod.rs | 6 ++++-- backend/src/context/rpc.rs | 6 +++++- backend/src/context/setup.rs | 4 ++++ backend/src/disk/mod.rs | 17 ++++++++++++++++- backend/src/disk/util.rs | 27 ++++++++++++++++++--------- backend/src/setup.rs | 4 ++-- 7 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 DEBIAN/control diff --git a/DEBIAN/control b/DEBIAN/control new file mode 100644 index 000000000..ffe35ef57 --- /dev/null +++ b/DEBIAN/control @@ -0,0 +1,26 @@ +Package: embassy-os +Version: 0.3.2 +Section: base +Priority: required +Architecture: amd64 +Depends: nginx, + libavahi-client3, + avahi-daemon, + avahi-utils, + iotop, + bmon, + lvm2, + cryptsetup, + exfat-utils, + sqlite3, + wireless-tools, + net-tools, + ecryptfs-utils, + cifs-utils, + samba-common-bin, + network-manager, + vim, + jq, + ncdu, + postgresql, + pgloader diff --git a/backend/src/backup/target/mod.rs b/backend/src/backup/target/mod.rs index c87e914a9..501eb6ed8 100644 --- a/backend/src/backup/target/mod.rs +++ b/backend/src/backup/target/mod.rs @@ -139,8 +139,10 @@ pub async fn list( #[context] ctx: RpcContext, ) -> Result, Error> { let mut sql_handle = ctx.secret_store.acquire().await?; - let (disks_res, cifs) = - tokio::try_join!(crate::disk::util::list(), cifs::list(&mut sql_handle),)?; + let (disks_res, cifs) = tokio::try_join!( + crate::disk::util::list(&ctx.os_partitions), + cifs::list(&mut sql_handle), + )?; Ok(disks_res .into_iter() .flat_map(|mut disk| { diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index 55e64f0d0..933ff398f 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -23,6 +23,7 @@ use tracing::instrument; use crate::core::rpc_continuations::{RequestGuid, RestHandler, RpcContinuation}; use crate::db::model::{Database, InstalledPackageDataEntry, PackageDataEntry}; +use crate::disk::OsPartitionInfo; use crate::hostname::HostNameReceipt; use crate::init::{init_postgres, pgloader}; use crate::install::cleanup::{cleanup_failed, uninstall, CleanupFailedReceipts}; @@ -42,6 +43,7 @@ use crate::{Error, ErrorKind, ResultExt}; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct RpcContextConfig { + pub os_partitions: OsPartitionInfo, pub migration_batch_rows: Option, pub migration_prefetch_rows: Option, pub bind_rpc: Option, @@ -116,6 +118,7 @@ impl RpcContextConfig { pub struct RpcContextSeed { is_closed: AtomicBool, + pub os_partitions: OsPartitionInfo, pub bind_rpc: SocketAddr, pub bind_ws: SocketAddr, pub bind_static: SocketAddr, @@ -248,10 +251,11 @@ impl RpcContext { tracing::info!("Initialized Notification Manager"); let seed = Arc::new(RpcContextSeed { is_closed: AtomicBool::new(false), + datadir: base.datadir().to_path_buf(), + os_partitions: base.os_partitions, bind_rpc: base.bind_rpc.unwrap_or(([127, 0, 0, 1], 5959).into()), bind_ws: base.bind_ws.unwrap_or(([127, 0, 0, 1], 5960).into()), bind_static: base.bind_static.unwrap_or(([127, 0, 0, 1], 5961).into()), - datadir: base.datadir().to_path_buf(), disk_guid, db, secret_store, diff --git a/backend/src/context/setup.rs b/backend/src/context/setup.rs index 005ee471b..ec28560b1 100644 --- a/backend/src/context/setup.rs +++ b/backend/src/context/setup.rs @@ -18,6 +18,7 @@ use tracing::instrument; use url::Host; use crate::db::model::Database; +use crate::disk::OsPartitionInfo; use crate::init::{init_postgres, pgloader}; use crate::net::tor::os_key; use crate::setup::{password_hash, RecoveryStatus}; @@ -36,6 +37,7 @@ pub struct SetupResult { #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct SetupContextConfig { + pub os_partitions: OsPartitionInfo, pub migration_batch_rows: Option, pub migration_prefetch_rows: Option, pub bind_rpc: Option, @@ -65,6 +67,7 @@ impl SetupContextConfig { } pub struct SetupContextSeed { + pub os_partitions: OsPartitionInfo, pub config_path: Option, pub migration_batch_rows: usize, pub migration_prefetch_rows: usize, @@ -95,6 +98,7 @@ impl SetupContext { let (shutdown, _) = tokio::sync::broadcast::channel(1); let datadir = cfg.datadir().to_owned(); Ok(Self(Arc::new(SetupContextSeed { + os_partitions: cfg.os_partitions, config_path: path.as_ref().map(|p| p.as_ref().to_owned()), migration_batch_rows: cfg.migration_batch_rows.unwrap_or(25000), migration_prefetch_rows: cfg.migration_prefetch_rows.unwrap_or(100_000), diff --git a/backend/src/disk/mod.rs b/backend/src/disk/mod.rs index 91e63f21b..520f8fd72 100644 --- a/backend/src/disk/mod.rs +++ b/backend/src/disk/mod.rs @@ -1,6 +1,10 @@ +use std::path::PathBuf; + use clap::ArgMatches; use rpc_toolkit::command; +use serde::Deserialize; +use crate::context::RpcContext; use crate::disk::util::DiskInfo; use crate::util::display_none; use crate::util::serde::{display_serializable, IoFormat}; @@ -14,6 +18,16 @@ pub mod util; pub const BOOT_RW_PATH: &str = "/media/boot-rw"; pub const REPAIR_DISK_PATH: &str = "/embassy-os/repair-disk"; +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct OsPartitionInfo { + pub disk: PathBuf, + pub boot: PathBuf, + pub embassy: PathBuf, + pub green: PathBuf, + pub blue: PathBuf, +} + #[command(subcommands(list, repair))] pub fn disk() -> Result<(), Error> { Ok(()) @@ -75,11 +89,12 @@ fn display_disk_info(info: Vec, matches: &ArgMatches) { #[command(display(display_disk_info))] pub async fn list( + #[context] ctx: RpcContext, #[allow(unused_variables)] #[arg] format: Option, ) -> Result, Error> { - crate::disk::util::list().await + crate::disk::util::list(&ctx.os_partitions).await } #[command(display(display_none))] diff --git a/backend/src/disk/util.rs b/backend/src/disk/util.rs index a082da21a..9f02a3e5a 100644 --- a/backend/src/disk/util.rs +++ b/backend/src/disk/util.rs @@ -19,6 +19,7 @@ use tracing::instrument; use super::mount::filesystem::block_dev::BlockDev; use super::mount::filesystem::ReadOnly; use super::mount::guard::TmpMountGuard; +use crate::disk::OsPartitionInfo; use crate::util::io::from_yaml_async_reader; use crate::util::serde::IoFormat; use crate::util::{Invoke, Version}; @@ -232,7 +233,7 @@ pub async fn recovery_info( } #[instrument] -pub async fn list() -> Result, Error> { +pub async fn list(os: &OsPartitionInfo) -> Result, Error> { let disk_guids = pvscan().await?; let disks = tokio_stream::wrappers::ReadDirStream::new( tokio::fs::read_dir(DISK_PATH) @@ -262,13 +263,7 @@ pub async fn list() -> Result, Error> { disk_path.display().to_string(), ) })?; - if &*disk == Path::new("/dev/mmcblk0") { - return Ok(disks); - } - if !disks.contains_key(&disk) { - disks.insert(disk.clone(), IndexSet::new()); - } - if let Some(part_path) = part_path { + let part = if let Some(part_path) = part_path { let part_path = Path::new(DISK_PATH).join(part_path); let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| { ( @@ -276,7 +271,21 @@ pub async fn list() -> Result, Error> { part_path.display().to_string(), ) })?; - disks.get_mut(&disk).unwrap().insert(part); + Some(part) + } else { + None + }; + if disk == os.disk { + if let Some(part) = part { + disks.insert(part.clone(), IndexSet::new()); + } + } else { + if !disks.contains_key(&disk) { + disks.insert(disk.clone(), IndexSet::new()); + } + if let Some(part) = part { + disks.get_mut(&disk).unwrap().insert(part); + } } } Ok(disks) diff --git a/backend/src/setup.rs b/backend/src/setup.rs index 684eeecb6..9f3f73c9e 100644 --- a/backend/src/setup.rs +++ b/backend/src/setup.rs @@ -88,8 +88,8 @@ pub fn disk() -> Result<(), Error> { } #[command(rename = "list", rpc_only, metadata(authenticated = false))] -pub async fn list_disks() -> Result, Error> { - crate::disk::list(None).await +pub async fn list_disks(#[context] ctx: SetupContext) -> Result, Error> { + crate::disk::util::list(&ctx.os_partitions).await } #[command(rpc_only)] From 1ca46d11ea56523087cd0c20ab66658484a3c1d9 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 12 Sep 2022 13:26:50 -0600 Subject: [PATCH 02/53] wip --- .gitignore | 4 +++- DEBIAN/control | 36 ++++++++++++++++++------------------ Makefile | 5 +++++ build/make-deb.sh | 30 ++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 19 deletions(-) create mode 100755 build/make-deb.sh diff --git a/.gitignore b/.gitignore index 9e63eb749..2a9dd7193 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ secrets.db /cargo-deps/**/* /ENVIRONMENT.txt /GIT_HASH.txt -/eos.tar.gz \ No newline at end of file +/eos.tar.gz +/debian +/*.deb \ No newline at end of file diff --git a/DEBIAN/control b/DEBIAN/control index ffe35ef57..5c6e30da1 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -6,21 +6,21 @@ Architecture: amd64 Depends: nginx, libavahi-client3, avahi-daemon, - avahi-utils, - iotop, - bmon, - lvm2, - cryptsetup, - exfat-utils, - sqlite3, - wireless-tools, - net-tools, - ecryptfs-utils, - cifs-utils, - samba-common-bin, - network-manager, - vim, - jq, - ncdu, - postgresql, - pgloader + avahi-utils, + iotop, + bmon, + lvm2, + cryptsetup, + exfat-utils, + sqlite3, + wireless-tools, + net-tools, + ecryptfs-utils, + cifs-utils, + samba-common-bin, + network-manager, + vim, + jq, + ncdu, + postgresql, + pgloader diff --git a/Makefile b/Makefile index 15d8987cb..e5523adba 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ all: eos.img gzip: eos.tar.gz +deb: embassy-os.deb + eos.tar.gz: eos.img tar --format=posix -cS -f- eos.img | $(GZIP_BIN) > eos.tar.gz @@ -50,6 +52,9 @@ eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/util ! test -f eos.img || rm eos.img if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi +embassy-os.deb: DEBIAN/control + ./build/make-deb.sh + system-images/compat/compat.tar: $(COMPAT_SRC) cd system-images/compat && make diff --git a/build/make-deb.sh b/build/make-deb.sh new file mode 100755 index 000000000..ab8c9b856 --- /dev/null +++ b/build/make-deb.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +rm -rf debian && mkdir debian +cp -R DEBIAN debian/ + +mkdir -p debian/etc/embassy +cp ENVIRONMENT.txt debian/etc/embassy/ +cp GIT_HASH.txt debian/etc/embassy/ + +mkdir -p debian/usr/local/bin +cp backend/target/aarch64-unknown-linux-gnu/release/embassy-init debian/usr/local/bin/ +cp backend/target/aarch64-unknown-linux-gnu/release/embassyd debian/usr/local/bin/ +cp backend/target/aarch64-unknown-linux-gnu/release/embassy-cli debian/usr/local/bin/ + +mkdir -p debian/etc/systemd/system +cp backend/embassy-init.service debian/etc/systemd/system/ +cp backend/embassyd.service debian/etc/systemd/system/ + +mkdir -p debian/var/lib/embassy/system-images +cp system-images/**/*.tar debian/var/lib/embassy/system-images/ + +mkdir -p debian/var/www/html +cp -r frontend/dist/diagnostic-ui debian/var/www/html/diagnostic +cp -r frontend/dist/setup-wizard debian/var/www/html/setup +cp -r frontend/dist/ui debian/var/www/html/main +cp index.html debian/var/www/html/ + +dpkg-deb --root-owner-group --build debian embassy-os.deb \ No newline at end of file From 5916ef878db4f3f76e6d2925e5d7656b5067ab01 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 14 Sep 2022 11:24:57 -0600 Subject: [PATCH 03/53] wip --- DEBIAN/control | 3 ++- libs/js_engine/src/lib.rs | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/DEBIAN/control b/DEBIAN/control index 5c6e30da1..e20a93dfc 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -23,4 +23,5 @@ Depends: nginx, jq, ncdu, postgresql, - pgloader + pgloader, + openssh-server diff --git a/libs/js_engine/src/lib.rs b/libs/js_engine/src/lib.rs index 6f5810d0c..887e21fed 100644 --- a/libs/js_engine/src/lib.rs +++ b/libs/js_engine/src/lib.rs @@ -808,6 +808,15 @@ mod fns { let parent = tokio::fs::canonicalize(parent).await?; Ok(child.starts_with(parent)) } + + #[tokio::test] + async fn test_is_subset() { + assert!( + !is_subset("/home/drbonez", "/home/drbonez/code/fakedir/../../..") + .await + .unwrap() + ) + } } fn system_time_as_unix_ms(system_time: &SystemTime) -> Option { From ab4d74bc4dfdccea9828328c135a63aff9b9c54a Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 19 Sep 2022 15:34:36 -0600 Subject: [PATCH 04/53] wip --- DEBIAN/README | 6 ++++++ DEBIAN/copyright | 39 +++++++++++++++++++++++++++++++++++++ DEBIAN/embassy-os-docs.docs | 1 + DEBIAN/postinst.ex | 39 +++++++++++++++++++++++++++++++++++++ DEBIAN/postrm.ex | 37 +++++++++++++++++++++++++++++++++++ DEBIAN/preinst.ex | 35 +++++++++++++++++++++++++++++++++ DEBIAN/prerm.ex | 38 ++++++++++++++++++++++++++++++++++++ DEBIAN/rules | 25 ++++++++++++++++++++++++ DEBIAN/source/format | 1 + 9 files changed, 221 insertions(+) create mode 100644 DEBIAN/README create mode 100644 DEBIAN/copyright create mode 100644 DEBIAN/embassy-os-docs.docs create mode 100644 DEBIAN/postinst.ex create mode 100644 DEBIAN/postrm.ex create mode 100644 DEBIAN/preinst.ex create mode 100644 DEBIAN/prerm.ex create mode 100755 DEBIAN/rules create mode 100644 DEBIAN/source/format diff --git a/DEBIAN/README b/DEBIAN/README new file mode 100644 index 000000000..b4557eea1 --- /dev/null +++ b/DEBIAN/README @@ -0,0 +1,6 @@ +The Debian Package embassy-os +---------------------------- + +Comments regarding the Package + + -- Aiden Mon, 19 Sep 2022 14:34:19 -0600 diff --git a/DEBIAN/copyright b/DEBIAN/copyright new file mode 100644 index 000000000..3e5e3c4ec --- /dev/null +++ b/DEBIAN/copyright @@ -0,0 +1,39 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: embassy-os +Upstream-Contact: +Source: + +Files: * +Copyright: + +License: GPL-3.0+ + +Files: debian/* +Copyright: 2022 Aiden +License: GPL-3.0+ + +License: GPL-3.0+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". + +# Please also look if there are files or directories which have a +# different copyright/license attached and list them here. +# Please avoid picking licenses with terms that are more restrictive than the +# packaged work, as it may make Debian's contributions unacceptable upstream. +# +# If you need, there are some extra license texts available in two places: +# /usr/share/debhelper/dh_make/licenses/ +# /usr/share/common-licenses/ diff --git a/DEBIAN/embassy-os-docs.docs b/DEBIAN/embassy-os-docs.docs new file mode 100644 index 000000000..e845566c0 --- /dev/null +++ b/DEBIAN/embassy-os-docs.docs @@ -0,0 +1 @@ +README diff --git a/DEBIAN/postinst.ex b/DEBIAN/postinst.ex new file mode 100644 index 000000000..9f0ff7151 --- /dev/null +++ b/DEBIAN/postinst.ex @@ -0,0 +1,39 @@ +#!/bin/sh +# postinst script for embassy-os +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/DEBIAN/postrm.ex b/DEBIAN/postrm.ex new file mode 100644 index 000000000..26446c868 --- /dev/null +++ b/DEBIAN/postrm.ex @@ -0,0 +1,37 @@ +#!/bin/sh +# postrm script for embassy-os +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/DEBIAN/preinst.ex b/DEBIAN/preinst.ex new file mode 100644 index 000000000..2af27b047 --- /dev/null +++ b/DEBIAN/preinst.ex @@ -0,0 +1,35 @@ +#!/bin/sh +# preinst script for embassy-os +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `install' +# * `install' +# * `upgrade' +# * `abort-upgrade' +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + install|upgrade) + ;; + + abort-upgrade) + ;; + + *) + echo "preinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/DEBIAN/prerm.ex b/DEBIAN/prerm.ex new file mode 100644 index 000000000..9f64759bb --- /dev/null +++ b/DEBIAN/prerm.ex @@ -0,0 +1,38 @@ +#!/bin/sh +# prerm script for embassy-os +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `remove' +# * `upgrade' +# * `failed-upgrade' +# * `remove' `in-favour' +# * `deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + remove|upgrade|deconfigure) + ;; + + failed-upgrade) + ;; + + *) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/DEBIAN/rules b/DEBIAN/rules new file mode 100755 index 000000000..59ea751e6 --- /dev/null +++ b/DEBIAN/rules @@ -0,0 +1,25 @@ +#!/usr/bin/make -f +# See debhelper(7) (uncomment to enable) +# output every command that modifies files on the build system. +#export DH_VERBOSE = 1 + + +# see FEATURE AREAS in dpkg-buildflags(1) +#export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +# see ENVIRONMENT in dpkg-buildflags(1) +# package maintainers to append CFLAGS +#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic +# package maintainers to append LDFLAGS +#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed + + +%: + dh $@ + + +# dh_make generated override targets +# This is example for Cmake (See https://bugs.debian.org/641051 ) +#override_dh_auto_configure: +# dh_auto_configure -- \ +# -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH) diff --git a/DEBIAN/source/format b/DEBIAN/source/format new file mode 100644 index 000000000..89ae9db8f --- /dev/null +++ b/DEBIAN/source/format @@ -0,0 +1 @@ +3.0 (native) From 48e227850bb2dc0de6751d001e3325f3537097a6 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 20 Sep 2022 16:53:12 -0600 Subject: [PATCH 05/53] wip --- .github/workflows/backend.yaml | 2 +- .gitignore | 2 +- DEBIAN/copyright | 39 ---- Makefile | 31 ++- backend/build-dev.sh | 4 +- backend/build-portable-dev.sh | 4 +- backend/build-portable.sh | 4 +- backend/build-prod.sh | 4 +- build-cargo-dep.sh | 4 +- debian/.gitignore | 1 + {DEBIAN => debian}/README | 0 debian/README.Debian | 6 + debian/README.source | 10 + debian/changelog | 5 + {DEBIAN => debian}/control | 9 +- debian/copyright | 43 ++++ {DEBIAN => debian}/embassy-os-docs.docs | 0 debian/embassy-os.cron.d.ex | 4 + debian/embassy-os.doc-base.ex | 20 ++ debian/manpage.1.ex | 56 +++++ debian/manpage.md.ex | 126 ++++++++++ debian/manpage.sgml.ex | 154 +++++++++++++ debian/manpage.xml.ex | 291 ++++++++++++++++++++++++ {DEBIAN => debian}/postinst.ex | 0 {DEBIAN => debian}/postrm.ex | 0 {DEBIAN => debian}/preinst.ex | 0 {DEBIAN => debian}/prerm.ex | 0 {DEBIAN => debian}/rules | 0 debian/salsa-ci.yml.ex | 11 + {DEBIAN => debian}/source/format | 0 libs/build-arm-v8-snapshot.sh | 4 +- libs/build-v8-snapshot.sh | 4 +- system-images/compat/build.sh | 3 + 33 files changed, 785 insertions(+), 56 deletions(-) delete mode 100644 DEBIAN/copyright create mode 100644 debian/.gitignore rename {DEBIAN => debian}/README (100%) create mode 100644 debian/README.Debian create mode 100644 debian/README.source create mode 100644 debian/changelog rename {DEBIAN => debian}/control (67%) create mode 100644 debian/copyright rename {DEBIAN => debian}/embassy-os-docs.docs (100%) create mode 100644 debian/embassy-os.cron.d.ex create mode 100644 debian/embassy-os.doc-base.ex create mode 100644 debian/manpage.1.ex create mode 100644 debian/manpage.md.ex create mode 100644 debian/manpage.sgml.ex create mode 100644 debian/manpage.xml.ex rename {DEBIAN => debian}/postinst.ex (100%) rename {DEBIAN => debian}/postrm.ex (100%) rename {DEBIAN => debian}/preinst.ex (100%) rename {DEBIAN => debian}/prerm.ex (100%) rename {DEBIAN => debian}/rules (100%) create mode 100644 debian/salsa-ci.yml.ex rename {DEBIAN => debian}/source/format (100%) diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index 9a80105ec..904cd351b 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -163,7 +163,7 @@ jobs: if: ${{ matrix.target == 'aarch64' }} - name: Reset permissions - run: sudo chown -R $USER target + run: sudo chown -R $(whoami) target working-directory: backend if: ${{ matrix.target == 'aarch64' }} diff --git a/.gitignore b/.gitignore index 2a9dd7193..24c80865b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,5 @@ secrets.db /ENVIRONMENT.txt /GIT_HASH.txt /eos.tar.gz -/debian +/embassy-os-*.tar.gz /*.deb \ No newline at end of file diff --git a/DEBIAN/copyright b/DEBIAN/copyright deleted file mode 100644 index 3e5e3c4ec..000000000 --- a/DEBIAN/copyright +++ /dev/null @@ -1,39 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: embassy-os -Upstream-Contact: -Source: - -Files: * -Copyright: - -License: GPL-3.0+ - -Files: debian/* -Copyright: 2022 Aiden -License: GPL-3.0+ - -License: GPL-3.0+ - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - . - This package is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - . - You should have received a copy of the GNU General Public License - along with this program. If not, see . - . - On Debian systems, the complete text of the GNU General - Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". - -# Please also look if there are files or directories which have a -# different copyright/license attached and list them here. -# Please avoid picking licenses with terms that are more restrictive than the -# packaged work, as it may make Debian's contributions unacceptable upstream. -# -# If you need, there are some extra license texts available in two places: -# /usr/share/debhelper/dh_make/licenses/ -# /usr/share/common-licenses/ diff --git a/Makefile b/Makefile index e5523adba..f31b0ce2a 100644 --- a/Makefile +++ b/Makefile @@ -29,14 +29,19 @@ eos.tar.gz: eos.img tar --format=posix -cS -f- eos.img | $(GZIP_BIN) > eos.tar.gz clean: + rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip + rm -f raspios.img rm -f eos.img rm -f ubuntu.img rm -f product_key.txt rm -f system-images/**/*.tar - sudo rm -f $(EMBASSY_BINS) + rm -rf system-images/compat/target + rm -rf backend/target + rm -rf frontend/.angular rm -f frontend/config.json rm -rf frontend/node_modules rm -rf frontend/dist + rm -rf libs/target rm -rf patch-db/client/node_modules rm -rf patch-db/client/dist sudo rm -rf cargo-deps @@ -52,6 +57,30 @@ eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/util ! test -f eos.img || rm eos.img if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi +# For creating dpkg. DO NOT USE +install: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) + mkdir -p $(DESTDIR)/etc/embassy + cp ENVIRONMENT.txt $(DESTDIR)/etc/embassy/ + cp GIT_HASH.txt $(DESTDIR)/etc/embassy/ + + mkdir -p $(DESTDIR)/usr/local/bin + cp backend/target/aarch64-unknown-linux-gnu/release/embassy-init $(DESTDIR)/usr/local/bin/ + cp backend/target/aarch64-unknown-linux-gnu/release/embassyd $(DESTDIR)/usr/local/bin/ + cp backend/target/aarch64-unknown-linux-gnu/release/embassy-cli $(DESTDIR)/usr/local/bin/ + + mkdir -p $(DESTDIR)/etc/systemd/system + cp backend/embassy-init.service $(DESTDIR)/etc/systemd/system/ + cp backend/embassyd.service $(DESTDIR)/etc/systemd/system/ + + mkdir -p $(DESTDIR)/var/lib/embassy/system-images + cp system-images/**/*.tar $(DESTDIR)/var/lib/embassy/system-images/ + + mkdir -p $(DESTDIR)/var/www/html + cp -r frontend/dist/diagnostic-ui $(DESTDIR)/var/www/html/diagnostic + cp -r frontend/dist/setup-wizard $(DESTDIR)/var/www/html/setup + cp -r frontend/dist/ui $(DESTDIR)/var/www/html/main + cp index.html $(DESTDIR)/var/www/html/ + embassy-os.deb: DEBIAN/control ./build/make-deb.sh diff --git a/backend/build-dev.sh b/backend/build-dev.sh index 80bfb44c0..6736ffbef 100755 --- a/backend/build-dev.sh +++ b/backend/build-dev.sh @@ -19,6 +19,6 @@ cd .. rust-arm64-builder sh -c "(cd backend && cargo build --locked)" cd backend -sudo chown -R $USER target -sudo chown -R $USER ~/.cargo +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/backend/build-portable-dev.sh b/backend/build-portable-dev.sh index cdeb2b6d3..bacd19c6f 100755 --- a/backend/build-portable-dev.sh +++ b/backend/build-portable-dev.sh @@ -19,5 +19,5 @@ cd .. rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features --locked)" cd backend -sudo chown -R $USER target -sudo chown -R $USER ~/.cargo \ No newline at end of file +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo \ No newline at end of file diff --git a/backend/build-portable.sh b/backend/build-portable.sh index 9b1b96f34..146b795dd 100755 --- a/backend/build-portable.sh +++ b/backend/build-portable.sh @@ -19,5 +19,5 @@ cd .. rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features --locked)" cd backend -sudo chown -R $USER target -sudo chown -R $USER ~/.cargo \ No newline at end of file +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo \ No newline at end of file diff --git a/backend/build-prod.sh b/backend/build-prod.sh index 3ac0b0453..d5b27b584 100755 --- a/backend/build-prod.sh +++ b/backend/build-prod.sh @@ -31,7 +31,7 @@ else fi cd backend -sudo chown -R $USER target -sudo chown -R $USER ~/.cargo +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/build-cargo-dep.sh b/build-cargo-dep.sh index 08fad4799..d537432c0 100755 --- a/build-cargo-dep.sh +++ b/build-cargo-dep.sh @@ -17,5 +17,5 @@ mkdir -p cargo-deps alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64' rust-arm64-builder cargo install "$1" --target-dir /home/rust/src -sudo chown -R $USER cargo-deps -sudo chown -R $USER ~/.cargo \ No newline at end of file +sudo chown -R $(whoami) cargo-deps +sudo chown -R $(whoami) ~/.cargo \ No newline at end of file diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 000000000..d21933f1c --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1 @@ +/.debhelper diff --git a/DEBIAN/README b/debian/README similarity index 100% rename from DEBIAN/README rename to debian/README diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 000000000..a70b5bd63 --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,6 @@ +embassy-os for Debian +-------------------- + + + + -- Aiden Mon, 19 Sep 2022 16:40:40 -0600 diff --git a/debian/README.source b/debian/README.source new file mode 100644 index 000000000..6fafd1bf5 --- /dev/null +++ b/debian/README.source @@ -0,0 +1,10 @@ +embassy-os for Debian +-------------------- + + + + + + -- Aiden Mon, 19 Sep 2022 16:40:40 -0600 + diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 000000000..df736f99f --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +embassy-os (0.3.2-1) UNRELEASED; urgency=medium + + * Initial release (Closes: #nnnn) + + -- Aiden Mon, 19 Sep 2022 16:40:40 -0600 diff --git a/DEBIAN/control b/debian/control similarity index 67% rename from DEBIAN/control rename to debian/control index e20a93dfc..0c83930e1 100644 --- a/DEBIAN/control +++ b/debian/control @@ -1,6 +1,11 @@ +Source: embassy-os +Maintainer: Aiden McClelland +Build-Depends: debhelper-compat (= 13) +Standards-Version: 3.9.3 +Section: unknown + Package: embassy-os -Version: 0.3.2 -Section: base +Section: unknown Priority: required Architecture: amd64 Depends: nginx, diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 000000000..821932eab --- /dev/null +++ b/debian/copyright @@ -0,0 +1,43 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: embassy-os +Upstream-Contact: support@start9labs.com +Source: https://github.com/Start9Labs/embassy-os + +Files: * +Copyright: 2022 Start9Labs +License: START9 PERSONAL USE LICENSE v1.0 + +License: MIT + This license governs the use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software. + + 1. Definitions. + 1. “Licensor” means the copyright owner, Start9 Labs, Inc, or its successor(s) in interest, or a future assignee of the copyright. + 2. “Source Code” means the preferred form of the Software for making modifications to it. + 3. “Object Code” means any non-source form of the Software, including the machine-language output by a compiler or assembler. + 4. “Distribute” means to convey or to publish and generally has the same meaning here as under U.S. Copyright law. + 5. “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including without limitation fees for hosting or consulting/support services related to the Software), a product or service whose value derives, entirely or substantially, from the functionality of the Software. + + 2. Grant of Rights. Subject to the terms of this license, the Licensor grants you, the licensee, a non-exclusive, worldwide, royalty-free copyright license to: + 1. Access, audit, copy, modify, compile, or distribute the Source Code or modifications to the Source Code. + 2. Run, test, or otherwise use the Object Code. + + 3. Limitations. + 1. The grant of rights under the License will NOT include, and the License does NOT grant you the right to: + 1. Sell the Software or any derivative works based thereon. + 2. Distribute the Object Code. + 2. If you Distribute the Source Code, or if permission is separately granted to Distribute the Object Code, you expressly undertake not to remove, or modify, in any manner, the copyright notices attached to the Source Code, and displayed in any output of the Object Code when run, and to reproduce these notices, in an identical manner, in any distributed copies of the Software together with a copy of this license. If you Distribute a modified copy of the Software, or a derivative work based thereon, the work must carry prominent notices stating that you modified it, and giving a relevant date. + 3. The terms of this license will apply to anyone who comes into possession of a copy of the Software, and any modifications or derivative works based thereon, made by anyone. + + 4. Contributions. You hereby grant to Licensor a perpetual, irrevocable, worldwide, non-exclusive, royalty-free license to use and exploit any modifications or derivative works based on the Source Code of which you are the author. + + 5. Disclaimer. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LICENSOR HAS NO OBLIGATION TO SUPPORT RECIPIENTS OF THE SOFTWARE. + + +# Please also look if there are files or directories which have a +# different copyright/license attached and list them here. +# Please avoid picking licenses with terms that are more restrictive than the +# packaged work, as it may make Debian's contributions unacceptable upstream. +# +# If you need, there are some extra license texts available in two places: +# /usr/share/debhelper/dh_make/licenses/ +# /usr/share/common-licenses/ diff --git a/DEBIAN/embassy-os-docs.docs b/debian/embassy-os-docs.docs similarity index 100% rename from DEBIAN/embassy-os-docs.docs rename to debian/embassy-os-docs.docs diff --git a/debian/embassy-os.cron.d.ex b/debian/embassy-os.cron.d.ex new file mode 100644 index 000000000..843e7dc9e --- /dev/null +++ b/debian/embassy-os.cron.d.ex @@ -0,0 +1,4 @@ +# +# Regular cron jobs for the embassy-os package +# +0 4 * * * root [ -x /usr/bin/embassy-os_maintenance ] && /usr/bin/embassy-os_maintenance diff --git a/debian/embassy-os.doc-base.ex b/debian/embassy-os.doc-base.ex new file mode 100644 index 000000000..a70dc68f3 --- /dev/null +++ b/debian/embassy-os.doc-base.ex @@ -0,0 +1,20 @@ +Document: embassy-os +Title: Debian embassy-os Manual +Author: +Abstract: This manual describes what embassy-os is + and how it can be used to + manage online manuals on Debian systems. +Section: unknown + +Format: debiandoc-sgml +Files: /usr/share/doc/embassy-os/embassy-os.sgml.gz + +Format: postscript +Files: /usr/share/doc/embassy-os/embassy-os.ps.gz + +Format: text +Files: /usr/share/doc/embassy-os/embassy-os.text.gz + +Format: HTML +Index: /usr/share/doc/embassy-os/html/index.html +Files: /usr/share/doc/embassy-os/html/*.html diff --git a/debian/manpage.1.ex b/debian/manpage.1.ex new file mode 100644 index 000000000..e549b4ce7 --- /dev/null +++ b/debian/manpage.1.ex @@ -0,0 +1,56 @@ +.\" Hey, EMACS: -*- nroff -*- +.\" (C) Copyright 2022 Aiden , +.\" +.\" First parameter, NAME, should be all caps +.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection +.\" other parameters are allowed: see man(7), man(1) +.TH Embassy-os SECTION "September 19 2022" +.\" Please adjust this date whenever revising the manpage. +.\" +.\" Some roff macros, for reference: +.\" .nh disable hyphenation +.\" .hy enable hyphenation +.\" .ad l left justify +.\" .ad b justify to both left and right margins +.\" .nf disable filling +.\" .fi enable filling +.\" .br insert line break +.\" .sp insert n+1 empty lines +.\" for manpage-specific macros, see man(7) +.SH NAME +embassy-os \- program to do something +.SH SYNOPSIS +.B embassy-os +.RI [ options ] " files" ... +.br +.B bar +.RI [ options ] " files" ... +.SH DESCRIPTION +This manual page documents briefly the +.B embassy-os +and +.B bar +commands. +.PP +.\" TeX users may be more comfortable with the \fB\fP and +.\" \fI\fP escape sequences to invode bold face and italics, +.\" respectively. +\fBembassy-os\fP is a program that... +.SH OPTIONS +These programs follow the usual GNU command line syntax, with long +options starting with two dashes (`-'). +A summary of options is included below. +For a complete description, see the Info files. +.TP +.B \-h, \-\-help +Show summary of options. +.TP +.B \-v, \-\-version +Show version of program. +.SH SEE ALSO +.BR bar (1), +.BR baz (1). +.br +The programs are documented fully by +.IR "The Rise and Fall of a Fooish Bar" , +available via the Info system. diff --git a/debian/manpage.md.ex b/debian/manpage.md.ex new file mode 100644 index 000000000..f2c2c51e4 --- /dev/null +++ b/debian/manpage.md.ex @@ -0,0 +1,126 @@ +% embassy-os(SECTION) | User Commands +% +% "September 19 2022" + +[comment]: # The lines above form a Pandoc metadata block. They must be +[comment]: # the first ones in the file. +[comment]: # See https://pandoc.org/MANUAL.html#metadata-blocks for details. + +[comment]: # pandoc -s -f markdown -t man package.md -o package.1 +[comment]: # +[comment]: # A manual page package.1 will be generated. You may view the +[comment]: # manual page with: nroff -man package.1 | less. A typical entry +[comment]: # in a Makefile or Makefile.am is: +[comment]: # +[comment]: # package.1: package.md +[comment]: # pandoc --standalone --from=markdown --to=man $< --output=$@ +[comment]: # +[comment]: # The pandoc binary is found in the pandoc package. Please remember +[comment]: # that if you create the nroff version in one of the debian/rules +[comment]: # file targets, such as build, you will need to include pandoc in +[comment]: # your Build-Depends control field. + +[comment]: # Remove the lines starting with `[comment]:' in this file in order +[comment]: # to avoid warning messages from pandoc. + +# NAME + +embassy-os - program to do something + +# SYNOPSIS + +**embassy-os** **-e** _this_ [**\-\-example=that**] [{**-e** | **\-\-example**} _this_] + [{**-e** | **\-\-example**} {_this_ | _that_}] + +**embassy-os** [{**-h** | *\-\-help**} | {**-v** | **\-\-version**}] + +# DESCRIPTION + +This manual page documents briefly the **embassy-os** and **bar** commands. + +This manual page was written for the Debian distribution because the +original program does not have a manual page. Instead, it has documentation +in the GNU info(1) format; see below. + +**embassy-os** is a program that... + +# OPTIONS + +The program follows the usual GNU command line syntax, with long options +starting with two dashes (`-'). A summary of options is included below. For +a complete description, see the **info**(1) files. + +**-e** _this_, **\-\-example=**_that_ +: Does this and that. + +**-h**, **\-\-help** +: Show summary of options. + +**-v**, **\-\-version** +: Show version of program. + +# FILES + +/etc/foo.conf +: The system-wide configuration file to control the behaviour of + embassy-os. See **foo.conf**(5) for further details. + +${HOME}/.foo.conf +: The per-user configuration file to control the behaviour of + embassy-os. See **foo.conf**(5) for further details. + +# ENVIRONMENT + +**FOO_CONF** +: If used, the defined file is used as configuration file (see also + the section called “FILES”). + +# DIAGNOSTICS + +The following diagnostics may be issued on stderr: + +Bad configuration file. Exiting. +: The configuration file seems to contain a broken configuration + line. Use the **\-\-verbose** option, to get more info. + +**embassy-os** provides some return codes, that can be used in scripts: + + Code Diagnostic + 0 Program exited successfully. + 1 The configuration file seems to be broken. + +# BUGS + +The program is currently limited to only work with the foobar library. + +The upstream BTS can be found at http://bugzilla.foo.tld. + +# SEE ALSO + +**bar**(1), **baz**(1), **foo.conf**(5) + +The programs are documented fully by The Rise and Fall of a Fooish Bar +available via the **info**(1) system. + +# AUTHOR + +Aiden +: Wrote this manpage for the Debian system. + +# COPYRIGHT + +Copyright © 2007 Aiden + +This manual page was written for the Debian system (and may be used by +others). + +Permission is granted to copy, distribute and/or modify this document under +the terms of the GNU General Public License, Version 2 or (at your option) +any later version published by the Free Software Foundation. + +On Debian systems, the complete text of the GNU General Public License +can be found in /usr/share/common-licenses/GPL. + +[comment]: # Local Variables: +[comment]: # mode: markdown +[comment]: # End: diff --git a/debian/manpage.sgml.ex b/debian/manpage.sgml.ex new file mode 100644 index 000000000..1d2eb429b --- /dev/null +++ b/debian/manpage.sgml.ex @@ -0,0 +1,154 @@ + manpage.1'. You may view + the manual page with: `docbook-to-man manpage.sgml | nroff -man | + less'. A typical entry in a Makefile or Makefile.am is: + +manpage.1: manpage.sgml + docbook-to-man $< > $@ + + + The docbook-to-man binary is found in the docbook-to-man package. + Please remember that if you create the nroff version in one of the + debian/rules file targets (such as build), you will need to include + docbook-to-man in your Build-Depends control field. + + --> + + + FIRSTNAME"> + SURNAME"> + + September 19 2022"> + + SECTION"> + aiden@start9labs.com"> + + Embassy-os"> + + + Debian"> + GNU"> + GPL"> +]> + + + +
+ &dhemail; +
+ + &dhfirstname; + &dhsurname; + + + 2003 + &dhusername; + + &dhdate; + + + &dhucpackage; + + &dhsection; + + + &dhpackage; + + program to do something + + + + &dhpackage; + + + + + + + + DESCRIPTION + + This manual page documents briefly the + &dhpackage; and bar + commands. + + This manual page was written for the &debian; distribution + because the original program does not have a manual page. + Instead, it has documentation in the &gnu; + Info format; see below. + + &dhpackage; is a program that... + + + + OPTIONS + + These programs follow the usual &gnu; command line syntax, + with long options starting with two dashes (`-'). A summary of + options is included below. For a complete description, see the + Info files. + + + + + + + + Show summary of options. + + + + + + + + Show version of program. + + + + + + SEE ALSO + + bar (1), baz (1). + + The programs are documented fully by The Rise and + Fall of a Fooish Bar available via the + Info system. + + + AUTHOR + + This manual page was written by &dhusername; &dhemail; for + the &debian; system (and may be used by others). Permission is + granted to copy, distribute and/or modify this document under + the terms of the &gnu; General Public License, Version 2 any + later version published by the Free Software Foundation. + + + On Debian systems, the complete text of the GNU General Public + License can be found in /usr/share/common-licenses/GPL. + + + + + + diff --git a/debian/manpage.xml.ex b/debian/manpage.xml.ex new file mode 100644 index 000000000..286a25e7d --- /dev/null +++ b/debian/manpage.xml.ex @@ -0,0 +1,291 @@ + +.
will be generated. You may view the +manual page with: nroff -man .
| less'. A typical entry +in a Makefile or Makefile.am is: + +DB2MAN = /usr/share/sgml/docbook/stylesheet/xsl/docbook-xsl/manpages/docbook.xsl +XP = xsltproc -''-nonet -''-param man.charmap.use.subset "0" + +manpage.1: manpage.xml + $(XP) $(DB2MAN) $< + +The xsltproc binary is found in the xsltproc package. The XSL files are in +docbook-xsl. A description of the parameters you can use can be found in the +docbook-xsl-doc-* packages. Please remember that if you create the nroff +version in one of the debian/rules file targets (such as build), you will need +to include xsltproc and docbook-xsl in your Build-Depends control field. +Alternatively use the xmlto command/package. That will also automatically +pull in xsltproc and docbook-xsl. + +Notes for using docbook2x: docbook2x-man does not automatically create the +AUTHOR(S) and COPYRIGHT sections. In this case, please add them manually as + ... . + +To disable the automatic creation of the AUTHOR(S) and COPYRIGHT sections +read /usr/share/doc/docbook-xsl/doc/manpages/authors.html. This file can be +found in the docbook-xsl-doc-html package. + +Validation can be done using: `xmllint -''-noout -''-valid manpage.xml` + +General documentation about man-pages and man-page-formatting: +man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ + +--> + + + + + + + + + + + + + +]> + + + + &dhtitle; + &dhpackage; + + + &dhfirstname; + &dhsurname; + Wrote this manpage for the Debian system. +
+ &dhemail; +
+ + + + 2007 + &dhusername; + + + This manual page was written for the Debian system + (and may be used by others). + Permission is granted to copy, distribute and/or modify this + document under the terms of the GNU General Public License, + Version 2 or (at your option) any later version published by + the Free Software Foundation. + On Debian systems, the complete text of the GNU General Public + License can be found in + /usr/share/common-licenses/GPL. + + + + &dhucpackage; + &dhsection; + + + &dhpackage; + program to do something + + + + &dhpackage; + + + + + + + + + this + + + + + + + + this + that + + + + + &dhpackage; + + + + + + + + + + + + + + + + + + + DESCRIPTION + This manual page documents briefly the + &dhpackage; and bar + commands. + This manual page was written for the Debian distribution + because the original program does not have a manual page. + Instead, it has documentation in the GNU + info + 1 + format; see below. + &dhpackage; is a program that... + + + OPTIONS + The program follows the usual GNU command line syntax, + with long options starting with two dashes (`-'). A summary of + options is included below. For a complete description, see the + + info + 1 + files. + + + + + + + Does this and that. + + + + + + + Show summary of options. + + + + + + + Show version of program. + + + + + + FILES + + + /etc/foo.conf + + The system-wide configuration file to control the + behaviour of &dhpackage;. See + + foo.conf + 5 + for further details. + + + + ${HOME}/.foo.conf + + The per-user configuration file to control the + behaviour of &dhpackage;. See + + foo.conf + 5 + for further details. + + + + + + ENVIRONMENT + + + FOO_CONF + + If used, the defined file is used as configuration + file (see also ). + + + + + + DIAGNOSTICS + The following diagnostics may be issued + on stderr: + + + Bad configuration file. Exiting. + + The configuration file seems to contain a broken configuration + line. Use the option, to get more info. + + + + + &dhpackage; provides some return codes, that can + be used in scripts: + + Code + Diagnostic + + 0 + Program exited successfully. + + + 1 + The configuration file seems to be broken. + + + + + + BUGS + The program is currently limited to only work + with the foobar library. + The upstreams BTS can be found + at . + + + SEE ALSO + + + bar + 1 + , + baz + 1 + , + foo.conf + 5 + + The programs are documented fully by The Rise and + Fall of a Fooish Bar available via the + info + 1 + system. + + + diff --git a/DEBIAN/postinst.ex b/debian/postinst.ex similarity index 100% rename from DEBIAN/postinst.ex rename to debian/postinst.ex diff --git a/DEBIAN/postrm.ex b/debian/postrm.ex similarity index 100% rename from DEBIAN/postrm.ex rename to debian/postrm.ex diff --git a/DEBIAN/preinst.ex b/debian/preinst.ex similarity index 100% rename from DEBIAN/preinst.ex rename to debian/preinst.ex diff --git a/DEBIAN/prerm.ex b/debian/prerm.ex similarity index 100% rename from DEBIAN/prerm.ex rename to debian/prerm.ex diff --git a/DEBIAN/rules b/debian/rules similarity index 100% rename from DEBIAN/rules rename to debian/rules diff --git a/debian/salsa-ci.yml.ex b/debian/salsa-ci.yml.ex new file mode 100644 index 000000000..a6fb8bdad --- /dev/null +++ b/debian/salsa-ci.yml.ex @@ -0,0 +1,11 @@ +# For more information on what jobs are run see: +# https://salsa.debian.org/salsa-ci-team/pipeline +# +# To enable the jobs, go to your repository (at salsa.debian.org) +# and click over Settings > CI/CD > Expand (in General pipelines). +# In "CI/CD configuration file" write debian/salsa-ci.yml and click +# in "Save Changes". The CI tests will run after the next commit. +--- +include: + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml + - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml diff --git a/DEBIAN/source/format b/debian/source/format similarity index 100% rename from DEBIAN/source/format rename to debian/source/format diff --git a/libs/build-arm-v8-snapshot.sh b/libs/build-arm-v8-snapshot.sh index f4f72c04a..5d936bd16 100755 --- a/libs/build-arm-v8-snapshot.sh +++ b/libs/build-arm-v8-snapshot.sh @@ -20,7 +20,9 @@ cd - echo "Creating Arm v8 Snapshot" docker run --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot-creator" -sudo chown ${whoami}:${whoami} JS_SNAPSHOT.bin +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo +sudo chown $USER JS_SNAPSHOT.bin sudo chmod 0644 JS_SNAPSHOT.bin sudo mv -f JS_SNAPSHOT.bin ./js_engine/src/artifacts/ARM_JS_SNAPSHOT.bin \ No newline at end of file diff --git a/libs/build-v8-snapshot.sh b/libs/build-v8-snapshot.sh index fb482bc10..30dedc8e1 100755 --- a/libs/build-v8-snapshot.sh +++ b/libs/build-v8-snapshot.sh @@ -10,7 +10,9 @@ fi echo "Creating v8 Snapshot" cargo run -p snapshot-creator --release -sudo chown ${whoami}:${whoami} JS_SNAPSHOT.bin +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo +sudo chown $USER JS_SNAPSHOT.bin sudo chmod 0644 JS_SNAPSHOT.bin sudo mv -f JS_SNAPSHOT.bin ./js_engine/src/artifacts/JS_SNAPSHOT.bin \ No newline at end of file diff --git a/system-images/compat/build.sh b/system-images/compat/build.sh index 7e2c38c4c..aa7a87e10 100755 --- a/system-images/compat/build.sh +++ b/system-images/compat/build.sh @@ -18,3 +18,6 @@ alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/ cd ../../.. rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd embassy-os/system-images/compat && cargo +beta build --release --target=aarch64-unknown-linux-musl --no-default-features)" cd embassy-os/system-images/compat + +sudo chown -R $(whoami) target +sudo chown -R $(whoami) ~/.cargo \ No newline at end of file From a659b6e875b2225ead26d8a9ccff96cdb3151866 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 23 Sep 2022 10:00:20 -0600 Subject: [PATCH 06/53] wip --- .github/workflows/backend.yaml | 2 +- Makefile | 31 +++++++++++++++++++++---------- backend/build-dev.sh | 4 ++-- backend/build-portable-dev.sh | 4 ++-- backend/build-portable.sh | 4 ++-- backend/build-prod-native.sh | 28 ++++++++++++++++++++++++++++ backend/build-prod.sh | 4 ++-- backend/src/util/http_reader.rs | 7 ++++--- build-cargo-dep.sh | 4 ++-- build/make-deb.sh | 27 +-------------------------- debian/.gitignore | 2 ++ debian/control | 25 ++----------------------- debian/debhelper-build-stamp | 1 + debian/files | 3 +++ debian/rules | 2 ++ libs/build-arm-v8-snapshot.sh | 4 ++-- libs/build-v8-snapshot.sh | 4 ++-- system-images/compat/build.sh | 4 ++-- 18 files changed, 81 insertions(+), 79 deletions(-) create mode 100755 backend/build-prod-native.sh create mode 100644 debian/debhelper-build-stamp create mode 100644 debian/files diff --git a/.github/workflows/backend.yaml b/.github/workflows/backend.yaml index 904cd351b..9a80105ec 100644 --- a/.github/workflows/backend.yaml +++ b/.github/workflows/backend.yaml @@ -163,7 +163,7 @@ jobs: if: ${{ matrix.target == 'aarch64' }} - name: Reset permissions - run: sudo chown -R $(whoami) target + run: sudo chown -R $USER target working-directory: backend if: ${{ matrix.target == 'aarch64' }} diff --git a/Makefile b/Makefile index f31b0ce2a..c29dbbff9 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ ARCH = aarch64 ENVIRONMENT_FILE := $(shell ./check-environment.sh) GIT_HASH_FILE := $(shell ./check-git-hash.sh) EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias +EMBASSY_NATIVE_BINS := backend/target/release/embassyd backend/target/release/embassy-init backend/target/release/embassy-cli backend/target/release/embassy-sdk backend/target/release/avahi-alias EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui -EMBASSY_SRC := raspios.img product_key.txt $(EMBASSY_BINS) backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build) +EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build) COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name compat.tar -and -not -name target) UTILS_SRC := $(shell find system-images/utils/ -not -name utils.tar) BINFMT_SRC := $(shell find system-images/binfmt/ -not -name binfmt.tar) @@ -32,6 +33,7 @@ clean: rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip rm -f raspios.img rm -f eos.img + rm -f eos.tar.gz rm -f ubuntu.img rm -f product_key.txt rm -f system-images/**/*.tar @@ -44,29 +46,34 @@ clean: rm -rf libs/target rm -rf patch-db/client/node_modules rm -rf patch-db/client/dist - sudo rm -rf cargo-deps + rm -rf patch-db/target + rm -rf cargo-deps + rm -rf debian/.debhelper format: cd backend && cargo +nightly fmt cd libs && cargo +nightly fmt -sdk: +sdk: cd backend/ && ./install-sdk.sh -eos.img: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar system-images/binfmt/binfmt.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) +eos.img: raspios.img $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/compat.tar system-images/utils/utils.tar system-images/binfmt/binfmt.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) ! test -f eos.img || rm eos.img if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi +dpkg_deps: $(EMBASSY_SRC) $(EMBASSY_NATIVE_BINS) system-images/compat/compat.tar system-images/utils/utils.tar system-images/binfmt/binfmt.tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) + # For creating dpkg. DO NOT USE -install: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/utils.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) +install: dpkg_deps mkdir -p $(DESTDIR)/etc/embassy cp ENVIRONMENT.txt $(DESTDIR)/etc/embassy/ cp GIT_HASH.txt $(DESTDIR)/etc/embassy/ - mkdir -p $(DESTDIR)/usr/local/bin - cp backend/target/aarch64-unknown-linux-gnu/release/embassy-init $(DESTDIR)/usr/local/bin/ - cp backend/target/aarch64-unknown-linux-gnu/release/embassyd $(DESTDIR)/usr/local/bin/ - cp backend/target/aarch64-unknown-linux-gnu/release/embassy-cli $(DESTDIR)/usr/local/bin/ + mkdir -p $(DESTDIR)/usr/bin + cp backend/target/release/embassy-init $(DESTDIR)/usr/bin/ + cp backend/target/release/embassyd $(DESTDIR)/usr/bin/ + cp backend/target/release/embassy-cli $(DESTDIR)/usr/bin/ + cp backend/target/release/avahi-alias $(DESTDIR)/usr/bin/ mkdir -p $(DESTDIR)/etc/systemd/system cp backend/embassy-init.service $(DESTDIR)/etc/systemd/system/ @@ -81,7 +88,7 @@ install: $(EMBASSY_SRC) system-images/compat/compat.tar system-images/utils/util cp -r frontend/dist/ui $(DESTDIR)/var/www/html/main cp index.html $(DESTDIR)/var/www/html/ -embassy-os.deb: DEBIAN/control +embassy-os.deb: debian/control ./build/make-deb.sh system-images/compat/compat.tar: $(COMPAT_SRC) @@ -112,6 +119,10 @@ $(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/pa cd backend && ./build-prod.sh touch $(EMBASSY_BINS) +$(EMBASSY_NATIVE_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) + cd backend && ./build-prod-native.sh + touch $(EMBASSY_NATIVE_BINS) + frontend/node_modules: frontend/package.json npm --prefix frontend ci diff --git a/backend/build-dev.sh b/backend/build-dev.sh index 6736ffbef..80bfb44c0 100755 --- a/backend/build-dev.sh +++ b/backend/build-dev.sh @@ -19,6 +19,6 @@ cd .. rust-arm64-builder sh -c "(cd backend && cargo build --locked)" cd backend -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/backend/build-portable-dev.sh b/backend/build-portable-dev.sh index bacd19c6f..cdeb2b6d3 100755 --- a/backend/build-portable-dev.sh +++ b/backend/build-portable-dev.sh @@ -19,5 +19,5 @@ cd .. rust-musl-builder sh -c "(cd backend && cargo +beta build --target=x86_64-unknown-linux-musl --no-default-features --locked)" cd backend -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo \ No newline at end of file +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo \ No newline at end of file diff --git a/backend/build-portable.sh b/backend/build-portable.sh index 146b795dd..9b1b96f34 100755 --- a/backend/build-portable.sh +++ b/backend/build-portable.sh @@ -19,5 +19,5 @@ cd .. rust-musl-builder sh -c "(cd backend && cargo +beta build --release --target=x86_64-unknown-linux-musl --no-default-features --locked)" cd backend -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo \ No newline at end of file +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo \ No newline at end of file diff --git a/backend/build-prod-native.sh b/backend/build-prod-native.sh new file mode 100755 index 000000000..c63b32879 --- /dev/null +++ b/backend/build-prod-native.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -e +shopt -s expand_aliases + +if [ "$0" != "./build-prod-native.sh" ]; then + >&2 echo "Must be run from backend directory" + exit 1 +fi + +FLAGS="" +if [[ "$ENVIRONMENT" =~ (^|-)unstable($|-) ]]; then + FLAGS="unstable,$FLAGS" +fi +if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then + FLAGS="dev,$FLAGS" +fi +if [[ "$FLAGS" = "" ]]; then + cargo +stable build --release --locked +else + echo "FLAGS=$FLAGS" + cargo +stable build --release --features $FLAGS --locked +fi + +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo + +#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/backend/build-prod.sh b/backend/build-prod.sh index d5b27b584..3ac0b0453 100755 --- a/backend/build-prod.sh +++ b/backend/build-prod.sh @@ -31,7 +31,7 @@ else fi cd backend -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo #rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/backend/src/util/http_reader.rs b/backend/src/util/http_reader.rs index f37d51d2d..a9921f5c9 100644 --- a/backend/src/util/http_reader.rs +++ b/backend/src/util/http_reader.rs @@ -62,6 +62,7 @@ impl Display for RangeUnit { impl HttpReader { pub async fn new(http_url: Url) -> Result { let http_client = Client::builder() + .proxy(reqwest::Proxy::all("socks5h://127.0.0.1:9050").unwrap()) .build() .with_kind(crate::ErrorKind::TLSInit)?; @@ -342,16 +343,16 @@ async fn main_test() { async fn s9pk_test() { use tokio::io::BufReader; - let http_url = Url::parse("https://github.com/Start9Labs/hello-world-wrapper/releases/download/v0.3.0/hello-world.s9pk").unwrap(); + let http_url = Url::parse("http://qhc6ac47cytstejcepk2ia3ipadzjhlkc5qsktsbl4e7u2krfmfuaqqd.onion/content/files/2022/09/ghost.s9pk").unwrap(); println!("Getting this resource: {}", http_url); let test_reader = BufReader::with_capacity(1024 * 1024, HttpReader::new(http_url).await.unwrap()); - let mut s9pk = crate::s9pk::reader::S9pkReader::from_reader(test_reader, true) + let mut s9pk = crate::s9pk::reader::S9pkReader::from_reader(test_reader, false) .await .unwrap(); let manifest = s9pk.manifest().await.unwrap(); - assert_eq!(&**manifest.id, "hello-world"); + assert_eq!(&**manifest.id, "ghost"); } diff --git a/build-cargo-dep.sh b/build-cargo-dep.sh index d537432c0..08fad4799 100755 --- a/build-cargo-dep.sh +++ b/build-cargo-dep.sh @@ -17,5 +17,5 @@ mkdir -p cargo-deps alias 'rust-arm64-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)"/cargo-deps:/home/rust/src -P start9/rust-arm-cross:aarch64' rust-arm64-builder cargo install "$1" --target-dir /home/rust/src -sudo chown -R $(whoami) cargo-deps -sudo chown -R $(whoami) ~/.cargo \ No newline at end of file +sudo chown -R $USER cargo-deps +sudo chown -R $USER ~/.cargo \ No newline at end of file diff --git a/build/make-deb.sh b/build/make-deb.sh index ab8c9b856..c29fa25b1 100755 --- a/build/make-deb.sh +++ b/build/make-deb.sh @@ -2,29 +2,4 @@ set -e -rm -rf debian && mkdir debian -cp -R DEBIAN debian/ - -mkdir -p debian/etc/embassy -cp ENVIRONMENT.txt debian/etc/embassy/ -cp GIT_HASH.txt debian/etc/embassy/ - -mkdir -p debian/usr/local/bin -cp backend/target/aarch64-unknown-linux-gnu/release/embassy-init debian/usr/local/bin/ -cp backend/target/aarch64-unknown-linux-gnu/release/embassyd debian/usr/local/bin/ -cp backend/target/aarch64-unknown-linux-gnu/release/embassy-cli debian/usr/local/bin/ - -mkdir -p debian/etc/systemd/system -cp backend/embassy-init.service debian/etc/systemd/system/ -cp backend/embassyd.service debian/etc/systemd/system/ - -mkdir -p debian/var/lib/embassy/system-images -cp system-images/**/*.tar debian/var/lib/embassy/system-images/ - -mkdir -p debian/var/www/html -cp -r frontend/dist/diagnostic-ui debian/var/www/html/diagnostic -cp -r frontend/dist/setup-wizard debian/var/www/html/setup -cp -r frontend/dist/ui debian/var/www/html/main -cp index.html debian/var/www/html/ - -dpkg-deb --root-owner-group --build debian embassy-os.deb \ No newline at end of file +DEB_BUILD_OPTIONS="parallel=1" debuild --preserve-envvar USER --preserve-envvar PATH --no-lintian -i -I \ No newline at end of file diff --git a/debian/.gitignore b/debian/.gitignore index d21933f1c..10e5da3dc 100644 --- a/debian/.gitignore +++ b/debian/.gitignore @@ -1 +1,3 @@ /.debhelper +/embassy-os +/embassy-os.* \ No newline at end of file diff --git a/debian/control b/debian/control index 0c83930e1..78ec5306d 100644 --- a/debian/control +++ b/debian/control @@ -1,6 +1,6 @@ Source: embassy-os Maintainer: Aiden McClelland -Build-Depends: debhelper-compat (= 13) +Build-Depends: debhelper-compat (= 12) Standards-Version: 3.9.3 Section: unknown @@ -8,25 +8,4 @@ Package: embassy-os Section: unknown Priority: required Architecture: amd64 -Depends: nginx, - libavahi-client3, - avahi-daemon, - avahi-utils, - iotop, - bmon, - lvm2, - cryptsetup, - exfat-utils, - sqlite3, - wireless-tools, - net-tools, - ecryptfs-utils, - cifs-utils, - samba-common-bin, - network-manager, - vim, - jq, - ncdu, - postgresql, - pgloader, - openssh-server +Depends: ${shlibs:Depends}, ${misc:Depends} diff --git a/debian/debhelper-build-stamp b/debian/debhelper-build-stamp new file mode 100644 index 000000000..8d73416c9 --- /dev/null +++ b/debian/debhelper-build-stamp @@ -0,0 +1 @@ +embassy-os diff --git a/debian/files b/debian/files new file mode 100644 index 000000000..c94477afa --- /dev/null +++ b/debian/files @@ -0,0 +1,3 @@ +embassy-os-dbgsym_0.3.2-1_amd64.ddeb debug optional automatic=yes +embassy-os_0.3.2-1_amd64.buildinfo unknown - +embassy-os_0.3.2-1_amd64.deb unknown required diff --git a/debian/rules b/debian/rules index 59ea751e6..e27b7f821 100755 --- a/debian/rules +++ b/debian/rules @@ -13,6 +13,8 @@ # package maintainers to append LDFLAGS #export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed +override_dh_auto_build: + dh_auto_build -- dpkg_deps %: dh $@ diff --git a/libs/build-arm-v8-snapshot.sh b/libs/build-arm-v8-snapshot.sh index 5d936bd16..896a0dccf 100755 --- a/libs/build-arm-v8-snapshot.sh +++ b/libs/build-arm-v8-snapshot.sh @@ -20,8 +20,8 @@ cd - echo "Creating Arm v8 Snapshot" docker run --platform linux/arm64/v8 --mount type=bind,src=$(pwd),dst=/mnt arm64v8/ubuntu:20.04 /bin/sh -c "cd /mnt && /mnt/target/aarch64-unknown-linux-gnu/release/snapshot-creator" -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo sudo chown $USER JS_SNAPSHOT.bin sudo chmod 0644 JS_SNAPSHOT.bin diff --git a/libs/build-v8-snapshot.sh b/libs/build-v8-snapshot.sh index 30dedc8e1..33a5dea48 100755 --- a/libs/build-v8-snapshot.sh +++ b/libs/build-v8-snapshot.sh @@ -10,8 +10,8 @@ fi echo "Creating v8 Snapshot" cargo run -p snapshot-creator --release -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo sudo chown $USER JS_SNAPSHOT.bin sudo chmod 0644 JS_SNAPSHOT.bin diff --git a/system-images/compat/build.sh b/system-images/compat/build.sh index aa7a87e10..b82f4d095 100755 --- a/system-images/compat/build.sh +++ b/system-images/compat/build.sh @@ -19,5 +19,5 @@ cd ../../.. rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd embassy-os/system-images/compat && cargo +beta build --release --target=aarch64-unknown-linux-musl --no-default-features)" cd embassy-os/system-images/compat -sudo chown -R $(whoami) target -sudo chown -R $(whoami) ~/.cargo \ No newline at end of file +sudo chown -R $USER target +sudo chown -R $USER ~/.cargo \ No newline at end of file From 594667db73e4a3dffeb71bfb5be1fea6396d1c7b Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 10:15:44 -0600 Subject: [PATCH 07/53] remove debian dir --- debian/.gitignore | 3 - debian/README | 6 - debian/README.Debian | 6 - debian/README.source | 10 -- debian/changelog | 5 - debian/control | 11 -- debian/copyright | 43 ----- debian/debhelper-build-stamp | 1 - debian/embassy-os-docs.docs | 1 - debian/embassy-os.cron.d.ex | 4 - debian/embassy-os.doc-base.ex | 20 --- debian/files | 3 - debian/manpage.1.ex | 56 ------- debian/manpage.md.ex | 126 --------------- debian/manpage.sgml.ex | 154 ------------------ debian/manpage.xml.ex | 291 ---------------------------------- debian/postinst.ex | 39 ----- debian/postrm.ex | 37 ----- debian/preinst.ex | 35 ---- debian/prerm.ex | 38 ----- debian/rules | 27 ---- debian/salsa-ci.yml.ex | 11 -- debian/source/format | 1 - 23 files changed, 928 deletions(-) delete mode 100644 debian/.gitignore delete mode 100644 debian/README delete mode 100644 debian/README.Debian delete mode 100644 debian/README.source delete mode 100644 debian/changelog delete mode 100644 debian/control delete mode 100644 debian/copyright delete mode 100644 debian/debhelper-build-stamp delete mode 100644 debian/embassy-os-docs.docs delete mode 100644 debian/embassy-os.cron.d.ex delete mode 100644 debian/embassy-os.doc-base.ex delete mode 100644 debian/files delete mode 100644 debian/manpage.1.ex delete mode 100644 debian/manpage.md.ex delete mode 100644 debian/manpage.sgml.ex delete mode 100644 debian/manpage.xml.ex delete mode 100644 debian/postinst.ex delete mode 100644 debian/postrm.ex delete mode 100644 debian/preinst.ex delete mode 100644 debian/prerm.ex delete mode 100755 debian/rules delete mode 100644 debian/salsa-ci.yml.ex delete mode 100644 debian/source/format diff --git a/debian/.gitignore b/debian/.gitignore deleted file mode 100644 index 10e5da3dc..000000000 --- a/debian/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/.debhelper -/embassy-os -/embassy-os.* \ No newline at end of file diff --git a/debian/README b/debian/README deleted file mode 100644 index b4557eea1..000000000 --- a/debian/README +++ /dev/null @@ -1,6 +0,0 @@ -The Debian Package embassy-os ----------------------------- - -Comments regarding the Package - - -- Aiden Mon, 19 Sep 2022 14:34:19 -0600 diff --git a/debian/README.Debian b/debian/README.Debian deleted file mode 100644 index a70b5bd63..000000000 --- a/debian/README.Debian +++ /dev/null @@ -1,6 +0,0 @@ -embassy-os for Debian --------------------- - - - - -- Aiden Mon, 19 Sep 2022 16:40:40 -0600 diff --git a/debian/README.source b/debian/README.source deleted file mode 100644 index 6fafd1bf5..000000000 --- a/debian/README.source +++ /dev/null @@ -1,10 +0,0 @@ -embassy-os for Debian --------------------- - - - - - - -- Aiden Mon, 19 Sep 2022 16:40:40 -0600 - diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index df736f99f..000000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -embassy-os (0.3.2-1) UNRELEASED; urgency=medium - - * Initial release (Closes: #nnnn) - - -- Aiden Mon, 19 Sep 2022 16:40:40 -0600 diff --git a/debian/control b/debian/control deleted file mode 100644 index 78ec5306d..000000000 --- a/debian/control +++ /dev/null @@ -1,11 +0,0 @@ -Source: embassy-os -Maintainer: Aiden McClelland -Build-Depends: debhelper-compat (= 12) -Standards-Version: 3.9.3 -Section: unknown - -Package: embassy-os -Section: unknown -Priority: required -Architecture: amd64 -Depends: ${shlibs:Depends}, ${misc:Depends} diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index 821932eab..000000000 --- a/debian/copyright +++ /dev/null @@ -1,43 +0,0 @@ -Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: embassy-os -Upstream-Contact: support@start9labs.com -Source: https://github.com/Start9Labs/embassy-os - -Files: * -Copyright: 2022 Start9Labs -License: START9 PERSONAL USE LICENSE v1.0 - -License: MIT - This license governs the use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software. - - 1. Definitions. - 1. “Licensor” means the copyright owner, Start9 Labs, Inc, or its successor(s) in interest, or a future assignee of the copyright. - 2. “Source Code” means the preferred form of the Software for making modifications to it. - 3. “Object Code” means any non-source form of the Software, including the machine-language output by a compiler or assembler. - 4. “Distribute” means to convey or to publish and generally has the same meaning here as under U.S. Copyright law. - 5. “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including without limitation fees for hosting or consulting/support services related to the Software), a product or service whose value derives, entirely or substantially, from the functionality of the Software. - - 2. Grant of Rights. Subject to the terms of this license, the Licensor grants you, the licensee, a non-exclusive, worldwide, royalty-free copyright license to: - 1. Access, audit, copy, modify, compile, or distribute the Source Code or modifications to the Source Code. - 2. Run, test, or otherwise use the Object Code. - - 3. Limitations. - 1. The grant of rights under the License will NOT include, and the License does NOT grant you the right to: - 1. Sell the Software or any derivative works based thereon. - 2. Distribute the Object Code. - 2. If you Distribute the Source Code, or if permission is separately granted to Distribute the Object Code, you expressly undertake not to remove, or modify, in any manner, the copyright notices attached to the Source Code, and displayed in any output of the Object Code when run, and to reproduce these notices, in an identical manner, in any distributed copies of the Software together with a copy of this license. If you Distribute a modified copy of the Software, or a derivative work based thereon, the work must carry prominent notices stating that you modified it, and giving a relevant date. - 3. The terms of this license will apply to anyone who comes into possession of a copy of the Software, and any modifications or derivative works based thereon, made by anyone. - - 4. Contributions. You hereby grant to Licensor a perpetual, irrevocable, worldwide, non-exclusive, royalty-free license to use and exploit any modifications or derivative works based on the Source Code of which you are the author. - - 5. Disclaimer. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LICENSOR HAS NO OBLIGATION TO SUPPORT RECIPIENTS OF THE SOFTWARE. - - -# Please also look if there are files or directories which have a -# different copyright/license attached and list them here. -# Please avoid picking licenses with terms that are more restrictive than the -# packaged work, as it may make Debian's contributions unacceptable upstream. -# -# If you need, there are some extra license texts available in two places: -# /usr/share/debhelper/dh_make/licenses/ -# /usr/share/common-licenses/ diff --git a/debian/debhelper-build-stamp b/debian/debhelper-build-stamp deleted file mode 100644 index 8d73416c9..000000000 --- a/debian/debhelper-build-stamp +++ /dev/null @@ -1 +0,0 @@ -embassy-os diff --git a/debian/embassy-os-docs.docs b/debian/embassy-os-docs.docs deleted file mode 100644 index e845566c0..000000000 --- a/debian/embassy-os-docs.docs +++ /dev/null @@ -1 +0,0 @@ -README diff --git a/debian/embassy-os.cron.d.ex b/debian/embassy-os.cron.d.ex deleted file mode 100644 index 843e7dc9e..000000000 --- a/debian/embassy-os.cron.d.ex +++ /dev/null @@ -1,4 +0,0 @@ -# -# Regular cron jobs for the embassy-os package -# -0 4 * * * root [ -x /usr/bin/embassy-os_maintenance ] && /usr/bin/embassy-os_maintenance diff --git a/debian/embassy-os.doc-base.ex b/debian/embassy-os.doc-base.ex deleted file mode 100644 index a70dc68f3..000000000 --- a/debian/embassy-os.doc-base.ex +++ /dev/null @@ -1,20 +0,0 @@ -Document: embassy-os -Title: Debian embassy-os Manual -Author: -Abstract: This manual describes what embassy-os is - and how it can be used to - manage online manuals on Debian systems. -Section: unknown - -Format: debiandoc-sgml -Files: /usr/share/doc/embassy-os/embassy-os.sgml.gz - -Format: postscript -Files: /usr/share/doc/embassy-os/embassy-os.ps.gz - -Format: text -Files: /usr/share/doc/embassy-os/embassy-os.text.gz - -Format: HTML -Index: /usr/share/doc/embassy-os/html/index.html -Files: /usr/share/doc/embassy-os/html/*.html diff --git a/debian/files b/debian/files deleted file mode 100644 index c94477afa..000000000 --- a/debian/files +++ /dev/null @@ -1,3 +0,0 @@ -embassy-os-dbgsym_0.3.2-1_amd64.ddeb debug optional automatic=yes -embassy-os_0.3.2-1_amd64.buildinfo unknown - -embassy-os_0.3.2-1_amd64.deb unknown required diff --git a/debian/manpage.1.ex b/debian/manpage.1.ex deleted file mode 100644 index e549b4ce7..000000000 --- a/debian/manpage.1.ex +++ /dev/null @@ -1,56 +0,0 @@ -.\" Hey, EMACS: -*- nroff -*- -.\" (C) Copyright 2022 Aiden , -.\" -.\" First parameter, NAME, should be all caps -.\" Second parameter, SECTION, should be 1-8, maybe w/ subsection -.\" other parameters are allowed: see man(7), man(1) -.TH Embassy-os SECTION "September 19 2022" -.\" Please adjust this date whenever revising the manpage. -.\" -.\" Some roff macros, for reference: -.\" .nh disable hyphenation -.\" .hy enable hyphenation -.\" .ad l left justify -.\" .ad b justify to both left and right margins -.\" .nf disable filling -.\" .fi enable filling -.\" .br insert line break -.\" .sp insert n+1 empty lines -.\" for manpage-specific macros, see man(7) -.SH NAME -embassy-os \- program to do something -.SH SYNOPSIS -.B embassy-os -.RI [ options ] " files" ... -.br -.B bar -.RI [ options ] " files" ... -.SH DESCRIPTION -This manual page documents briefly the -.B embassy-os -and -.B bar -commands. -.PP -.\" TeX users may be more comfortable with the \fB\fP and -.\" \fI\fP escape sequences to invode bold face and italics, -.\" respectively. -\fBembassy-os\fP is a program that... -.SH OPTIONS -These programs follow the usual GNU command line syntax, with long -options starting with two dashes (`-'). -A summary of options is included below. -For a complete description, see the Info files. -.TP -.B \-h, \-\-help -Show summary of options. -.TP -.B \-v, \-\-version -Show version of program. -.SH SEE ALSO -.BR bar (1), -.BR baz (1). -.br -The programs are documented fully by -.IR "The Rise and Fall of a Fooish Bar" , -available via the Info system. diff --git a/debian/manpage.md.ex b/debian/manpage.md.ex deleted file mode 100644 index f2c2c51e4..000000000 --- a/debian/manpage.md.ex +++ /dev/null @@ -1,126 +0,0 @@ -% embassy-os(SECTION) | User Commands -% -% "September 19 2022" - -[comment]: # The lines above form a Pandoc metadata block. They must be -[comment]: # the first ones in the file. -[comment]: # See https://pandoc.org/MANUAL.html#metadata-blocks for details. - -[comment]: # pandoc -s -f markdown -t man package.md -o package.1 -[comment]: # -[comment]: # A manual page package.1 will be generated. You may view the -[comment]: # manual page with: nroff -man package.1 | less. A typical entry -[comment]: # in a Makefile or Makefile.am is: -[comment]: # -[comment]: # package.1: package.md -[comment]: # pandoc --standalone --from=markdown --to=man $< --output=$@ -[comment]: # -[comment]: # The pandoc binary is found in the pandoc package. Please remember -[comment]: # that if you create the nroff version in one of the debian/rules -[comment]: # file targets, such as build, you will need to include pandoc in -[comment]: # your Build-Depends control field. - -[comment]: # Remove the lines starting with `[comment]:' in this file in order -[comment]: # to avoid warning messages from pandoc. - -# NAME - -embassy-os - program to do something - -# SYNOPSIS - -**embassy-os** **-e** _this_ [**\-\-example=that**] [{**-e** | **\-\-example**} _this_] - [{**-e** | **\-\-example**} {_this_ | _that_}] - -**embassy-os** [{**-h** | *\-\-help**} | {**-v** | **\-\-version**}] - -# DESCRIPTION - -This manual page documents briefly the **embassy-os** and **bar** commands. - -This manual page was written for the Debian distribution because the -original program does not have a manual page. Instead, it has documentation -in the GNU info(1) format; see below. - -**embassy-os** is a program that... - -# OPTIONS - -The program follows the usual GNU command line syntax, with long options -starting with two dashes (`-'). A summary of options is included below. For -a complete description, see the **info**(1) files. - -**-e** _this_, **\-\-example=**_that_ -: Does this and that. - -**-h**, **\-\-help** -: Show summary of options. - -**-v**, **\-\-version** -: Show version of program. - -# FILES - -/etc/foo.conf -: The system-wide configuration file to control the behaviour of - embassy-os. See **foo.conf**(5) for further details. - -${HOME}/.foo.conf -: The per-user configuration file to control the behaviour of - embassy-os. See **foo.conf**(5) for further details. - -# ENVIRONMENT - -**FOO_CONF** -: If used, the defined file is used as configuration file (see also - the section called “FILES”). - -# DIAGNOSTICS - -The following diagnostics may be issued on stderr: - -Bad configuration file. Exiting. -: The configuration file seems to contain a broken configuration - line. Use the **\-\-verbose** option, to get more info. - -**embassy-os** provides some return codes, that can be used in scripts: - - Code Diagnostic - 0 Program exited successfully. - 1 The configuration file seems to be broken. - -# BUGS - -The program is currently limited to only work with the foobar library. - -The upstream BTS can be found at http://bugzilla.foo.tld. - -# SEE ALSO - -**bar**(1), **baz**(1), **foo.conf**(5) - -The programs are documented fully by The Rise and Fall of a Fooish Bar -available via the **info**(1) system. - -# AUTHOR - -Aiden -: Wrote this manpage for the Debian system. - -# COPYRIGHT - -Copyright © 2007 Aiden - -This manual page was written for the Debian system (and may be used by -others). - -Permission is granted to copy, distribute and/or modify this document under -the terms of the GNU General Public License, Version 2 or (at your option) -any later version published by the Free Software Foundation. - -On Debian systems, the complete text of the GNU General Public License -can be found in /usr/share/common-licenses/GPL. - -[comment]: # Local Variables: -[comment]: # mode: markdown -[comment]: # End: diff --git a/debian/manpage.sgml.ex b/debian/manpage.sgml.ex deleted file mode 100644 index 1d2eb429b..000000000 --- a/debian/manpage.sgml.ex +++ /dev/null @@ -1,154 +0,0 @@ - manpage.1'. You may view - the manual page with: `docbook-to-man manpage.sgml | nroff -man | - less'. A typical entry in a Makefile or Makefile.am is: - -manpage.1: manpage.sgml - docbook-to-man $< > $@ - - - The docbook-to-man binary is found in the docbook-to-man package. - Please remember that if you create the nroff version in one of the - debian/rules file targets (such as build), you will need to include - docbook-to-man in your Build-Depends control field. - - --> - - - FIRSTNAME"> - SURNAME"> - - September 19 2022"> - - SECTION"> - aiden@start9labs.com"> - - Embassy-os"> - - - Debian"> - GNU"> - GPL"> -]> - - - -
- &dhemail; -
- - &dhfirstname; - &dhsurname; - - - 2003 - &dhusername; - - &dhdate; - - - &dhucpackage; - - &dhsection; - - - &dhpackage; - - program to do something - - - - &dhpackage; - - - - - - - - DESCRIPTION - - This manual page documents briefly the - &dhpackage; and bar - commands. - - This manual page was written for the &debian; distribution - because the original program does not have a manual page. - Instead, it has documentation in the &gnu; - Info format; see below. - - &dhpackage; is a program that... - - - - OPTIONS - - These programs follow the usual &gnu; command line syntax, - with long options starting with two dashes (`-'). A summary of - options is included below. For a complete description, see the - Info files. - - - - - - - - Show summary of options. - - - - - - - - Show version of program. - - - - - - SEE ALSO - - bar (1), baz (1). - - The programs are documented fully by The Rise and - Fall of a Fooish Bar available via the - Info system. - - - AUTHOR - - This manual page was written by &dhusername; &dhemail; for - the &debian; system (and may be used by others). Permission is - granted to copy, distribute and/or modify this document under - the terms of the &gnu; General Public License, Version 2 any - later version published by the Free Software Foundation. - - - On Debian systems, the complete text of the GNU General Public - License can be found in /usr/share/common-licenses/GPL. - - - - - - diff --git a/debian/manpage.xml.ex b/debian/manpage.xml.ex deleted file mode 100644 index 286a25e7d..000000000 --- a/debian/manpage.xml.ex +++ /dev/null @@ -1,291 +0,0 @@ - -.
will be generated. You may view the -manual page with: nroff -man .
| less'. A typical entry -in a Makefile or Makefile.am is: - -DB2MAN = /usr/share/sgml/docbook/stylesheet/xsl/docbook-xsl/manpages/docbook.xsl -XP = xsltproc -''-nonet -''-param man.charmap.use.subset "0" - -manpage.1: manpage.xml - $(XP) $(DB2MAN) $< - -The xsltproc binary is found in the xsltproc package. The XSL files are in -docbook-xsl. A description of the parameters you can use can be found in the -docbook-xsl-doc-* packages. Please remember that if you create the nroff -version in one of the debian/rules file targets (such as build), you will need -to include xsltproc and docbook-xsl in your Build-Depends control field. -Alternatively use the xmlto command/package. That will also automatically -pull in xsltproc and docbook-xsl. - -Notes for using docbook2x: docbook2x-man does not automatically create the -AUTHOR(S) and COPYRIGHT sections. In this case, please add them manually as - ... . - -To disable the automatic creation of the AUTHOR(S) and COPYRIGHT sections -read /usr/share/doc/docbook-xsl/doc/manpages/authors.html. This file can be -found in the docbook-xsl-doc-html package. - -Validation can be done using: `xmllint -''-noout -''-valid manpage.xml` - -General documentation about man-pages and man-page-formatting: -man(1), man(7), http://www.tldp.org/HOWTO/Man-Page/ - ---> - - - - - - - - - - - - - -]> - - - - &dhtitle; - &dhpackage; - - - &dhfirstname; - &dhsurname; - Wrote this manpage for the Debian system. -
- &dhemail; -
- - - - 2007 - &dhusername; - - - This manual page was written for the Debian system - (and may be used by others). - Permission is granted to copy, distribute and/or modify this - document under the terms of the GNU General Public License, - Version 2 or (at your option) any later version published by - the Free Software Foundation. - On Debian systems, the complete text of the GNU General Public - License can be found in - /usr/share/common-licenses/GPL. - - - - &dhucpackage; - &dhsection; - - - &dhpackage; - program to do something - - - - &dhpackage; - - - - - - - - - this - - - - - - - - this - that - - - - - &dhpackage; - - - - - - - - - - - - - - - - - - - DESCRIPTION - This manual page documents briefly the - &dhpackage; and bar - commands. - This manual page was written for the Debian distribution - because the original program does not have a manual page. - Instead, it has documentation in the GNU - info - 1 - format; see below. - &dhpackage; is a program that... - - - OPTIONS - The program follows the usual GNU command line syntax, - with long options starting with two dashes (`-'). A summary of - options is included below. For a complete description, see the - - info - 1 - files. - - - - - - - Does this and that. - - - - - - - Show summary of options. - - - - - - - Show version of program. - - - - - - FILES - - - /etc/foo.conf - - The system-wide configuration file to control the - behaviour of &dhpackage;. See - - foo.conf - 5 - for further details. - - - - ${HOME}/.foo.conf - - The per-user configuration file to control the - behaviour of &dhpackage;. See - - foo.conf - 5 - for further details. - - - - - - ENVIRONMENT - - - FOO_CONF - - If used, the defined file is used as configuration - file (see also ). - - - - - - DIAGNOSTICS - The following diagnostics may be issued - on stderr: - - - Bad configuration file. Exiting. - - The configuration file seems to contain a broken configuration - line. Use the option, to get more info. - - - - - &dhpackage; provides some return codes, that can - be used in scripts: - - Code - Diagnostic - - 0 - Program exited successfully. - - - 1 - The configuration file seems to be broken. - - - - - - BUGS - The program is currently limited to only work - with the foobar library. - The upstreams BTS can be found - at . - - - SEE ALSO - - - bar - 1 - , - baz - 1 - , - foo.conf - 5 - - The programs are documented fully by The Rise and - Fall of a Fooish Bar available via the - info - 1 - system. - - - diff --git a/debian/postinst.ex b/debian/postinst.ex deleted file mode 100644 index 9f0ff7151..000000000 --- a/debian/postinst.ex +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# postinst script for embassy-os -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `configure' -# * `abort-upgrade' -# * `abort-remove' `in-favour' -# -# * `abort-remove' -# * `abort-deconfigure' `in-favour' -# `removing' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - configure) - ;; - - abort-upgrade|abort-remove|abort-deconfigure) - ;; - - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/postrm.ex b/debian/postrm.ex deleted file mode 100644 index 26446c868..000000000 --- a/debian/postrm.ex +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# postrm script for embassy-os -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `purge' -# * `upgrade' -# * `failed-upgrade' -# * `abort-install' -# * `abort-install' -# * `abort-upgrade' -# * `disappear' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - purge|remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) - ;; - - *) - echo "postrm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/preinst.ex b/debian/preinst.ex deleted file mode 100644 index 2af27b047..000000000 --- a/debian/preinst.ex +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# preinst script for embassy-os -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `install' -# * `install' -# * `upgrade' -# * `abort-upgrade' -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - install|upgrade) - ;; - - abort-upgrade) - ;; - - *) - echo "preinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/prerm.ex b/debian/prerm.ex deleted file mode 100644 index 9f64759bb..000000000 --- a/debian/prerm.ex +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# prerm script for embassy-os -# -# see: dh_installdeb(1) - -set -e - -# summary of how this script can be called: -# * `remove' -# * `upgrade' -# * `failed-upgrade' -# * `remove' `in-favour' -# * `deconfigure' `in-favour' -# `removing' -# -# for details, see https://www.debian.org/doc/debian-policy/ or -# the debian-policy package - - -case "$1" in - remove|upgrade|deconfigure) - ;; - - failed-upgrade) - ;; - - *) - echo "prerm called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - -# dh_installdeb will replace this with shell code automatically -# generated by other debhelper scripts. - -#DEBHELPER# - -exit 0 diff --git a/debian/rules b/debian/rules deleted file mode 100755 index e27b7f821..000000000 --- a/debian/rules +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/make -f -# See debhelper(7) (uncomment to enable) -# output every command that modifies files on the build system. -#export DH_VERBOSE = 1 - - -# see FEATURE AREAS in dpkg-buildflags(1) -#export DEB_BUILD_MAINT_OPTIONS = hardening=+all - -# see ENVIRONMENT in dpkg-buildflags(1) -# package maintainers to append CFLAGS -#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic -# package maintainers to append LDFLAGS -#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed - -override_dh_auto_build: - dh_auto_build -- dpkg_deps - -%: - dh $@ - - -# dh_make generated override targets -# This is example for Cmake (See https://bugs.debian.org/641051 ) -#override_dh_auto_configure: -# dh_auto_configure -- \ -# -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH) diff --git a/debian/salsa-ci.yml.ex b/debian/salsa-ci.yml.ex deleted file mode 100644 index a6fb8bdad..000000000 --- a/debian/salsa-ci.yml.ex +++ /dev/null @@ -1,11 +0,0 @@ -# For more information on what jobs are run see: -# https://salsa.debian.org/salsa-ci-team/pipeline -# -# To enable the jobs, go to your repository (at salsa.debian.org) -# and click over Settings > CI/CD > Expand (in General pipelines). -# In "CI/CD configuration file" write debian/salsa-ci.yml and click -# in "Save Changes". The CI tests will run after the next commit. ---- -include: - - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/salsa-ci.yml - - https://salsa.debian.org/salsa-ci-team/pipeline/raw/master/pipeline-jobs.yml diff --git a/debian/source/format b/debian/source/format deleted file mode 100644 index 89ae9db8f..000000000 --- a/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) From 145a62f0bcc42e6a77437370b6eb25e4c4fed5b6 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 11:34:36 -0600 Subject: [PATCH 08/53] lazy env and git hash --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c29dbbff9..abf953193 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ ARCH = aarch64 -ENVIRONMENT_FILE := $(shell ./check-environment.sh) -GIT_HASH_FILE := $(shell ./check-git-hash.sh) +ENVIRONMENT_FILE = $(shell ./check-environment.sh) +GIT_HASH_FILE = $(shell ./check-git-hash.sh) EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias EMBASSY_NATIVE_BINS := backend/target/release/embassyd backend/target/release/embassy-init backend/target/release/embassy-cli backend/target/release/embassy-sdk backend/target/release/avahi-alias EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui From 6ed7476e911f8369ca38a089a5d93a58b8bdc910 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 11:39:31 -0600 Subject: [PATCH 09/53] remove env and git hash on clean --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index abf953193..5062aa2fb 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,8 @@ clean: rm -rf patch-db/client/dist rm -rf patch-db/target rm -rf cargo-deps - rm -rf debian/.debhelper + rm ENVIRONMENT.txt + rm GIT_HASH.txt format: cd backend && cargo +nightly fmt From 70c462173ece16a0164aba0eef63eae6211825f0 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 11:55:25 -0600 Subject: [PATCH 10/53] don't leave project dir --- system-images/compat/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system-images/compat/build.sh b/system-images/compat/build.sh index b82f4d095..752e3c05c 100755 --- a/system-images/compat/build.sh +++ b/system-images/compat/build.sh @@ -15,9 +15,9 @@ fi alias 'rust-musl-builder'='docker run $USE_TTY --rm -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-musl-cross:aarch64-musl' -cd ../../.. -rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd embassy-os/system-images/compat && cargo +beta build --release --target=aarch64-unknown-linux-musl --no-default-features)" -cd embassy-os/system-images/compat +cd ../.. +rust-musl-builder sh -c "(git config --global --add safe.directory '*'; cd system-images/compat && cargo +beta build --release --target=aarch64-unknown-linux-musl --no-default-features)" +cd system-images/compat sudo chown -R $USER target sudo chown -R $USER ~/.cargo \ No newline at end of file From e75fb3e44e657cc07150386a9f8a4b433b4cfd14 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 13:28:03 -0600 Subject: [PATCH 11/53] use docker for native builds --- backend/build-prod-native.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/build-prod-native.sh b/backend/build-prod-native.sh index c63b32879..57ddf058e 100755 --- a/backend/build-prod-native.sh +++ b/backend/build-prod-native.sh @@ -8,6 +8,14 @@ if [ "$0" != "./build-prod-native.sh" ]; then exit 1 fi +USE_TTY= +if tty -s; then + USE_TTY="-it" +fi + +alias 'rust-native-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/root -P rust' + +cd .. FLAGS="" if [[ "$ENVIRONMENT" =~ (^|-)unstable($|-) ]]; then FLAGS="unstable,$FLAGS" @@ -16,11 +24,12 @@ if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then FLAGS="dev,$FLAGS" fi if [[ "$FLAGS" = "" ]]; then - cargo +stable build --release --locked + rust-native-builder sh -c "cd ~/backend && cargo build --release --locked" else echo "FLAGS=$FLAGS" - cargo +stable build --release --features $FLAGS --locked + rust-native-builder sh -c "cd ~/backend && cargo build --release --features $FLAGS --locked" fi +cd backend sudo chown -R $USER target sudo chown -R $USER ~/.cargo From f5f66a8335089a4b55457dd319249f8ba1742e7b Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 13:32:58 -0600 Subject: [PATCH 12/53] start9 rust --- backend/build-prod-native.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/build-prod-native.sh b/backend/build-prod-native.sh index 57ddf058e..253ffe5e6 100755 --- a/backend/build-prod-native.sh +++ b/backend/build-prod-native.sh @@ -13,7 +13,7 @@ if tty -s; then USE_TTY="-it" fi -alias 'rust-native-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/root -P rust' +alias 'rust-native-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/root -P start9/rust' cd .. FLAGS="" From 0925a698c034118cd37453c2a1c796142b5d0fd6 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 13:52:39 -0600 Subject: [PATCH 13/53] correctly mount registry --- backend/build-prod-native.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/build-prod-native.sh b/backend/build-prod-native.sh index 253ffe5e6..a876a1202 100755 --- a/backend/build-prod-native.sh +++ b/backend/build-prod-native.sh @@ -13,7 +13,7 @@ if tty -s; then USE_TTY="-it" fi -alias 'rust-native-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/root -P start9/rust' +alias 'rust-native-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/root -P start9/rust' cd .. FLAGS="" From 26b61f08c57f37fed2f5cf51db6d0e645cf83c97 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 16:06:39 -0600 Subject: [PATCH 14/53] remove systemd config --- Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 5062aa2fb..cb9a8bf3a 100644 --- a/Makefile +++ b/Makefile @@ -75,11 +75,7 @@ install: dpkg_deps cp backend/target/release/embassyd $(DESTDIR)/usr/bin/ cp backend/target/release/embassy-cli $(DESTDIR)/usr/bin/ cp backend/target/release/avahi-alias $(DESTDIR)/usr/bin/ - - mkdir -p $(DESTDIR)/etc/systemd/system - cp backend/embassy-init.service $(DESTDIR)/etc/systemd/system/ - cp backend/embassyd.service $(DESTDIR)/etc/systemd/system/ - + mkdir -p $(DESTDIR)/var/lib/embassy/system-images cp system-images/**/*.tar $(DESTDIR)/var/lib/embassy/system-images/ From 1ac4b33695b6d130b291ee876ca20536a721542d Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 17:06:44 -0600 Subject: [PATCH 15/53] switch to /usr/bin --- backend/embassy-init.service | 2 +- backend/embassyd.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/embassy-init.service b/backend/embassy-init.service index f43518abd..7a9b9ce66 100644 --- a/backend/embassy-init.service +++ b/backend/embassy-init.service @@ -7,7 +7,7 @@ Wants=avahi-daemon.service nginx.service tor.service [Service] Type=oneshot Environment=RUST_LOG=embassy_init=debug,embassy=debug,js_engine=debug,patch_db=warn -ExecStart=/usr/local/bin/embassy-init +ExecStart=/usr/bin/embassy-init RemainAfterExit=true StandardOutput=append:/var/log/embassy-init.log diff --git a/backend/embassyd.service b/backend/embassyd.service index 55251a756..f9bec7a7f 100644 --- a/backend/embassyd.service +++ b/backend/embassyd.service @@ -6,7 +6,7 @@ Requires=embassy-init.service [Service] Type=simple Environment=RUST_LOG=embassyd=debug,embassy=debug,js_engine=debug,patch_db=warn -ExecStart=/usr/local/bin/embassyd +ExecStart=/usr/bin/embassyd Restart=always RestartSec=3 ManagedOOMPreference=avoid From 5c79e3f8dd432bbf81eeac80a7a01aa6ed6206bc Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 17:09:47 -0600 Subject: [PATCH 16/53] disable sound for now --- backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2fb86e719..3e048df2c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -42,7 +42,7 @@ path = "src/bin/avahi-alias.rs" [features] avahi = ["avahi-sys"] -default = ["avahi", "sound", "metal", "js_engine"] +default = ["avahi", "metal", "js_engine"] dev = [] metal = [] sound = [] From ad4a3ff645250bc4731f86833298b46d3d08c6b2 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 28 Sep 2022 21:23:12 -0600 Subject: [PATCH 17/53] wip --- backend/src/disk/mod.rs | 10 +++++++++- backend/src/disk/util.rs | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/src/disk/mod.rs b/backend/src/disk/mod.rs index 520f8fd72..6fb9afbb3 100644 --- a/backend/src/disk/mod.rs +++ b/backend/src/disk/mod.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use clap::ArgMatches; use rpc_toolkit::command; @@ -27,6 +27,14 @@ pub struct OsPartitionInfo { pub green: PathBuf, pub blue: PathBuf, } +impl OsPartitionInfo { + pub fn contains(&self, logicalname: impl AsRef) -> bool { + &*self.boot == logicalname.as_ref() + || &*self.embassy == logicalname.as_ref() + || &*self.green == logicalname.as_ref() + || &*self.blue == logicalname.as_ref() + } +} #[command(subcommands(list, repair))] pub fn disk() -> Result<(), Error> { diff --git a/backend/src/disk/util.rs b/backend/src/disk/util.rs index 9f02a3e5a..9460667f6 100644 --- a/backend/src/disk/util.rs +++ b/backend/src/disk/util.rs @@ -277,7 +277,9 @@ pub async fn list(os: &OsPartitionInfo) -> Result, Error> { }; if disk == os.disk { if let Some(part) = part { - disks.insert(part.clone(), IndexSet::new()); + if !os.contains(&part) { + disks.insert(part.clone(), IndexSet::new()); + } } } else { if !disks.contains_key(&disk) { From 4aad8439c1eb33fa7075b89f646b12bb5e583154 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 29 Sep 2022 10:20:32 -0600 Subject: [PATCH 18/53] change disk list --- backend/src/disk/util.rs | 119 ++++++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 40 deletions(-) diff --git a/backend/src/disk/util.rs b/backend/src/disk/util.rs index 9460667f6..b26849f3a 100644 --- a/backend/src/disk/util.rs +++ b/backend/src/disk/util.rs @@ -235,7 +235,7 @@ pub async fn recovery_info( #[instrument] pub async fn list(os: &OsPartitionInfo) -> Result, Error> { let disk_guids = pvscan().await?; - let disks = tokio_stream::wrappers::ReadDirStream::new( + let (disks, internal) = tokio_stream::wrappers::ReadDirStream::new( tokio::fs::read_dir(DISK_PATH) .await .with_ctx(|_| (crate::ErrorKind::Filesystem, DISK_PATH))?, @@ -246,52 +246,55 @@ pub async fn list(os: &OsPartitionInfo) -> Result, Error> { crate::ErrorKind::Filesystem, ) }) - .try_fold(BTreeMap::new(), |mut disks, dir_entry| async move { - if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) { - let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) { - ( - disk_path.strip_suffix(end.as_str()).unwrap_or_default(), - Some(disk_path), - ) - } else { - (disk_path, None) - }; - let disk_path = Path::new(DISK_PATH).join(disk_path); - let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - disk_path.display().to_string(), - ) - })?; - let part = if let Some(part_path) = part_path { - let part_path = Path::new(DISK_PATH).join(part_path); - let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| { + .try_fold( + (BTreeMap::new(), Vec::new()), + |(mut disks, mut internal), dir_entry| async move { + if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) { + let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) { + ( + disk_path.strip_suffix(end.as_str()).unwrap_or_default(), + Some(disk_path), + ) + } else { + (disk_path, None) + }; + let disk_path = Path::new(DISK_PATH).join(disk_path); + let disk = tokio::fs::canonicalize(&disk_path).await.with_ctx(|_| { ( crate::ErrorKind::Filesystem, - part_path.display().to_string(), + disk_path.display().to_string(), ) })?; - Some(part) - } else { - None - }; - if disk == os.disk { - if let Some(part) = part { - if !os.contains(&part) { - disks.insert(part.clone(), IndexSet::new()); + let part = if let Some(part_path) = part_path { + let part_path = Path::new(DISK_PATH).join(part_path); + let part = tokio::fs::canonicalize(&part_path).await.with_ctx(|_| { + ( + crate::ErrorKind::Filesystem, + part_path.display().to_string(), + ) + })?; + Some(part) + } else { + None + }; + if disk == os.disk { + if let Some(part) = part { + if !os.contains(&part) { + internal.push(part.clone()); + } + } + } else { + if !disks.contains_key(&disk) { + disks.insert(disk.clone(), IndexSet::new()); + } + if let Some(part) = part { + disks.get_mut(&disk).unwrap().insert(part); } - } - } else { - if !disks.contains_key(&disk) { - disks.insert(disk.clone(), IndexSet::new()); - } - if let Some(part) = part { - disks.get_mut(&disk).unwrap().insert(part); } } - } - Ok(disks) - }) + Ok((disks, internal)) + }, + ) .await?; let mut res = Vec::with_capacity(disks.len()); @@ -373,6 +376,42 @@ pub async fn list(os: &OsPartitionInfo) -> Result, Error> { guid, }) } + for disk in internal { + let mut guid: Option = None; + let vendor = get_vendor(&os.disk) + .await + .map_err(|e| { + tracing::warn!( + "Could not get vendor of {}: {}", + os.disk.display(), + e.source + ) + }) + .unwrap_or_default(); + let model = get_model(&os.disk) + .await + .map_err(|e| { + tracing::warn!("Could not get model of {}: {}", os.disk.display(), e.source) + }) + .unwrap_or_default(); + let capacity = get_capacity(&disk) + .await + .map_err(|e| { + tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source) + }) + .unwrap_or_default(); + if let Some(g) = disk_guids.get(&disk) { + guid = g.clone(); + } + res.push(DiskInfo { + logicalname: disk, + vendor, + model, + partitions: Vec::new(), + capacity, + guid, + }) + } Ok(res) } From 5656040b45f02c22ebd66ad050384d79fd942292 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 29 Sep 2022 16:53:16 -0600 Subject: [PATCH 19/53] multi-arch images --- backend/src/action.rs | 6 +- backend/src/backup/mod.rs | 3 +- backend/src/config/action.rs | 3 +- backend/src/config/mod.rs | 3 +- backend/src/config/spec.rs | 3 +- backend/src/dependencies.rs | 6 +- backend/src/error.rs | 6 +- backend/src/install/cleanup.rs | 8 +-- backend/src/install/mod.rs | 2 +- backend/src/manager/mod.rs | 6 +- backend/src/middleware/db.rs | 10 ++- backend/src/migration.rs | 3 +- backend/src/s9pk/docker.rs | 94 ++++++++++++++++++++++++++++ backend/src/s9pk/manifest.rs | 5 +- backend/src/s9pk/mod.rs | 99 ++++++++++++++++++++++-------- backend/src/s9pk/reader.rs | 65 +++++++++++++++----- backend/src/status/health_check.rs | 3 +- backend/src/util/io.rs | 55 ++++++++++++++++- backend/src/util/mod.rs | 2 +- backend/src/version/v0_3_2.rs | 3 +- system-images/binfmt/Makefile | 2 +- system-images/binfmt/build.sh | 15 +++++ system-images/binfmt/manifest.json | 1 + 23 files changed, 317 insertions(+), 86 deletions(-) create mode 100644 backend/src/s9pk/docker.rs create mode 100755 system-images/binfmt/build.sh create mode 100644 system-images/binfmt/manifest.json diff --git a/backend/src/action.rs b/backend/src/action.rs index c372f9427..8d0f4ddf3 100644 --- a/backend/src/action.rs +++ b/backend/src/action.rs @@ -8,17 +8,15 @@ use rpc_toolkit::command; use serde::{Deserialize, Serialize}; use tracing::instrument; +use crate::config::{Config, ConfigSpec}; use crate::context::RpcContext; use crate::id::ImageId; +use crate::procedure::docker::DockerContainer; use crate::procedure::{PackageProcedure, ProcedureName}; use crate::s9pk::manifest::PackageId; use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat}; use crate::util::Version; use crate::volume::Volumes; -use crate::{ - config::{Config, ConfigSpec}, - procedure::docker::DockerContainer, -}; use crate::{Error, ResultExt}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct Actions(pub BTreeMap); diff --git a/backend/src/backup/mod.rs b/backend/src/backup/mod.rs index 129f7b35d..d667bb33a 100644 --- a/backend/src/backup/mod.rs +++ b/backend/src/backup/mod.rs @@ -14,17 +14,18 @@ use tokio::io::AsyncWriteExt; use tracing::instrument; use self::target::PackageBackupInfo; +use crate::context::RpcContext; use crate::dependencies::reconfigure_dependents_with_live_pointers; use crate::id::ImageId; use crate::install::PKG_ARCHIVE_DIR; use crate::net::interface::{InterfaceId, Interfaces}; +use crate::procedure::docker::DockerContainer; use crate::procedure::{NoOutput, PackageProcedure, ProcedureName}; use crate::s9pk::manifest::PackageId; use crate::util::serde::IoFormat; use crate::util::Version; use crate::version::{Current, VersionT}; use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR}; -use crate::{context::RpcContext, procedure::docker::DockerContainer}; use crate::{Error, ErrorKind, ResultExt}; pub mod backup_bulk; diff --git a/backend/src/config/action.rs b/backend/src/config/action.rs index 5d1cce45b..b1e674731 100644 --- a/backend/src/config/action.rs +++ b/backend/src/config/action.rs @@ -7,14 +7,15 @@ use serde::{Deserialize, Serialize}; use tracing::instrument; use super::{Config, ConfigSpec}; +use crate::context::RpcContext; use crate::dependencies::Dependencies; use crate::id::ImageId; +use crate::procedure::docker::DockerContainer; use crate::procedure::{PackageProcedure, ProcedureName}; use crate::s9pk::manifest::PackageId; use crate::status::health_check::HealthCheckId; use crate::util::Version; use crate::volume::Volumes; -use crate::{context::RpcContext, procedure::docker::DockerContainer}; use crate::{Error, ResultExt}; #[derive(Debug, Deserialize, Serialize, HasModel)] diff --git a/backend/src/config/mod.rs b/backend/src/config/mod.rs index dd4bc45c1..94be02a59 100644 --- a/backend/src/config/mod.rs +++ b/backend/src/config/mod.rs @@ -13,6 +13,7 @@ use rpc_toolkit::command; use serde_json::Value; use tracing::instrument; +use crate::context::RpcContext; use crate::db::model::{CurrentDependencies, CurrentDependencyInfo, CurrentDependents}; use crate::dependencies::{ add_dependent_to_current_dependents_lists, break_transitive, heal_all_dependents_transitive, @@ -20,11 +21,11 @@ use crate::dependencies::{ DependencyErrors, DependencyReceipt, TaggedDependencyError, TryHealReceipts, }; use crate::install::cleanup::{remove_from_current_dependents_lists, UpdateDependencyReceipts}; +use crate::procedure::docker::DockerContainer; use crate::s9pk::manifest::{Manifest, PackageId}; use crate::util::display_none; use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat}; use crate::Error; -use crate::{context::RpcContext, procedure::docker::DockerContainer}; pub mod action; pub mod spec; diff --git a/backend/src/config/spec.rs b/backend/src/config/spec.rs index 90a34e8a9..cea9ce1ea 100644 --- a/backend/src/config/spec.rs +++ b/backend/src/config/spec.rs @@ -22,11 +22,12 @@ use sqlx::PgPool; use super::util::{self, CharSet, NumRange, UniqueBy, STATIC_NULL}; use super::{Config, MatchError, NoMatchWithPath, TimeoutError, TypeOf}; +use crate::config::ConfigurationError; use crate::context::RpcContext; use crate::net::interface::InterfaceId; +use crate::procedure::docker::DockerContainer; use crate::s9pk::manifest::{Manifest, PackageId}; use crate::Error; -use crate::{config::ConfigurationError, procedure::docker::DockerContainer}; // Config Value Specifications #[async_trait] diff --git a/backend/src/dependencies.rs b/backend/src/dependencies.rs index 2b6c76197..34a8bdfae 100644 --- a/backend/src/dependencies.rs +++ b/backend/src/dependencies.rs @@ -14,10 +14,12 @@ use rpc_toolkit::command; use serde::{Deserialize, Serialize}; use tracing::instrument; +use crate::config::action::{ConfigActions, ConfigRes}; use crate::config::spec::PackagePointerSpec; use crate::config::{not_found, Config, ConfigReceipts, ConfigSpec}; use crate::context::RpcContext; use crate::db::model::{CurrentDependencies, CurrentDependents, InstalledPackageDataEntry}; +use crate::procedure::docker::DockerContainer; use crate::procedure::{NoOutput, PackageProcedure, ProcedureName}; use crate::s9pk::manifest::{Manifest, PackageId}; use crate::status::health_check::{HealthCheckId, HealthCheckResult}; @@ -26,10 +28,6 @@ use crate::util::serde::display_serializable; use crate::util::{display_none, Version}; use crate::volume::Volumes; use crate::Error; -use crate::{ - config::action::{ConfigActions, ConfigRes}, - procedure::docker::DockerContainer, -}; #[command(subcommands(configure))] pub fn dependency() -> Result<(), Error> { diff --git a/backend/src/error.rs b/backend/src/error.rs index 82b14dcac..5e7fa9d80 100644 --- a/backend/src/error.rs +++ b/backend/src/error.rs @@ -70,7 +70,7 @@ pub enum ErrorKind { TLSInit = 61, HttpRange = 62, ContentLength = 63, - BytesError = 64 + BytesError = 64, } impl ErrorKind { pub fn as_str(&self) -> &'static str { @@ -139,7 +139,7 @@ impl ErrorKind { TLSInit => "TLS Backend Initialize Error", HttpRange => "No Support for Web Server HTTP Ranges", ContentLength => "Request has no content length header", - BytesError => "Could not get the bytes for this request" + BytesError => "Could not get the bytes for this request", } } } @@ -244,8 +244,6 @@ impl From for Error { fn from(e: openssl::error::ErrorStack) -> Self { Error::new(eyre!("OpenSSL ERROR:\n{}", e), ErrorKind::OpenSsl) } - - } impl From for RpcError { fn from(e: Error) -> Self { diff --git a/backend/src/install/cleanup.rs b/backend/src/install/cleanup.rs index e9b5e0579..72fbdf432 100644 --- a/backend/src/install/cleanup.rs +++ b/backend/src/install/cleanup.rs @@ -1,8 +1,6 @@ -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use bollard::image::ListImagesOptions; use color_eyre::Report; diff --git a/backend/src/install/mod.rs b/backend/src/install/mod.rs index efb7c286b..76bff5652 100644 --- a/backend/src/install/mod.rs +++ b/backend/src/install/mod.rs @@ -896,7 +896,7 @@ impl InstallS9Receipts { } #[instrument(skip(ctx, rdr))] -pub async fn install_s9pk( +pub async fn install_s9pk( ctx: &RpcContext, pkg_id: &PackageId, version: &Version, diff --git a/backend/src/manager/mod.rs b/backend/src/manager/mod.rs index 9d1654815..c33fc579a 100644 --- a/backend/src/manager/mod.rs +++ b/backend/src/manager/mod.rs @@ -22,17 +22,17 @@ use tokio::task::JoinHandle; use torut::onion::TorSecretKeyV3; use tracing::instrument; +use crate::context::RpcContext; +use crate::manager::sync::synchronizer; use crate::net::interface::InterfaceId; use crate::net::GeneratedCertificateMountPoint; use crate::notifications::NotificationLevel; -use crate::procedure::docker::DockerProcedure; +use crate::procedure::docker::{DockerContainer, DockerInject, DockerProcedure}; use crate::procedure::{NoOutput, PackageProcedure, ProcedureName}; use crate::s9pk::manifest::{Manifest, PackageId}; use crate::status::MainStatus; use crate::util::{Container, NonDetachingJoinHandle, Version}; use crate::Error; -use crate::{context::RpcContext, procedure::docker::DockerContainer}; -use crate::{manager::sync::synchronizer, procedure::docker::DockerInject}; pub mod health; mod sync; diff --git a/backend/src/middleware/db.rs b/backend/src/middleware/db.rs index 232e05086..eeeeb299a 100644 --- a/backend/src/middleware/db.rs +++ b/backend/src/middleware/db.rs @@ -59,12 +59,10 @@ pub fn db(ctx: RpcContext) -> DynMiddleware { .append("X-Patch-Updates", HeaderValue::from_str(&a)?), Err(e) => res.headers.append( "X-Patch-Error", - HeaderValue::from_str( - &base64::encode_config( - &e.to_string(), - base64::URL_SAFE, - ), - )?, + HeaderValue::from_str(&base64::encode_config( + &e.to_string(), + base64::URL_SAFE, + ))?, ), }; } diff --git a/backend/src/migration.rs b/backend/src/migration.rs index 2b65df438..03e8fde0b 100644 --- a/backend/src/migration.rs +++ b/backend/src/migration.rs @@ -8,12 +8,13 @@ use patch_db::HasModel; use serde::{Deserialize, Serialize}; use tracing::instrument; +use crate::context::RpcContext; use crate::id::ImageId; +use crate::procedure::docker::DockerContainer; use crate::procedure::{PackageProcedure, ProcedureName}; use crate::s9pk::manifest::PackageId; use crate::util::Version; use crate::volume::Volumes; -use crate::{context::RpcContext, procedure::docker::DockerContainer}; use crate::{Error, ResultExt}; #[derive(Clone, Debug, Default, Deserialize, Serialize, HasModel)] diff --git a/backend/src/s9pk/docker.rs b/backend/src/s9pk/docker.rs new file mode 100644 index 000000000..8d623fb9a --- /dev/null +++ b/backend/src/s9pk/docker.rs @@ -0,0 +1,94 @@ +use std::borrow::Cow; +use std::collections::BTreeSet; +use std::io::SeekFrom; +use std::path::Path; + +use color_eyre::eyre::eyre; +use futures::{FutureExt, TryStreamExt}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncSeek, AsyncSeekExt}; +use tokio_tar::{Archive, Entry}; + +use crate::util::io::from_cbor_async_reader; +use crate::{Error, ErrorKind, ARCH}; + +#[derive(Default, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct DockerMultiArch { + pub default: String, + pub available: BTreeSet, +} + +#[pin_project::pin_project(project = DockerReaderProject)] +pub enum DockerReader { + SingleArch(#[pin] R), + MultiArch(#[pin] Entry>), +} +impl DockerReader { + pub async fn new(mut rdr: R) -> Result { + let arch = if let Some(multiarch) = tokio_tar::Archive::new(&mut rdr) + .entries()? + .try_filter_map(|e| { + async move { + Ok(if &*e.path()? == Path::new("multiarch.cbor") { + Some(e) + } else { + None + }) + } + .boxed() + }) + .try_next() + .await? + { + let multiarch: DockerMultiArch = from_cbor_async_reader(multiarch).await?; + Some(if multiarch.available.contains(&**ARCH) { + Cow::Borrowed(&**ARCH) + } else { + Cow::Owned(multiarch.default) + }) + } else { + None + }; + rdr.seek(SeekFrom::Start(0)).await?; + if let Some(arch) = arch { + if let Some(image) = tokio_tar::Archive::new(rdr) + .entries()? + .try_filter_map(|e| { + let arch = arch.clone(); + async move { + Ok(if &*e.path()? == Path::new(&format!("{}.tar", arch)) { + Some(e) + } else { + None + }) + } + .boxed() + }) + .try_next() + .await? + { + Ok(Self::MultiArch(image)) + } else { + Err(Error::new( + eyre!("Docker image section does not contain tarball for architecture"), + ErrorKind::ParseS9pk, + )) + } + } else { + Ok(Self::SingleArch(rdr)) + } + } +} +impl AsyncRead for DockerReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + match self.project() { + DockerReaderProject::SingleArch(r) => r.poll_read(cx, buf), + DockerReaderProject::MultiArch(r) => r.poll_read(cx, buf), + } + } +} diff --git a/backend/src/s9pk/manifest.rs b/backend/src/s9pk/manifest.rs index 3abae432c..f836b2348 100644 --- a/backend/src/s9pk/manifest.rs +++ b/backend/src/s9pk/manifest.rs @@ -6,18 +6,19 @@ use patch_db::HasModel; use serde::{Deserialize, Serialize}; use url::Url; +use crate::action::Actions; use crate::backup::BackupActions; use crate::config::action::ConfigActions; use crate::dependencies::Dependencies; use crate::migration::Migrations; use crate::net::interface::Interfaces; +use crate::procedure::docker::DockerContainer; use crate::procedure::PackageProcedure; use crate::status::health_check::HealthChecks; use crate::util::Version; use crate::version::{Current, VersionT}; use crate::volume::Volumes; use crate::Error; -use crate::{action::Actions, procedure::docker::DockerContainer}; fn current_version() -> Version { Current::new().semver().into() @@ -143,7 +144,7 @@ impl Assets { self.docker_images .as_ref() .map(|a| a.as_path()) - .unwrap_or(Path::new("image.tar")) + .unwrap_or(Path::new("docker-images")) } pub fn assets_path(&self) -> &Path { self.assets diff --git a/backend/src/s9pk/mod.rs b/backend/src/s9pk/mod.rs index a0dadb161..66062cb9d 100644 --- a/backend/src/s9pk/mod.rs +++ b/backend/src/s9pk/mod.rs @@ -1,22 +1,27 @@ +use std::ffi::OsStr; use std::path::PathBuf; use color_eyre::eyre::eyre; +use futures::TryStreamExt; use imbl::OrdMap; -use patch_db::{LockReceipt, LockType}; use rpc_toolkit::command; use serde_json::Value; +use tokio::io::AsyncRead; use tracing::instrument; +use crate::context::SdkContext; use crate::s9pk::builder::S9pkPacker; +use crate::s9pk::docker::DockerMultiArch; use crate::s9pk::manifest::Manifest; use crate::s9pk::reader::S9pkReader; use crate::util::display_none; +use crate::util::io::BufferedWriteReader; use crate::util::serde::IoFormat; use crate::volume::Volume; -use crate::{context::SdkContext, procedure::docker::DockerContainer}; use crate::{Error, ErrorKind, ResultExt}; pub mod builder; +pub mod docker; pub mod header; pub mod manifest; pub mod reader; @@ -94,33 +99,73 @@ pub async fn pack(#[context] ctx: SdkContext, #[arg] path: Option) -> R ) })?, ) - .docker_images( - File::open(path.join(manifest.assets.docker_images_path())) - .await - .with_ctx(|_| { - ( - crate::ErrorKind::Filesystem, - manifest.assets.docker_images_path().display().to_string(), - ) - })?, - ) + .docker_images({ + let docker_images_path = path.join(manifest.assets.docker_images_path()); + let res: Box = if tokio::fs::metadata(&docker_images_path).await?.is_dir() { + let tars: Vec<_> = tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(&docker_images_path).await?).try_collect().await?; + let mut arch_info = DockerMultiArch::default(); + for tar in &tars { + if tar.path().extension() == Some(OsStr::new("tar")) { + arch_info.available.insert(tar.path().file_stem().unwrap_or_default().to_str().unwrap_or_default().to_owned()); + } + } + if arch_info.available.contains("arm64") { + arch_info.default = "arm64".to_owned(); + } else { + arch_info.default = arch_info.available.iter().next().cloned().unwrap_or_default(); + } + let arch_info_cbor = IoFormat::Cbor.to_vec(&arch_info)?; + Box::new(BufferedWriteReader::new(|w| async move { + let mut docker_images = tokio_tar::Builder::new(w); + let mut multiarch_header = tokio_tar::Header::new_gnu(); + multiarch_header.set_path("multiarch.cbor"); + multiarch_header.set_size(arch_info_cbor.len() as u64); + multiarch_header.set_cksum(); + docker_images.append(&multiarch_header, std::io::Cursor::new(arch_info_cbor)).await?; + for tar in tars + { + docker_images + .append_path_with_name( + tar.path(), + tar.file_name(), + ) + .await?; + } + Ok::<_, std::io::Error>(()) + }, 1024 * 1024)) + } else { + Box::new(File::open(docker_images_path) + .await + .with_ctx(|_| { + ( + crate::ErrorKind::Filesystem, + manifest.assets.docker_images_path().display().to_string(), + ) + })?) + }; + res + }) .assets({ - let mut assets = tokio_tar::Builder::new(Vec::new()); // TODO: Ideally stream this? best not to buffer in memory + let asset_volumes = manifest + .volumes + .iter() + .filter(|(_, v)| matches!(v, &&Volume::Assets {})).map(|(id, _)| id.clone()).collect::>(); + let assets_path = manifest.assets.assets_path().to_owned(); + let path = path.clone(); - for (asset_volume, _) in manifest - .volumes - .iter() - .filter(|(_, v)| matches!(v, &&Volume::Assets {})) - { - assets - .append_dir_all( - asset_volume, - path.join(manifest.assets.assets_path()).join(asset_volume), - ) - .await?; - } - - std::io::Cursor::new(assets.into_inner().await?) + BufferedWriteReader::new(|w| async move { + let mut assets = tokio_tar::Builder::new(w); + for asset_volume in asset_volumes + { + assets + .append_dir_all( + &asset_volume, + path.join(&assets_path).join(&asset_volume), + ) + .await?; + } + Ok::<_, std::io::Error>(()) + }, 1024 * 1024) }) .scripts({ let script_path = path.join(manifest.assets.scripts_path()).join("embassy.js"); diff --git a/backend/src/s9pk/reader.rs b/backend/src/s9pk/reader.rs index a1591deec..5e27c283b 100644 --- a/backend/src/s9pk/reader.rs +++ b/backend/src/s9pk/reader.rs @@ -1,5 +1,6 @@ use std::collections::BTreeSet; use std::io::SeekFrom; +use std::ops::Range; use std::path::Path; use std::pin::Pin; use std::str::FromStr; @@ -8,34 +9,36 @@ use std::task::{Context, Poll}; use color_eyre::eyre::eyre; use digest_old::Output; use ed25519_dalek::PublicKey; -use futures::TryStreamExt; +use futures::{FutureExt, TryStreamExt}; use sha2_old::{Digest, Sha512}; use tokio::fs::File; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf, Take}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf}; use tracing::instrument; use super::header::{FileSection, Header, TableOfContents}; use super::manifest::{Manifest, PackageId}; use super::SIG_CONTEXT; +use crate::id::ImageId; use crate::install::progress::InstallProgressTracker; +use crate::s9pk::docker::DockerReader; use crate::util::Version; -use crate::{id::ImageId, procedure::docker::DockerContainer}; use crate::{Error, ResultExt}; #[pin_project::pin_project] -pub struct ReadHandle<'a, R: AsyncRead + AsyncSeek + Unpin = File> { +pub struct ReadHandle<'a, R = File> { pos: &'a mut u64, + range: Range, #[pin] - rdr: Take<&'a mut R>, + rdr: &'a mut R, } -impl<'a, R: AsyncRead + AsyncSeek + Unpin> ReadHandle<'a, R> { +impl<'a, R: AsyncRead + Unpin> ReadHandle<'a, R> { pub async fn to_vec(mut self) -> std::io::Result> { - let mut buf = vec![0; self.rdr.limit() as usize]; + let mut buf = vec![0; (self.range.end - self.range.start) as usize]; self.read_exact(&mut buf).await?; Ok(buf) } } -impl<'a, R: AsyncRead + AsyncSeek + Unpin> AsyncRead for ReadHandle<'a, R> { +impl<'a, R: AsyncRead + Unpin> AsyncRead for ReadHandle<'a, R> { fn poll_read( self: Pin<&mut Self>, cx: &mut Context<'_>, @@ -43,12 +46,39 @@ impl<'a, R: AsyncRead + AsyncSeek + Unpin> AsyncRead for ReadHandle<'a, R> { ) -> Poll> { let start = buf.filled().len(); let this = self.project(); - let pos = this.pos; - let res = AsyncRead::poll_read(this.rdr, cx, buf); - **pos += (buf.filled().len() - start) as u64; + let res = AsyncRead::poll_read( + this.rdr, + cx, + &mut buf.take(this.range.end.saturating_sub(**this.pos) as usize), + ); + **this.pos += (buf.filled().len() - start) as u64; res } } +impl<'a, R: AsyncSeek + Unpin> AsyncSeek for ReadHandle<'a, R> { + fn start_seek(self: Pin<&mut Self>, position: SeekFrom) -> std::io::Result<()> { + let this = self.project(); + AsyncSeek::start_seek( + this.rdr, + match position { + SeekFrom::Current(n) => SeekFrom::Current(n), + SeekFrom::End(n) => SeekFrom::Start((this.range.end as i64 + n) as u64), + SeekFrom::Start(n) => SeekFrom::Start(this.range.start + n), + }, + ) + } + fn poll_complete(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + match AsyncSeek::poll_complete(this.rdr, cx) { + Poll::Ready(Ok(n)) => { + let res = n.saturating_sub(this.range.start); + **this.pos = this.range.start + res; + Poll::Ready(Ok(res)) + } + a => a, + } + } +} #[derive(Debug)] pub struct ImageTag { @@ -110,7 +140,7 @@ impl FromStr for ImageTag { } } -pub struct S9pkReader { +pub struct S9pkReader { hash: Option>, hash_string: Option, developer_key: PublicKey, @@ -128,12 +158,12 @@ impl S9pkReader { Self::from_reader(rdr, check_sig).await } } -impl S9pkReader> { +impl S9pkReader> { pub fn validated(&mut self) { self.rdr.validated() } } -impl S9pkReader { +impl S9pkReader { #[instrument(skip(self))] pub async fn validate(&mut self) -> Result<(), Error> { if self.toc.icon.length > 102_400 { @@ -309,8 +339,9 @@ impl S9pkReader { self.pos = section.position; } Ok(ReadHandle { + range: self.pos..(self.pos + section.length), pos: &mut self.pos, - rdr: (&mut self.rdr).take(section.length), + rdr: &mut self.rdr, }) } @@ -336,8 +367,8 @@ impl S9pkReader { Ok(self.read_handle(self.toc.icon).await?) } - pub async fn docker_images<'a>(&'a mut self) -> Result, Error> { - Ok(self.read_handle(self.toc.docker_images).await?) + pub async fn docker_images<'a>(&'a mut self) -> Result>, Error> { + DockerReader::new(self.read_handle(self.toc.docker_images).await?).await } pub async fn assets<'a>(&'a mut self) -> Result, Error> { diff --git a/backend/src/status/health_check.rs b/backend/src/status/health_check.rs index 2cb0fad4b..9b1a27b85 100644 --- a/backend/src/status/health_check.rs +++ b/backend/src/status/health_check.rs @@ -5,13 +5,14 @@ pub use models::HealthCheckId; use serde::{Deserialize, Serialize}; use tracing::instrument; +use crate::context::RpcContext; use crate::id::ImageId; +use crate::procedure::docker::DockerContainer; use crate::procedure::{NoOutput, PackageProcedure, ProcedureName}; use crate::s9pk::manifest::PackageId; use crate::util::serde::Duration; use crate::util::Version; use crate::volume::Volumes; -use crate::{context::RpcContext, procedure::docker::DockerContainer}; use crate::{Error, ResultExt}; #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/backend/src/util/io.rs b/backend/src/util/io.rs index ba8e10196..d28c974f3 100644 --- a/backend/src/util/io.rs +++ b/backend/src/util/io.rs @@ -1,11 +1,19 @@ +use std::future::Future; use std::path::Path; +use std::task::Poll; -use futures::future::BoxFuture; -use futures::{FutureExt, TryStreamExt}; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}; +use futures::future::{BoxFuture, Fuse}; +use futures::{AsyncSeek, FutureExt, TryStreamExt}; +use helpers::NonDetachingJoinHandle; +use tokio::io::{ + duplex, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf, WriteHalf, +}; use crate::ResultExt; +pub trait AsyncReadSeek: AsyncRead + AsyncSeek {} +impl AsyncReadSeek for T {} + #[derive(Clone, Debug)] pub struct AsyncCompat(pub T); impl futures::io::AsyncRead for AsyncCompat @@ -246,3 +254,44 @@ pub fn response_to_reader(response: reqwest::Response) -> impl AsyncRead + Unpin ) })) } + +#[pin_project::pin_project] +pub struct BufferedWriteReader { + #[pin] + hdl: Fuse>>, + #[pin] + rdr: DuplexStream, +} +impl BufferedWriteReader { + pub fn new< + W: FnOnce(WriteHalf) -> Fut, + Fut: Future> + Send + Sync, + >( + write_fn: W, + max_buf_size: usize, + ) -> Self { + let (w, rdr) = duplex(max_buf_size); + let (_, w) = tokio::io::split(w); + BufferedWriteReader { + hdl: NonDetachingJoinHandle::from(tokio::spawn(write_fn(w))).fuse(), + rdr, + } + } +} +impl AsyncRead for BufferedWriteReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.project(); + let res = this.rdr.poll_read(cx, buf); + match this.hdl.poll(cx) { + Poll::Ready(Ok(Err(e))) => return Poll::Ready(Err(e)), + Poll::Ready(Err(e)) => { + return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, e))) + } + _ => res, + } + } +} diff --git a/backend/src/util/mod.rs b/backend/src/util/mod.rs index abb637df0..ebfb14192 100644 --- a/backend/src/util/mod.rs +++ b/backend/src/util/mod.rs @@ -24,8 +24,8 @@ use tracing::instrument; use crate::shutdown::Shutdown; use crate::{Error, ErrorKind, ResultExt as _}; pub mod config; -pub mod io; pub mod http_reader; +pub mod io; pub mod logger; pub mod serde; diff --git a/backend/src/version/v0_3_2.rs b/backend/src/version/v0_3_2.rs index f26d83039..ed0cb37ea 100644 --- a/backend/src/version/v0_3_2.rs +++ b/backend/src/version/v0_3_2.rs @@ -1,9 +1,8 @@ use emver::VersionRange; -use crate::hostname::{generate_id, sync_hostname}; - use super::v0_3_0::V0_3_0_COMPAT; use super::*; +use crate::hostname::{generate_id, sync_hostname}; const V0_3_2: emver::Version = emver::Version::new(0, 3, 2, 0); diff --git a/system-images/binfmt/Makefile b/system-images/binfmt/Makefile index 543906079..fdc354727 100644 --- a/system-images/binfmt/Makefile +++ b/system-images/binfmt/Makefile @@ -3,4 +3,4 @@ all: binfmt.tar binfmt.tar: Dockerfile - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/binfmt --platform=linux/arm64 -o type=docker,dest=binfmt.tar . + DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/binfmt --platform=linux/arm64,linux/amd64 -o type=docker,dest=binfmt.tar . diff --git a/system-images/binfmt/build.sh b/system-images/binfmt/build.sh new file mode 100755 index 000000000..1a6e887f6 --- /dev/null +++ b/system-images/binfmt/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +docker run -d -p5000:5000 --rm --name registry registry +docker buildx build --platform linux/amd64 --push -t 172.17.0.1:5000/binfmt:amd64 . +docker buildx build --platform linux/arm64 --push -t 172.17.0.1:5000/binfmt:arm64 . +docker manifest create --amend --insecure 172.17.0.1:5000/binfmt:latest 172.17.0.1:5000/binfmt:amd64 172.17.0.1:5000/binfmt:arm64 +docker manifest push --insecure 172.17.0.1:5000/binfmt:latest +docker pull --platform=linux/arm64 --platform=linux/amd64 172.17.0.1:5000/binfmt:latest +docker image save -o binfmt.tar 172.17.0.1:5000/binfmt:amd64 172.17.0.1:5000/binfmt:arm64 +tar -xvf binfmt.tar manifest.json +sed -i 's/172\.17\.0\.1:5000/start9\/x_system/g' manifest.json +tar -uf binfmt.tar manifest.json +docker stop registry \ No newline at end of file diff --git a/system-images/binfmt/manifest.json b/system-images/binfmt/manifest.json new file mode 100644 index 000000000..ebdfc259d --- /dev/null +++ b/system-images/binfmt/manifest.json @@ -0,0 +1 @@ +[{"Config":"3f7fd8db05afd7a5fa804ebfb0e40038068af52b45d8f9a04c1c9fa8d316e511.json","RepoTags":["multitest/binfmt:latest"],"Layers":["753827173463aa2b7471a5be8ac2323a1cf78ccedfdfcc7bb21831d9e0141732/layer.tar","e0747212aee318099c729893bb4cca2ff164f6bd154841d5777636f82b7f18d4/layer.tar"]},{"Config":"143329dfbcc6abb472070a2f0bae5cf9865f178e5391a15592e75b349c803c69.json","RepoTags":["multitest/binfmt:latest"],"Layers":["de6e72d83667be34f782cf1f0a376e0507d189872142e1934f14d3be4813715a/layer.tar","12670876b666af35f388c3d2160436257b390a163a4c7178863badd05394577a/layer.tar"]}] From 47496810e7d578de829ded35bbfd299ebfa372c8 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 29 Sep 2022 17:24:00 -0600 Subject: [PATCH 20/53] multi-arch system images --- Makefile | 22 ++++++++++++---------- backend/src/util/io.rs | 2 +- system-images/binfmt/.gitignore | 2 +- system-images/binfmt/Makefile | 15 ++++++++++++--- system-images/binfmt/build.sh | 15 --------------- system-images/compat/.gitignore | 2 +- system-images/compat/Makefile | 13 ++++++++++--- system-images/utils/.gitignore | 2 +- system-images/utils/Makefile | 15 ++++++++++++--- 9 files changed, 50 insertions(+), 38 deletions(-) delete mode 100755 system-images/binfmt/build.sh diff --git a/Makefile b/Makefile index cb9a8bf3a..a83a7825e 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,9 @@ EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backen EMBASSY_NATIVE_BINS := backend/target/release/embassyd backend/target/release/embassy-init backend/target/release/embassy-cli backend/target/release/embassy-sdk backend/target/release/avahi-alias EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build) -COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name compat.tar -and -not -name target) -UTILS_SRC := $(shell find system-images/utils/ -not -name utils.tar) -BINFMT_SRC := $(shell find system-images/binfmt/ -not -name binfmt.tar) +COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name *.tar -and -not -name target) +UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar) +BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar) BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) backend/Cargo.toml backend/Cargo.lock FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell find frontend/assets) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json FRONTEND_UI_SRC := $(shell find frontend/projects/ui) @@ -58,14 +58,14 @@ format: sdk: cd backend/ && ./install-sdk.sh -eos.img: raspios.img $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/compat.tar system-images/utils/utils.tar system-images/binfmt/binfmt.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) +eos.img: raspios.img $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/aarch64.tar system-images/binfmt/docker-images/aarch64.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) ! test -f eos.img || rm eos.img if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi -dpkg_deps: $(EMBASSY_SRC) $(EMBASSY_NATIVE_BINS) system-images/compat/compat.tar system-images/utils/utils.tar system-images/binfmt/binfmt.tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) +install-dependencies: $(EMBASSY_SRC) $(EMBASSY_NATIVE_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/$(shell uname -m).tar system-images/binfmt/docker-images/$(shell uname -m).tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) # For creating dpkg. DO NOT USE -install: dpkg_deps +install: install-dependencies mkdir -p $(DESTDIR)/etc/embassy cp ENVIRONMENT.txt $(DESTDIR)/etc/embassy/ cp GIT_HASH.txt $(DESTDIR)/etc/embassy/ @@ -77,7 +77,9 @@ install: dpkg_deps cp backend/target/release/avahi-alias $(DESTDIR)/usr/bin/ mkdir -p $(DESTDIR)/var/lib/embassy/system-images - cp system-images/**/*.tar $(DESTDIR)/var/lib/embassy/system-images/ + cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/var/lib/embassy/system-images/compat.tar + cp system-images/utils/docker-images/$(shell uname -m).tar $(DESTDIR)/var/lib/embassy/system-images/utils.tar + cp system-images/binfmt/docker-images/$(shell uname -m).tar $(DESTDIR)/var/lib/embassy/system-images/binfmt.tar mkdir -p $(DESTDIR)/var/www/html cp -r frontend/dist/diagnostic-ui $(DESTDIR)/var/www/html/diagnostic @@ -88,13 +90,13 @@ install: dpkg_deps embassy-os.deb: debian/control ./build/make-deb.sh -system-images/compat/compat.tar: $(COMPAT_SRC) +system-images/compat/docker-images/aarch64.tar: $(COMPAT_SRC) cd system-images/compat && make -system-images/utils/utils.tar: $(UTILS_SRC) +system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/aarch64.tar: $(UTILS_SRC) cd system-images/utils && make -system-images/binfmt/binfmt.tar: $(BINFMT_SRC) +system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/aarch64.tar: $(BINFMT_SRC) cd system-images/binfmt && make raspios.img: diff --git a/backend/src/util/io.rs b/backend/src/util/io.rs index d28c974f3..af3b5b7da 100644 --- a/backend/src/util/io.rs +++ b/backend/src/util/io.rs @@ -265,7 +265,7 @@ pub struct BufferedWriteReader { impl BufferedWriteReader { pub fn new< W: FnOnce(WriteHalf) -> Fut, - Fut: Future> + Send + Sync, + Fut: Future> + Send + Sync + 'static, >( write_fn: W, max_buf_size: usize, diff --git a/system-images/binfmt/.gitignore b/system-images/binfmt/.gitignore index e0480321e..9ea8ef8d2 100644 --- a/system-images/binfmt/.gitignore +++ b/system-images/binfmt/.gitignore @@ -1 +1 @@ -binfmt.tar \ No newline at end of file +/docker-images \ No newline at end of file diff --git a/system-images/binfmt/Makefile b/system-images/binfmt/Makefile index fdc354727..141b50fc1 100644 --- a/system-images/binfmt/Makefile +++ b/system-images/binfmt/Makefile @@ -1,6 +1,15 @@ .DELETE_ON_ERROR: -all: binfmt.tar +all: docker-images/aarch64.tar docker-images/x86_64.tar -binfmt.tar: Dockerfile - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/binfmt --platform=linux/arm64,linux/amd64 -o type=docker,dest=binfmt.tar . +clean: + rm -rf docker-images + +docker-images: + mkdir docker-images + +docker-images/aarch64.tar: Dockerfile docker-images + docker buildx build --tag start9/x_system/binfmt --platform=linux/arm64 -o type=docker,dest=docker-images/aarch64.tar . + +docker-images/x86_64.tar: Dockerfile docker-images + docker buildx build --tag start9/x_system/binfmt --platform=linux/amd64 -o type=docker,dest=docker-images/x86_64.tar . \ No newline at end of file diff --git a/system-images/binfmt/build.sh b/system-images/binfmt/build.sh deleted file mode 100755 index 1a6e887f6..000000000 --- a/system-images/binfmt/build.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -docker run -d -p5000:5000 --rm --name registry registry -docker buildx build --platform linux/amd64 --push -t 172.17.0.1:5000/binfmt:amd64 . -docker buildx build --platform linux/arm64 --push -t 172.17.0.1:5000/binfmt:arm64 . -docker manifest create --amend --insecure 172.17.0.1:5000/binfmt:latest 172.17.0.1:5000/binfmt:amd64 172.17.0.1:5000/binfmt:arm64 -docker manifest push --insecure 172.17.0.1:5000/binfmt:latest -docker pull --platform=linux/arm64 --platform=linux/amd64 172.17.0.1:5000/binfmt:latest -docker image save -o binfmt.tar 172.17.0.1:5000/binfmt:amd64 172.17.0.1:5000/binfmt:arm64 -tar -xvf binfmt.tar manifest.json -sed -i 's/172\.17\.0\.1:5000/start9\/x_system/g' manifest.json -tar -uf binfmt.tar manifest.json -docker stop registry \ No newline at end of file diff --git a/system-images/compat/.gitignore b/system-images/compat/.gitignore index e3123ca00..0a8fcff98 100644 --- a/system-images/compat/.gitignore +++ b/system-images/compat/.gitignore @@ -2,4 +2,4 @@ **/*.rs.bk .DS_Store .vscode -compat.tar \ No newline at end of file +/docker-images \ No newline at end of file diff --git a/system-images/compat/Makefile b/system-images/compat/Makefile index 065bbf0ef..39321b8fe 100644 --- a/system-images/compat/Makefile +++ b/system-images/compat/Makefile @@ -2,10 +2,17 @@ .DELETE_ON_ERROR: -all: compat.tar +all: docker-images/aarch64.tar -compat.tar: Dockerfile target/aarch64-unknown-linux-musl/release/compat - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/compat --platform=linux/arm64 -o type=docker,dest=compat.tar . +clean: + cargo clean + rm -rf docker-images + +docker-images: + mkdir docker-images + +docker-images/aarch64.tar: Dockerfile target/aarch64-unknown-linux-musl/release/compat docker-images + docker buildx build --tag start9/x_system/compat --platform=linux/arm64 -o type=docker,dest=docker-images/aarch64.tar . target/aarch64-unknown-linux-musl/release/compat: $(COMPAT_SRC) ./build.sh diff --git a/system-images/utils/.gitignore b/system-images/utils/.gitignore index 83c210c12..9ea8ef8d2 100644 --- a/system-images/utils/.gitignore +++ b/system-images/utils/.gitignore @@ -1 +1 @@ -utils.tar \ No newline at end of file +/docker-images \ No newline at end of file diff --git a/system-images/utils/Makefile b/system-images/utils/Makefile index fdae14b4d..ffb8c0779 100644 --- a/system-images/utils/Makefile +++ b/system-images/utils/Makefile @@ -1,6 +1,15 @@ .DELETE_ON_ERROR: -all: utils.tar +all: docker-images/aarch64.tar docker-images/x86_64.tar -utils.tar: Dockerfile - DOCKER_CLI_EXPERIMENTAL=enabled docker buildx build --tag start9/x_system/utils --platform=linux/arm64 -o type=docker,dest=utils.tar . +clean: + rm -rf docker-images + +docker-images: + mkdir docker-images + +docker-images/aarch64.tar: Dockerfile docker-images + docker buildx build --tag start9/x_system/utils --platform=linux/arm64 -o type=docker,dest=docker-images/aarch64.tar . + +docker-images/x86_64.tar: Dockerfile docker-images + docker buildx build --tag start9/x_system/utils --platform=linux/amd64 -o type=docker,dest=docker-images/x86_64.tar . \ No newline at end of file From e4e3b393abe0c87fe8dd68f644ef954a28bc5a24 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 29 Sep 2022 17:28:25 -0600 Subject: [PATCH 21/53] default aarch64 --- Makefile | 4 ++-- backend/src/s9pk/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a83a7825e..bfbdc3db7 100644 --- a/Makefile +++ b/Makefile @@ -93,10 +93,10 @@ embassy-os.deb: debian/control system-images/compat/docker-images/aarch64.tar: $(COMPAT_SRC) cd system-images/compat && make -system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/aarch64.tar: $(UTILS_SRC) +system-images/utils/docker-images/aarch64.tar system-images/utils/docker-images/x86_64.tar: $(UTILS_SRC) cd system-images/utils && make -system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/aarch64.tar: $(BINFMT_SRC) +system-images/binfmt/docker-images/aarch64.tar system-images/binfmt/docker-images/x86_64.tar: $(BINFMT_SRC) cd system-images/binfmt && make raspios.img: diff --git a/backend/src/s9pk/mod.rs b/backend/src/s9pk/mod.rs index 66062cb9d..d39b4214a 100644 --- a/backend/src/s9pk/mod.rs +++ b/backend/src/s9pk/mod.rs @@ -109,8 +109,8 @@ pub async fn pack(#[context] ctx: SdkContext, #[arg] path: Option) -> R arch_info.available.insert(tar.path().file_stem().unwrap_or_default().to_str().unwrap_or_default().to_owned()); } } - if arch_info.available.contains("arm64") { - arch_info.default = "arm64".to_owned(); + if arch_info.available.contains("aarch64") { + arch_info.default = "aarch64".to_owned(); } else { arch_info.default = arch_info.available.iter().next().cloned().unwrap_or_default(); } From ae06790d9d4e322d65545e95480d131f74bc19a9 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 29 Sep 2022 17:36:53 -0600 Subject: [PATCH 22/53] edition 2021 --- backend/Cargo.toml | 2 +- backend/src/s9pk/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 3e048df2c..2dab81a48 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Aiden McClelland "] description = "The core of the Start9 Embassy Operating System" documentation = "https://docs.rs/embassy-os" -edition = "2018" +edition = "2021" keywords = [ "self-hosted", "raspberry-pi", diff --git a/backend/src/s9pk/mod.rs b/backend/src/s9pk/mod.rs index d39b4214a..399ac0a7e 100644 --- a/backend/src/s9pk/mod.rs +++ b/backend/src/s9pk/mod.rs @@ -118,7 +118,7 @@ pub async fn pack(#[context] ctx: SdkContext, #[arg] path: Option) -> R Box::new(BufferedWriteReader::new(|w| async move { let mut docker_images = tokio_tar::Builder::new(w); let mut multiarch_header = tokio_tar::Header::new_gnu(); - multiarch_header.set_path("multiarch.cbor"); + multiarch_header.set_path("multiarch.cbor")?; multiarch_header.set_size(arch_info_cbor.len() as u64); multiarch_header.set_cksum(); docker_images.append(&multiarch_header, std::io::Cursor::new(arch_info_cbor)).await?; From e0afd248a99c1e3a4f0f4795a04d690c7f18b5b8 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Thu, 29 Sep 2022 18:02:04 -0600 Subject: [PATCH 23/53] dynamic wifi interface name --- backend/src/context/rpc.rs | 3 ++- backend/src/net/wifi.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index 933ff398f..b2eb0cbde 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -43,6 +43,7 @@ use crate::{Error, ErrorKind, ResultExt}; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct RpcContextConfig { + pub wifi_interface: String, pub os_partitions: OsPartitionInfo, pub migration_batch_rows: Option, pub migration_prefetch_rows: Option, @@ -270,7 +271,7 @@ impl RpcContext { notification_manager, open_authed_websockets: Mutex::new(BTreeMap::new()), rpc_stream_continuations: Mutex::new(BTreeMap::new()), - wifi_manager: Arc::new(RwLock::new(WpaCli::init("wlan0".to_string()))), + wifi_manager: Arc::new(RwLock::new(WpaCli::init(base.wifi_interface))), }); let res = Self(seed); diff --git a/backend/src/net/wifi.rs b/backend/src/net/wifi.rs index 90ed8d0b7..b16e14ff8 100644 --- a/backend/src/net/wifi.rs +++ b/backend/src/net/wifi.rs @@ -776,6 +776,7 @@ pub fn country_code_parse(code: &str, _matches: &ArgMatches) -> Result>( main_datadir: P, + wifi_iface: &str, last_country_code: &Option, ) -> Result<(), Error> { let persistent = main_datadir.as_ref().join("system-connections"); @@ -797,7 +798,7 @@ pub async fn synchronize_wpa_supplicant_conf>( .invoke(ErrorKind::Wifi) .await?; Command::new("ifconfig") - .arg("wlan0") + .arg(wifi_iface) .arg("up") .invoke(ErrorKind::Wifi) .await?; From 84e8163188f72641d459ef05a2ed29e4d1b64031 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 30 Sep 2022 14:46:10 -0600 Subject: [PATCH 24/53] use wifi interface from config --- backend/src/init.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/init.rs b/backend/src/init.rs index 5d18dd4fc..bfaa01d40 100644 --- a/backend/src/init.rs +++ b/backend/src/init.rs @@ -334,6 +334,7 @@ pub async fn init(cfg: &RpcContextConfig) -> Result { crate::net::wifi::synchronize_wpa_supplicant_conf( &cfg.datadir().join("main"), + &cfg.wifi_interface, &receipts.last_wifi_region.get(&mut handle).await?, ) .await?; From f1f9ee4e697bedcd4da4fdded7ec5c7608561e92 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Fri, 30 Sep 2022 17:16:02 -0600 Subject: [PATCH 25/53] bugfixes --- backend/src/context/rpc.rs | 5 +++++ backend/src/install/progress.rs | 1 + backend/src/net/wifi.rs | 6 +++--- backend/src/s9pk/docker.rs | 1 + backend/src/s9pk/reader.rs | 15 ++++++++------- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index b2eb0cbde..0b0270efd 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -44,6 +44,7 @@ use crate::{Error, ErrorKind, ResultExt}; #[serde(rename_all = "kebab-case")] pub struct RpcContextConfig { pub wifi_interface: String, + pub ethernet_interface: String, pub os_partitions: OsPartitionInfo, pub migration_batch_rows: Option, pub migration_prefetch_rows: Option, @@ -120,6 +121,8 @@ impl RpcContextConfig { pub struct RpcContextSeed { is_closed: AtomicBool, pub os_partitions: OsPartitionInfo, + pub wifi_interface: String, + pub ethernet_interface: String, pub bind_rpc: SocketAddr, pub bind_ws: SocketAddr, pub bind_static: SocketAddr, @@ -254,6 +257,8 @@ impl RpcContext { is_closed: AtomicBool::new(false), datadir: base.datadir().to_path_buf(), os_partitions: base.os_partitions, + wifi_interface: base.wifi_interface.clone(), + ethernet_interface: base.ethernet_interface, bind_rpc: base.bind_rpc.unwrap_or(([127, 0, 0, 1], 5959).into()), bind_ws: base.bind_ws.unwrap_or(([127, 0, 0, 1], 5960).into()), bind_static: base.bind_static.unwrap_or(([127, 0, 0, 1], 5961).into()), diff --git a/backend/src/install/progress.rs b/backend/src/install/progress.rs index 4d160748b..80bee0675 100644 --- a/backend/src/install/progress.rs +++ b/backend/src/install/progress.rs @@ -113,6 +113,7 @@ impl InstallProgress { } #[pin_project::pin_project] +#[derive(Debug)] pub struct InstallProgressTracker { #[pin] inner: RW, diff --git a/backend/src/net/wifi.rs b/backend/src/net/wifi.rs index b16e14ff8..af44e1acc 100644 --- a/backend/src/net/wifi.rs +++ b/backend/src/net/wifi.rs @@ -155,7 +155,7 @@ pub async fn delete(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<( let ssid = Ssid(ssid); let is_current_being_removed = matches!(current, Some(current) if current == ssid); let is_current_removed_and_no_hardwire = - is_current_being_removed && !interface_connected("eth0").await?; + is_current_being_removed && !interface_connected(&ctx.ethernet_interface).await?; if is_current_removed_and_no_hardwire { return Err(Error::new(color_eyre::eyre::eyre!("Forbidden: Deleting this Network would make your Embassy Unreachable. Either connect to ethernet or connect to a different WiFi network to remedy this."), ErrorKind::Wifi)); } @@ -289,7 +289,7 @@ pub async fn get( wpa_supplicant.list_networks_low(), wpa_supplicant.get_current_network(), wpa_supplicant.get_country_low(), - interface_connected("eth0"), // TODO: pull from config + interface_connected(&ctx.ethernet_interface), wpa_supplicant.list_wifi_low() ); let signal_strengths = signal_strengths?; @@ -365,7 +365,7 @@ pub async fn set_country( #[context] ctx: RpcContext, #[arg(parse(country_code_parse))] country: CountryCode, ) -> Result<(), Error> { - if !interface_connected("eth0").await? { + if !interface_connected(&ctx.ethernet_interface).await? { return Err(Error::new( color_eyre::eyre::eyre!("Won't change country without hardwire connection"), crate::ErrorKind::Wifi, diff --git a/backend/src/s9pk/docker.rs b/backend/src/s9pk/docker.rs index 8d623fb9a..be93905fb 100644 --- a/backend/src/s9pk/docker.rs +++ b/backend/src/s9pk/docker.rs @@ -20,6 +20,7 @@ pub struct DockerMultiArch { } #[pin_project::pin_project(project = DockerReaderProject)] +#[derive(Debug)] pub enum DockerReader { SingleArch(#[pin] R), MultiArch(#[pin] Entry>), diff --git a/backend/src/s9pk/reader.rs b/backend/src/s9pk/reader.rs index 5e27c283b..6026a37f1 100644 --- a/backend/src/s9pk/reader.rs +++ b/backend/src/s9pk/reader.rs @@ -25,6 +25,7 @@ use crate::util::Version; use crate::{Error, ResultExt}; #[pin_project::pin_project] +#[derive(Debug)] pub struct ReadHandle<'a, R = File> { pos: &'a mut u64, range: Range, @@ -44,14 +45,14 @@ impl<'a, R: AsyncRead + Unpin> AsyncRead for ReadHandle<'a, R> { cx: &mut Context<'_>, buf: &mut ReadBuf<'_>, ) -> Poll> { - let start = buf.filled().len(); let this = self.project(); - let res = AsyncRead::poll_read( - this.rdr, - cx, - &mut buf.take(this.range.end.saturating_sub(**this.pos) as usize), - ); - **this.pos += (buf.filled().len() - start) as u64; + let start = buf.filled().len(); + let mut take_buf = buf.take(this.range.end.saturating_sub(**this.pos) as usize); + let res = AsyncRead::poll_read(this.rdr, cx, &mut take_buf); + let n = take_buf.filled().len(); + unsafe { buf.assume_init(start + n) }; + buf.advance(n); + **this.pos += n as u64; res } } From e2b8640de5590ede8c2ef2d87910f0b4ff31dbf6 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 3 Oct 2022 11:21:22 -0600 Subject: [PATCH 26/53] add beep based sound --- backend/Cargo.toml | 2 +- backend/src/sound.rs | 112 +++++++++++++++++++++++++++---------------- 2 files changed, 71 insertions(+), 43 deletions(-) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2dab81a48..5f032d56f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -42,7 +42,7 @@ path = "src/bin/avahi-alias.rs" [features] avahi = ["avahi-sys"] -default = ["avahi", "metal", "js_engine"] +default = ["avahi", "metal", "sound", "js_engine"] dev = [] metal = [] sound = [] diff --git a/backend/src/sound.rs b/backend/src/sound.rs index 83cb542bf..20be898f4 100644 --- a/backend/src/sound.rs +++ b/backend/src/sound.rs @@ -4,9 +4,10 @@ use std::time::{Duration, Instant}; use divrem::DivRem; use proptest_derive::Arbitrary; +use tokio::process::Command; use tracing::instrument; -use crate::util::FileLock; +use crate::util::{FileLock, Invoke}; use crate::{Error, ErrorKind, ResultExt}; lazy_static::lazy_static! { @@ -22,31 +23,49 @@ lazy_static::lazy_static! { pub const SOUND_LOCK_FILE: &'static str = "/etc/embassy/sound.lock"; -struct SoundInterface(Option); +struct SoundInterface { + use_beep: bool, + guard: Option, +} impl SoundInterface { #[instrument] pub async fn lease() -> Result { let guard = FileLock::new(SOUND_LOCK_FILE, true).await?; - tokio::fs::write(&*EXPORT_FILE, "0") + if Command::new("which") + .arg("beep") + .invoke(ErrorKind::NotFound) .await - .or_else(|e| { - if e.raw_os_error() == Some(16) { - Ok(()) - } else { - Err(e) - } - }) - .with_ctx(|_| (ErrorKind::SoundError, EXPORT_FILE.to_string_lossy()))?; - let instant = Instant::now(); - while tokio::fs::metadata(&*PERIOD_FILE).await.is_err() - && instant.elapsed() < Duration::from_secs(1) + .is_ok() { - tokio::time::sleep(Duration::from_millis(1)).await; + Ok(SoundInterface { + use_beep: true, + guard: Some(guard), + }) + } else { + tokio::fs::write(&*EXPORT_FILE, "0") + .await + .or_else(|e| { + if e.raw_os_error() == Some(16) { + Ok(()) + } else { + Err(e) + } + }) + .with_ctx(|_| (ErrorKind::SoundError, EXPORT_FILE.to_string_lossy()))?; + let instant = Instant::now(); + while tokio::fs::metadata(&*PERIOD_FILE).await.is_err() + && instant.elapsed() < Duration::from_secs(1) + { + tokio::time::sleep(Duration::from_millis(1)).await; + } + Ok(SoundInterface { + use_beep: false, + guard: Some(guard), + }) } - Ok(SoundInterface(Some(guard))) } #[instrument(skip(self))] - pub async fn play(&mut self, note: &Note) -> Result<(), Error> { + async fn play_pwm(&mut self, note: &Note) -> Result<(), Error> { let curr_period = tokio::fs::read_to_string(&*PERIOD_FILE) .await .with_ctx(|_| (ErrorKind::SoundError, PERIOD_FILE.to_string_lossy()))?; @@ -71,38 +90,47 @@ impl SoundInterface { Ok(()) } #[instrument(skip(self))] + async fn stop_pwm(&mut self) -> Result<(), Error> { + tokio::fs::write(&*SWITCH_FILE, "0") + .await + .with_ctx(|_| (ErrorKind::SoundError, SWITCH_FILE.to_string_lossy())) + } + #[instrument(skip(self))] + pub async fn close(mut self) -> Result<(), Error> { + if let Some(lock) = self.guard.take() { + lock.unlock().await?; + } + Ok(()) + } + #[instrument(skip(self))] pub async fn play_for_time_slice( &mut self, tempo_qpm: u16, note: &Note, time_slice: &TimeSlice, ) -> Result<(), Error> { - if let Err(e) = async { - self.play(note).await?; - tokio::time::sleep(time_slice.to_duration(tempo_qpm) * 19 / 20).await; - self.stop().await?; - tokio::time::sleep(time_slice.to_duration(tempo_qpm) / 20).await; - Ok::<_, Error>(()) - } - .await - { - // we could catch this error and propagate but I'd much prefer the original error bubble up - let _mute = self.stop().await; - Err(e) + if self.use_beep { + Command::new("beep") + .arg("-f") + .arg(note.frequency().to_string()) + .arg("-l") + .arg(time_slice.to_duration(tempo_qpm).as_millis().to_string()) + .invoke(ErrorKind::SoundError) + .await?; } else { - Ok(()) - } - } - #[instrument(skip(self))] - pub async fn stop(&mut self) -> Result<(), Error> { - tokio::fs::write(&*SWITCH_FILE, "0") + if let Err(e) = async { + self.play_pwm(note).await?; + tokio::time::sleep(time_slice.to_duration(tempo_qpm) * 19 / 20).await; + self.stop_pwm().await?; + tokio::time::sleep(time_slice.to_duration(tempo_qpm) / 20).await; + Ok::<_, Error>(()) + } .await - .with_ctx(|_| (ErrorKind::SoundError, SWITCH_FILE.to_string_lossy())) - } - #[instrument(skip(self))] - pub async fn close(mut self) -> Result<(), Error> { - if let Some(lock) = self.0.take() { - lock.unlock().await?; + { + // we could catch this error and propagate but I'd much prefer the original error bubble up + let _mute = self.stop_pwm().await; + return Err(e); + } } Ok(()) } @@ -139,7 +167,7 @@ where impl Drop for SoundInterface { fn drop(&mut self) { - let guard = self.0.take(); + let guard = self.guard.take(); tokio::spawn(async move { if let Err(e) = tokio::fs::write(&*UNEXPORT_FILE, "0").await { tracing::error!("Failed to Unexport Sound Interface: {}", e); From f39b87e3f57005f0118f9691d4ea7ae45d06f51b Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 17 Oct 2022 17:05:02 -0600 Subject: [PATCH 27/53] wip --- .../src/disk/mount/filesystem/httpdirfs.rs | 52 ++++ backend/src/disk/mount/filesystem/mod.rs | 1 + backend/src/s9pk/reader.rs | 2 +- backend/src/update/mod.rs | 294 +++++------------- backend/src/util/http_reader.rs | 45 ++- backend/src/util/io.rs | 28 ++ backend/src/util/mod.rs | 1 + backend/src/util/rsync.rs | 55 ++++ 8 files changed, 242 insertions(+), 236 deletions(-) create mode 100644 backend/src/disk/mount/filesystem/httpdirfs.rs create mode 100644 backend/src/util/rsync.rs diff --git a/backend/src/disk/mount/filesystem/httpdirfs.rs b/backend/src/disk/mount/filesystem/httpdirfs.rs new file mode 100644 index 000000000..fda437ec3 --- /dev/null +++ b/backend/src/disk/mount/filesystem/httpdirfs.rs @@ -0,0 +1,52 @@ +use std::path::Path; + +use async_trait::async_trait; +use digest::generic_array::GenericArray; +use digest::{Digest, OutputSizeUser}; +use reqwest::Url; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; + +use super::{FileSystem, MountType}; +use crate::util::Invoke; +use crate::Error; + +pub async fn mount_httpdirfs(url: &Url, mountpoint: impl AsRef) -> Result<(), Error> { + tokio::fs::create_dir_all(mountpoint.as_ref()).await?; + let mut cmd = tokio::process::Command::new("httpdirfs"); + cmd.arg("--cache") + .arg("--single-file-mode") + .arg(url.as_str()) + .arg(mountpoint.as_ref()); + cmd.invoke(crate::ErrorKind::Filesystem).await?; + Ok(()) +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub struct HttpDirFS { + url: Url, +} +impl HttpDirFS { + pub fn new(url: Url) -> Self { + HttpDirFS { url } + } +} +#[async_trait] +impl FileSystem for HttpDirFS { + async fn mount + Send + Sync>( + &self, + mountpoint: P, + _mount_type: MountType, + ) -> Result<(), Error> { + mount_httpdirfs(&self.url, mountpoint).await + } + async fn source_hash( + &self, + ) -> Result::OutputSize>, Error> { + let mut sha = Sha256::new(); + sha.update("HttpDirFS"); + sha.update(self.url.as_str()); + Ok(sha.finalize()) + } +} diff --git a/backend/src/disk/mount/filesystem/mod.rs b/backend/src/disk/mount/filesystem/mod.rs index 01fe48d8c..5d3c431d4 100644 --- a/backend/src/disk/mount/filesystem/mod.rs +++ b/backend/src/disk/mount/filesystem/mod.rs @@ -10,6 +10,7 @@ use crate::Error; pub mod block_dev; pub mod cifs; pub mod ecryptfs; +pub mod httpdirfs; pub mod label; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/backend/src/s9pk/reader.rs b/backend/src/s9pk/reader.rs index 6026a37f1..9c7d60e7b 100644 --- a/backend/src/s9pk/reader.rs +++ b/backend/src/s9pk/reader.rs @@ -9,7 +9,7 @@ use std::task::{Context, Poll}; use color_eyre::eyre::eyre; use digest_old::Output; use ed25519_dalek::PublicKey; -use futures::{FutureExt, TryStreamExt}; +use futures::TryStreamExt; use sha2_old::{Digest, Sha512}; use tokio::fs::File; use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, ReadBuf}; diff --git a/backend/src/update/mod.rs b/backend/src/update/mod.rs index 5faa96835..b4dc7c830 100644 --- a/backend/src/update/mod.rs +++ b/backend/src/update/mod.rs @@ -9,6 +9,7 @@ use color_eyre::eyre::{eyre, Result}; use digest::Digest; use emver::Version; use futures::Stream; +use helpers::AtomicFile; use lazy_static::lazy_static; use patch_db::{DbHandle, LockType, Revision}; use regex::Regex; @@ -25,6 +26,8 @@ use tracing::instrument; use crate::context::RpcContext; use crate::db::model::UpdateProgress; use crate::disk::mount::filesystem::block_dev::BlockDev; +use crate::disk::mount::filesystem::httpdirfs::HttpDirFS; +use crate::disk::mount::filesystem::ReadOnly; use crate::disk::mount::filesystem::{FileSystem, ReadWrite}; use crate::disk::mount::guard::TmpMountGuard; use crate::disk::BOOT_RW_PATH; @@ -33,6 +36,7 @@ use crate::sound::{ CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4, }; use crate::update::latest_information::LatestInformation; +use crate::util::rsync::Rsync; use crate::util::Invoke; use crate::version::{Current, VersionT}; use crate::{Error, ErrorKind, ResultExt}; @@ -86,46 +90,6 @@ fn display_update_result(status: UpdateResult, _: &ArgMatches) { const HEADER_KEY: &str = "x-eos-hash"; -#[derive(Debug, Clone, Copy)] -pub enum WritableDrives { - Green, - Blue, -} -impl WritableDrives { - pub fn label(&self) -> &'static str { - match self { - Self::Green => "green", - Self::Blue => "blue", - } - } - pub fn block_dev(&self) -> &'static Path { - Path::new(match self { - Self::Green => "/dev/mmcblk0p3", - Self::Blue => "/dev/mmcblk0p4", - }) - } - pub fn part_uuid(&self) -> &'static str { - match self { - Self::Green => "cb15ae4d-03", - Self::Blue => "cb15ae4d-04", - } - } - pub fn as_fs(&self) -> impl FileSystem { - BlockDev::new(self.block_dev()) - } -} - -/// This will be where we are going to be putting the new update -#[derive(Debug, Clone, Copy)] -pub struct NewLabel(pub WritableDrives); - -/// This is our current label where the os is running -pub struct CurrentLabel(pub WritableDrives); - -lazy_static! { - static ref PARSE_COLOR: Regex = Regex::new("LABEL=(\\w+)[ \t]+/").unwrap(); -} - #[instrument(skip(ctx))] async fn maybe_do_update( ctx: RpcContext, @@ -172,26 +136,41 @@ async fn maybe_do_update( return Ok(None); } - let (new_label, _current_label) = query_mounted_label().await?; - let (size, download) = download_file( - ctx.db.handle(), - &EosUrl { - base: marketplace_url, - version: latest_version.clone(), - }, - new_label, + // mount httpdirfs + // losetup remote fs + // BEGIN TASK + // rsync fs + // validate (hash) fs + // kernel update? + // swap selected fs + let new_block_dev = TmpMountGuard::mount( + &HttpDirFS::new( + EosUrl { + base: marketplace_url, + version: latest_version, + } + .to_string() + .parse()?, + ), + ReadOnly, + ) + .await?; + let new_fs = TmpMountGuard::mount( + &BlockDev::new(new_block_dev.as_ref().join("eos.img")), + ReadOnly, ) .await?; + status.update_progress = Some(UpdateProgress { - size, + size: Some(100), downloaded: 0, }); status.save(&mut tx).await?; let rev = tx.commit().await?; tokio::spawn(async move { + let res = do_update(ctx.clone(), new_fs, new_block_dev).await; let mut db = ctx.db.handle(); - let res = do_update(download, new_label).await; let mut status = crate::db::DatabaseModel::new() .server_info() .status_info() @@ -245,47 +224,35 @@ async fn maybe_do_update( Ok(rev) } -#[instrument(skip(download))] +#[instrument(skip(ctx, new_fs, new_block_dev))] async fn do_update( - download: impl Future>, - new_label: NewLabel, + ctx: RpcContext, + new_fs: TmpMountGuard, + new_block_dev: TmpMountGuard, ) -> Result<(), Error> { - download.await?; - copy_machine_id(new_label).await?; - copy_ssh_host_keys(new_label).await?; - swap_boot_label(new_label).await?; - - Ok(()) -} + let mut rsync = Rsync::new(new_fs.as_ref().join(""), "/gboverlay/unselected")?; + while let Some(progress) = rsync.progress.next().await { + crate::db::DatabaseModel::new() + .server_info() + .status_info() + .update_progress() + .put( + &mut ctx.db.handle(), + &UpdateProgress { + size: Some(100), + downloaded: (100.0 * progress) as u64, + }, + ) + .await?; + } + new_fs.unmount().await?; + new_block_dev.unmount().await?; -#[instrument] -pub async fn query_mounted_label() -> Result<(NewLabel, CurrentLabel), Error> { - let output = tokio::fs::read_to_string("/etc/fstab") - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, "/etc/fstab"))?; + copy_machine_id().await?; + copy_ssh_host_keys().await?; + swap_boot_label().await?; - match &PARSE_COLOR.captures(&output).ok_or_else(|| { - Error::new( - eyre!("Can't find pattern in {}", output), - crate::ErrorKind::Filesystem, - ) - })?[1] - { - x if x == WritableDrives::Green.label() => Ok(( - NewLabel(WritableDrives::Blue), - CurrentLabel(WritableDrives::Green), - )), - x if x == WritableDrives::Blue.label() => Ok(( - NewLabel(WritableDrives::Green), - CurrentLabel(WritableDrives::Blue), - )), - e => { - return Err(Error::new( - eyre!("Could not find a mounted resource for {}", e), - crate::ErrorKind::Filesystem, - )) - } - } + Ok(()) } #[derive(Debug)] @@ -306,173 +273,54 @@ impl std::fmt::Display for EosUrl { } } -#[instrument(skip(db))] -async fn download_file<'a, Db: DbHandle + 'a>( - mut db: Db, - eos_url: &EosUrl, - new_label: NewLabel, -) -> Result<(Option, impl Future> + 'a), Error> { - let download_request = reqwest::get(eos_url.to_string()) - .await - .with_kind(ErrorKind::Network)?; - let size = download_request - .headers() - .get("content-length") - .and_then(|a| a.to_str().ok()) - .map(|l| l.parse()) - .transpose()?; - Ok((size, async move { - let hash_from_header: String = download_request - .headers() - .get(HEADER_KEY) - .ok_or_else(|| Error::new(eyre!("No {} in headers", HEADER_KEY), ErrorKind::Network))? - .to_str() - .with_kind(ErrorKind::InvalidRequest)? - .to_owned(); - let stream_download = download_request.bytes_stream(); - let file_sum = write_stream_to_label(&mut db, size, stream_download, new_label).await?; - check_download(&hash_from_header, file_sum).await?; - Ok(()) - })) -} - -#[instrument(skip(db, stream_download))] -async fn write_stream_to_label( - db: &mut Db, - size: Option, - stream_download: impl Stream>, - file: NewLabel, -) -> Result, Error> { - let block_dev = file.0.block_dev(); - let mut file = tokio::fs::OpenOptions::new() - .write(true) - .open(&block_dev) - .await - .with_kind(ErrorKind::Filesystem)?; - let mut hasher = Sha256::new(); - pin!(stream_download); - let mut downloaded = 0; - let mut last_progress_update = Instant::now(); - while let Some(item) = stream_download - .next() - .await - .transpose() - .with_kind(ErrorKind::Network)? - { - file.write_all(&item) - .await - .with_kind(ErrorKind::Filesystem)?; - downloaded += item.len() as u64; - if last_progress_update.elapsed() > Duration::from_secs(1) { - last_progress_update = Instant::now(); - crate::db::DatabaseModel::new() - .server_info() - .status_info() - .update_progress() - .put(db, &UpdateProgress { size, downloaded }) - .await?; - } - hasher.update(item); - } - file.flush().await.with_kind(ErrorKind::Filesystem)?; - file.shutdown().await.with_kind(ErrorKind::Filesystem)?; - file.sync_all().await.with_kind(ErrorKind::Filesystem)?; - drop(file); - Ok(hasher.finalize().to_vec()) -} - -#[instrument] -async fn check_download(hash_from_header: &str, file_digest: Vec) -> Result<(), Error> { - if hex::decode(hash_from_header).with_kind(ErrorKind::Network)? != file_digest { - return Err(Error::new( - eyre!("Hash sum does not match source"), - ErrorKind::Network, - )); - } - Ok(()) -} - -async fn copy_machine_id(new_label: NewLabel) -> Result<(), Error> { - let new_guard = TmpMountGuard::mount(&new_label.0.as_fs(), ReadWrite).await?; - tokio::fs::copy("/etc/machine-id", new_guard.as_ref().join("etc/machine-id")).await?; - new_guard.unmount().await?; +async fn copy_machine_id() -> Result<(), Error> { + tokio::fs::copy("/etc/machine-id", "/gboverlay/unselected/etc/machine-id").await?; Ok(()) } -async fn copy_ssh_host_keys(new_label: NewLabel) -> Result<(), Error> { - let new_guard = TmpMountGuard::mount(&new_label.0.as_fs(), ReadWrite).await?; +async fn copy_ssh_host_keys() -> Result<(), Error> { tokio::fs::copy( "/etc/ssh/ssh_host_rsa_key", - new_guard.as_ref().join("etc/ssh/ssh_host_rsa_key"), + "/gboverlay/unselected/etc/ssh/ssh_host_rsa_key", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_rsa_key.pub", - new_guard.as_ref().join("etc/ssh/ssh_host_rsa_key.pub"), + "/gboverlay/unselected/etc/ssh/ssh_host_rsa_key.pub", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ecdsa_key", - new_guard.as_ref().join("etc/ssh/ssh_host_ecdsa_key"), + "/gboverlay/unselected/etc/ssh/ssh_host_ecdsa_key", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ecdsa_key.pub", - new_guard.as_ref().join("etc/ssh/ssh_host_ecdsa_key.pub"), + "/gboverlay/unselected/etc/ssh/ssh_host_ecdsa_key.pub", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ed25519_key", - new_guard.as_ref().join("etc/ssh/ssh_host_ed25519_key"), + "/gboverlay/unselected/etc/ssh/ssh_host_ed25519_key", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ed25519_key.pub", - new_guard.as_ref().join("etc/ssh/ssh_host_ed25519_key.pub"), + "/gboverlay/unselected/etc/ssh/ssh_host_ed25519_key.pub", ) .await?; - new_guard.unmount().await?; Ok(()) } #[instrument] -async fn swap_boot_label(new_label: NewLabel) -> Result<(), Error> { - let block_dev = new_label.0.block_dev(); - Command::new("e2label") - .arg(block_dev) - .arg(new_label.0.label()) - .invoke(crate::ErrorKind::BlockDevice) - .await?; - let mounted = TmpMountGuard::mount(&new_label.0.as_fs(), ReadWrite).await?; - Command::new("sed") - .arg("-i") - .arg(&format!( - "s/LABEL=\\(blue\\|green\\)/LABEL={}/g", - new_label.0.label() - )) - .arg(mounted.as_ref().join("etc/fstab")) - .invoke(crate::ErrorKind::Filesystem) - .await?; - mounted.unmount().await?; - Command::new("sed") - .arg("-i") - .arg(&format!( - "s/PARTUUID=cb15ae4d-\\(03\\|04\\)/PARTUUID={}/g", - new_label.0.part_uuid() - )) - .arg(Path::new(BOOT_RW_PATH).join("cmdline.txt.orig")) - .invoke(crate::ErrorKind::Filesystem) - .await?; - Command::new("sed") - .arg("-i") - .arg(&format!( - "s/PARTUUID=cb15ae4d-\\(03\\|04\\)/PARTUUID={}/g", - new_label.0.part_uuid() - )) - .arg(Path::new(BOOT_RW_PATH).join("cmdline.txt")) - .invoke(crate::ErrorKind::Filesystem) - .await?; - +async fn swap_boot_label() -> Result<(), Error> { + let current = tokio::fs::read_to_string("/gboverlay/config/selected").await?; + let target = if current == "green" { "blue" } else { "green" }; + let mut selected = AtomicFile::new("/gboverlay/config/selected", None::<&str>) + .await + .with_kind(ErrorKind::Filesystem)?; + selected.write_all(target.as_bytes()).await?; + selected.save().await.with_kind(ErrorKind::Filesystem)?; UPDATED.store(true, Ordering::SeqCst); Ok(()) } diff --git a/backend/src/util/http_reader.rs b/backend/src/util/http_reader.rs index a9921f5c9..b4b06b1ca 100644 --- a/backend/src/util/http_reader.rs +++ b/backend/src/util/http_reader.rs @@ -1,14 +1,13 @@ use std::cmp::min; use std::convert::TryFrom; use std::fmt::Display; +use std::future::Future; use std::io::Error as StdIOError; use std::pin::Pin; use std::task::{Context, Poll}; use color_eyre::eyre::eyre; -use futures::future::BoxFuture; -use futures::stream::BoxStream; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, Stream}; use http::header::{ACCEPT_RANGES, CONTENT_LENGTH, RANGE}; use hyper::body::Bytes; use pin_project::pin_project; @@ -30,9 +29,27 @@ pub struct HttpReader { enum ReadInProgress { None, InProgress( - BoxFuture<'static, Result>, Error>>, + Pin< + Box< + dyn Future< + Output = Result< + Pin< + Box< + dyn Stream> + + Send + + Sync + + 'static, + >, + >, + Error, + >, + > + Send + + Sync + + 'static, + >, + >, ), - Complete(BoxStream<'static, Result>), + Complete(Pin> + Send + Sync + 'static>>), } impl ReadInProgress { fn take(&mut self) -> Self { @@ -142,11 +159,14 @@ impl HttpReader { start: usize, len: usize, total_bytes: usize, - ) -> Result>, Error> { + ) -> Result< + Pin> + Send + Sync + 'static>>, + Error, + > { let end = min(start + len, total_bytes) - 1; if start > end { - return Ok(futures::stream::empty().boxed()); + return Ok(Box::pin(futures::stream::empty())); } let data_range = format!("{}={}-{} ", range_unit.unwrap_or_default(), start, end); @@ -160,7 +180,7 @@ impl HttpReader { .error_for_status() .with_kind(crate::ErrorKind::Network)?; - Ok(data_resp.bytes_stream().boxed()) + Ok(Box::pin(data_resp.bytes_stream())) } } @@ -171,7 +191,9 @@ impl AsyncRead for HttpReader { buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { fn poll_complete( - body: &mut BoxStream<'static, Result>, + body: &mut Pin< + Box> + Send + Sync + 'static>, + >, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll>> { @@ -221,15 +243,14 @@ impl AsyncRead for HttpReader { continue; } }, - ReadInProgress::None => HttpReader::get_range( + ReadInProgress::None => Box::pin(HttpReader::get_range( *this.range_unit, this.http_client.clone(), this.http_url.clone(), *this.cursor_pos, buf.remaining(), *this.total_bytes, - ) - .boxed(), + )), ReadInProgress::InProgress(fut) => fut, }; diff --git a/backend/src/util/io.rs b/backend/src/util/io.rs index af3b5b7da..d30bdaaf5 100644 --- a/backend/src/util/io.rs +++ b/backend/src/util/io.rs @@ -295,3 +295,31 @@ impl AsyncRead for BufferedWriteReader { } } } + +#[pin_project::pin_project] +pub struct ByteReplacementReader { + pub replace: u8, + pub with: u8, + #[pin] + pub inner: R, +} +impl AsyncRead for ByteReplacementReader { + fn poll_read( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> std::task::Poll> { + let this = self.project(); + match this.inner.poll_read(cx, buf) { + Poll::Ready(Ok(())) => { + for idx in 0..buf.filled().len() { + if buf.filled()[idx] == *this.replace { + buf.filled_mut()[idx] = *this.with; + } + } + Poll::Ready(Ok(())) + } + a => a, + } + } +} diff --git a/backend/src/util/mod.rs b/backend/src/util/mod.rs index ebfb14192..afa65baa2 100644 --- a/backend/src/util/mod.rs +++ b/backend/src/util/mod.rs @@ -27,6 +27,7 @@ pub mod config; pub mod http_reader; pub mod io; pub mod logger; +pub mod rsync; pub mod serde; #[derive(Clone, Copy, Debug)] diff --git a/backend/src/util/rsync.rs b/backend/src/util/rsync.rs new file mode 100644 index 000000000..437635c64 --- /dev/null +++ b/backend/src/util/rsync.rs @@ -0,0 +1,55 @@ +use std::path::Path; + +use helpers::NonDetachingJoinHandle; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::{Child, Command}; +use tokio::sync::watch; +use tokio_stream::wrappers::WatchStream; + +use crate::util::io::ByteReplacementReader; +use crate::Error; + +pub struct Rsync { + pub command: Child, + _progress_task: NonDetachingJoinHandle>, + pub progress: WatchStream, +} +impl Rsync { + pub fn new(src: impl AsRef, dst: impl AsRef) -> Result { + let mut command = Command::new("rsync") + .arg("-a") + .arg("--no-devices") + .arg("--info=progress2") + .arg(src.as_ref()) + .arg(dst.as_ref()) + .kill_on_drop(true) + .stdout(std::process::Stdio::piped()) + .spawn()?; + let cmd_stdout = command.stdout.take().unwrap(); + let (send, recv) = watch::channel(0.0); + let progress_task = tokio::spawn(async move { + let mut lines = BufReader::new(ByteReplacementReader { + replace: b'\r', + with: b'\n', + inner: cmd_stdout, + }) + .lines(); + while let Some(line) = lines.next_line().await? { + if let Some(percentage) = line + .split_ascii_whitespace() + .find_map(|col| col.strip_suffix("%")) + { + send.send(percentage.parse::()? / 100.0).unwrap(); + } + } + Ok(()) + }) + .into(); + Ok(Rsync { + command, + _progress_task: progress_task, + progress: WatchStream::new(recv), + }) + } + // pub fn +} From 175fc6139e34689debee835f4f15f583ad8956c9 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Mon, 17 Oct 2022 17:06:32 -0600 Subject: [PATCH 28/53] wip --- backend/src/version/v0_3_0_1.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/src/version/v0_3_0_1.rs b/backend/src/version/v0_3_0_1.rs index 577eb9f66..23cc9bc58 100644 --- a/backend/src/version/v0_3_0_1.rs +++ b/backend/src/version/v0_3_0_1.rs @@ -5,7 +5,6 @@ use tokio::process::Command; use super::*; use crate::disk::BOOT_RW_PATH; -use crate::update::query_mounted_label; use crate::util::Invoke; const V0_3_0_1: emver::Version = emver::Version::new(0, 3, 0, 1); @@ -25,16 +24,6 @@ impl VersionT for Version { &*v0_3_0::V0_3_0_COMPAT } async fn up(&self, _db: &mut Db) -> Result<(), Error> { - let (_, current) = query_mounted_label().await?; - Command::new("sed") - .arg("-i") - .arg(&format!( - "s/PARTUUID=cb15ae4d-\\(03\\|04\\)/PARTUUID={}/g", - current.0.part_uuid() - )) - .arg(Path::new(BOOT_RW_PATH).join("cmdline.txt.orig")) - .invoke(crate::ErrorKind::Filesystem) - .await?; Ok(()) } async fn down(&self, _db: &mut Db) -> Result<(), Error> { From 7dc8ba39b0d7f8891da245ccea7c4312bfe9ffe9 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 14:49:40 -0600 Subject: [PATCH 29/53] wip --- backend/src/context/rpc.rs | 10 +- backend/src/disk/mod.rs | 10 +- backend/src/disk/util.rs | 232 +++++++++++++++++-------------------- backend/src/init.rs | 16 +-- backend/src/net/wifi.rs | 37 ++++-- 5 files changed, 153 insertions(+), 152 deletions(-) diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index 0b0270efd..5cb5ad678 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -43,7 +43,7 @@ use crate::{Error, ErrorKind, ResultExt}; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct RpcContextConfig { - pub wifi_interface: String, + pub wifi_interface: Option, pub ethernet_interface: String, pub os_partitions: OsPartitionInfo, pub migration_batch_rows: Option, @@ -121,7 +121,7 @@ impl RpcContextConfig { pub struct RpcContextSeed { is_closed: AtomicBool, pub os_partitions: OsPartitionInfo, - pub wifi_interface: String, + pub wifi_interface: Option, pub ethernet_interface: String, pub bind_rpc: SocketAddr, pub bind_ws: SocketAddr, @@ -141,7 +141,7 @@ pub struct RpcContextSeed { pub notification_manager: NotificationManager, pub open_authed_websockets: Mutex>>>, pub rpc_stream_continuations: Mutex>, - pub wifi_manager: Arc>, + pub wifi_manager: Option>>, } pub struct RpcCleanReceipts { @@ -276,7 +276,9 @@ impl RpcContext { notification_manager, open_authed_websockets: Mutex::new(BTreeMap::new()), rpc_stream_continuations: Mutex::new(BTreeMap::new()), - wifi_manager: Arc::new(RwLock::new(WpaCli::init(base.wifi_interface))), + wifi_manager: base + .wifi_interface + .map(|i| Arc::new(RwLock::new(WpaCli::init(i)))), }); let res = Self(seed); diff --git a/backend/src/disk/mod.rs b/backend/src/disk/mod.rs index 6fb9afbb3..e734c8e55 100644 --- a/backend/src/disk/mod.rs +++ b/backend/src/disk/mod.rs @@ -21,18 +21,12 @@ pub const REPAIR_DISK_PATH: &str = "/embassy-os/repair-disk"; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct OsPartitionInfo { - pub disk: PathBuf, pub boot: PathBuf, - pub embassy: PathBuf, - pub green: PathBuf, - pub blue: PathBuf, + pub root: PathBuf, } impl OsPartitionInfo { pub fn contains(&self, logicalname: impl AsRef) -> bool { - &*self.boot == logicalname.as_ref() - || &*self.embassy == logicalname.as_ref() - || &*self.green == logicalname.as_ref() - || &*self.blue == logicalname.as_ref() + &*self.boot == logicalname.as_ref() || &*self.root == logicalname.as_ref() } } diff --git a/backend/src/disk/util.rs b/backend/src/disk/util.rs index b26849f3a..79d21ef36 100644 --- a/backend/src/disk/util.rs +++ b/backend/src/disk/util.rs @@ -234,8 +234,12 @@ pub async fn recovery_info( #[instrument] pub async fn list(os: &OsPartitionInfo) -> Result, Error> { + struct DiskIndex { + parts: IndexSet, + internal: bool, + } let disk_guids = pvscan().await?; - let (disks, internal) = tokio_stream::wrappers::ReadDirStream::new( + let disks = tokio_stream::wrappers::ReadDirStream::new( tokio::fs::read_dir(DISK_PATH) .await .with_ctx(|_| (crate::ErrorKind::Filesystem, DISK_PATH))?, @@ -247,8 +251,8 @@ pub async fn list(os: &OsPartitionInfo) -> Result, Error> { ) }) .try_fold( - (BTreeMap::new(), Vec::new()), - |(mut disks, mut internal), dir_entry| async move { + BTreeMap::::new(), + |mut disks, dir_entry| async move { if let Some(disk_path) = dir_entry.path().file_name().and_then(|s| s.to_str()) { let (disk_path, part_path) = if let Some(end) = PARTITION_REGEX.find(disk_path) { ( @@ -277,143 +281,123 @@ pub async fn list(os: &OsPartitionInfo) -> Result, Error> { } else { None }; - if disk == os.disk { - if let Some(part) = part { - if !os.contains(&part) { - internal.push(part.clone()); - } - } - } else { - if !disks.contains_key(&disk) { - disks.insert(disk.clone(), IndexSet::new()); - } - if let Some(part) = part { - disks.get_mut(&disk).unwrap().insert(part); + if !disks.contains_key(&disk) { + disks.insert( + disk.clone(), + DiskIndex { + parts: IndexSet::new(), + internal: false, + }, + ); + } + if let Some(part) = part { + if os.contains(&part) { + disks.get_mut(&disk).unwrap().internal = true; + } else { + disks.get_mut(&disk).unwrap().parts.insert(part); } } } - Ok((disks, internal)) + Ok(disks) }, ) .await?; let mut res = Vec::with_capacity(disks.len()); - for (disk, parts) in disks { - let mut guid: Option = None; - let mut partitions = Vec::with_capacity(parts.len()); - let vendor = get_vendor(&disk) - .await - .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source)) - .unwrap_or_default(); - let model = get_model(&disk) - .await - .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source)) - .unwrap_or_default(); - let capacity = get_capacity(&disk) - .await - .map_err(|e| { - tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source) - }) - .unwrap_or_default(); - if let Some(g) = disk_guids.get(&disk) { - guid = g.clone(); + for (disk, index) in disks { + if index.internal { + for part in index.parts { + let mut disk_info = disk_info(disk.clone()).await; + if let Some(g) = disk_guids.get(&disk) { + disk_info.guid = g.clone(); + } else { + disk_info.partitions = vec![part_info(part).await]; + } + res.push(disk_info); + } } else { - for part in parts { - let mut embassy_os = None; - let label = get_label(&part).await?; - let capacity = get_capacity(&part) - .await - .map_err(|e| { - tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source) - }) - .unwrap_or_default(); - let mut used = None; - - match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await { - Err(e) => tracing::warn!("Could not collect usage information: {}", e.source), - Ok(mount_guard) => { - used = get_used(&mount_guard) - .await - .map_err(|e| { - tracing::warn!( - "Could not get usage of {}: {}", - part.display(), - e.source - ) - }) - .ok(); - if let Some(recovery_info) = match recovery_info(&mount_guard).await { - Ok(a) => a, - Err(e) => { - tracing::error!( - "Error fetching unencrypted backup metadata: {}", - e - ); - None - } - } { - embassy_os = Some(recovery_info) - } - mount_guard.unmount().await?; - } + let mut disk_info = disk_info(disk).await; + disk_info.partitions = Vec::with_capacity(index.parts.len()); + if let Some(g) = disk_guids.get(&disk_info.logicalname) { + disk_info.guid = g.clone(); + } else { + for part in index.parts { + disk_info.partitions.push(part_info(part).await); } - - partitions.push(PartitionInfo { - logicalname: part, - label, - capacity, - used, - embassy_os, - }); } + res.push(disk_info); } - res.push(DiskInfo { - logicalname: disk, - vendor, - model, - partitions, - capacity, - guid, - }) } - for disk in internal { - let mut guid: Option = None; - let vendor = get_vendor(&os.disk) - .await - .map_err(|e| { - tracing::warn!( - "Could not get vendor of {}: {}", - os.disk.display(), - e.source - ) - }) - .unwrap_or_default(); - let model = get_model(&os.disk) - .await - .map_err(|e| { - tracing::warn!("Could not get model of {}: {}", os.disk.display(), e.source) - }) - .unwrap_or_default(); - let capacity = get_capacity(&disk) - .await - .map_err(|e| { - tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source) - }) - .unwrap_or_default(); - if let Some(g) = disk_guids.get(&disk) { - guid = g.clone(); + + Ok(res) +} + +async fn disk_info(disk: PathBuf) -> DiskInfo { + let vendor = get_vendor(&disk) + .await + .map_err(|e| tracing::warn!("Could not get vendor of {}: {}", disk.display(), e.source)) + .unwrap_or_default(); + let model = get_model(&disk) + .await + .map_err(|e| tracing::warn!("Could not get model of {}: {}", disk.display(), e.source)) + .unwrap_or_default(); + let capacity = get_capacity(&disk) + .await + .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", disk.display(), e.source)) + .unwrap_or_default(); + DiskInfo { + logicalname: disk, + vendor, + model, + partitions: Vec::new(), + capacity, + guid: None, + } +} + +async fn part_info(part: PathBuf) -> PartitionInfo { + let mut embassy_os = None; + let label = get_label(&part) + .await + .map_err(|e| tracing::warn!("Could not get label of {}: {}", part.display(), e.source)) + .unwrap_or_default(); + let capacity = get_capacity(&part) + .await + .map_err(|e| tracing::warn!("Could not get capacity of {}: {}", part.display(), e.source)) + .unwrap_or_default(); + let mut used = None; + + match TmpMountGuard::mount(&BlockDev::new(&part), ReadOnly).await { + Err(e) => tracing::warn!("Could not collect usage information: {}", e.source), + Ok(mount_guard) => { + used = get_used(&mount_guard) + .await + .map_err(|e| { + tracing::warn!("Could not get usage of {}: {}", part.display(), e.source) + }) + .ok(); + if let Some(recovery_info) = match recovery_info(&mount_guard).await { + Ok(a) => a, + Err(e) => { + tracing::error!("Error fetching unencrypted backup metadata: {}", e); + None + } + } { + embassy_os = Some(recovery_info) + } + if let Err(e) = mount_guard.unmount().await { + tracing::error!("Error unmounting partition {}: {}", part.display(), e); + } } - res.push(DiskInfo { - logicalname: disk, - vendor, - model, - partitions: Vec::new(), - capacity, - guid, - }) } - Ok(res) + PartitionInfo { + logicalname: part, + label, + capacity, + used, + embassy_os, + } } fn parse_pvscan_output(pvscan_output: &str) -> BTreeMap> { diff --git a/backend/src/init.rs b/backend/src/init.rs index bfaa01d40..e193c59a8 100644 --- a/backend/src/init.rs +++ b/backend/src/init.rs @@ -332,13 +332,15 @@ pub async fn init(cfg: &RpcContextConfig) -> Result { crate::ssh::sync_keys_from_db(&secret_store, "/home/start9/.ssh/authorized_keys").await?; tracing::info!("Synced SSH Keys"); - crate::net::wifi::synchronize_wpa_supplicant_conf( - &cfg.datadir().join("main"), - &cfg.wifi_interface, - &receipts.last_wifi_region.get(&mut handle).await?, - ) - .await?; - tracing::info!("Synchronized wpa_supplicant.conf"); + if let Some(wifi_interface) = &cfg.wifi_interface { + crate::net::wifi::synchronize_wpa_supplicant_conf( + &cfg.datadir().join("main"), + wifi_interface, + &receipts.last_wifi_region.get(&mut handle).await?, + ) + .await?; + tracing::info!("Synchronized WiFi"); + } receipts .status_info .set( diff --git a/backend/src/net/wifi.rs b/backend/src/net/wifi.rs index af44e1acc..5931e26ac 100644 --- a/backend/src/net/wifi.rs +++ b/backend/src/net/wifi.rs @@ -18,6 +18,19 @@ use crate::util::serde::{display_serializable, IoFormat}; use crate::util::{display_none, Invoke}; use crate::{Error, ErrorKind}; +type WifiManager = Arc>; + +pub fn wifi_manager(ctx: &RpcContext) -> Result<&WifiManager, Error> { + if let Some(wifi_manager) = ctx.wifi_manager.as_ref() { + Ok(wifi_manager) + } else { + Err(Error::new( + color_eyre::eyre::eyre!("No WiFi interface available"), + ErrorKind::Wifi, + )) + } +} + #[command(subcommands(add, connect, delete, get, country, available))] pub async fn wifi() -> Result<(), Error> { Ok(()) @@ -42,6 +55,7 @@ pub async fn add( #[arg] priority: isize, #[arg] connect: bool, ) -> Result<(), Error> { + let wifi_manager = wifi_manager(&ctx)?; if !ssid.is_ascii() { return Err(Error::new( color_eyre::eyre::eyre!("SSID may not have special characters"), @@ -56,7 +70,7 @@ pub async fn add( } async fn add_procedure( db: impl DbHandle, - wifi_manager: Arc>, + wifi_manager: WifiManager, ssid: &Ssid, password: &Psk, priority: isize, @@ -71,7 +85,7 @@ pub async fn add( } if let Err(err) = add_procedure( &mut ctx.db.handle(), - ctx.wifi_manager.clone(), + wifi_manager.clone(), &Ssid(ssid.clone()), &Psk(password.clone()), priority, @@ -91,6 +105,7 @@ pub async fn add( #[command(display(display_none))] #[instrument(skip(ctx))] pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<(), Error> { + let wifi_manager = wifi_manager(&ctx)?; if !ssid.is_ascii() { return Err(Error::new( color_eyre::eyre::eyre!("SSID may not have special characters"), @@ -99,7 +114,7 @@ pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result< } async fn connect_procedure( mut db: impl DbHandle, - wifi_manager: Arc>, + wifi_manager: WifiManager, ssid: &Ssid, ) -> Result<(), Error> { let wpa_supplicant = wifi_manager.read().await; @@ -125,7 +140,7 @@ pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result< if let Err(err) = connect_procedure( &mut ctx.db.handle(), - ctx.wifi_manager.clone(), + wifi_manager.clone(), &Ssid(ssid.clone()), ) .await @@ -142,16 +157,17 @@ pub async fn connect(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result< #[command(display(display_none))] #[instrument(skip(ctx))] pub async fn delete(#[context] ctx: RpcContext, #[arg] ssid: String) -> Result<(), Error> { + let wifi_manager = wifi_manager(&ctx)?; if !ssid.is_ascii() { return Err(Error::new( color_eyre::eyre::eyre!("SSID may not have special characters"), ErrorKind::Wifi, )); } - let wpa_supplicant = ctx.wifi_manager.read().await; + let wpa_supplicant = wifi_manager.read().await; let current = wpa_supplicant.get_current_network().await?; drop(wpa_supplicant); - let mut wpa_supplicant = ctx.wifi_manager.write().await; + let mut wpa_supplicant = wifi_manager.write().await; let ssid = Ssid(ssid); let is_current_being_removed = matches!(current, Some(current) if current == ssid); let is_current_removed_and_no_hardwire = @@ -284,7 +300,8 @@ pub async fn get( #[arg(long = "format")] format: Option, ) -> Result { - let wpa_supplicant = ctx.wifi_manager.read().await; + let wifi_manager = wifi_manager(&ctx)?; + let wpa_supplicant = wifi_manager.read().await; let (list_networks, current_res, country_res, ethernet_res, signal_strengths) = tokio::join!( wpa_supplicant.list_networks_low(), wpa_supplicant.get_current_network(), @@ -337,7 +354,8 @@ pub async fn get_available( #[arg(long = "format")] format: Option, ) -> Result, Error> { - let wpa_supplicant = ctx.wifi_manager.read().await; + let wifi_manager = wifi_manager(&ctx)?; + let wpa_supplicant = wifi_manager.read().await; let (wifi_list, network_list) = tokio::join!( wpa_supplicant.list_wifi_low(), wpa_supplicant.list_networks_low() @@ -365,13 +383,14 @@ pub async fn set_country( #[context] ctx: RpcContext, #[arg(parse(country_code_parse))] country: CountryCode, ) -> Result<(), Error> { + let wifi_manager = wifi_manager(&ctx)?; if !interface_connected(&ctx.ethernet_interface).await? { return Err(Error::new( color_eyre::eyre::eyre!("Won't change country without hardwire connection"), crate::ErrorKind::Wifi, )); } - let mut wpa_supplicant = ctx.wifi_manager.write().await; + let mut wpa_supplicant = wifi_manager.write().await; wpa_supplicant.set_country_low(country.alpha2()).await?; for (network_id, _wifi_info) in wpa_supplicant.list_networks_low().await? { wpa_supplicant.remove_network_low(network_id).await?; From 7e5cbf5f1a6fe34065b9ce3015f6b043bc35aba2 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 19:20:45 -0600 Subject: [PATCH 30/53] separate out raspberry pi specific files --- .gitignore | 3 +- Makefile | 56 +++--- backend/build-prod.sh | 8 +- backend/src/bin/embassy-init.rs | 18 +- backend/src/bin/embassyd.rs | 9 +- backend/src/diagnostic.rs | 2 +- backend/src/disk/fsck.rs | 2 +- backend/src/disk/mod.rs | 2 +- backend/src/disk/mount/guard.rs | 2 +- backend/src/init.rs | 6 +- backend/src/setup.rs | 2 +- backend/src/volume.rs | 2 +- build/fstab | 4 - build/initialization.sh | 161 ------------------ build/lib/conflicts | 3 + build/lib/depends | 31 ++++ build/{ => lib}/docker-engine.slice | 0 build/{ => lib}/journald.conf | 0 build/{00-embassy => lib/motd} | 0 build/lib/scripts/add-apt-sources | 9 + build/lib/scripts/embassy-initramfs-module | 93 ++++++++++ build/lib/scripts/grub-probe-eos | 34 ++++ build/lib/scripts/install | 94 ++++++++++ build/lib/scripts/postinst | 90 ++++++++++ build/make-deb.sh | 5 - build/make-image.sh | 42 ----- build/partitioning.sh | 6 - build/quick-flash.sh | 132 -------------- build/{ => raspberry-pi}/init-with-sound.sh | 0 .../{ => raspberry-pi}/initialization.service | 0 build/raspberry-pi/initialization.sh | 55 ++++++ build/raspberry-pi/make-image.sh | 18 ++ build/{ => raspberry-pi}/nc-broadcast.service | 0 build/{ => raspberry-pi}/rip-image.sh | 0 build/raspberry-pi/write-image.sh | 56 ++++++ build/write-image.sh | 105 ------------ 36 files changed, 540 insertions(+), 510 deletions(-) delete mode 100644 build/fstab delete mode 100755 build/initialization.sh create mode 100644 build/lib/conflicts create mode 100644 build/lib/depends rename build/{ => lib}/docker-engine.slice (100%) rename build/{ => lib}/journald.conf (100%) rename build/{00-embassy => lib/motd} (100%) create mode 100644 build/lib/scripts/add-apt-sources create mode 100644 build/lib/scripts/embassy-initramfs-module create mode 100644 build/lib/scripts/grub-probe-eos create mode 100644 build/lib/scripts/install create mode 100644 build/lib/scripts/postinst delete mode 100755 build/make-deb.sh delete mode 100755 build/make-image.sh delete mode 100755 build/partitioning.sh delete mode 100755 build/quick-flash.sh rename build/{ => raspberry-pi}/init-with-sound.sh (100%) rename build/{ => raspberry-pi}/initialization.service (100%) create mode 100755 build/raspberry-pi/initialization.sh create mode 100755 build/raspberry-pi/make-image.sh rename build/{ => raspberry-pi}/nc-broadcast.service (100%) rename build/{ => raspberry-pi}/rip-image.sh (100%) create mode 100755 build/raspberry-pi/write-image.sh delete mode 100755 build/write-image.sh diff --git a/.gitignore b/.gitignore index 24c80865b..7b926711e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,5 @@ secrets.db /cargo-deps/**/* /ENVIRONMENT.txt /GIT_HASH.txt -/eos.tar.gz -/embassy-os-*.tar.gz +/embassyos-*.tar.gz /*.deb \ No newline at end of file diff --git a/Makefile b/Makefile index bfbdc3db7..ce5dfa14f 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ -ARCH = aarch64 +ARCH = $(shell uname -m) ENVIRONMENT_FILE = $(shell ./check-environment.sh) GIT_HASH_FILE = $(shell ./check-git-hash.sh) EMBASSY_BINS := backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-sdk backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias -EMBASSY_NATIVE_BINS := backend/target/release/embassyd backend/target/release/embassy-init backend/target/release/embassy-cli backend/target/release/embassy-sdk backend/target/release/avahi-alias EMBASSY_UIS := frontend/dist/ui frontend/dist/setup-wizard frontend/dist/diagnostic-ui EMBASSY_SRC := backend/embassyd.service backend/embassy-init.service $(EMBASSY_UIS) $(shell find build) COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compat/target/*' -and -not -name *.tar -and -not -name target) @@ -19,21 +18,20 @@ $(shell sudo true) .DELETE_ON_ERROR: -.PHONY: all gzip clean format sdk snapshots frontends ui backend -all: eos.img +.PHONY: all gzip install clean format sdk snapshots frontends ui backend -gzip: eos.tar.gz +all: $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/$(ARCH).tar system-images/binfmt/docker-images/$(ARCH).tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) -deb: embassy-os.deb +gzip: embassyos-raspi.tar.gz -eos.tar.gz: eos.img - tar --format=posix -cS -f- eos.img | $(GZIP_BIN) > eos.tar.gz +embassyos-raspi.tar.gz: embassyos-raspi.img + tar --format=posix -cS -f- eos.img | $(GZIP_BIN) > embassyos-raspi.tar.gz clean: rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip rm -f raspios.img - rm -f eos.img - rm -f eos.tar.gz + rm -f embassyos-raspi.img + rm -f embassyos-raspi.tar.gz rm -f ubuntu.img rm -f product_key.txt rm -f system-images/**/*.tar @@ -58,24 +56,23 @@ format: sdk: cd backend/ && ./install-sdk.sh -eos.img: raspios.img $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/aarch64.tar system-images/binfmt/docker-images/aarch64.tar cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) - ! test -f eos.img || rm eos.img - if [ "$(NO_KEY)" = "1" ]; then NO_KEY=1 ./build/make-image.sh; else ./build/make-image.sh; fi - -install-dependencies: $(EMBASSY_SRC) $(EMBASSY_NATIVE_BINS) system-images/compat/docker-images/aarch64.tar system-images/utils/docker-images/$(shell uname -m).tar system-images/binfmt/docker-images/$(shell uname -m).tar $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) - -# For creating dpkg. DO NOT USE -install: install-dependencies - mkdir -p $(DESTDIR)/etc/embassy - cp ENVIRONMENT.txt $(DESTDIR)/etc/embassy/ - cp GIT_HASH.txt $(DESTDIR)/etc/embassy/ +embassyos-raspi.img: all raspios.img cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast + ! test -f embassyos-raspi.img || rm embassyos-raspi.img + ./build/raspberry-pi/make-image.sh +# For creating os images. DO NOT USE +install: all mkdir -p $(DESTDIR)/usr/bin - cp backend/target/release/embassy-init $(DESTDIR)/usr/bin/ - cp backend/target/release/embassyd $(DESTDIR)/usr/bin/ - cp backend/target/release/embassy-cli $(DESTDIR)/usr/bin/ - cp backend/target/release/avahi-alias $(DESTDIR)/usr/bin/ + cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-init $(DESTDIR)/usr/bin/ + cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassyd $(DESTDIR)/usr/bin/ + cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli $(DESTDIR)/usr/bin/ + cp backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias $(DESTDIR)/usr/bin/ + cp -r build/lib $(DESTDIR)/usr/lib/embassy + + cp ENVIRONMENT.txt $(DESTDIR)/usr/lib/embassy/ + cp GIT_HASH.txt $(DESTDIR)/usr/lib/embassy/ + mkdir -p $(DESTDIR)/var/lib/embassy/system-images cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/var/lib/embassy/system-images/compat.tar cp system-images/utils/docker-images/$(shell uname -m).tar $(DESTDIR)/var/lib/embassy/system-images/utils.tar @@ -87,9 +84,6 @@ install: install-dependencies cp -r frontend/dist/ui $(DESTDIR)/var/www/html/main cp index.html $(DESTDIR)/var/www/html/ -embassy-os.deb: debian/control - ./build/make-deb.sh - system-images/compat/docker-images/aarch64.tar: $(COMPAT_SRC) cd system-images/compat && make @@ -115,13 +109,9 @@ snapshots: libs/snapshot-creator/Cargo.toml cd libs/ && ./build-arm-v8-snapshot.sh $(EMBASSY_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) frontend/patchdb-ui-seed.json - cd backend && ./build-prod.sh + cd backend && ARCH=$(ARCH) ./build-prod.sh touch $(EMBASSY_BINS) -$(EMBASSY_NATIVE_BINS): $(BACKEND_SRC) $(ENVIRONMENT_FILE) $(GIT_HASH_FILE) - cd backend && ./build-prod-native.sh - touch $(EMBASSY_NATIVE_BINS) - frontend/node_modules: frontend/package.json npm --prefix frontend ci diff --git a/backend/build-prod.sh b/backend/build-prod.sh index 3ac0b0453..9c1aacf0c 100755 --- a/backend/build-prod.sh +++ b/backend/build-prod.sh @@ -3,6 +3,10 @@ set -e shopt -s expand_aliases +if [ -z "$ARCH" ]; then + ARCH=$(uname -m) +fi + if [ "$0" != "./build-prod.sh" ]; then >&2 echo "Must be run from backend directory" exit 1 @@ -24,10 +28,10 @@ if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then FLAGS="dev,$FLAGS" fi if [[ "$FLAGS" = "" ]]; then - rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked)" + rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --locked --target=$ARCH-unknown-linux-gnu)" else echo "FLAGS=$FLAGS" - rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked)" + rust-arm64-builder sh -c "(git config --global --add safe.directory '*'; cd backend && cargo build --release --features $FLAGS --locked --target=$ARCH-unknown-linux-gnu)" fi cd backend diff --git a/backend/src/bin/embassy-init.rs b/backend/src/bin/embassy-init.rs index 4e8455cec..463baee93 100644 --- a/backend/src/bin/embassy-init.rs +++ b/backend/src/bin/embassy-init.rs @@ -28,7 +28,10 @@ fn status_fn(_: i32) -> StatusCode { #[instrument] async fn setup_or_init(cfg_path: Option<&str>) -> Result<(), Error> { - if tokio::fs::metadata("/embassy-os/disk.guid").await.is_err() { + if tokio::fs::metadata("/media/embassy/config/disk.guid") + .await + .is_err() + { #[cfg(feature = "avahi")] let _mdns = MdnsController::init(); tokio::fs::write( @@ -68,7 +71,7 @@ async fn setup_or_init(cfg_path: Option<&str>) -> Result<(), Error> { .with_kind(embassy::ErrorKind::Network)?; } else { let cfg = RpcContextConfig::load(cfg_path).await?; - let guid_string = tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy + let guid_string = tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy .await?; let guid = guid_string.trim(); let requires_reboot = embassy::disk::main::import( @@ -129,7 +132,7 @@ async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { embassy::sound::BEP.play().await?; - run_script_if_exists("/embassy-os/preinit.sh").await; + run_script_if_exists("/media/embassy/config/preinit.sh").await; let res = if let Err(e) = setup_or_init(cfg_path).await { async { @@ -156,9 +159,12 @@ async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { .await?; let ctx = DiagnosticContext::init( cfg_path, - if tokio::fs::metadata("/embassy-os/disk.guid").await.is_ok() { + if tokio::fs::metadata("/media/embassy/config/disk.guid") + .await + .is_ok() + { Some(Arc::new( - tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy + tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy .await? .trim() .to_owned(), @@ -200,7 +206,7 @@ async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { Ok(None) }; - run_script_if_exists("/embassy-os/postinit.sh").await; + run_script_if_exists("/media/embassy/config/postinit.sh").await; res } diff --git a/backend/src/bin/embassyd.rs b/backend/src/bin/embassyd.rs index 0a227077f..ac5527d32 100644 --- a/backend/src/bin/embassyd.rs +++ b/backend/src/bin/embassyd.rs @@ -44,7 +44,7 @@ async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { let rpc_ctx = RpcContext::init( cfg_path, Arc::new( - tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy + tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy .await? .trim() .to_owned(), @@ -327,9 +327,12 @@ fn main() { .await?; let ctx = DiagnosticContext::init( cfg_path, - if tokio::fs::metadata("/embassy-os/disk.guid").await.is_ok() { + if tokio::fs::metadata("/media/embassy/config/disk.guid") + .await + .is_ok() + { Some(Arc::new( - tokio::fs::read_to_string("/embassy-os/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy + tokio::fs::read_to_string("/media/embassy/config/disk.guid") // unique identifier for volume group - keeps track of the disk that goes with your embassy .await? .trim() .to_owned(), diff --git a/backend/src/diagnostic.rs b/backend/src/diagnostic.rs index 9b5417a6f..47ad2339f 100644 --- a/backend/src/diagnostic.rs +++ b/backend/src/diagnostic.rs @@ -58,7 +58,7 @@ pub fn disk() -> Result<(), Error> { #[command(rename = "forget", display(display_none))] pub async fn forget_disk() -> Result<(), Error> { - let disk_guid = Path::new("/embassy-os/disk.guid"); + let disk_guid = Path::new("/media/embassy/config/disk.guid"); if tokio::fs::metadata(disk_guid).await.is_ok() { tokio::fs::remove_file(disk_guid).await?; } diff --git a/backend/src/disk/fsck.rs b/backend/src/disk/fsck.rs index 492b9ea83..1ae0be512 100644 --- a/backend/src/disk/fsck.rs +++ b/backend/src/disk/fsck.rs @@ -63,7 +63,7 @@ fn backup_existing_undo_file<'a>(path: &'a Path) -> BoxFuture<'a, Result<(), Err pub async fn e2fsck_aggressive( logicalname: impl AsRef + std::fmt::Debug, ) -> Result { - let undo_path = Path::new("/embassy-os") + let undo_path = Path::new("/media/embassy/config") .join( logicalname .as_ref() diff --git a/backend/src/disk/mod.rs b/backend/src/disk/mod.rs index e734c8e55..1911894d1 100644 --- a/backend/src/disk/mod.rs +++ b/backend/src/disk/mod.rs @@ -16,7 +16,7 @@ pub mod mount; pub mod util; pub const BOOT_RW_PATH: &str = "/media/boot-rw"; -pub const REPAIR_DISK_PATH: &str = "/embassy-os/repair-disk"; +pub const REPAIR_DISK_PATH: &str = "/media/embassy/config/repair-disk"; #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/backend/src/disk/mount/guard.rs b/backend/src/disk/mount/guard.rs index 883a39f06..18e93f6d7 100644 --- a/backend/src/disk/mount/guard.rs +++ b/backend/src/disk/mount/guard.rs @@ -11,7 +11,7 @@ use super::util::unmount; use crate::util::Invoke; use crate::Error; -pub const TMP_MOUNTPOINT: &'static str = "/media/embassy-os/tmp"; +pub const TMP_MOUNTPOINT: &'static str = "/media/embassy/tmp"; #[async_trait::async_trait] pub trait GenericMountGuard: AsRef + std::fmt::Debug + Send + Sync + 'static { diff --git a/backend/src/init.rs b/backend/src/init.rs index e193c59a8..fc4d65940 100644 --- a/backend/src/init.rs +++ b/backend/src/init.rs @@ -17,8 +17,8 @@ use crate::util::Invoke; use crate::version::VersionT; use crate::Error; -pub const SYSTEM_REBUILD_PATH: &str = "/embassy-os/system-rebuild"; -pub const STANDBY_MODE_PATH: &str = "/embassy-os/standby"; +pub const SYSTEM_REBUILD_PATH: &str = "/media/embassy/config/system-rebuild"; +pub const STANDBY_MODE_PATH: &str = "/media/embassy/config/standby"; pub async fn check_time_is_synchronized() -> Result { Ok(String::from_utf8( @@ -309,7 +309,7 @@ pub async fn init(cfg: &RpcContextConfig) -> Result { tracing::info!("Created Docker Network"); tracing::info!("Loading System Docker Images"); - crate::install::load_images("/var/lib/embassy/system-images").await?; + crate::install::load_images("/usr/lib/embassy/system-images").await?; tracing::info!("Loaded System Docker Images"); tracing::info!("Loading Package Docker Images"); diff --git a/backend/src/setup.rs b/backend/src/setup.rs index 9f3f73c9e..0c4f79fa8 100644 --- a/backend/src/setup.rs +++ b/backend/src/setup.rs @@ -331,7 +331,7 @@ pub async fn complete(#[context] ctx: SetupContext) -> Result); diff --git a/build/fstab b/build/fstab deleted file mode 100644 index 66dd713e6..000000000 --- a/build/fstab +++ /dev/null @@ -1,4 +0,0 @@ -LABEL=green / ext4 discard,errors=remount-ro 0 1 -LABEL=system-boot /media/boot-rw vfat defaults 0 1 -/media/boot-rw /boot none defaults,bind,ro 0 0 -LABEL=EMBASSY /embassy-os vfat defaults 0 1 diff --git a/build/initialization.sh b/build/initialization.sh deleted file mode 100755 index 44a7118b6..000000000 --- a/build/initialization.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash -set -e - -# introduce start9 username and embassy as default password -if ! awk -F: '{ print $1 }' /etc/passwd | grep start9 -then - usermod -l start9 -d /home/start9 -m pi - groupmod --new-name start9 pi - echo start9:embassy | chpasswd -fi - -passwd -l start9 - -START=$(date +%s) -while ! ping -q -w 1 -c 1 `ip r | grep default | cut -d ' ' -f 3` > /dev/null; do - >&2 echo "Waiting for internet connection..." - sleep 1 - if [ "$[$START + 60]" -lt $(date +%s) ]; then - >&2 echo "Timed out waiting for internet connection..." - exit 1 - fi -done -echo "Connected to network" - -# change timezone -timedatectl set-timezone Etc/UTC - -! test -f /etc/docker/daemon.json || rm /etc/docker/daemon.json -mount -o remount,rw /boot - -apt-mark hold raspberrypi-bootloader -apt-mark hold raspberrypi-kernel - -# Convert all repos to use https:// before apt update -sed -i "s/http:/https:/g" /etc/apt/sources.list /etc/apt/sources.list.d/*.list - -apt-get update -apt-get install -y \ - nginx \ - libavahi-client3 \ - avahi-daemon \ - avahi-utils \ - iotop \ - bmon \ - lvm2 \ - cryptsetup \ - exfat-utils \ - sqlite3 \ - network-manager \ - wireless-tools \ - net-tools \ - ecryptfs-utils \ - cifs-utils \ - samba-common-bin \ - vim \ - jq \ - ncdu \ - postgresql \ - pgloader \ - dnsutils - -# switch to systemd-resolved & network-manager -systemctl enable systemd-resolved -systemctl start systemd-resolved -apt-get remove --purge openresolv dhcpcd5 -y -echo "#" > /etc/network/interfaces -systemctl disable wpa_supplicant.service -ln -rsf /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf -cat << EOF > /etc/NetworkManager/NetworkManager.conf -[main] -plugins=ifupdown,keyfile -dns=systemd-resolved - -[ifupdown] -managed=true -EOF -sudo systemctl restart NetworkManager -nmcli device modify eth0 ipv4.ignore-auto-dns no - -START=$(date +%s) -while ! ping -q -w 1 -c 1 start9.com > /dev/null; do - >&2 echo "Waiting for network to reinitialize..." - sleep 1 - if [ "$[$START + 60]" -lt $(date +%s) ]; then - >&2 echo "Timed out waiting for network to reinitialize..." - exit 1 - fi -done -echo "Network reinitialized" - -# Setup repository from The Guardian Project and install latest stable Tor daemon -wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg >/dev/null -echo "deb [arch=arm64 signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org bullseye main" > /etc/apt/sources.list.d/tor.list -apt-get update && apt-get install -y tor deb.torproject.org-keyring - -curl -fsSL https://get.docker.com | sh # TODO: commit this script into git instead of live fetching it - -systemctl disable postgresql.service -systemctl disable bluetooth.service -systemctl disable hciuart.service -systemctl disable triggerhappy.service - -apt-get autoremove -y -apt-get upgrade -y - -sed -i 's/Restart=on-failure/Restart=always/g' /lib/systemd/system/tor@default.service -sed -i 's/ExecStart=\/usr\/bin\/dockerd/ExecStart=\/usr\/bin\/dockerd --exec-opt native.cgroupdriver=systemd/g' /lib/systemd/system/docker.service -sed -i '/}/i \ \ \ \ application\/wasm \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ wasm;' /etc/nginx/mime.types -sed -i 's/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/g' /etc/nginx/nginx.conf -sed -i 's/#allow-interfaces=eth0/allow-interfaces=eth0,wlan0/g' /etc/avahi/avahi-daemon.conf -sed -i '/\(^\|#\)entries-per-entry-group-max=/c\entries-per-entry-group-max=128' /etc/avahi/avahi-daemon.conf -echo '{ "cgroup-parent": "docker-engine.slice" }' > /etc/docker/daemon.json -mkdir -p /etc/nginx/ssl - -# fix to suppress docker warning, fixed in 21.xx release of docker cli: https://github.com/docker/cli/pull/2934 -mkdir -p /root/.docker -touch /root/.docker/config.json - -mkdir -p /etc/embassy -systemctl enable embassyd.service embassy-init.service -cat << EOF > /etc/tor/torrc -SocksPort 0.0.0.0:9050 -SocksPolicy accept 127.0.0.1 -SocksPolicy accept 172.18.0.0/16 -SocksPolicy reject * -ControlPort 9051 -CookieAuthentication 1 -EOF - - - -if [ -f /embassy-os/product_key.txt ] -then - cat /embassy-os/product_key.txt | tr -d '\n' | sha256sum | head -c 32 | sed 's/$/\n/' > /etc/machine-id -else - head -c 16 /dev/urandom | xxd -p | sed 's/$/\n/' > /etc/machine-id -fi - -systemctl stop tor -rm -rf /var/lib/tor/* - -raspi-config nonint enable_overlayfs - -# create a copy of the cmdline *without* the quirk string, so that it can be easily amended -sed -i 's/usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u //g' /boot/cmdline.txt -cp /boot/cmdline.txt /boot/cmdline.txt.orig -sed -i 's/^/usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u /g' /boot/cmdline.txt - -# making that *sudo docker stats* command fulfil its purpose by displaying all metrics -sed -i 's/rootwait quiet.*/rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory quiet/g' /boot/cmdline.txt - -systemctl disable nc-broadcast.service -systemctl disable initialization.service - -echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/97-embassy.conf - - -sync - -reboot - diff --git a/build/lib/conflicts b/build/lib/conflicts new file mode 100644 index 000000000..58d721ef5 --- /dev/null +++ b/build/lib/conflicts @@ -0,0 +1,3 @@ +openresolv +dhcpcd5 +firewalld \ No newline at end of file diff --git a/build/lib/depends b/build/lib/depends new file mode 100644 index 000000000..da355dd56 --- /dev/null +++ b/build/lib/depends @@ -0,0 +1,31 @@ +tor +nginx +avahi-daemon +avahi-utils +iotop +bmon +lvm2 +cryptsetup +exfat-utils +sqlite3 +wireless-tools +net-tools +ecryptfs-utils +cifs-utils +samba-common-bin +network-manager +vim +jq +ncdu +postgresql +pgloader +openssh-server +docker-ce +docker-ce-cli +containerd.io +docker-compose-plugin +beep +httpdirfs +iw +squashfs-tools +rsync \ No newline at end of file diff --git a/build/docker-engine.slice b/build/lib/docker-engine.slice similarity index 100% rename from build/docker-engine.slice rename to build/lib/docker-engine.slice diff --git a/build/journald.conf b/build/lib/journald.conf similarity index 100% rename from build/journald.conf rename to build/lib/journald.conf diff --git a/build/00-embassy b/build/lib/motd similarity index 100% rename from build/00-embassy rename to build/lib/motd diff --git a/build/lib/scripts/add-apt-sources b/build/lib/scripts/add-apt-sources new file mode 100644 index 000000000..f304a6ad5 --- /dev/null +++ b/build/lib/scripts/add-apt-sources @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +curl -fsSL https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor -o /usr/share/keyrings/tor-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org bullseye main" > /etc/apt/sources.list.d/tor.list + +curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" > /etc/apt/sources.list.d/docker.list diff --git a/build/lib/scripts/embassy-initramfs-module b/build/lib/scripts/embassy-initramfs-module new file mode 100644 index 000000000..16940070c --- /dev/null +++ b/build/lib/scripts/embassy-initramfs-module @@ -0,0 +1,93 @@ +# Local filesystem mounting -*- shell-script -*- + +# +# This script overrides local_mount_root() in /scripts/local +# and mounts root as a read-only filesystem with a temporary (rw) +# overlay filesystem. +# + +. /scripts/local + +local_mount_root() +{ + local_top + local_device_setup "${ROOT}" "root file system" + ROOT="${DEV}" + + # Get the root filesystem type if not set + if [ -z "${ROOTFSTYPE}" ]; then + FSTYPE=$(get_fstype "${ROOT}") + else + FSTYPE=${ROOTFSTYPE} + fi + + local_premount + + # CHANGES TO THE ORIGINAL FUNCTION BEGIN HERE + # N.B. this code still lacks error checking + + modprobe ${FSTYPE} + checkfs ${ROOT} root "${FSTYPE}" + + if [ "${FSTYPE}" != "unknown" ]; then + mount -t ${FSTYPE} ${ROOTFLAGS} ${ROOT} ${rootmnt} + else + mount ${ROOTFLAGS} ${ROOT} ${rootmnt} + fi + + if ! [ -f ${rootmnt}/embassyfs ]; then + return + fi + + mkdir /embassyfs + + mount --move ${rootmnt} /embassyfs + + if ! [ -d /embassyfs/current ] && [ -d /embassyfs/prev ]; then + mv /embassyfs/prev /embassyfs/current + fi + + if ! [ -d /embassyfs/current ]; then + mkdir /embassyfs/current + for FILE in $(ls /embassyfs); do + if [ "$FILE" != current ] && [ "$FILE" != embassyfs ]; then + mv /embassyfs/$FILE /embassyfs/current/ + fi + done + fi + + mkdir -p /embassyfs/config + + if [ -f /embassyfs/config/upgrade ] && [ -d /embassyfs/next ]; then + mv /embassyfs/current /embassyfs/prev + mv /embassyfs/next /embassyfs/current + fi + + if ! [ -d /embassyfs/next ]; then + if [ -d /embassyfs/prev ]; then + mv /embassyfs/prev /embassyfs/next + else + mkdir /embassyfs/next + fi + fi + + mount -r --bind /embassyfs/current /lower + + modprobe overlay || insmod "/lower/lib/modules/$(uname -r)/kernel/fs/overlayfs/overlay.ko" + + # Mount a tmpfs for the overlay in /upper + mount -t tmpfs tmpfs /upper + mkdir /upper/data /upper/work + + # Mount the final overlay-root in $rootmnt + mount -t overlay \ + -olowerdir=/lower,upperdir=/upper/data,workdir=/upper/work \ + overlay ${rootmnt} + + mkdir -p ${rootmnt}/media/embassy/config + mount --bind /embassyfs/config ${rootmnt}/media/embassy/config + mkdir -p ${rootmnt}/media/embassy/next + mount --bind /embassyfs/next ${rootmnt}/media/embassy/next + mkdir -p ${rootmnt}/media/embassy/embassyfs + mount -r --bind /embassyfs ${rootmnt}/media/embassy/embassyfs +} \ No newline at end of file diff --git a/build/lib/scripts/grub-probe-eos b/build/lib/scripts/grub-probe-eos new file mode 100644 index 000000000..7ebdff009 --- /dev/null +++ b/build/lib/scripts/grub-probe-eos @@ -0,0 +1,34 @@ +#!/bin/sh + +ARGS= + +for ARG in $@; do + if [ "${ARG%%[!/]*}" = "/" ]; then + + OPTIONS= + + path="$ARG" + while true; do + if FSTYPE=$( findmnt -n -o FSTYPE "$path" ); then + if [ "$FSTYPE" = "overlay" ]; then + OPTIONS=$(findmnt -n -o OPTIONS "$path") + break + else + break + fi + fi + if [ "$path" = "/" ]; then break; fi + path=$(dirname "$path") + done + + if LOWERDIR=$(echo "$OPTIONS" | grep -m 1 -oP 'lowerdir=\K[^,]+'); then + #echo "[DEBUG] Overlay filesystem detected ${ARG} --> ${LOWERDIR}${ARG%*/}" 1>&2 + ARG=/media/embassy/embassyfs"${ARG%*/}" + fi + fi + ARGS="$ARGS $ARG" +done + +grub-probe-default $ARGS + +exit $? \ No newline at end of file diff --git a/build/lib/scripts/install b/build/lib/scripts/install new file mode 100644 index 000000000..333b0b654 --- /dev/null +++ b/build/lib/scripts/install @@ -0,0 +1,94 @@ +#!/bin/bash + +set -e + +function partition_for () { + if [[ "$1" =~ [0-9]+$ ]]; then + echo "$1p$2" + else + echo "$1$2" + fi +} + +OSDISK=$1 +if [ -z "$OSDISK" ]; then + >&2 echo 'usage: embassy-install ' + exit 1 +fi + +WIFI_IFACE= +for IFACE in $(ls /sys/class/net); do + if [ -d /sys/class/net/$IFACE/wireless ]; then + WIFI_IFACE=$IFACE + break + fi +done + +ETH_IFACE= +for IFACE in $(ls /sys/class/net); do + if ! [ -d /sys/class/net/$IFACE/wireless ] && [ -d /sys/class/net/$IFACE/device ]; then + ETH_IFACE=$IFACE + break + fi +done +if [ -z "$ETH_IFACE" ]; then + >&2 echo 'Could not detect ethernet interface' + exit 1 +fi + +( + echo g # GPT + echo n # New Partition + echo 1 # Index #1 + echo # Default Starting Position + echo '+1G' # 1GB + echo n # New Partition + echo 2 # Index #2 + echo # Default Starting Position + echo '+15G' # 15GB + echo n # New Partition + echo 3 # Index #3 + echo # Default Starting Position + echo # Use Full Remaining + echo t # Change Type + echo 3 # (Still Index #3) + echo lvm # Linux LVM + echo w # Write Changes +) | fdisk $OSDISK + +BOOTPART=`partition_for $OSDISK 1` +ROOTPART=`partition_for $OSDISK 2` + +mkfs.vfat $BOOTPART +fatlabel $BOOTPART BOOT + +mkfs.ext4 $ROOTPART +e2label $ROOTPART embassyos + +mount $ROOTPART /mnt +mkdir /mnt/config +mkdir /mnt/current +mkdir /mnt/next + +mkdir /mnt/current/boot + +unsquashfs -f -d /mnt/current /cdrom/casper/filesystem.squashfs + +cat > /mnt/config/config.yaml << EOF +os-partitions: + boot: $BOOTPART + root: $ROOTPART +ethernet-interface: $ETH_IFACE +EOF + +if [ -n "$WIFI_IFACE" ]; then + echo "wifi-interface: $WIFI_IFACE" >> /mnt/config/config.yaml +fi + +mount --bind /dev /mnt/current/dev +mount --bind /sys /mnt/current/sys +mount --bind /proc /mnt/current/proc + +chroot /mnt/current update-grub + +# TODO update grub to use eos initramfs \ No newline at end of file diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst new file mode 100644 index 000000000..fe17a510c --- /dev/null +++ b/build/lib/scripts/postinst @@ -0,0 +1,90 @@ +#!/bin/sh +set -e + +SYSTEMCTL=systemctl +if [ -n "DPKG_MAINTSCRIPT_PACKAGE" ]; then + SYSTEMCTL=deb-systemd-helper +fi + +mv /usr/sbin/grub-probe /usr/sbin/grub-probe-default +ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe + +ln -sf /usr/lib/embassy/scripts/embassy-initramfs-modules /etc/initramfs-tools/scripts/embassy + + # add the overlay to the list of modules +if ! grep embassy /etc/initramfs-tools/modules > /dev/null; then + echo embassy >> /etc/initramfs-tools/modules +fi + +for KERN in $(ls /boot/vmlinuz-* | sed 's/\/boot\/vmlinuz-//g'); do + mv /boot/initrd.img-"$KERN" /boot/initrd.img-"$KERN"-old + + # build the new initramfs + update-initramfs -c -k "$KERN" +done + +# change timezone +rm -f /etc/localtime +ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime + +# switch to systemd-resolved & network-manager +echo "#" > /etc/network/interfaces +ln -rsf /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf +cat << EOF > /etc/NetworkManager/NetworkManager.conf +[main] +plugins=ifupdown,keyfile +dns=systemd-resolved + +[ifupdown] +managed=true +EOF +$SYSTEMCTL enable systemd-resolved.service +$SYSTEMCTL disable wpa_supplicant.service + +$SYSTEMCTL disable postgresql.service +$SYSTEMCTL disable bluetooth.service +$SYSTEMCTL disable hciuart.service +$SYSTEMCTL disable triggerhappy.service + +$SYSTEMCTL mask sleep.target +$SYSTEMCTL mask suspend.target +$SYSTEMCTL mask hibernate.target +$SYSTEMCTL mask hybrid-sleep.target + +if which gsettings > /dev/null; then + gsettings set org.gnome.settings-daemon.plugins.power sleep-inactive-ac-timeout '0' + gsettings set org.gnome.settings-daemon.plugins.power sleep-inactive-battery-timeout '0' +fi + +sed -i 's/Restart=on-failure/Restart=always/g' /lib/systemd/system/tor@default.service +sed -i 's/ExecStart=\/usr\/bin\/dockerd/ExecStart=\/usr\/bin\/dockerd --exec-opt native.cgroupdriver=systemd/g' /lib/systemd/system/docker.service +sed -i '/}/i \ \ \ \ application\/wasm \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ wasm;' /etc/nginx/mime.types +sed -i 's/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/g' /etc/nginx/nginx.conf +sed -i 's/#allow-interfaces=eth0/allow-interfaces=eno0,wlp1s0/g' /etc/avahi/avahi-daemon.conf +sed -i '/\(^\|#\)entries-per-entry-group-max=/c\entries-per-entry-group-max=128' /etc/avahi/avahi-daemon.conf +mkdir -p /etc/docker +ln -s /usr/lib/embassy/docker-engine.slice /etc/systemd/system/docker-engine.slice +echo '{ "cgroup-parent": "docker-engine.slice" }' > /etc/docker/daemon.json +mkdir -p /etc/nginx/ssl + +# fix to suppress docker warning, fixed in 21.xx release of docker cli: https://github.com/docker/cli/pull/2934 +mkdir -p /root/.docker +touch /root/.docker/config.json + +cat << EOF > /etc/tor/torrc +SocksPort 0.0.0.0:9050 +SocksPolicy accept 127.0.0.1 +SocksPolicy accept 172.18.0.0/16 +SocksPolicy reject * +ControlPort 9051 +CookieAuthentication 1 +EOF + +rm -rf /var/lib/tor/* + +echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/97-embassy.conf + +rm /etc/motd +ln -s /usr/lib/embassy/motd /etc/update-motd.d/00-embassy +chmod -x /etc/update-motd.d/* +chmod +x /etc/update-motd.d/00-embassy \ No newline at end of file diff --git a/build/make-deb.sh b/build/make-deb.sh deleted file mode 100755 index c29fa25b1..000000000 --- a/build/make-deb.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -e - -DEB_BUILD_OPTIONS="parallel=1" debuild --preserve-envvar USER --preserve-envvar PATH --no-lintian -i -I \ No newline at end of file diff --git a/build/make-image.sh b/build/make-image.sh deleted file mode 100755 index 01fd86010..000000000 --- a/build/make-image.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -e - -function partition_for () { - if [[ "$1" =~ [0-9]+$ ]]; then - echo "$1p$2" - else - echo "$1$2" - fi -} - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" -cd "$DIR/.." - -truncate --size=$[(31116287+1)*512] eos.img -if [ -z "$OUTPUT_DEVICE" ]; then - export OUTPUT_DEVICE=$(sudo losetup --show -fP eos.img) - export DETACH_OUTPUT_DEVICE=1 -else - export DETACH_OUTPUT_DEVICE=0 - sudo dd if=/dev/zero of=$OUTPUT_DEVICE bs=1M count=1 -fi -export LOOPDEV=$(sudo losetup --show -fP raspios.img) -./build/partitioning.sh -./build/write-image.sh -sudo e2fsck -f -y `partition_for ${OUTPUT_DEVICE} 3` -sudo resize2fs -M `partition_for ${OUTPUT_DEVICE} 3` -BLOCK_INFO=$(sudo dumpe2fs `partition_for ${OUTPUT_DEVICE} 3`) -BLOCK_COUNT=$(echo "$BLOCK_INFO" | grep "Block count:" | sed 's/Block count:\s\+//g') -BLOCK_SIZE=$(echo "$BLOCK_INFO" | grep "Block size:" | sed 's/Block size:\s\+//g') -echo "YOUR GREEN FILESYSTEM is '$[$BLOCK_COUNT*$BLOCK_SIZE]' BYTES" -echo "IF YOU ARE QUICK-FLASHING FROM MAC-OS, NOTE THIS NUMBER FOR LATER" -if [ "$DETACH_OUTPUT_DEVICE" -eq "1" ]; then - sudo losetup -d $OUTPUT_DEVICE -fi diff --git a/build/partitioning.sh b/build/partitioning.sh deleted file mode 100755 index 2c86ea6f3..000000000 --- a/build/partitioning.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -# Use fdisk to create DOS partition table with 4 primary partitions, set 1 as bootable, write, and quite -(echo o; echo x; echo i; echo "0xcb15ae4d"; echo r; echo n; echo p; echo 1; echo 2048; echo 526335; echo t; echo c; echo n; echo p; echo 2; echo 526336; echo 1050623; echo t; echo 2; echo c; echo n; echo p; echo 3; echo 1050624; echo 16083455; echo n; echo p; echo 16083456; echo 31116287; echo a; echo 1; echo w) | sudo fdisk ${OUTPUT_DEVICE} > /dev/null diff --git a/build/quick-flash.sh b/build/quick-flash.sh deleted file mode 100755 index 1035fccb2..000000000 --- a/build/quick-flash.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash - -set -e - -function mktmpfifo () { - TMP_PATH=$(mktemp) - rm $TMP_PATH - mkfifo $TMP_PATH - echo $TMP_PATH -} - -echo 'This script will only work on a card that has previously had a full image written to it.' -echo 'It will *only* flash the ext4 portion (`green` partition) of the img file onto the card.' -echo 'The product key, disk guid, and kernel data will *not* be affected.' -read -p "Continue? [y/N]" -n 1 -r -echo -if ! [[ "$REPLY" =~ ^[Yy]$ ]]; then - exit 1 -fi - -if ! which pv > /dev/null; then - >&2 echo 'This script would like to use `pv` to show a progress indicator, but it is not installed.' - if which apt-get > /dev/null; then - read -p "Install? [y/N]" -n 1 -r - echo - if [[ "$REPLY" =~ ^[Yy]$ ]]; then - sudo apt-get install pv - fi - elif which pacman > /dev/null; then - read -p "Install? [y/N]" -n 1 -r - echo - if [[ "$REPLY" =~ ^[Yy]$ ]]; then - sudo pacman -S pv - fi - elif which brew > /dev/null; then - read -p "Install? [y/N]" -n 1 -r - echo - if [[ "$REPLY" =~ ^[Yy]$ ]]; then - brew install pv - fi - else - >&2 echo 'This script does not recognize what package manager you have available on your system.' - >&2 echo 'Please go install the utility manually if you want progress reporting.' - fi -fi - -if [[ "$(uname)" == "Darwin" ]]; then - export TARGET_PARTITION="/dev/disk$(diskutil list | grep EMBASSY | head -1 | rev | cut -b 3)s3" - if ! test -e $TARGET_PARTITION; then - >&2 echo '`green` partition not found' - exit 1 - fi - export SOURCE_DEVICE="$(hdiutil attach -nomount eos.img | head -n1 | sed -E 's/([^ ]+).*$/\1/g')" - export SOURCE_PARTITION="${SOURCE_DEVICE}s3" - function detach () { - hdiutil detach $SOURCE_DEVICE - } -else - if ! test -e /dev/disk/by-label/green; then - >&2 echo '`green` partition not found' - exit 1 - fi - export TARGET_PARTITION=$(readlink -f /dev/disk/by-label/green) - export SOURCE_DEVICE="$(sudo losetup --show -fP eos.img)" - export SOURCE_PARTITION="${SOURCE_DEVICE}p3" - function detach () { - sudo losetup -d ${SOURCE_DEVICE} - } -fi - -if [[ "$TARGET_PARTITION" =~ ^/dev/loop ]]; then - >&2 echo 'You are currently flashing onto a loop device.' - >&2 echo 'This is probably a mistake, and usually means you failed to detach a .img file.' - read -p "Continue anyway? [y/N]" -n 1 -r - echo - if ! [[ "$REPLY" =~ ^[Yy]$ ]]; then - exit 1 - fi -fi - -if [[ "$(uname)" == "Darwin" ]]; then - if test -z "$FS_SIZE"; then - read -p "Enter FS Size (shown during make of eos.img)" -r - export FS_SIZE=$REPLY - fi -else - sudo e2fsck -f ${SOURCE_PARTITION} - sudo resize2fs -M ${SOURCE_PARTITION} - export BLOCK_INFO=$(sudo dumpe2fs ${SOURCE_PARTITION}) - export BLOCK_COUNT=$(echo "$BLOCK_INFO" | grep "Block count:" | sed 's/Block count:\s\+//g') - export BLOCK_SIZE=$(echo "$BLOCK_INFO" | grep "Block size:" | sed 's/Block size:\s\+//g') - export FS_SIZE=$[$BLOCK_COUNT*$BLOCK_SIZE] -fi -echo "Flashing $FS_SIZE bytes to $TARGET_PARTITION" -if [[ "$(uname)" == "Darwin" ]]; then - if which pv > /dev/null; then - sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | pv -s $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1m 2>/dev/null - else - sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1m - fi -else - if which pv > /dev/null; then - sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | pv -s $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct conv=fsync 2>/dev/null - else - sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | sudo dd of=${TARGET_PARTITION} bs=1M iflag=fullblock oflag=direct conv=fsync - fi -fi -echo Verifying... -export INPUT_HASH=$(mktemp) -export OUTPUT_HASH=$(mktemp) -if which pv > /dev/null; then - export PV_IN=$(mktmpfifo) -fi -sudo cat ${SOURCE_PARTITION} | head -c $FS_SIZE | tee -a $PV_IN | sha256sum > $INPUT_HASH & -export INPUT_CHILD=$! -sudo cat ${TARGET_PARTITION} | head -c $FS_SIZE | tee -a $PV_IN | sha256sum > $OUTPUT_HASH & -export OUTPUT_CHILD=$! -if which pv > /dev/null; then - pv -s $[$FS_SIZE*2] < $PV_IN > /dev/null & -fi -wait $INPUT_CHILD $OUTPUT_CHILD -if which pv > /dev/null; then - rm $PV_IN -fi -detach -if ! [[ "$(cat $INPUT_HASH)" == "$(cat $OUTPUT_HASH)" ]]; then - rm $INPUT_HASH $OUTPUT_HASH - >&2 echo Verification Failed - exit 1 -fi -rm $INPUT_HASH $OUTPUT_HASH -echo "Verification Succeeded" diff --git a/build/init-with-sound.sh b/build/raspberry-pi/init-with-sound.sh similarity index 100% rename from build/init-with-sound.sh rename to build/raspberry-pi/init-with-sound.sh diff --git a/build/initialization.service b/build/raspberry-pi/initialization.service similarity index 100% rename from build/initialization.service rename to build/raspberry-pi/initialization.service diff --git a/build/raspberry-pi/initialization.sh b/build/raspberry-pi/initialization.sh new file mode 100755 index 000000000..c48fdefc7 --- /dev/null +++ b/build/raspberry-pi/initialization.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e + +# introduce start9 username and embassy as default password +if ! awk -F: '{ print $1 }' /etc/passwd | grep start9 +then + usermod -l start9 -d /home/start9 -m pi + groupmod --new-name start9 pi + echo start9:embassy | chpasswd +fi + +START=$(date +%s) +while ! ping -q -w 1 -c 1 `ip r | grep default | cut -d ' ' -f 3` > /dev/null; do + >&2 echo "Waiting for internet connection..." + sleep 1 + if [ "$[$START + 60]" -lt $(date +%s) ]; then + >&2 echo "Timed out waiting for internet connection..." + exit 1 + fi +done +echo "Connected to network" + +# Convert all repos to use https:// before apt update +sed -i "s/http:/https:/g" /etc/apt/sources.list /etc/apt/sources.list.d/*.list + +. /usr/lib/embassy/scripts/add-apt-sources + +apt-get update +apt-get install -y $(cat /usr/lib/embassy/dependencies) +apt-get remove --purge -y $(cat /usr/lib/embassy/conflicts) +apt-get autoremove -y +apt-get upgrade -y + +systemctl stop tor + +. /usr/lib/embassy/scripts/postinst + +systemctl enable embassyd.service embassy-init.service + +sed -i 's/^/usb-storage.quirks=152d:0562:u,14cd:121c:u,0781:cfcb:u /g' /boot/cmdline.txt + +# making that *sudo docker stats* command fulfil its purpose by displaying all metrics +sed -i 's/rootwait quiet.*/rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory quiet/g' /boot/cmdline.txt + +systemctl disable nc-broadcast.service +systemctl disable initialization.service + +passwd -l start9 + +touch /embassyfs + +sync + +reboot + diff --git a/build/raspberry-pi/make-image.sh b/build/raspberry-pi/make-image.sh new file mode 100755 index 000000000..66eb6b1c5 --- /dev/null +++ b/build/raspberry-pi/make-image.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +function partition_for () { + if [[ "$1" =~ [0-9]+$ ]]; then + echo "$1p$2" + else + echo "$1$2" + fi +} + +cp raspios.img embassyos-raspi.img +export OUTPUT_DEVICE=$(sudo losetup --show -fP embassyos-raspi.img) +./build/raspberry-pi/write-image.sh +sudo e2fsck -f -y `partition_for ${OUTPUT_DEVICE} 2` +sudo resize2fs -M `partition_for ${OUTPUT_DEVICE} 2` +sudo losetup -d $OUTPUT_DEVICE diff --git a/build/nc-broadcast.service b/build/raspberry-pi/nc-broadcast.service similarity index 100% rename from build/nc-broadcast.service rename to build/raspberry-pi/nc-broadcast.service diff --git a/build/rip-image.sh b/build/raspberry-pi/rip-image.sh similarity index 100% rename from build/rip-image.sh rename to build/raspberry-pi/rip-image.sh diff --git a/build/raspberry-pi/write-image.sh b/build/raspberry-pi/write-image.sh new file mode 100755 index 000000000..da57e0d3d --- /dev/null +++ b/build/raspberry-pi/write-image.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -e + +function partition_for () { + if [[ "$1" =~ [0-9]+$ ]]; then + echo "$1p$2" + else + echo "$1$2" + fi +} + +# Mount the boot partition and config +mkdir -p /tmp/eos-mnt +sudo mount `partition_for ${OUTPUT_DEVICE} 1` /tmp/eos-mnt + +sudo sed -i 's/PARTUUID=cb15ae4d-02/PARTUUID=cb15ae4d-03/g' /tmp/eos-mnt/cmdline.txt +sudo sed -i 's/ init=\/usr\/lib\/raspi-config\/init_resize.sh//g' /tmp/eos-mnt/cmdline.txt + +cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp > /dev/null +echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null +echo "gpu_mem=16" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null +sudo mv /tmp/eos-mnt/config.txt.tmp /tmp/eos-mnt/config.txt +sudo touch /tmp/eos-mnt/ssh + +sudo umount /tmp/eos-mnt + +sudo mount `partition_for ${OUTPUT_DEVICE} 2` /tmp/eos-mnt + +sudo mkdir /tmp/eos-mnt/media/embassy/ +sudo make install ARCH=aarch64 DESTDIR=/tmp/eos-mnt +sudo sed -i 's/raspberrypi/embassy/g' /tmp/eos-mnt/etc/hostname +sudo sed -i 's/raspberrypi/embassy/g' /tmp/eos-mnt/etc/hosts +sudo cp cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast /tmp/eos-mnt/usr/local/bin +sudo cp backend/*.service /tmp/eos-mnt/etc/systemd/system/ + +# Make the .ssh directory for UID 1000 user +sudo mkdir -p /tmp/eos-mnt/home/$(awk -v val=1000 -F ":" '$3==val{print $1}' /tmp/eos-mnt/etc/passwd)/.ssh +sudo mv /tmp/eos-mnt/etc/sudoers.d/010_pi-nopasswd /tmp/eos-mnt/etc/sudoers.d/010_start9-nopasswd +sudo sed -i 's/pi/start9/g' /tmp/eos-mnt/etc/sudoers.d/010_start9-nopasswd +sudo sed -i 's/ pi / start9 /g' /tmp/eos-mnt/etc/systemd/system/autologin@.service + +if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then + cat ./build/raspberry-pi/initialization.sh | grep -v "passwd -l start9" | sudo tee /tmp/eos-mnt/usr/local/bin/initialization.sh > /dev/null + sudo chmod +x /tmp/eos-mnt/usr/local/bin/initialization.sh +else + sudo cp ./build/raspberry-pi/initialization.sh /tmp/eos-mnt/usr/local/bin +fi +sudo cp ./build/raspberry-pi/init-with-sound.sh /tmp/eos-mnt/usr/local/bin + +sudo cp ./build/raspberry-pi/initialization.service /tmp/eos-mnt/etc/systemd/system/initialization.service +sudo ln -s /etc/systemd/system/initialization.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/initialization.service +sudo cp ./build/raspberry-pi/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/nc-broadcast.service +sudo ln -s /etc/systemd/system/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/nc-broadcast.service + +sudo umount /tmp/eos-mnt diff --git a/build/write-image.sh b/build/write-image.sh deleted file mode 100755 index a95a210bc..000000000 --- a/build/write-image.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -set -e - -function partition_for () { - if [[ "$1" =~ [0-9]+$ ]]; then - echo "$1p$2" - else - echo "$1$2" - fi -} - -# Write contents of LOOPDEV (Ubuntu image) to sd card and make filesystems, then detach the loop device -echo USING $LOOPDEV TO IMAGE $OUTPUT_DEVICE WITH ENVIRONMENT $ENVIRONMENT -sudo dd if=${LOOPDEV}p1 of=`partition_for ${OUTPUT_DEVICE} 1` bs=1M iflag=fullblock oflag=direct conv=fsync status=progress -sudo mkfs.vfat -F 32 `partition_for ${OUTPUT_DEVICE} 2` -sudo dd if=${LOOPDEV}p2 of=`partition_for ${OUTPUT_DEVICE} 3` bs=1M iflag=fullblock oflag=direct conv=fsync status=progress -sudo mkfs.ext4 `partition_for ${OUTPUT_DEVICE} 4` - -sudo losetup -d $LOOPDEV - -# Label the filesystems -sudo fatlabel `partition_for ${OUTPUT_DEVICE} 1` system-boot -sudo fatlabel `partition_for ${OUTPUT_DEVICE} 2` EMBASSY -sudo e2label `partition_for ${OUTPUT_DEVICE} 3` green -sudo e2label `partition_for ${OUTPUT_DEVICE} 4` blue - -# Mount the boot partition and config -mkdir -p /tmp/eos-mnt -sudo mount `partition_for ${OUTPUT_DEVICE} 1` /tmp/eos-mnt - -sudo sed -i 's/PARTUUID=cb15ae4d-02/PARTUUID=cb15ae4d-03/g' /tmp/eos-mnt/cmdline.txt -sudo sed -i 's/ init=\/usr\/lib\/raspi-config\/init_resize.sh//g' /tmp/eos-mnt/cmdline.txt - -cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp > /dev/null -echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null -echo "gpu_mem=16" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null -sudo mv /tmp/eos-mnt/config.txt.tmp /tmp/eos-mnt/config.txt -sudo touch /tmp/eos-mnt/ssh - -sudo umount /tmp/eos-mnt - -sudo mount `partition_for ${OUTPUT_DEVICE} 3` /tmp/eos-mnt - -sudo mkdir /tmp/eos-mnt/media/boot-rw -sudo mkdir /tmp/eos-mnt/embassy-os -sudo mkdir /tmp/eos-mnt/etc/embassy -sudo cp ENVIRONMENT.txt /tmp/eos-mnt/etc/embassy -sudo cp GIT_HASH.txt /tmp/eos-mnt/etc/embassy -sudo cp build/fstab /tmp/eos-mnt/etc/fstab -sudo cp build/journald.conf /tmp/eos-mnt/etc/systemd/journald.conf -sudo sed -i 's/raspberrypi/embassy/g' /tmp/eos-mnt/etc/hostname -sudo sed -i 's/raspberrypi/embassy/g' /tmp/eos-mnt/etc/hosts - -# copy over cargo dependencies -sudo cp cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast /tmp/eos-mnt/usr/local/bin - -# Enter the backend directory, copy over the built embassyOS binaries and systemd services, edit the nginx config, then create the .ssh directory -cd backend/ - -sudo cp target/aarch64-unknown-linux-gnu/release/embassy-init /tmp/eos-mnt/usr/local/bin -sudo cp target/aarch64-unknown-linux-gnu/release/embassyd /tmp/eos-mnt/usr/local/bin -sudo cp target/aarch64-unknown-linux-gnu/release/embassy-cli /tmp/eos-mnt/usr/local/bin -sudo cp target/aarch64-unknown-linux-gnu/release/avahi-alias /tmp/eos-mnt/usr/local/bin -sudo cp *.service /tmp/eos-mnt/etc/systemd/system/ - -cd .. - -# Copy system images -sudo mkdir -p /tmp/eos-mnt/var/lib/embassy/system-images -sudo cp system-images/**/*.tar /tmp/eos-mnt/var/lib/embassy/system-images - -# after performing npm run build -sudo mkdir -p /tmp/eos-mnt/var/www/html -sudo cp -R frontend/dist/diagnostic-ui /tmp/eos-mnt/var/www/html/diagnostic -sudo cp -R frontend/dist/setup-wizard /tmp/eos-mnt/var/www/html/setup -sudo cp -R frontend/dist/ui /tmp/eos-mnt/var/www/html/main -sudo cp index.html /tmp/eos-mnt/var/www/html/index.html - -# Make the .ssh directory for UID 1000 user -sudo mkdir -p /tmp/eos-mnt/home/$(awk -v val=1000 -F ":" '$3==val{print $1}' /tmp/eos-mnt/etc/passwd)/.ssh -sudo mv /tmp/eos-mnt/etc/sudoers.d/010_pi-nopasswd /tmp/eos-mnt/etc/sudoers.d/010_start9-nopasswd -sudo sed -i 's/pi/start9/g' /tmp/eos-mnt/etc/sudoers.d/010_start9-nopasswd -sudo sed -i 's/ pi / start9 /g' /tmp/eos-mnt/etc/systemd/system/autologin@.service - -# Custom MOTD -sudo rm /tmp/eos-mnt/etc/motd -sudo cp ./build/00-embassy /tmp/eos-mnt/etc/update-motd.d -sudo chmod -x /tmp/eos-mnt/etc/update-motd.d/* -sudo chmod +x /tmp/eos-mnt/etc/update-motd.d/00-embassy - -if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then - cat ./build/initialization.sh | grep -v "passwd -l start9" | sudo tee /tmp/eos-mnt/usr/local/bin/initialization.sh > /dev/null - sudo chmod +x /tmp/eos-mnt/usr/local/bin/initialization.sh -else - sudo cp ./build/initialization.sh /tmp/eos-mnt/usr/local/bin -fi -sudo cp ./build/init-with-sound.sh /tmp/eos-mnt/usr/local/bin - -sudo cp ./build/initialization.service /tmp/eos-mnt/etc/systemd/system/initialization.service -sudo ln -s /etc/systemd/system/initialization.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/initialization.service -sudo cp ./build/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/nc-broadcast.service -sudo ln -s /etc/systemd/system/nc-broadcast.service /tmp/eos-mnt/etc/systemd/system/multi-user.target.wants/nc-broadcast.service - -sudo umount /tmp/eos-mnt From ade1ebf413da751822cb8ae9a5ea1dd134901f13 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 19:33:28 -0600 Subject: [PATCH 31/53] fixes --- Makefile | 2 ++ build/raspberry-pi/write-image.sh | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ce5dfa14f..6985b6f63 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,8 @@ install: all cp backend/target/$(ARCH)-unknown-linux-gnu/release/embassy-cli $(DESTDIR)/usr/bin/ cp backend/target/$(ARCH)-unknown-linux-gnu/release/avahi-alias $(DESTDIR)/usr/bin/ + mkdir -p $(DESTDIR)/usr/lib + rm -rf $(DESTDIR)/usr/lib/embassy cp -r build/lib $(DESTDIR)/usr/lib/embassy cp ENVIRONMENT.txt $(DESTDIR)/usr/lib/embassy/ diff --git a/build/raspberry-pi/write-image.sh b/build/raspberry-pi/write-image.sh index da57e0d3d..83fa4758c 100755 --- a/build/raspberry-pi/write-image.sh +++ b/build/raspberry-pi/write-image.sh @@ -14,9 +14,6 @@ function partition_for () { mkdir -p /tmp/eos-mnt sudo mount `partition_for ${OUTPUT_DEVICE} 1` /tmp/eos-mnt -sudo sed -i 's/PARTUUID=cb15ae4d-02/PARTUUID=cb15ae4d-03/g' /tmp/eos-mnt/cmdline.txt -sudo sed -i 's/ init=\/usr\/lib\/raspi-config\/init_resize.sh//g' /tmp/eos-mnt/cmdline.txt - cat /tmp/eos-mnt/config.txt | grep -v "dtoverlay=" | sudo tee /tmp/eos-mnt/config.txt.tmp > /dev/null echo "dtoverlay=pwm-2chan,disable-bt" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null echo "gpu_mem=16" | sudo tee -a /tmp/eos-mnt/config.txt.tmp > /dev/null From 93355f256bd263a113c46a11929a1f29f5f8f566 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 20:07:33 -0600 Subject: [PATCH 32/53] use new initramfs always --- build/lib/scripts/postinst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index fe17a510c..0e3b6cd73 100644 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -16,12 +16,7 @@ if ! grep embassy /etc/initramfs-tools/modules > /dev/null; then echo embassy >> /etc/initramfs-tools/modules fi -for KERN in $(ls /boot/vmlinuz-* | sed 's/\/boot\/vmlinuz-//g'); do - mv /boot/initrd.img-"$KERN" /boot/initrd.img-"$KERN"-old - - # build the new initramfs - update-initramfs -c -k "$KERN" -done +update-initramfs -u -k all # change timezone rm -f /etc/localtime From 5c78ff16d73ef7ef2c018b9302be92cb86a656c2 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 20:14:54 -0600 Subject: [PATCH 33/53] switch journald conf to sed script --- build/lib/journald.conf | 44 -------------------------------------- build/lib/scripts/postinst | 6 +++++- 2 files changed, 5 insertions(+), 45 deletions(-) delete mode 100644 build/lib/journald.conf diff --git a/build/lib/journald.conf b/build/lib/journald.conf deleted file mode 100644 index 5b4836c2e..000000000 --- a/build/lib/journald.conf +++ /dev/null @@ -1,44 +0,0 @@ -# This file is part of systemd. -# -# systemd is free software; you can redistribute it and/or modify it -# under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation; either version 2.1 of the License, or -# (at your option) any later version. -# -# Entries in this file show the compile time defaults. -# You can change settings by editing this file. -# Defaults can be restored by simply deleting this file. -# -# See journald.conf(5) for details. - -[Journal] -Storage=persistent -Compress=yes -#Seal=yes -#SplitMode=uid -#SyncIntervalSec=5m -#RateLimitIntervalSec=30s -#RateLimitBurst=10000 -SystemMaxUse=1G -#SystemKeepFree= -#SystemMaxFileSize= -#SystemMaxFiles=100 -#RuntimeMaxUse= -#RuntimeKeepFree= -#RuntimeMaxFileSize= -#RuntimeMaxFiles=100 -#MaxRetentionSec= -#MaxFileSec=1month -ForwardToSyslog=no -#ForwardToKMsg=no -#ForwardToConsole=no -#ForwardToWall=yes -#TTYPath=/dev/console -#MaxLevelStore=debug -#MaxLevelSyslog=debug -#MaxLevelKMsg=notice -#MaxLevelConsole=info -#MaxLevelWall=emerg -#LineMax=48K -#ReadKMsg=yes -#Audit=no diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index 0e3b6cd73..5fee8dfdc 100644 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -57,6 +57,10 @@ sed -i '/}/i \ \ \ \ application\/wasm \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sed -i 's/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/g' /etc/nginx/nginx.conf sed -i 's/#allow-interfaces=eth0/allow-interfaces=eno0,wlp1s0/g' /etc/avahi/avahi-daemon.conf sed -i '/\(^\|#\)entries-per-entry-group-max=/c\entries-per-entry-group-max=128' /etc/avahi/avahi-daemon.conf +sed -i '/\(^\|#\)Storage=/c\Storage=persistent' /etc/systemd/journald.conf +sed -i '/\(^\|#\)Compress=/c\Compress=yes' /etc/systemd/journald.conf +sed -i '/\(^\|#\)SystemMaxUse=/c\SystemMaxUse=1G' /etc/systemd/journald.conf +sed -i '/\(^\|#\)ForwardToSyslog=/c\ForwardToSyslog=no' /etc/systemd/journald.conf mkdir -p /etc/docker ln -s /usr/lib/embassy/docker-engine.slice /etc/systemd/system/docker-engine.slice echo '{ "cgroup-parent": "docker-engine.slice" }' > /etc/docker/daemon.json @@ -82,4 +86,4 @@ echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/97-embassy.conf rm /etc/motd ln -s /usr/lib/embassy/motd /etc/update-motd.d/00-embassy chmod -x /etc/update-motd.d/* -chmod +x /etc/update-motd.d/00-embassy \ No newline at end of file +chmod +x /etc/update-motd.d/00-embassy From 8afe09ff2b1d1a6433f947f5500965f8f0140b61 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 20:26:56 -0600 Subject: [PATCH 34/53] fixes --- backend/build-prod-native.sh | 37 ------------------- backend/src/update/mod.rs | 41 +++++----------------- backend/src/util/rsync.rs | 8 +++-- build/lib/scripts/embassy-initramfs-module | 1 + 4 files changed, 16 insertions(+), 71 deletions(-) delete mode 100755 backend/build-prod-native.sh diff --git a/backend/build-prod-native.sh b/backend/build-prod-native.sh deleted file mode 100755 index a876a1202..000000000 --- a/backend/build-prod-native.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e -shopt -s expand_aliases - -if [ "$0" != "./build-prod-native.sh" ]; then - >&2 echo "Must be run from backend directory" - exit 1 -fi - -USE_TTY= -if tty -s; then - USE_TTY="-it" -fi - -alias 'rust-native-builder'='docker run $USE_TTY --rm -v "$HOME/.cargo/registry":/usr/local/cargo/registry -v "$(pwd)":/root -P start9/rust' - -cd .. -FLAGS="" -if [[ "$ENVIRONMENT" =~ (^|-)unstable($|-) ]]; then - FLAGS="unstable,$FLAGS" -fi -if [[ "$ENVIRONMENT" =~ (^|-)dev($|-) ]]; then - FLAGS="dev,$FLAGS" -fi -if [[ "$FLAGS" = "" ]]; then - rust-native-builder sh -c "cd ~/backend && cargo build --release --locked" -else - echo "FLAGS=$FLAGS" - rust-native-builder sh -c "cd ~/backend && cargo build --release --features $FLAGS --locked" -fi -cd backend - -sudo chown -R $USER target -sudo chown -R $USER ~/.cargo - -#rust-arm64-builder aarch64-linux-gnu-strip target/aarch64-unknown-linux-gnu/release/embassyd diff --git a/backend/src/update/mod.rs b/backend/src/update/mod.rs index b4dc7c830..1e9b7d153 100644 --- a/backend/src/update/mod.rs +++ b/backend/src/update/mod.rs @@ -88,8 +88,6 @@ fn display_update_result(status: UpdateResult, _: &ArgMatches) { } } -const HEADER_KEY: &str = "x-eos-hash"; - #[instrument(skip(ctx))] async fn maybe_do_update( ctx: RpcContext, @@ -230,7 +228,7 @@ async fn do_update( new_fs: TmpMountGuard, new_block_dev: TmpMountGuard, ) -> Result<(), Error> { - let mut rsync = Rsync::new(new_fs.as_ref().join(""), "/gboverlay/unselected")?; + let mut rsync = Rsync::new(new_fs.as_ref().join(""), "/media/embassy/next", true)?; while let Some(progress) = rsync.progress.next().await { crate::db::DatabaseModel::new() .server_info() @@ -274,39 +272,39 @@ impl std::fmt::Display for EosUrl { } async fn copy_machine_id() -> Result<(), Error> { - tokio::fs::copy("/etc/machine-id", "/gboverlay/unselected/etc/machine-id").await?; + tokio::fs::copy("/etc/machine-id", "/media/embassy/next/etc/machine-id").await?; Ok(()) } async fn copy_ssh_host_keys() -> Result<(), Error> { tokio::fs::copy( "/etc/ssh/ssh_host_rsa_key", - "/gboverlay/unselected/etc/ssh/ssh_host_rsa_key", + "/media/embassy/next/etc/ssh/ssh_host_rsa_key", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_rsa_key.pub", - "/gboverlay/unselected/etc/ssh/ssh_host_rsa_key.pub", + "/media/embassy/next/etc/ssh/ssh_host_rsa_key.pub", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ecdsa_key", - "/gboverlay/unselected/etc/ssh/ssh_host_ecdsa_key", + "/media/embassy/next/etc/ssh/ssh_host_ecdsa_key", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ecdsa_key.pub", - "/gboverlay/unselected/etc/ssh/ssh_host_ecdsa_key.pub", + "/media/embassy/next/etc/ssh/ssh_host_ecdsa_key.pub", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ed25519_key", - "/gboverlay/unselected/etc/ssh/ssh_host_ed25519_key", + "/media/embassy/next/etc/ssh/ssh_host_ed25519_key", ) .await?; tokio::fs::copy( "/etc/ssh/ssh_host_ed25519_key.pub", - "/gboverlay/unselected/etc/ssh/ssh_host_ed25519_key.pub", + "/media/embassy/next/etc/ssh/ssh_host_ed25519_key.pub", ) .await?; Ok(()) @@ -314,27 +312,6 @@ async fn copy_ssh_host_keys() -> Result<(), Error> { #[instrument] async fn swap_boot_label() -> Result<(), Error> { - let current = tokio::fs::read_to_string("/gboverlay/config/selected").await?; - let target = if current == "green" { "blue" } else { "green" }; - let mut selected = AtomicFile::new("/gboverlay/config/selected", None::<&str>) - .await - .with_kind(ErrorKind::Filesystem)?; - selected.write_all(target.as_bytes()).await?; - selected.save().await.with_kind(ErrorKind::Filesystem)?; - UPDATED.store(true, Ordering::SeqCst); + tokio::fs::write("/media/embassy/config/upgrade", b"").await?; Ok(()) } - -/// Captured from doing an fstab with an embassy box and the cat from the /etc/fstab -#[test] -fn test_capture() { - let output = r#" -LABEL=blue / ext4 discard,errors=remount-ro 0 1 -LABEL=system-boot /media/boot-rw vfat defaults 0 1 -/media/boot-rw /boot none defaults,bind,ro 0 0 -LABEL=EMBASSY /embassy-os vfat defaults 0 1 -# a swapfile is not a swap partition, no line here -# use dphys-swapfile swap[on|off] for that -"#; - assert_eq!(&PARSE_COLOR.captures(&output).unwrap()[1], "blue"); -} diff --git a/backend/src/util/rsync.rs b/backend/src/util/rsync.rs index 437635c64..978833bf0 100644 --- a/backend/src/util/rsync.rs +++ b/backend/src/util/rsync.rs @@ -15,8 +15,12 @@ pub struct Rsync { pub progress: WatchStream, } impl Rsync { - pub fn new(src: impl AsRef, dst: impl AsRef) -> Result { - let mut command = Command::new("rsync") + pub fn new(src: impl AsRef, dst: impl AsRef, delete: bool) -> Result { + let mut cmd = Command::new("rsync"); + if delete { + cmd.arg("--delete"); + } + let mut command = cmd .arg("-a") .arg("--no-devices") .arg("--info=progress2") diff --git a/build/lib/scripts/embassy-initramfs-module b/build/lib/scripts/embassy-initramfs-module index 16940070c..d652257ac 100644 --- a/build/lib/scripts/embassy-initramfs-module +++ b/build/lib/scripts/embassy-initramfs-module @@ -61,6 +61,7 @@ local_mount_root() if [ -f /embassyfs/config/upgrade ] && [ -d /embassyfs/next ]; then mv /embassyfs/current /embassyfs/prev mv /embassyfs/next /embassyfs/current + rm /embassyfs/config/upgrade fi if ! [ -d /embassyfs/next ]; then From 3bd29a171bf2b3326e95e8becd180f5139b39318 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 21:27:23 -0600 Subject: [PATCH 35/53] fix permissions --- build/lib/scripts/add-apt-sources | 0 build/lib/scripts/embassy-initramfs-module | 0 build/lib/scripts/grub-probe-eos | 0 build/lib/scripts/install | 0 build/lib/scripts/postinst | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build/lib/scripts/add-apt-sources mode change 100644 => 100755 build/lib/scripts/embassy-initramfs-module mode change 100644 => 100755 build/lib/scripts/grub-probe-eos mode change 100644 => 100755 build/lib/scripts/install mode change 100644 => 100755 build/lib/scripts/postinst diff --git a/build/lib/scripts/add-apt-sources b/build/lib/scripts/add-apt-sources old mode 100644 new mode 100755 diff --git a/build/lib/scripts/embassy-initramfs-module b/build/lib/scripts/embassy-initramfs-module old mode 100644 new mode 100755 diff --git a/build/lib/scripts/grub-probe-eos b/build/lib/scripts/grub-probe-eos old mode 100644 new mode 100755 diff --git a/build/lib/scripts/install b/build/lib/scripts/install old mode 100644 new mode 100755 diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst old mode 100644 new mode 100755 From 62988ef7a245f16a1d3cbfd3a780e6552167efab Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 21:38:47 -0600 Subject: [PATCH 36/53] talking about kernel modules not scripts --- build/lib/scripts/postinst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index 5fee8dfdc..7d3ad1adf 100755 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -12,8 +12,8 @@ ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe ln -sf /usr/lib/embassy/scripts/embassy-initramfs-modules /etc/initramfs-tools/scripts/embassy # add the overlay to the list of modules -if ! grep embassy /etc/initramfs-tools/modules > /dev/null; then - echo embassy >> /etc/initramfs-tools/modules +if ! grep overlay /etc/initramfs-tools/modules > /dev/null; then + echo overlay >> /etc/initramfs-tools/modules fi update-initramfs -u -k all From 7d383287b9aadc1b577e9460fd5290274e427078 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Tue, 18 Oct 2022 23:56:21 -0600 Subject: [PATCH 37/53] fix --- build/lib/scripts/install | 1 + 1 file changed, 1 insertion(+) diff --git a/build/lib/scripts/install b/build/lib/scripts/install index 333b0b654..9f63158c0 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -71,6 +71,7 @@ mkdir /mnt/current mkdir /mnt/next mkdir /mnt/current/boot +mount $BOOTPART /mnt/current/boot unsquashfs -f -d /mnt/current /cdrom/casper/filesystem.squashfs From cc176e44ef84345cf965a609663b1b4f10ce4050 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 00:07:15 -0600 Subject: [PATCH 38/53] fix --- build/lib/scripts/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index 7d3ad1adf..fdf8301fb 100755 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -9,7 +9,7 @@ fi mv /usr/sbin/grub-probe /usr/sbin/grub-probe-default ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe -ln -sf /usr/lib/embassy/scripts/embassy-initramfs-modules /etc/initramfs-tools/scripts/embassy +ln -sf /usr/lib/embassy/scripts/embassy-initramfs-module /etc/initramfs-tools/scripts/embassy # add the overlay to the list of modules if ! grep overlay /etc/initramfs-tools/modules > /dev/null; then From 8877a100dfab11f5f4cb9c8f503ac47245cb34af Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 00:42:01 -0600 Subject: [PATCH 39/53] switch to MBR --- build/lib/scripts/install | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build/lib/scripts/install b/build/lib/scripts/install index 9f63158c0..482395ab0 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -37,22 +37,30 @@ if [ -z "$ETH_IFACE" ]; then fi ( - echo g # GPT + echo o # GPT echo n # New Partition + echo p # Primary echo 1 # Index #1 echo # Default Starting Position echo '+1G' # 1GB + echo t # Change Type + echo 1 # (Still Index #1) + echo 0b # W95 FAT32 + echo a # Set Bootable + echo 1 # (Still Index #1) echo n # New Partition + echo p # Primary echo 2 # Index #2 echo # Default Starting Position echo '+15G' # 15GB echo n # New Partition + echo p # Primary echo 3 # Index #3 echo # Default Starting Position echo # Use Full Remaining echo t # Change Type echo 3 # (Still Index #3) - echo lvm # Linux LVM + echo 8e # Linux LVM echo w # Write Changes ) | fdisk $OSDISK From a05fc2be7113cb2f21b0486984216eb0931f11af Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 01:23:15 -0600 Subject: [PATCH 40/53] install to /usr/lib --- Makefile | 8 ++++---- build/lib/scripts/install | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 6985b6f63..39e18415e 100644 --- a/Makefile +++ b/Makefile @@ -75,10 +75,10 @@ install: all cp ENVIRONMENT.txt $(DESTDIR)/usr/lib/embassy/ cp GIT_HASH.txt $(DESTDIR)/usr/lib/embassy/ - mkdir -p $(DESTDIR)/var/lib/embassy/system-images - cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/var/lib/embassy/system-images/compat.tar - cp system-images/utils/docker-images/$(shell uname -m).tar $(DESTDIR)/var/lib/embassy/system-images/utils.tar - cp system-images/binfmt/docker-images/$(shell uname -m).tar $(DESTDIR)/var/lib/embassy/system-images/binfmt.tar + mkdir -p $(DESTDIR)/usr/lib/embassy/system-images + cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/usr/lib/embassy/system-images/compat.tar + cp system-images/utils/docker-images/$(shell uname -m).tar $(DESTDIR)/usr/lib/embassy/system-images/utils.tar + cp system-images/binfmt/docker-images/$(shell uname -m).tar $(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar mkdir -p $(DESTDIR)/var/www/html cp -r frontend/dist/diagnostic-ui $(DESTDIR)/var/www/html/diagnostic diff --git a/build/lib/scripts/install b/build/lib/scripts/install index 482395ab0..67d2bbd6a 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -44,10 +44,8 @@ fi echo # Default Starting Position echo '+1G' # 1GB echo t # Change Type - echo 1 # (Still Index #1) echo 0b # W95 FAT32 echo a # Set Bootable - echo 1 # (Still Index #1) echo n # New Partition echo p # Primary echo 2 # Index #2 @@ -100,4 +98,10 @@ mount --bind /proc /mnt/current/proc chroot /mnt/current update-grub -# TODO update grub to use eos initramfs \ No newline at end of file +umount /mnt/current/dev +umount /mnt/current/sys +umount /mnt/current/proc + +umount /mnt/current/boot + +umount /mnt From b4e4cd59fc00146f0bc3ece84b360dfaa092c7d4 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 01:37:30 -0600 Subject: [PATCH 41/53] fixes --- build/lib/scripts/embassy-initramfs-module | 7 +++++++ build/lib/scripts/install | 2 ++ 2 files changed, 9 insertions(+) diff --git a/build/lib/scripts/embassy-initramfs-module b/build/lib/scripts/embassy-initramfs-module index d652257ac..c52e7c58d 100755 --- a/build/lib/scripts/embassy-initramfs-module +++ b/build/lib/scripts/embassy-initramfs-module @@ -10,6 +10,8 @@ local_mount_root() { + echo 'using embassy initramfs module' + local_top local_device_setup "${ROOT}" "root file system" ROOT="${DEV}" @@ -36,9 +38,12 @@ local_mount_root() fi if ! [ -f ${rootmnt}/embassyfs ]; then + echo 'non embassyfs disk found - mounting normally' return fi + echo 'mounting embassyfs' + mkdir /embassyfs mount --move ${rootmnt} /embassyfs @@ -72,6 +77,8 @@ local_mount_root() fi fi + mkdir /lower /upper + mount -r --bind /embassyfs/current /lower modprobe overlay || insmod "/lower/lib/modules/$(uname -r)/kernel/fs/overlayfs/overlay.ko" diff --git a/build/lib/scripts/install b/build/lib/scripts/install index 67d2bbd6a..3d8847e70 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -80,6 +80,7 @@ mkdir /mnt/current/boot mount $BOOTPART /mnt/current/boot unsquashfs -f -d /mnt/current /cdrom/casper/filesystem.squashfs +touch /mnt/embassyfs cat > /mnt/config/config.yaml << EOF os-partitions: @@ -97,6 +98,7 @@ mount --bind /sys /mnt/current/sys mount --bind /proc /mnt/current/proc chroot /mnt/current update-grub +chroot /mnt/current grub-install $OSDISK umount /mnt/current/dev umount /mnt/current/sys From 2ba7b689887697de6250c7f2cf554217d29b0887 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 02:25:18 -0600 Subject: [PATCH 42/53] fixes --- build/lib/scripts/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index fdf8301fb..06aeb55e8 100755 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -9,7 +9,7 @@ fi mv /usr/sbin/grub-probe /usr/sbin/grub-probe-default ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe -ln -sf /usr/lib/embassy/scripts/embassy-initramfs-module /etc/initramfs-tools/scripts/embassy +cp /usr/lib/embassy/scripts/embassy-initramfs-module /etc/initramfs-tools/scripts/embassy # add the overlay to the list of modules if ! grep overlay /etc/initramfs-tools/modules > /dev/null; then From bb755343172e392b95e02215647f68e16dcbdfca Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 02:38:13 -0600 Subject: [PATCH 43/53] fixes --- backend/src/disk/util.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/disk/util.rs b/backend/src/disk/util.rs index 79d21ef36..cf9c453e3 100644 --- a/backend/src/disk/util.rs +++ b/backend/src/disk/util.rs @@ -308,10 +308,11 @@ pub async fn list(os: &OsPartitionInfo) -> Result, Error> { if index.internal { for part in index.parts { let mut disk_info = disk_info(disk.clone()).await; - if let Some(g) = disk_guids.get(&disk) { + disk_info.logicalname = part; + if let Some(g) = disk_guids.get(&disk_info.logicalname) { disk_info.guid = g.clone(); } else { - disk_info.partitions = vec![part_info(part).await]; + disk_info.partitions = vec![part_info(disk_info.logicalname.clone()).await]; } res.push(disk_info); } From d42a2245862bdcb6bf8127f57ba57cf8e822eb26 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 02:45:34 -0600 Subject: [PATCH 44/53] fixes --- build/lib/scripts/postinst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index 06aeb55e8..7f8b4a500 100755 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -3,7 +3,7 @@ set -e SYSTEMCTL=systemctl if [ -n "DPKG_MAINTSCRIPT_PACKAGE" ]; then - SYSTEMCTL=deb-systemd-helper + SYSTEMCTL=deb-systemd-helper fi mv /usr/sbin/grub-probe /usr/sbin/grub-probe-default @@ -11,13 +11,16 @@ ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe cp /usr/lib/embassy/scripts/embassy-initramfs-module /etc/initramfs-tools/scripts/embassy - # add the overlay to the list of modules if ! grep overlay /etc/initramfs-tools/modules > /dev/null; then - echo overlay >> /etc/initramfs-tools/modules + echo overlay >> /etc/initramfs-tools/modules fi update-initramfs -u -k all +if [ -f /etc/default/grub ]; then + sed -i '/\(^\|#\)GRUB_CMDLINE_LINUX=/c\GRUB_CMDLINE_LINUX="boot=embassy"' /etc/default/grub +fi + # change timezone rm -f /etc/localtime ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime From 401ebd4dec28044c2acd85d663bc19718e3f6aa4 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 03:03:40 -0600 Subject: [PATCH 45/53] add media config to cfg path --- backend/src/context/rpc.rs | 34 ++++++++++----------- backend/src/context/setup.rs | 35 +++++++++++----------- build/lib/scripts/embassy-initramfs-module | 7 +---- build/lib/scripts/install | 1 - 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/backend/src/context/rpc.rs b/backend/src/context/rpc.rs index 5cb5ad678..4e295c8b4 100644 --- a/backend/src/context/rpc.rs +++ b/backend/src/context/rpc.rs @@ -16,7 +16,6 @@ use rpc_toolkit::Context; use serde::Deserialize; use sqlx::postgres::PgConnectOptions; use sqlx::PgPool; -use tokio::fs::File; use tokio::process::Command; use tokio::sync::{broadcast, oneshot, Mutex, RwLock}; use tracing::instrument; @@ -36,8 +35,8 @@ use crate::notifications::NotificationManager; use crate::setup::password_hash; use crate::shutdown::Shutdown; use crate::status::{MainStatus, Status}; -use crate::util::io::from_yaml_async_reader; -use crate::util::{AsyncFileExt, Invoke}; +use crate::util::config::load_config_from_paths; +use crate::util::Invoke; use crate::{Error, ErrorKind, ResultExt}; #[derive(Debug, Default, Deserialize)] @@ -59,19 +58,20 @@ pub struct RpcContextConfig { pub log_server: Option, } impl RpcContextConfig { - pub async fn load>(path: Option

) -> Result { - let cfg_path = path - .as_ref() - .map(|p| p.as_ref()) - .unwrap_or(Path::new(crate::util::config::CONFIG_PATH)); - if let Some(f) = File::maybe_open(cfg_path) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))? - { - from_yaml_async_reader(f).await - } else { - Ok(Self::default()) - } + pub async fn load + Send + 'static>(path: Option

) -> Result { + tokio::task::spawn_blocking(move || { + load_config_from_paths( + path.as_ref() + .into_iter() + .map(|p| p.as_ref()) + .chain(std::iter::once(Path::new( + "/media/embassy/config/config.yaml", + ))) + .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))), + ) + }) + .await + .unwrap() } pub fn datadir(&self) -> &Path { self.datadir @@ -216,7 +216,7 @@ impl RpcSetNginxReceipts { pub struct RpcContext(Arc); impl RpcContext { #[instrument(skip(cfg_path))] - pub async fn init>( + pub async fn init + Send + 'static>( cfg_path: Option

, disk_guid: Arc, ) -> Result { diff --git a/backend/src/context/setup.rs b/backend/src/context/setup.rs index ec28560b1..e2a76fbc1 100644 --- a/backend/src/context/setup.rs +++ b/backend/src/context/setup.rs @@ -11,7 +11,6 @@ use rpc_toolkit::Context; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgConnectOptions; use sqlx::PgPool; -use tokio::fs::File; use tokio::sync::broadcast::Sender; use tokio::sync::RwLock; use tracing::instrument; @@ -22,8 +21,7 @@ use crate::disk::OsPartitionInfo; use crate::init::{init_postgres, pgloader}; use crate::net::tor::os_key; use crate::setup::{password_hash, RecoveryStatus}; -use crate::util::io::from_yaml_async_reader; -use crate::util::AsyncFileExt; +use crate::util::config::load_config_from_paths; use crate::{Error, ResultExt}; #[derive(Clone, Serialize, Deserialize)] @@ -45,19 +43,20 @@ pub struct SetupContextConfig { } impl SetupContextConfig { #[instrument(skip(path))] - pub async fn load>(path: Option

) -> Result { - let cfg_path = path - .as_ref() - .map(|p| p.as_ref()) - .unwrap_or(Path::new(crate::util::config::CONFIG_PATH)); - if let Some(f) = File::maybe_open(cfg_path) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, cfg_path.display().to_string()))? - { - from_yaml_async_reader(f).await - } else { - Ok(Self::default()) - } + pub async fn load + Send + 'static>(path: Option

) -> Result { + tokio::task::spawn_blocking(move || { + load_config_from_paths( + path.as_ref() + .into_iter() + .map(|p| p.as_ref()) + .chain(std::iter::once(Path::new( + "/media/embassy/config/config.yaml", + ))) + .chain(std::iter::once(Path::new(crate::util::config::CONFIG_PATH))), + ) + }) + .await + .unwrap() } pub fn datadir(&self) -> &Path { self.datadir @@ -93,8 +92,8 @@ impl AsRef for SetupContextSeed { pub struct SetupContext(Arc); impl SetupContext { #[instrument(skip(path))] - pub async fn init>(path: Option

) -> Result { - let cfg = SetupContextConfig::load(path.as_ref()).await?; + pub async fn init + Send + 'static>(path: Option

) -> Result { + let cfg = SetupContextConfig::load(path.as_ref().map(|p| p.as_ref().to_owned())).await?; let (shutdown, _) = tokio::sync::broadcast::channel(1); let datadir = cfg.datadir().to_owned(); Ok(Self(Arc::new(SetupContextSeed { diff --git a/build/lib/scripts/embassy-initramfs-module b/build/lib/scripts/embassy-initramfs-module index c52e7c58d..65079a2b8 100755 --- a/build/lib/scripts/embassy-initramfs-module +++ b/build/lib/scripts/embassy-initramfs-module @@ -37,11 +37,6 @@ local_mount_root() mount ${ROOTFLAGS} ${ROOT} ${rootmnt} fi - if ! [ -f ${rootmnt}/embassyfs ]; then - echo 'non embassyfs disk found - mounting normally' - return - fi - echo 'mounting embassyfs' mkdir /embassyfs @@ -55,7 +50,7 @@ local_mount_root() if ! [ -d /embassyfs/current ]; then mkdir /embassyfs/current for FILE in $(ls /embassyfs); do - if [ "$FILE" != current ] && [ "$FILE" != embassyfs ]; then + if [ "$FILE" != current ]; then mv /embassyfs/$FILE /embassyfs/current/ fi done diff --git a/build/lib/scripts/install b/build/lib/scripts/install index 3d8847e70..c92cb8e51 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -80,7 +80,6 @@ mkdir /mnt/current/boot mount $BOOTPART /mnt/current/boot unsquashfs -f -d /mnt/current /cdrom/casper/filesystem.squashfs -touch /mnt/embassyfs cat > /mnt/config/config.yaml << EOF os-partitions: From a5959fadce105ceec4d6d7caed47ddd50a765288 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 03:10:42 -0600 Subject: [PATCH 46/53] fixes --- backend/src/backup/restore.rs | 2 +- backend/src/setup.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/backup/restore.rs b/backend/src/backup/restore.rs index 5c945fe12..baee2304c 100644 --- a/backend/src/backup/restore.rs +++ b/backend/src/backup/restore.rs @@ -236,7 +236,7 @@ pub async fn recover_full_embassy( os_backup.tor_key.public().get_onion_address(), os_backup.root_ca_cert, async move { - let rpc_ctx = RpcContext::init(ctx.config_path.as_ref(), disk_guid).await?; + let rpc_ctx = RpcContext::init(ctx.config_path.clone(), disk_guid).await?; let mut db = rpc_ctx.db.handle(); let ids = backup_guard diff --git a/backend/src/setup.rs b/backend/src/setup.rs index 0c4f79fa8..479441c6f 100644 --- a/backend/src/setup.rs +++ b/backend/src/setup.rs @@ -135,7 +135,7 @@ pub async fn attach( ErrorKind::DiskManagement, )); } - init(&RpcContextConfig::load(ctx.config_path.as_ref()).await?).await?; + init(&RpcContextConfig::load(ctx.config_path.clone()).await?).await?; let secrets = ctx.secret_store().await?; let db = ctx.db(&secrets).await?; let mut secrets_handle = secrets.acquire().await?; @@ -378,7 +378,7 @@ pub async fn execute_inner( recovery_password, ) .await?; - let db = init(&RpcContextConfig::load(ctx.config_path.as_ref()).await?) + let db = init(&RpcContextConfig::load(ctx.config_path.clone()).await?) .await? .db; let hostname = { @@ -416,7 +416,7 @@ pub async fn execute_inner( res } else { let (tor_addr, root_ca) = fresh_setup(&ctx, &embassy_password).await?; - let db = init(&RpcContextConfig::load(ctx.config_path.as_ref()).await?) + let db = init(&RpcContextConfig::load(ctx.config_path.clone()).await?) .await? .db; let mut handle = db.handle(); From 7680bdeb12a22de065fc92ee37285d1122cda2e1 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 03:14:25 -0600 Subject: [PATCH 47/53] fixes --- backend/src/bin/embassy-init.rs | 12 ++++++------ backend/src/bin/embassyd.rs | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/backend/src/bin/embassy-init.rs b/backend/src/bin/embassy-init.rs index 463baee93..848e900e1 100644 --- a/backend/src/bin/embassy-init.rs +++ b/backend/src/bin/embassy-init.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -27,7 +27,7 @@ fn status_fn(_: i32) -> StatusCode { } #[instrument] -async fn setup_or_init(cfg_path: Option<&str>) -> Result<(), Error> { +async fn setup_or_init(cfg_path: Option) -> Result<(), Error> { if tokio::fs::metadata("/media/embassy/config/disk.guid") .await .is_err() @@ -122,7 +122,7 @@ async fn run_script_if_exists>(path: P) { } #[instrument] -async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { +async fn inner_main(cfg_path: Option) -> Result, Error> { if tokio::fs::metadata(STANDBY_MODE_PATH).await.is_ok() { tokio::fs::remove_file(STANDBY_MODE_PATH).await?; Command::new("sync").invoke(ErrorKind::Filesystem).await?; @@ -134,8 +134,8 @@ async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { run_script_if_exists("/media/embassy/config/preinit.sh").await; - let res = if let Err(e) = setup_or_init(cfg_path).await { - async { + let res = if let Err(e) = setup_or_init(cfg_path.clone()).await { + async move { tracing::error!("{}", e.source); tracing::debug!("{}", e.source); embassy::sound::BEETHOVEN.play().await?; @@ -223,7 +223,7 @@ fn main() { EmbassyLogger::init(); - let cfg_path = matches.value_of("config"); + let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned()); let res = { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/backend/src/bin/embassyd.rs b/backend/src/bin/embassyd.rs index ac5527d32..2ca188de8 100644 --- a/backend/src/bin/embassyd.rs +++ b/backend/src/bin/embassyd.rs @@ -1,3 +1,4 @@ +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -39,7 +40,7 @@ fn err_to_500(e: Error) -> Response { } #[instrument] -async fn inner_main(cfg_path: Option<&str>) -> Result, Error> { +async fn inner_main(cfg_path: Option) -> Result, Error> { let (rpc_ctx, shutdown) = { let rpc_ctx = RpcContext::init( cfg_path, @@ -292,7 +293,7 @@ fn main() { EmbassyLogger::init(); - let cfg_path = matches.value_of("config"); + let cfg_path = matches.value_of("config").map(|p| Path::new(p).to_owned()); let res = { let rt = tokio::runtime::Builder::new_multi_thread() @@ -300,7 +301,7 @@ fn main() { .build() .expect("failed to initialize runtime"); rt.block_on(async { - match inner_main(cfg_path).await { + match inner_main(cfg_path.clone()).await { Ok(a) => Ok(a), Err(e) => { (|| async { From 81ffa3043b9b37068ce45870061169a51506423d Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 04:10:12 -0600 Subject: [PATCH 48/53] fixes --- Makefile | 2 +- build/lib/scripts/postinst | 1 - build/raspberry-pi/config.yaml | 5 +++++ build/raspberry-pi/initialization.sh | 2 +- build/raspberry-pi/write-image.sh | 2 ++ 5 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 build/raspberry-pi/config.yaml diff --git a/Makefile b/Makefile index 39e18415e..885aa6dea 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ all: $(EMBASSY_SRC) $(EMBASSY_BINS) system-images/compat/docker-images/aarch64.t gzip: embassyos-raspi.tar.gz embassyos-raspi.tar.gz: embassyos-raspi.img - tar --format=posix -cS -f- eos.img | $(GZIP_BIN) > embassyos-raspi.tar.gz + tar --format=posix -cS -f- embassyos-raspi.img | $(GZIP_BIN) > embassyos-raspi.tar.gz clean: rm -f 2022-01-28-raspios-bullseye-arm64-lite.zip diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index 7f8b4a500..2af383bca 100755 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -58,7 +58,6 @@ sed -i 's/Restart=on-failure/Restart=always/g' /lib/systemd/system/tor@default.s sed -i 's/ExecStart=\/usr\/bin\/dockerd/ExecStart=\/usr\/bin\/dockerd --exec-opt native.cgroupdriver=systemd/g' /lib/systemd/system/docker.service sed -i '/}/i \ \ \ \ application\/wasm \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ wasm;' /etc/nginx/mime.types sed -i 's/# server_names_hash_bucket_size 64;/server_names_hash_bucket_size 128;/g' /etc/nginx/nginx.conf -sed -i 's/#allow-interfaces=eth0/allow-interfaces=eno0,wlp1s0/g' /etc/avahi/avahi-daemon.conf sed -i '/\(^\|#\)entries-per-entry-group-max=/c\entries-per-entry-group-max=128' /etc/avahi/avahi-daemon.conf sed -i '/\(^\|#\)Storage=/c\Storage=persistent' /etc/systemd/journald.conf sed -i '/\(^\|#\)Compress=/c\Compress=yes' /etc/systemd/journald.conf diff --git a/build/raspberry-pi/config.yaml b/build/raspberry-pi/config.yaml new file mode 100644 index 000000000..f512f294c --- /dev/null +++ b/build/raspberry-pi/config.yaml @@ -0,0 +1,5 @@ +os-partitions: + boot: /dev/mmcblk0p1 + root: /dev/mmcblk0p2 +ethernet-interface: eth0 +wifi-interface: wlan0 \ No newline at end of file diff --git a/build/raspberry-pi/initialization.sh b/build/raspberry-pi/initialization.sh index c48fdefc7..20899e570 100755 --- a/build/raspberry-pi/initialization.sh +++ b/build/raspberry-pi/initialization.sh @@ -47,7 +47,7 @@ systemctl disable initialization.service passwd -l start9 -touch /embassyfs +sed -i /tmp/eos-mnt/cmdline.txt -e "s/^/boot=embassy /" sync diff --git a/build/raspberry-pi/write-image.sh b/build/raspberry-pi/write-image.sh index 83fa4758c..0046eacbb 100755 --- a/build/raspberry-pi/write-image.sh +++ b/build/raspberry-pi/write-image.sh @@ -30,6 +30,8 @@ sudo sed -i 's/raspberrypi/embassy/g' /tmp/eos-mnt/etc/hostname sudo sed -i 's/raspberrypi/embassy/g' /tmp/eos-mnt/etc/hosts sudo cp cargo-deps/aarch64-unknown-linux-gnu/release/nc-broadcast /tmp/eos-mnt/usr/local/bin sudo cp backend/*.service /tmp/eos-mnt/etc/systemd/system/ +sudo mkdir -p /tmp/eos-mnt/etc/embassy +sudo cp build/raspberry-pi/config.yaml /tmp/eos-mnt/etc/embassy/config.yaml # Make the .ssh directory for UID 1000 user sudo mkdir -p /tmp/eos-mnt/home/$(awk -v val=1000 -F ":" '$3==val{print $1}' /tmp/eos-mnt/etc/passwd)/.ssh From f4d764a2e4659bef12876f4488ebf8ec9338c4a0 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 14:12:25 -0600 Subject: [PATCH 49/53] raspi image fixes --- .github/workflows/product.yaml | 19 ++++++++----------- Makefile | 6 +++--- build/lib/scripts/add-apt-sources | 4 ++-- build/lib/scripts/install | 4 ++-- build/lib/scripts/postinst | 14 ++++++++------ build/raspberry-pi/initialization.sh | 13 +++++++++---- frontend/patchdb-ui-seed.json | 2 +- 7 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.github/workflows/product.yaml b/.github/workflows/product.yaml index 5a38da424..580447484 100644 --- a/.github/workflows/product.yaml +++ b/.github/workflows/product.yaml @@ -15,23 +15,23 @@ jobs: compat: uses: ./.github/workflows/reusable-workflow.yaml with: - build_command: make system-images/compat/compat.tar + build_command: make system-images/compat/docker-images/aarch64.tar artifact_name: compat.tar - artifact_path: system-images/compat/compat.tar + artifact_path: system-images/compat/docker-images/aarch64.tar utils: uses: ./.github/workflows/reusable-workflow.yaml with: - build_command: make system-images/utils/utils.tar + build_command: make system-images/utils/docker-images/aarch64.tar artifact_name: utils.tar - artifact_path: system-images/utils/utils.tar + artifact_path: system-images/utils/docker-images/aarch64.tar binfmt: uses: ./.github/workflows/reusable-workflow.yaml with: - build_command: make system-images/binfmt/binfmt.tar + build_command: make system-images/binfmt/docker-images/aarch64.tar artifact_name: binfmt.tar - artifact_path: system-images/binfmt/binfmt.tar + artifact_path: system-images/binfmt/docker-images/aarch64.tar nc-broadcast: uses: ./.github/workflows/reusable-workflow.yaml @@ -126,12 +126,9 @@ jobs: key: cache-raspios - name: Build image - run: "make V=1 NO_KEY=1 eos.img --debug" - - - name: Compress image - run: "make gzip" + run: "make V=1 ARCH=aarch64 embassyos-raspi.img --debug" - uses: actions/upload-artifact@v3 with: name: image - path: eos.tar.gz + path: embassyos-raspi.img diff --git a/Makefile b/Makefile index 885aa6dea..a789c2d4e 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ COMPAT_SRC := $(shell find system-images/compat/ -not -path 'system-images/compa UTILS_SRC := $(shell find system-images/utils/ -not -name *.tar) BINFMT_SRC := $(shell find system-images/binfmt/ -not -name *.tar) BACKEND_SRC := $(shell find backend/src) $(shell find backend/migrations) $(shell find patch-db/*/src) backend/Cargo.toml backend/Cargo.lock -FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell find frontend/assets) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json +FRONTEND_SHARED_SRC := $(shell find frontend/projects/shared) $(shell ls -p frontend/ | grep -v / | sed 's/^/frontend\//g') frontend/node_modules frontend/config.json patch-db/client/dist frontend/patchdb-ui-seed.json FRONTEND_UI_SRC := $(shell find frontend/projects/ui) FRONTEND_SETUP_WIZARD_SRC := $(shell find frontend/projects/setup-wizard) FRONTEND_DIAGNOSTIC_UI_SRC := $(shell find frontend/projects/diagnostic-ui) @@ -77,8 +77,8 @@ install: all mkdir -p $(DESTDIR)/usr/lib/embassy/system-images cp system-images/compat/docker-images/aarch64.tar $(DESTDIR)/usr/lib/embassy/system-images/compat.tar - cp system-images/utils/docker-images/$(shell uname -m).tar $(DESTDIR)/usr/lib/embassy/system-images/utils.tar - cp system-images/binfmt/docker-images/$(shell uname -m).tar $(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar + cp system-images/utils/docker-images/$(ARCH).tar $(DESTDIR)/usr/lib/embassy/system-images/utils.tar + cp system-images/binfmt/docker-images/$(ARCH).tar $(DESTDIR)/usr/lib/embassy/system-images/binfmt.tar mkdir -p $(DESTDIR)/var/www/html cp -r frontend/dist/diagnostic-ui $(DESTDIR)/var/www/html/diagnostic diff --git a/build/lib/scripts/add-apt-sources b/build/lib/scripts/add-apt-sources index f304a6ad5..638d8dad6 100755 --- a/build/lib/scripts/add-apt-sources +++ b/build/lib/scripts/add-apt-sources @@ -2,8 +2,8 @@ set -e -curl -fsSL https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor -o /usr/share/keyrings/tor-archive-keyring.gpg +curl -fsSL https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor -o- > /usr/share/keyrings/tor-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org bullseye main" > /etc/apt/sources.list.d/tor.list -curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg +curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o- > /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian bullseye stable" > /etc/apt/sources.list.d/docker.list diff --git a/build/lib/scripts/install b/build/lib/scripts/install index c92cb8e51..e978bbc49 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -66,10 +66,10 @@ BOOTPART=`partition_for $OSDISK 1` ROOTPART=`partition_for $OSDISK 2` mkfs.vfat $BOOTPART -fatlabel $BOOTPART BOOT +fatlabel $BOOTPART boot mkfs.ext4 $ROOTPART -e2label $ROOTPART embassyos +e2label $ROOTPART rootfs mount $ROOTPART /mnt mkdir /mnt/config diff --git a/build/lib/scripts/postinst b/build/lib/scripts/postinst index 2af383bca..58e418c9d 100755 --- a/build/lib/scripts/postinst +++ b/build/lib/scripts/postinst @@ -2,12 +2,14 @@ set -e SYSTEMCTL=systemctl -if [ -n "DPKG_MAINTSCRIPT_PACKAGE" ]; then +if [ -n "$DPKG_MAINTSCRIPT_PACKAGE" ]; then SYSTEMCTL=deb-systemd-helper fi -mv /usr/sbin/grub-probe /usr/sbin/grub-probe-default -ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe +if [ -f /usr/sbin/grub-probe ]; then + mv /usr/sbin/grub-probe /usr/sbin/grub-probe-default + ln -s /usr/lib/embassy/scripts/grub-probe-eos /usr/sbin/grub-probe +fi cp /usr/lib/embassy/scripts/embassy-initramfs-module /etc/initramfs-tools/scripts/embassy @@ -64,7 +66,7 @@ sed -i '/\(^\|#\)Compress=/c\Compress=yes' /etc/systemd/journald.conf sed -i '/\(^\|#\)SystemMaxUse=/c\SystemMaxUse=1G' /etc/systemd/journald.conf sed -i '/\(^\|#\)ForwardToSyslog=/c\ForwardToSyslog=no' /etc/systemd/journald.conf mkdir -p /etc/docker -ln -s /usr/lib/embassy/docker-engine.slice /etc/systemd/system/docker-engine.slice +ln -sf /usr/lib/embassy/docker-engine.slice /etc/systemd/system/docker-engine.slice echo '{ "cgroup-parent": "docker-engine.slice" }' > /etc/docker/daemon.json mkdir -p /etc/nginx/ssl @@ -85,7 +87,7 @@ rm -rf /var/lib/tor/* echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/97-embassy.conf -rm /etc/motd -ln -s /usr/lib/embassy/motd /etc/update-motd.d/00-embassy +rm -f /etc/motd +ln -sf /usr/lib/embassy/motd /etc/update-motd.d/00-embassy chmod -x /etc/update-motd.d/* chmod +x /etc/update-motd.d/00-embassy diff --git a/build/raspberry-pi/initialization.sh b/build/raspberry-pi/initialization.sh index 20899e570..a88d90077 100755 --- a/build/raspberry-pi/initialization.sh +++ b/build/raspberry-pi/initialization.sh @@ -26,10 +26,10 @@ sed -i "s/http:/https:/g" /etc/apt/sources.list /etc/apt/sources.list.d/*.list . /usr/lib/embassy/scripts/add-apt-sources apt-get update -apt-get install -y $(cat /usr/lib/embassy/dependencies) +apt-get upgrade -y +apt-get install -y $(cat /usr/lib/embassy/depends) apt-get remove --purge -y $(cat /usr/lib/embassy/conflicts) apt-get autoremove -y -apt-get upgrade -y systemctl stop tor @@ -45,9 +45,14 @@ sed -i 's/rootwait quiet.*/rootwait cgroup_enable=cpuset cgroup_memory=1 cgroup_ systemctl disable nc-broadcast.service systemctl disable initialization.service -passwd -l start9 +update-initramfs -c -k "$(uname -r)" -sed -i /tmp/eos-mnt/cmdline.txt -e "s/^/boot=embassy /" +sed -i /boot/config.txt -e "/initramfs.*/d" +echo initramfs "initrd.img-$(uname -r)" >> /boot/config.txt + +sed -i /boot/cmdline.txt -e "s/^/boot=embassy /" + +passwd -l start9 sync diff --git a/frontend/patchdb-ui-seed.json b/frontend/patchdb-ui-seed.json index ac93a8b88..019505795 100644 --- a/frontend/patchdb-ui-seed.json +++ b/frontend/patchdb-ui-seed.json @@ -1,7 +1,7 @@ { "name": null, "auto-check-updates": true, - "ack-welcome": "0.3.3", + "ack-welcome": "0.3.2.1", "marketplace": { "selected-url": "https://registry.start9.com/", "known-hosts": { From 75a0dce85223c918dc75e83082201d941dab889b Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 15:30:40 -0600 Subject: [PATCH 50/53] fix test --- backend/src/util/http_reader.rs | 3 ++- build/raspberry-pi/initialization.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/util/http_reader.rs b/backend/src/util/http_reader.rs index b4b06b1ca..425517761 100644 --- a/backend/src/util/http_reader.rs +++ b/backend/src/util/http_reader.rs @@ -79,7 +79,7 @@ impl Display for RangeUnit { impl HttpReader { pub async fn new(http_url: Url) -> Result { let http_client = Client::builder() - .proxy(reqwest::Proxy::all("socks5h://127.0.0.1:9050").unwrap()) + // .proxy(reqwest::Proxy::all("socks5h://127.0.0.1:9050").unwrap()) .build() .with_kind(crate::ErrorKind::TLSInit)?; @@ -361,6 +361,7 @@ async fn main_test() { } #[tokio::test] +#[ignore] async fn s9pk_test() { use tokio::io::BufReader; diff --git a/build/raspberry-pi/initialization.sh b/build/raspberry-pi/initialization.sh index a88d90077..c61b1a490 100755 --- a/build/raspberry-pi/initialization.sh +++ b/build/raspberry-pi/initialization.sh @@ -28,7 +28,7 @@ sed -i "s/http:/https:/g" /etc/apt/sources.list /etc/apt/sources.list.d/*.list apt-get update apt-get upgrade -y apt-get install -y $(cat /usr/lib/embassy/depends) -apt-get remove --purge -y $(cat /usr/lib/embassy/conflicts) +apt-get remove --purge -y $(cat /usr/lib/embassy/conflicts) beep apt-get autoremove -y systemctl stop tor From f9273f061fb75e84de3fba6c589802be65789096 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 16:50:18 -0600 Subject: [PATCH 51/53] fix workflow --- .github/workflows/product.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/product.yaml b/.github/workflows/product.yaml index 580447484..2a5e7d207 100644 --- a/.github/workflows/product.yaml +++ b/.github/workflows/product.yaml @@ -60,19 +60,19 @@ jobs: uses: actions/download-artifact@v3 with: name: compat.tar - path: system-images/compat + path: system-images/compat/docker-images/aarch64.tar - name: Download utils.tar artifact uses: actions/download-artifact@v3 with: name: utils.tar - path: system-images/utils + path: system-images/utils/docker-images/aarch64.tar - name: Download binfmt.tar artifact uses: actions/download-artifact@v3 with: name: binfmt.tar - path: system-images/binfmt + path: system-images/binfmt/docker-images/aarch64.tar - name: Download nc-broadcast.tar artifact uses: actions/download-artifact@v3 From cc94e6d2f4f4c2282e62fd2b9faeeab06c113c61 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 17:54:34 -0600 Subject: [PATCH 52/53] sync boot partition --- .github/workflows/product.yaml | 15 ++++-- backend/src/disk/mount/filesystem/bind.rs | 54 +++++++++++++++++++ backend/src/disk/mount/filesystem/mod.rs | 1 + backend/src/update/mod.rs | 65 +++++++++++++++++------ backend/src/util/rsync.rs | 58 +++++++++++++++++--- build/lib/scripts/install | 14 +++++ 6 files changed, 182 insertions(+), 25 deletions(-) create mode 100644 backend/src/disk/mount/filesystem/bind.rs diff --git a/.github/workflows/product.yaml b/.github/workflows/product.yaml index 2a5e7d207..c9c716f05 100644 --- a/.github/workflows/product.yaml +++ b/.github/workflows/product.yaml @@ -60,19 +60,28 @@ jobs: uses: actions/download-artifact@v3 with: name: compat.tar - path: system-images/compat/docker-images/aarch64.tar + path: system-images/compat/docker-images/ + + - name: Rename compat.tar artifact + run: mv system-images/compat/docker-images/compat.tar system-images/compat/docker-images/aarch64.tar - name: Download utils.tar artifact uses: actions/download-artifact@v3 with: name: utils.tar - path: system-images/utils/docker-images/aarch64.tar + path: system-images/utils/docker-images/ + + - name: Rename utils.tar artifact + run: mv system-images/utils/docker-images/utils.tar system-images/utils/docker-images/aarch64.tar - name: Download binfmt.tar artifact uses: actions/download-artifact@v3 with: name: binfmt.tar - path: system-images/binfmt/docker-images/aarch64.tar + path: system-images/binfmt/docker-images/ + + - name: Rename binfmt.tar artifact + run: mv system-images/binfmt/docker-images/binfmt.tar system-images/binfmt/docker-images/aarch64.tar - name: Download nc-broadcast.tar artifact uses: actions/download-artifact@v3 diff --git a/backend/src/disk/mount/filesystem/bind.rs b/backend/src/disk/mount/filesystem/bind.rs new file mode 100644 index 000000000..8799372e5 --- /dev/null +++ b/backend/src/disk/mount/filesystem/bind.rs @@ -0,0 +1,54 @@ +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +use async_trait::async_trait; +use digest::generic_array::GenericArray; +use digest::{Digest, OutputSizeUser}; +use sha2::Sha256; + +use super::{FileSystem, MountType, ReadOnly}; +use crate::disk::mount::util::bind; +use crate::{Error, ResultExt}; + +pub struct Bind> { + src_dir: SrcDir, +} +impl> Bind { + pub fn new(src_dir: SrcDir) -> Self { + Self { src_dir } + } +} +#[async_trait] +impl + Send + Sync> FileSystem for Bind { + async fn mount + Send + Sync>( + &self, + mountpoint: P, + mount_type: MountType, + ) -> Result<(), Error> { + bind( + self.src_dir.as_ref(), + mountpoint, + matches!(mount_type, ReadOnly), + ) + .await + } + async fn source_hash( + &self, + ) -> Result::OutputSize>, Error> { + let mut sha = Sha256::new(); + sha.update("Bind"); + sha.update( + tokio::fs::canonicalize(self.src_dir.as_ref()) + .await + .with_ctx(|_| { + ( + crate::ErrorKind::Filesystem, + self.src_dir.as_ref().display().to_string(), + ) + })? + .as_os_str() + .as_bytes(), + ); + Ok(sha.finalize()) + } +} diff --git a/backend/src/disk/mount/filesystem/mod.rs b/backend/src/disk/mount/filesystem/mod.rs index 5d3c431d4..fd87051ea 100644 --- a/backend/src/disk/mount/filesystem/mod.rs +++ b/backend/src/disk/mount/filesystem/mod.rs @@ -7,6 +7,7 @@ use sha2::Sha256; use crate::Error; +pub mod bind; pub mod block_dev; pub mod cifs; pub mod ecryptfs; diff --git a/backend/src/update/mod.rs b/backend/src/update/mod.rs index 1e9b7d153..948f27a86 100644 --- a/backend/src/update/mod.rs +++ b/backend/src/update/mod.rs @@ -1,42 +1,31 @@ -use std::future::Future; -use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use std::time::Duration; use clap::ArgMatches; use color_eyre::eyre::{eyre, Result}; -use digest::Digest; use emver::Version; -use futures::Stream; -use helpers::AtomicFile; use lazy_static::lazy_static; use patch_db::{DbHandle, LockType, Revision}; -use regex::Regex; use reqwest::Url; use rpc_toolkit::command; -use sha2::Sha256; -use tokio::io::AsyncWriteExt; -use tokio::pin; use tokio::process::Command; -use tokio::time::Instant; use tokio_stream::StreamExt; use tracing::instrument; use crate::context::RpcContext; use crate::db::model::UpdateProgress; +use crate::disk::mount::filesystem::bind::Bind; use crate::disk::mount::filesystem::block_dev::BlockDev; use crate::disk::mount::filesystem::httpdirfs::HttpDirFS; use crate::disk::mount::filesystem::ReadOnly; -use crate::disk::mount::filesystem::{FileSystem, ReadWrite}; -use crate::disk::mount::guard::TmpMountGuard; -use crate::disk::BOOT_RW_PATH; +use crate::disk::mount::filesystem::ReadWrite; +use crate::disk::mount::guard::{MountGuard, TmpMountGuard}; use crate::notifications::NotificationLevel; use crate::sound::{ CIRCLE_OF_5THS_SHORT, UPDATE_FAILED_1, UPDATE_FAILED_2, UPDATE_FAILED_3, UPDATE_FAILED_4, }; use crate::update::latest_information::LatestInformation; -use crate::util::rsync::Rsync; +use crate::util::rsync::{Rsync, RsyncOptions}; use crate::util::Invoke; use crate::version::{Current, VersionT}; use crate::{Error, ErrorKind, ResultExt}; @@ -228,7 +217,11 @@ async fn do_update( new_fs: TmpMountGuard, new_block_dev: TmpMountGuard, ) -> Result<(), Error> { - let mut rsync = Rsync::new(new_fs.as_ref().join(""), "/media/embassy/next", true)?; + let mut rsync = Rsync::new( + new_fs.as_ref().join(""), + "/media/embassy/next", + Default::default(), + )?; while let Some(progress) = rsync.progress.next().await { crate::db::DatabaseModel::new() .server_info() @@ -243,11 +236,14 @@ async fn do_update( ) .await?; } + rsync.wait().await?; new_fs.unmount().await?; new_block_dev.unmount().await?; + copy_fstab().await?; copy_machine_id().await?; copy_ssh_host_keys().await?; + sync_boot().await?; swap_boot_label().await?; Ok(()) @@ -271,6 +267,11 @@ impl std::fmt::Display for EosUrl { } } +async fn copy_fstab() -> Result<(), Error> { + tokio::fs::copy("/etc/fstab", "/media/embassy/next/etc/fstab").await?; + Ok(()) +} + async fn copy_machine_id() -> Result<(), Error> { tokio::fs::copy("/etc/machine-id", "/media/embassy/next/etc/machine-id").await?; Ok(()) @@ -310,6 +311,38 @@ async fn copy_ssh_host_keys() -> Result<(), Error> { Ok(()) } +async fn sync_boot() -> Result<(), Error> { + Rsync::new( + "/media/embassy/next/boot/", + "/boot", + RsyncOptions { + delete: false, + force: false, + ignore_existing: true, + }, + )? + .wait() + .await?; + let dev_mnt = + MountGuard::mount(&Bind::new("/dev"), "/media/embassy/next/dev", ReadWrite).await?; + let sys_mnt = + MountGuard::mount(&Bind::new("/sys"), "/media/embassy/next/sys", ReadWrite).await?; + let proc_mnt = + MountGuard::mount(&Bind::new("/proc"), "/media/embassy/next/proc", ReadWrite).await?; + let boot_mnt = + MountGuard::mount(&Bind::new("/boot"), "/media/embassy/next/boot", ReadWrite).await?; + Command::new("chroot") + .arg("/media/embassy/next") + .arg("update-grub") + .invoke(ErrorKind::MigrationFailed) + .await?; + boot_mnt.unmount().await?; + proc_mnt.unmount().await?; + sys_mnt.unmount().await?; + dev_mnt.unmount().await?; + Ok(()) +} + #[instrument] async fn swap_boot_label() -> Result<(), Error> { tokio::fs::write("/media/embassy/config/upgrade", b"").await?; diff --git a/backend/src/util/rsync.rs b/backend/src/util/rsync.rs index 978833bf0..377cfcbab 100644 --- a/backend/src/util/rsync.rs +++ b/backend/src/util/rsync.rs @@ -1,28 +1,54 @@ +use color_eyre::eyre::eyre; use std::path::Path; use helpers::NonDetachingJoinHandle; -use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::process::{Child, Command}; use tokio::sync::watch; use tokio_stream::wrappers::WatchStream; use crate::util::io::ByteReplacementReader; -use crate::Error; +use crate::{Error, ErrorKind}; + +pub struct RsyncOptions { + pub delete: bool, + pub force: bool, + pub ignore_existing: bool, +} +impl Default for RsyncOptions { + fn default() -> Self { + Self { + delete: true, + force: true, + ignore_existing: false, + } + } +} pub struct Rsync { pub command: Child, _progress_task: NonDetachingJoinHandle>, + stderr: NonDetachingJoinHandle>, pub progress: WatchStream, } impl Rsync { - pub fn new(src: impl AsRef, dst: impl AsRef, delete: bool) -> Result { + pub fn new( + src: impl AsRef, + dst: impl AsRef, + options: RsyncOptions, + ) -> Result { let mut cmd = Command::new("rsync"); - if delete { + if options.delete { cmd.arg("--delete"); } + if options.force { + cmd.arg("--force"); + } + if options.ignore_existing { + cmd.arg("--ignore-existing"); + } let mut command = cmd .arg("-a") - .arg("--no-devices") .arg("--info=progress2") .arg(src.as_ref()) .arg(dst.as_ref()) @@ -30,7 +56,14 @@ impl Rsync { .stdout(std::process::Stdio::piped()) .spawn()?; let cmd_stdout = command.stdout.take().unwrap(); + let mut cmd_stderr = command.stderr.take().unwrap(); let (send, recv) = watch::channel(0.0); + let stderr = tokio::spawn(async move { + let mut res = String::new(); + cmd_stderr.read_to_string(&mut res).await?; + Ok(res) + }) + .into(); let progress_task = tokio::spawn(async move { let mut lines = BufReader::new(ByteReplacementReader { replace: b'\r', @@ -52,8 +85,21 @@ impl Rsync { Ok(Rsync { command, _progress_task: progress_task, + stderr, progress: WatchStream::new(recv), }) } - // pub fn + pub async fn wait(mut self) -> Result<(), Error> { + let status = self.command.wait().await?; + let stderr = self.stderr.await.unwrap()?; + if status.success() { + tracing::info!("rsync: {}", stderr); + } else { + return Err(Error::new( + eyre!("rsync error: {}", stderr), + ErrorKind::Filesystem, + )); + } + Ok(()) + } } diff --git a/build/lib/scripts/install b/build/lib/scripts/install index e978bbc49..997e7ecfc 100755 --- a/build/lib/scripts/install +++ b/build/lib/scripts/install @@ -92,6 +92,20 @@ if [ -n "$WIFI_IFACE" ]; then echo "wifi-interface: $WIFI_IFACE" >> /mnt/config/config.yaml fi +# gen fstab +cat > /mnt/current/etc/fstab << EOF +$BOOTPART /boot vfat defaults 0 2 +$ROOTPART / ext4 defaults 0 1 +EOF + +# gen machine-id +chroot /mnt/current systemd-machine-id-setup + +# gen ssh host keys +ssh-keygen -t rsa /mnt/current/etc/ssh_host_rsa_key +ssh-keygen -t ecdsa /mnt/current/etc/ssh_host_ecdsa_key +ssh-keygen -t ed25519 /mnt/current/etc/ssh_host_ed25519_key + mount --bind /dev /mnt/current/dev mount --bind /sys /mnt/current/sys mount --bind /proc /mnt/current/proc From b5c25cc2cf7fc8326931b3eee00c767777201614 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 19 Oct 2022 21:54:20 -0600 Subject: [PATCH 53/53] gahhhhh --- .github/workflows/product.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/product.yaml b/.github/workflows/product.yaml index c9c716f05..a4bea3b77 100644 --- a/.github/workflows/product.yaml +++ b/.github/workflows/product.yaml @@ -62,27 +62,18 @@ jobs: name: compat.tar path: system-images/compat/docker-images/ - - name: Rename compat.tar artifact - run: mv system-images/compat/docker-images/compat.tar system-images/compat/docker-images/aarch64.tar - - name: Download utils.tar artifact uses: actions/download-artifact@v3 with: name: utils.tar path: system-images/utils/docker-images/ - - name: Rename utils.tar artifact - run: mv system-images/utils/docker-images/utils.tar system-images/utils/docker-images/aarch64.tar - - name: Download binfmt.tar artifact uses: actions/download-artifact@v3 with: name: binfmt.tar path: system-images/binfmt/docker-images/ - - name: Rename binfmt.tar artifact - run: mv system-images/binfmt/docker-images/binfmt.tar system-images/binfmt/docker-images/aarch64.tar - - name: Download nc-broadcast.tar artifact uses: actions/download-artifact@v3 with: