Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/long running #1676

Merged
merged 22 commits into from
Aug 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
19ac5f3
feat: Start the long running container
Blu-J Jul 15, 2022
cf99ab4
Merge remote-tracking branch 'origin/master' into feat/long-running
Blu-J Jul 18, 2022
c8beec7
Merge remote-tracking branch 'origin/master' into feat/long-running
Blu-J Jul 19, 2022
46c97c1
feat: Long running docker, running, stoping, and uninstalling
Blu-J Jul 19, 2022
f879a1d
Merge remote-tracking branch 'origin/master' into feat/long-running
Blu-J Jul 26, 2022
2e7ad3d
Merge remote-tracking branch 'origin/master' into feat/long-running
Blu-J Jul 28, 2022
e667d63
feat: Just make the folders that we would like to mount.
Blu-J Jul 28, 2022
52f1d5a
Merge remote-tracking branch 'origin/master' into feat/long-running
Blu-J Jul 31, 2022
84be459
fix: Uninstall not working
Blu-J Aug 1, 2022
60f75e1
chore: remove some logging
Blu-J Aug 1, 2022
bebf498
feat: Smarter cleanup
Blu-J Aug 2, 2022
f5ff316
feat: Wait for start
Blu-J Aug 3, 2022
10b9674
wip: Need to kill
Blu-J Aug 3, 2022
3475876
chore: Remove the bad tracing
Blu-J Aug 3, 2022
a8b838d
feat: Stopping the long running processes without killing the long
Blu-J Aug 10, 2022
05c6879
Mino Feat: Change the Manifest To have a new type (#1736)
Blu-J Aug 17, 2022
3f162b2
Merge remote-tracking branch 'origin/next' into feat/long-running
Blu-J Aug 17, 2022
95a9f88
chore: Fix a dbg!
Blu-J Aug 17, 2022
9148586
chore: Make the commands of the docker-inject do inject instead of exec
Blu-J Aug 17, 2022
49e9b45
chore: Fix compile mistake
Blu-J Aug 17, 2022
53d2975
chore: Change to use simpler
Blu-J Aug 18, 2022
2e1a659
Merge remote-tracking branch 'origin/next' into feat/long-running
Blu-J Aug 18, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions backend/src/action.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,17 @@ use rpc_toolkit::command;
use serde::{Deserialize, Serialize};
use tracing::instrument;

use crate::config::{Config, ConfigSpec};
use crate::context::RpcContext;
use crate::id::ImageId;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::util::Version;
use crate::volume::Volumes;
use crate::{
config::{Config, ConfigSpec},
procedure::docker::DockerContainer,
};
use crate::{Error, ResultExt};
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct Actions(pub BTreeMap<ActionId, Action>);
Expand Down Expand Up @@ -58,12 +61,13 @@ impl Action {
#[instrument]
pub fn validate(
&self,
container: &Option<DockerContainer>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.implementation
.validate(eos_version, volumes, image_ids, true)
.validate(container, eos_version, volumes, image_ids, true)
.with_ctx(|_| {
(
crate::ErrorKind::ValidateS9pk,
Expand All @@ -76,6 +80,7 @@ impl Action {
pub async fn execute(
&self,
ctx: &RpcContext,
container: &Option<DockerContainer>,
pkg_id: &PackageId,
pkg_version: &Version,
action_id: &ActionId,
Expand All @@ -90,12 +95,12 @@ impl Action {
self.implementation
.execute(
ctx,
container,
pkg_id,
pkg_version,
ProcedureName::Action(action_id.clone()),
volumes,
input,
true,
None,
)
.await?
Expand Down Expand Up @@ -141,10 +146,24 @@ pub async fn action(
.get(&mut db, true)
.await?
.to_owned();

let container = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&pkg_id)
.and_then(|p| p.installed())
.expect(&mut db)
.await
.with_kind(crate::ErrorKind::NotFound)?
.manifest()
.container()
.get(&mut db, false)
.await?
.to_owned();
if let Some(action) = manifest.actions.0.get(&action_id) {
action
.execute(
&ctx,
&container,
&manifest.id,
&manifest.version,
&action_id,
Expand Down
24 changes: 19 additions & 5 deletions backend/src/backup/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ use tokio::io::AsyncWriteExt;
use tracing::instrument;

use self::target::PackageBackupInfo;
use crate::context::RpcContext;
use crate::dependencies::reconfigure_dependents_with_live_pointers;
use crate::id::ImageId;
use crate::install::PKG_ARCHIVE_DIR;
Expand All @@ -25,6 +24,7 @@ use crate::util::serde::IoFormat;
use crate::util::Version;
use crate::version::{Current, VersionT};
use crate::volume::{backup_dir, Volume, VolumeId, Volumes, BACKUP_DIR};
use crate::{context::RpcContext, procedure::docker::DockerContainer};
use crate::{Error, ErrorKind, ResultExt};

pub mod backup_bulk;
Expand Down Expand Up @@ -73,15 +73,16 @@ pub struct BackupActions {
impl BackupActions {
pub fn validate(
&self,
container: &Option<DockerContainer>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.create
.validate(eos_version, volumes, image_ids, false)
.validate(container, eos_version, volumes, image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Create"))?;
self.restore
.validate(eos_version, volumes, image_ids, false)
.validate(container, eos_version, volumes, image_ids, false)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Backup Restore"))?;
Ok(())
}
Expand All @@ -100,18 +101,30 @@ impl BackupActions {
let mut volumes = volumes.to_readonly();
volumes.insert(VolumeId::Backup, Volume::Backup { readonly: false });
let backup_dir = backup_dir(pkg_id);
let container = crate::db::DatabaseModel::new()
.package_data()
.idx_model(&pkg_id)
.and_then(|p| p.installed())
.expect(db)
.await
.with_kind(crate::ErrorKind::NotFound)?
.manifest()
.container()
.get(db, false)
.await?
.to_owned();
if tokio::fs::metadata(&backup_dir).await.is_err() {
tokio::fs::create_dir_all(&backup_dir).await?
}
self.create
.execute::<(), NoOutput>(
ctx,
&container,
pkg_id,
pkg_version,
ProcedureName::CreateBackup,
&volumes,
None,
false,
None,
)
.await?
Expand Down Expand Up @@ -186,6 +199,7 @@ impl BackupActions {
#[instrument(skip(ctx, db, secrets))]
pub async fn restore<Ex, Db: DbHandle>(
&self,
container: &Option<DockerContainer>,
ctx: &RpcContext,
db: &mut Db,
secrets: &mut Ex,
Expand All @@ -202,12 +216,12 @@ impl BackupActions {
self.restore
.execute::<(), NoOutput>(
ctx,
container,
pkg_id,
pkg_version,
ProcedureName::RestoreBackup,
&volumes,
None,
false,
None,
)
.await?
Expand Down
13 changes: 8 additions & 5 deletions backend/src/config/action.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ use serde::{Deserialize, Serialize};
use tracing::instrument;

use super::{Config, ConfigSpec};
use crate::context::RpcContext;
use crate::dependencies::Dependencies;
use crate::id::ImageId;
use crate::procedure::{PackageProcedure, ProcedureName};
use crate::s9pk::manifest::PackageId;
use crate::status::health_check::HealthCheckId;
use crate::util::Version;
use crate::volume::Volumes;
use crate::{context::RpcContext, procedure::docker::DockerContainer};
use crate::{Error, ResultExt};

#[derive(Debug, Deserialize, Serialize, HasModel)]
Expand All @@ -33,35 +33,37 @@ impl ConfigActions {
#[instrument]
pub fn validate(
&self,
container: &Option<DockerContainer>,
eos_version: &Version,
volumes: &Volumes,
image_ids: &BTreeSet<ImageId>,
) -> Result<(), Error> {
self.get
.validate(eos_version, volumes, image_ids, true)
.validate(container, eos_version, volumes, image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Get"))?;
self.set
.validate(eos_version, volumes, image_ids, true)
.validate(container, eos_version, volumes, image_ids, true)
.with_ctx(|_| (crate::ErrorKind::ValidateS9pk, "Config Set"))?;
Ok(())
}
#[instrument(skip(ctx))]
pub async fn get(
&self,
ctx: &RpcContext,
container: &Option<DockerContainer>,
pkg_id: &PackageId,
pkg_version: &Version,
volumes: &Volumes,
) -> Result<ConfigRes, Error> {
self.get
.execute(
ctx,
container,
pkg_id,
pkg_version,
ProcedureName::GetConfig,
volumes,
None::<()>,
false,
None,
)
.await
Expand All @@ -74,6 +76,7 @@ impl ConfigActions {
pub async fn set(
&self,
ctx: &RpcContext,
container: &Option<DockerContainer>,
pkg_id: &PackageId,
pkg_version: &Version,
dependencies: &Dependencies,
Expand All @@ -84,12 +87,12 @@ impl ConfigActions {
.set
.execute(
ctx,
container,
pkg_id,
pkg_version,
ProcedureName::SetConfig,
volumes,
Some(input),
false,
None,
)
.await
Expand Down
41 changes: 37 additions & 4 deletions backend/src/config/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use rpc_toolkit::command;
use serde_json::Value;
use tracing::instrument;

use crate::context::RpcContext;
use crate::db::model::{CurrentDependencies, CurrentDependencyInfo, CurrentDependents};
use crate::db::util::WithRevision;
use crate::dependencies::{
Expand All @@ -26,6 +25,7 @@ use crate::s9pk::manifest::{Manifest, PackageId};
use crate::util::display_none;
use crate::util::serde::{display_serializable, parse_stdin_deserializable, IoFormat};
use crate::Error;
use crate::{context::RpcContext, procedure::docker::DockerContainer};

pub mod action;
pub mod spec;
Expand Down Expand Up @@ -168,6 +168,7 @@ pub struct ConfigGetReceipts {
manifest_volumes: LockReceipt<crate::volume::Volumes, ()>,
manifest_version: LockReceipt<crate::util::Version, ()>,
manifest_config: LockReceipt<Option<ConfigActions>, ()>,
docker_container: LockReceipt<DockerContainer, String>,
}

impl ConfigGetReceipts {
Expand Down Expand Up @@ -203,11 +204,19 @@ impl ConfigGetReceipts {
.map(|x| x.manifest().config())
.make_locker(LockType::Write)
.add_to_keys(locks);
let docker_container = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().container())
.make_locker(LockType::Write)
.add_to_keys(locks);
move |skeleton_key| {
Ok(Self {
manifest_volumes: manifest_volumes.verify(skeleton_key)?,
manifest_version: manifest_version.verify(skeleton_key)?,
manifest_config: manifest_config.verify(skeleton_key)?,
docker_container: docker_container.verify(skeleton_key)?,
})
}
}
Expand All @@ -230,9 +239,11 @@ pub async fn get(
.await?
.ok_or_else(|| Error::new(eyre!("{} has no config", id), crate::ErrorKind::NotFound))?;

let container = receipts.docker_container.get(&mut db, &id).await?;

let volumes = receipts.manifest_volumes.get(&mut db).await?;
let version = receipts.manifest_version.get(&mut db).await?;
action.get(&ctx, &id, &version, &volumes).await
action.get(&ctx, &container, &id, &version, &volumes).await
}

#[command(
Expand Down Expand Up @@ -275,6 +286,7 @@ pub struct ConfigReceipts {
pub current_dependencies: LockReceipt<CurrentDependencies, String>,
dependency_errors: LockReceipt<DependencyErrors, String>,
manifest_dependencies_config: LockReceipt<DependencyConfig, (String, String)>,
docker_container: LockReceipt<DockerContainer, String>,
}

impl ConfigReceipts {
Expand Down Expand Up @@ -379,6 +391,13 @@ impl ConfigReceipts {
.and_then(|x| x.manifest().dependencies().star().config())
.make_locker(LockType::Write)
.add_to_keys(locks);
let docker_container = crate::db::DatabaseModel::new()
.package_data()
.star()
.installed()
.and_then(|x| x.manifest().container())
.make_locker(LockType::Write)
.add_to_keys(locks);

move |skeleton_key| {
Ok(Self {
Expand All @@ -398,6 +417,7 @@ impl ConfigReceipts {
current_dependencies: current_dependencies.verify(skeleton_key)?,
dependency_errors: dependency_errors.verify(skeleton_key)?,
manifest_dependencies_config: manifest_dependencies_config.verify(skeleton_key)?,
docker_container: docker_container.verify(skeleton_key)?,
})
}
}
Expand Down Expand Up @@ -496,6 +516,8 @@ pub fn configure_rec<'a, Db: DbHandle>(
receipts: &'a ConfigReceipts,
) -> BoxFuture<'a, Result<(), Error>> {
async move {
let container = receipts.docker_container.get(db, &id).await?;
let container = &container;
// fetch data from db
let action = receipts
.config_actions
Expand All @@ -519,7 +541,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
let ConfigRes {
config: old_config,
spec,
} = action.get(ctx, id, &version, &volumes).await?;
} = action.get(ctx, container, id, &version, &volumes).await?;

// determine new config to use
let mut config = if let Some(config) = config.or_else(|| old_config.clone()) {
Expand Down Expand Up @@ -587,7 +609,15 @@ pub fn configure_rec<'a, Db: DbHandle>(
let signal = if !dry_run {
// run config action
let res = action
.set(ctx, id, &version, &dependencies, &volumes, &config)
.set(
ctx,
container,
id,
&version,
&dependencies,
&volumes,
&config,
)
.await?;

// track dependencies with no pointers
Expand Down Expand Up @@ -679,6 +709,8 @@ pub fn configure_rec<'a, Db: DbHandle>(
.unwrap_or_default();
let next = Value::Object(config.clone());
for (dependent, dep_info) in dependents.0.iter().filter(|(dep_id, _)| dep_id != &id) {
let dependent_container = receipts.docker_container.get(db, &dependent).await?;
let dependent_container = &dependent_container;
// check if config passes dependent check
if let Some(cfg) = receipts
.manifest_dependencies_config
Expand All @@ -693,6 +725,7 @@ pub fn configure_rec<'a, Db: DbHandle>(
if let Err(error) = cfg
.check(
ctx,
dependent_container,
dependent,
&manifest.version,
&manifest.volumes,
Expand Down
Loading