Merge #113

113: snapshots r=MarinPostma a=MarinPostma

 This pr adds support for snapshoting.

The snapshoting process for an index requires that no other update is processing at the same time. A mutex lock has been added to prevent a snapshot from occuring at the same time as an update, while still premitting updates to be pushed.

The list of the indexes to snapshot is first retrieved from the `UuidResolver` which also performs its snapshot.

This list is passed to the update store, which attempts to acquire a lock on the update store while it snaphots itself and it's associated index store.

 This means that a snapshot can only be completed once all indexes have finished their ongoing update.

This pr also adds refactoring of the code to allow unit testing and mocking, and unit test the snapshot creation.

Co-authored-by: mpostma <postma.marin@protonmail.com>
Co-authored-by: tamo <irevoire@protonmail.ch>
Co-authored-by: marin <postma.marin@protonmail.com>
Co-authored-by: Marin Postma <postma.marin@protonmail.com>

meilisearch-http/Cargo.toml

@@ -80,10 +80,11 @@ version = "0.18.1"
 
 
 [dev-dependencies]
+actix-rt = "2.1.0"
+assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
+mockall = "0.9.1"
 serde_url_params = "0.2.0"
 tempdir = "0.3.7"
-assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
-actix-rt = "2.1.0"
 urlencoding = "1.1.1"
 
 [features]

meilisearch-http/src/data/mod.rs

@@ -1,7 +1,6 @@
 pub mod search;
 mod updates;
 
-use std::fs::create_dir_all;
 use std::ops::Deref;
 use std::sync::Arc;
 
@@ -59,10 +58,7 @@ impl Data {
     pub fn new(options: Opt) -> anyhow::Result<Data> {
         let path = options.db_path.clone();
 
-        create_dir_all(&path)?;
-        let index_size = options.max_mdb_size.get_bytes() as usize;
-        let update_store_size = options.max_udb_size.get_bytes() as usize;
-        let index_controller = IndexController::new(&path, index_size, update_store_size)?;
+        let index_controller = IndexController::new(&path, &options)?;
 
         let mut api_keys = ApiKeys {
             master: options.clone().master_key,

meilisearch-http/src/helpers/compression.rs

@@ -1,27 +1,26 @@
-use flate2::read::GzDecoder;
-use flate2::write::GzEncoder;
-use flate2::Compression;
 use std::fs::{create_dir_all, File};
+use std::io::Write;
 use std::path::Path;
-use tar::{Archive, Builder};
 
-use crate::error::Error;
+use flate2::{read::GzDecoder, write::GzEncoder, Compression};
+use tar::{Archive, Builder};
 
-pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> {
-    let f = File::create(dest)?;
-    let gz_encoder = GzEncoder::new(f, Compression::default());
+pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut f = File::create(dest)?;
+    let gz_encoder = GzEncoder::new(&mut f, Compression::default());
     let mut tar_encoder = Builder::new(gz_encoder);
     tar_encoder.append_dir_all(".", src)?;
     let gz_encoder = tar_encoder.into_inner()?;
     gz_encoder.finish()?;
+    f.flush()?;
     Ok(())
 }
 
-pub fn from_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> {
-    let f = File::open(src)?;
+pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
+    let f = File::open(&src)?;
     let gz = GzDecoder::new(f);
     let mut ar = Archive::new(gz);
-    create_dir_all(dest)?;
-    ar.unpack(dest)?;
+    create_dir_all(&dest)?;
+    ar.unpack(&dest)?;
     Ok(())
 }

meilisearch-http/src/index/search.rs

@@ -20,7 +20,6 @@ const fn default_search_limit() -> usize {
 
 #[derive(Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
-#[allow(dead_code)]
 pub struct SearchQuery {
     pub q: Option<String>,
     pub offset: Option<usize>,