Skip to content
This repository has been archived by the owner on Jul 1, 2021. It is now read-only.

snapshots #113

Merged
merged 30 commits into from
Apr 1, 2021
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
ee838be
implement snapshot scheduler
MarinPostma Mar 17, 2021
c966b1d
use options to schedule snapshot
MarinPostma Mar 17, 2021
35a7b80
snapshot indexes
MarinPostma Mar 19, 2021
520f7c0
sequential index snapshot
MarinPostma Mar 20, 2021
7f6a54c
add lock to prevent snapshot during update
MarinPostma Mar 20, 2021
4847884
restore snapshots
MarinPostma Mar 22, 2021
a85e7ab
fix snapshot creation
MarinPostma Mar 22, 2021
44dcfe2
clean snapshot creation
MarinPostma Mar 22, 2021
d73fbde
remove from snapshot
MarinPostma Mar 22, 2021
e9da191
fix snapshot bugs
MarinPostma Mar 22, 2021
1f51fc8
create indexes snapshots concurrently
MarinPostma Mar 22, 2021
3cc3637
refactor for tests
MarinPostma Mar 23, 2021
4629354
add tests and mocks
MarinPostma Mar 23, 2021
eb53ed4
load snapshot
MarinPostma Mar 23, 2021
48d5f88
fix snapshot dir already exists
MarinPostma Mar 23, 2021
06f9dae
remove prints
MarinPostma Mar 23, 2021
1f16c8d
integration test snapshot
MarinPostma Mar 24, 2021
4041d9d
format code
MarinPostma Mar 24, 2021
83ffdc8
remove bad file name test
MarinPostma Mar 24, 2021
d892a26
fix clippy
MarinPostma Mar 24, 2021
79d0970
perform snapshot on startup
MarinPostma Mar 25, 2021
d029464
fix snapshot path
MarinPostma Mar 25, 2021
4850746
add snapshot tests
MarinPostma Mar 25, 2021
f3dc853
Merge remote-tracking branch 'origin/main' into snapshots
MarinPostma Mar 25, 2021
7d6ec7f
resolve merge
MarinPostma Mar 25, 2021
d7c077c
atomic snapshot import
MarinPostma Mar 25, 2021
0f2143e
remove the now useless dead_code flags
irevoire Mar 26, 2021
7050236
Merge pull request #123 from irevoire/snapshots
MarinPostma Mar 26, 2021
d8af4a7
ignore snapshot test (#130)
MarinPostma Mar 31, 2021
248e9b3
Merge remote-tracking branch 'origin/main' into snapshots
MarinPostma Apr 1, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions meilisearch-http/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,11 @@ version = "0.18.1"


[dev-dependencies]
actix-rt = "2.1.0"
assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
mockall = "0.9.1"
serde_url_params = "0.2.0"
tempdir = "0.3.7"
assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
actix-rt = "2.1.0"
urlencoding = "1.1.1"

[features]
Expand Down
6 changes: 1 addition & 5 deletions meilisearch-http/src/data/mod.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
pub mod search;
mod updates;

use std::fs::create_dir_all;
use std::ops::Deref;
use std::sync::Arc;

Expand Down Expand Up @@ -59,10 +58,7 @@ impl Data {
pub fn new(options: Opt) -> anyhow::Result<Data> {
let path = options.db_path.clone();

create_dir_all(&path)?;
let index_size = options.max_mdb_size.get_bytes() as usize;
let update_store_size = options.max_udb_size.get_bytes() as usize;
let index_controller = IndexController::new(&path, index_size, update_store_size)?;
let index_controller = IndexController::new(&path, &options)?;

let mut api_keys = ApiKeys {
master: options.clone().master_key,
Expand Down
23 changes: 11 additions & 12 deletions meilisearch-http/src/helpers/compression.rs
Original file line number Diff line number Diff line change
@@ -1,27 +1,26 @@
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use std::fs::{create_dir_all, File};
use std::io::Write;
use std::path::Path;
use tar::{Archive, Builder};

use crate::error::Error;
use flate2::{read::GzDecoder, write::GzEncoder, Compression};
use tar::{Archive, Builder};

pub fn to_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> {
let f = File::create(dest)?;
let gz_encoder = GzEncoder::new(f, Compression::default());
pub fn to_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
let mut f = File::create(dest)?;
let gz_encoder = GzEncoder::new(&mut f, Compression::default());
let mut tar_encoder = Builder::new(gz_encoder);
tar_encoder.append_dir_all(".", src)?;
let gz_encoder = tar_encoder.into_inner()?;
gz_encoder.finish()?;
f.flush()?;
Ok(())
}

pub fn from_tar_gz(src: &Path, dest: &Path) -> Result<(), Error> {
let f = File::open(src)?;
pub fn from_tar_gz(src: impl AsRef<Path>, dest: impl AsRef<Path>) -> anyhow::Result<()> {
let f = File::open(&src)?;
let gz = GzDecoder::new(f);
let mut ar = Archive::new(gz);
create_dir_all(dest)?;
ar.unpack(dest)?;
create_dir_all(&dest)?;
ar.unpack(&dest)?;
Ok(())
}
1 change: 0 additions & 1 deletion meilisearch-http/src/index/search.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ const fn default_search_limit() -> usize {

#[derive(Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
#[allow(dead_code)]
pub struct SearchQuery {
pub q: Option<String>,
pub offset: Option<usize>,
Expand Down
Loading