fix(core): Make TRIGGER backup work while stores are in use
`TRIGGER backup` answered `ERR internal_error` in two common cases: - a KV store was open in the pool (e.g. right after a `PUSH`): the backup opened the database a second time, which RocksDB refuses as its `LOCK` file is held. The pooled store is now backed up instead, after a flush. - the KV or FST store directory did not exist yet (nothing ingested, or nothing consolidated yet): listing it failed. There is nothing to back up in that case. Also enable the `trigger_backup` test, which covers both cases. Fixes #274
This commit is contained in:
1 parent
fdad02e264
commit
51a00dd099
3 files changed
+75
-9
No files matched your search
@@ -41,9 +41,52 @@ fn trigger_consolidate() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "Not supported by sonic_client yet (FIXME)"]
|
||||
fn trigger_backup() {
|
||||
todo!()
|
||||
let ctx = start_empty(|command| {
|
||||
command
|
||||
.env("SONIC_STORE__FST__GRAPH__CONSOLIDATE_AFTER", "3600")
|
||||
.env("SONIC_STORE__FST__POOL__INACTIVE_AFTER", "3700")
|
||||
.env("SONIC_STORE__KV__DATABASE__FLUSH_AFTER", "3600")
|
||||
.env("SONIC_STORE__KV__POOL__INACTIVE_AFTER", "3700")
|
||||
});
|
||||
|
||||
let multiplexer = SonicMultiplexer::new().unwrap();
|
||||
|
||||
let ingest =
|
||||
SonicChannelIngestBlocking::connect(ctx.addr, "SecretPassword", &multiplexer).unwrap();
|
||||
let control =
|
||||
SonicChannelControlBlocking::connect(ctx.addr, "SecretPassword", &multiplexer).unwrap();
|
||||
|
||||
let backup_path = std::path::Path::new(env!("CARGO_TARGET_TMPDIR"))
|
||||
.join("test-backups")
|
||||
.join(ctx.id.to_string());
|
||||
if backup_path.exists() {
|
||||
std::fs::remove_dir_all(&backup_path).unwrap();
|
||||
}
|
||||
|
||||
// NOTE: Nothing was pushed yet, so the store directories do not exist.
|
||||
() = control
|
||||
.trigger_backup(backup_path.to_str().unwrap())
|
||||
.unwrap();
|
||||
|
||||
// NOTE: Pushing opens the KV store, which then stays open in the pool
|
||||
// while the backup runs, and leaves the FST changes pending.
|
||||
ingest
|
||||
.push("collection", "bucket", "object", "foo bar")
|
||||
.unwrap();
|
||||
|
||||
() = control
|
||||
.trigger_backup(backup_path.to_str().unwrap())
|
||||
.unwrap();
|
||||
|
||||
let kv_backups = std::fs::read_dir(backup_path.join("kv"))
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(kv_backups.len(), 1);
|
||||
assert!(std::fs::read_dir(&kv_backups[0]).unwrap().next().is_some());
|
||||
|
||||
std::fs::remove_dir_all(&backup_path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in new issue
Block a user