[metasrv]: feature: import exported json to restore a metasrv state

.github/actions/test_metactl/action.yml

@@ -0,0 +1,27 @@
+name: 'Test metactl'
+inputs:
+  profile:
+    description: 'The profile for this test'
+    required: true
+    default: 'debug'
+runs:
+  using: "composite"
+  steps:
+
+    - name: Maximize build space
+      uses: ./.github/actions/cleanup
+
+    # If you need to reset the cache version, increment the number after `v`
+    - uses: Swatinem/rust-cache@v1
+      with:
+        sharedKey: unit-v1
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ inputs.profile }}-${{ github.sha }}-${{ matrix.config.os }}-${{ matrix.config.target }}
+        path: ./target/debug
+
+    - name: Test metactl
+      shell: bash
+      run: |
+        make metactl-test

.github/workflows/developing.yml

@@ -59,6 +59,17 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
 
+  test_metactl:
+    runs-on: ${{ matrix.config.os }}
+    needs: build
+    strategy:
+      matrix:
+        config:
+          - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu }
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/test_metactl
+
   test_stateless_standalone:
     runs-on: ${{ matrix.config.os }}
     needs: build

Makefile

@@ -129,7 +129,10 @@ stateless-cluster-test-tls: build-debug
 	rm -rf ./_meta*/
 	bash ./scripts/ci/ci-run-stateless-tests-cluster-tls.sh
 
-test: unit-test stateless-test
+metactl-test: build-debug
+	bash ./tests/metactl/test-metactl.sh
+
+test: unit-test stateless-test metactl-test
 
 docker:
 	docker build --network host -f docker/Dockerfile -t ${HUB}/databend-query:${TAG} .

metasrv/Cargo.toml

@@ -18,7 +18,7 @@ doctest = false
 test = false
 
 [[bin]]
-name = "metactl"
+name = "databend-metactl"
 path = "src/bin/metactl.rs"
 doctest = false
 test = false

metasrv/src/bin/metactl.rs

@@ -12,13 +12,38 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(stdin_forwarders)]
+
+use std::collections::BTreeMap;
+use std::io;
+
 use clap::Parser;
 use common_base::tokio;
 use common_meta_raft_store::config::RaftConfig;
+use common_meta_raft_store::sled_key_spaces::KeySpaceKV;
 use common_meta_sled_store::get_sled_db;
 use common_meta_sled_store::init_sled_db;
 use common_tracing::init_global_tracing;
 use databend_meta::export::deserialize_to_kv_variant;
+use databend_meta::export::serialize_kv_variant;
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Parser)]
+#[clap(about, version, author)]
+struct Config {
+    #[clap(long, default_value = "INFO")]
+    pub log_level: String,
+
+    #[clap(long)]
+    pub import: bool,
+
+    #[clap(long)]
+    pub export: bool,
+
+    #[clap(flatten)]
+    pub raft_config: RaftConfig,
+}
 
 /// Usage:
 /// - To dump a sled db: `$0 --raft-dir ./_your_meta_dir/`:
@@ -31,24 +56,97 @@ use databend_meta::export::deserialize_to_kv_variant;
 ///   ```
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    let raft_config: RaftConfig = RaftConfig::parse();
+    let config = Config::parse();
+    let raft_config = &config.raft_config;
+
+    let _guards = init_global_tracing("metactl", "./_metactl_log", &config.log_level);
 
-    let _guards = init_global_tracing("metactl", "./_metactl_log", "debug");
+    eprintln!("");
+    eprintln!("███╗   ███╗███████╗████████╗ █████╗        ██████╗████████╗██╗     ");
+    eprintln!("████╗ ████║██╔════╝╚══██╔══╝██╔══██╗      ██╔════╝╚══██╔══╝██║     ");
+    eprintln!("██╔████╔██║█████╗     ██║   ███████║█████╗██║        ██║   ██║     ");
+    eprintln!("██║╚██╔╝██║██╔══╝     ██║   ██╔══██║╚════╝██║        ██║   ██║     ");
+    eprintln!("██║ ╚═╝ ██║███████╗   ██║   ██║  ██║      ╚██████╗   ██║   ███████╗");
+    eprintln!("╚═╝     ╚═╝╚══════╝   ╚═╝   ╚═╝  ╚═╝       ╚═════╝   ╚═╝   ╚══════╝");
+    eprintln!("");
 
-    println!("raft_config: {:?}", raft_config);
+    eprintln!("raft_config: {}", pretty(raft_config)?);
 
     init_sled_db(raft_config.raft_dir.clone());
 
-    print_meta().await
+    if config.export {
+        eprintln!("export meta dir from: {}", raft_config.raft_dir);
+        print_meta()?;
+    } else if config.import {
+        eprintln!("import meta dir into: {}", raft_config.raft_dir);
+        clear()?;
+        import_from_stdin()?;
+    }
+
+    Ok(())
+}
+
+fn pretty<T>(v: &T) -> Result<String, serde_json::Error>
+where T: Serialize {
+    serde_json::to_string_pretty(v)
+}
+
+fn clear() -> anyhow::Result<()> {
+    let db = get_sled_db();
+
+    let tree_names = db.tree_names();
+    for n in tree_names.iter() {
+        let name = String::from_utf8(n.to_vec())?;
+        let tree = db.open_tree(&name)?;
+        tree.clear()?;
+        eprintln!("Clear sled tree {} Done", name);
+    }
+
+    Ok(())
+}
+
+/// Read every line from stdin, deserialize it into tree_name, key and value. Insert them into sled db and flush.
+fn import_from_stdin() -> anyhow::Result<()> {
+    let db = get_sled_db();
+
+    let mut trees = BTreeMap::new();
+
+    let lines = io::stdin().lines();
+    let mut n = 0;
+    for line in lines {
+        let l = line?;
+        let (tree_name, kv_variant): (String, KeySpaceKV) = serde_json::from_str(&l)?;
+        // eprintln!("line: {}", l);
+
+        if !trees.contains_key(&tree_name) {
+            let tree = db.open_tree(&tree_name)?;
+            trees.insert(tree_name.clone(), tree);
+        }
+
+        let tree = trees.get(&tree_name).unwrap();
+
+        let (k, v) = serialize_kv_variant(&kv_variant)?;
+
+        tree.insert(k, v)?;
+        n += 1;
+    }
+
+    for tree in trees.values() {
+        tree.flush()?;
+    }
+
+    eprintln!("Imported {} records", n);
+
+    Ok(())
 }
 
 /// Print the entire sled db.
 ///
 /// The output encodes every key-value into one line:
-/// `[sled_tree_name, subtree_prefix(u8), subtree_prefix(str), key, value]`
+/// `[sled_tree_name, {key_space: {key, value}}]`
 /// E.g.:
-/// `["global-local-kv/state_machine/0",7,"sledks::Sequences","databases",1]`
-async fn print_meta() -> anyhow::Result<()> {
+/// `["test-29000-state_machine/0",{"GenericKV":{"key":"wow","value":{"seq":3,"meta":null,"data":[119,111,119]}}}`
+fn print_meta() -> anyhow::Result<()> {
     let db = get_sled_db();
 
     let tree_names = db.tree_names();

metasrv/src/export.rs

@@ -36,6 +36,32 @@ macro_rules! to_kv_variant {
     };
 }
 
+pub fn serialize_kv_variant(kv: &KeySpaceKV) -> Result<(sled::IVec, sled::IVec), MetaStorageError> {
+    macro_rules! ser {
+        ($ks:tt, $key:expr, $value:expr) => {
+            Ok((
+                sled_key_spaces::$ks::serialize_key($key)?,
+                sled_key_spaces::$ks::serialize_value($value)?,
+            ))
+        };
+    }
+
+    match kv {
+        KeySpaceKV::Logs { key, value } => ser!(Logs, key, value),
+        KeySpaceKV::Nodes { key, value } => ser!(Nodes, key, value),
+        KeySpaceKV::StateMachineMeta { key, value } => ser!(StateMachineMeta, key, value),
+        KeySpaceKV::RaftStateKV { key, value } => ser!(RaftStateKV, key, value),
+        KeySpaceKV::GenericKV { key, value } => ser!(GenericKV, key, value),
+        KeySpaceKV::Sequences { key, value } => ser!(Sequences, key, value),
+        KeySpaceKV::Databases { key, value } => ser!(Databases, key, value),
+        KeySpaceKV::Tables { key, value } => ser!(Tables, key, value),
+        KeySpaceKV::ClientLastResps { key, value } => ser!(ClientLastResps, key, value),
+        KeySpaceKV::TableLookup { key, value } => ser!(TableLookup, key, value),
+        KeySpaceKV::DatabaseLookup { key, value } => ser!(DatabaseLookup, key, value),
+        KeySpaceKV::LogMeta { key, value } => ser!(LogMeta, key, value),
+    }
+}
+
 pub fn deserialize_to_kv_variant(d: &[Vec<u8>]) -> Result<KeySpaceKV, MetaStorageError> {
     let prefix_key = &d[0];
     let vec_value = &d[1];

tests/metactl/.gitignore

@@ -0,0 +1,2 @@
+/_meta_dir
+/exported

tests/metactl/README.md

@@ -0,0 +1 @@
+Data for testing metactl