docs(meta/upgrade): add doc for meta-upgrade

Rename upgrade program to `databend-meta-upgrade-09`.

.github/actions/test_meta-upgrade-2023-01-15/action.yml → .github/actions/test_meta-upgrade-09/action.yml

@@ -1,4 +1,4 @@
-name: "Test meta-upgrade-2023-01-15"
+name: "Test meta-upgrade-09"
 description: ""
 inputs:
   profile:
@@ -19,7 +19,7 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Test meta-upgrade-2023-01-15
+    - name: Test meta-upgrade-09
       shell: bash
       run: |
-        bash ./tests/meta-upgrade-2023-01-15/test-meta-upgrade-to-24.sh
+        bash ./tests/meta-upgrade-09/test-meta-upgrade-to-24.sh

.github/workflows/dev-linux.yml

@@ -99,13 +99,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_metactl
 
-  test_meta-upgrade-2023-01-15:
+  test_meta-upgrade-09:
     timeout-minutes: 10
     runs-on: [self-hosted, X64, Linux, development]
     needs: build_gnu
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_meta-upgrade-2023-01-15
+      - uses: ./.github/actions/test_meta-upgrade-09
 
   test_compat:
     timeout-minutes: 10

Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 members = [
     # Binaries
     "src/binaries",
-    "src/binaries/meta-upgrade-2023-01-15",
+    "src/binaries/meta-upgrade-09",
     # Common
     "src/common/arrow",
     "src/common/auth",

docs/doc/10-deploy/09-upgrade/90-upgrade-0.8-to-0.9.md

@@ -0,0 +1,50 @@
+---
+title: Upgrade Databend-query from 0.8 to 0.9
+sidebar_label: Upgrade Databend-query from 0.8 to 0.9 
+description:
+    Meta data migration when upgrading databend-query from 0.8 to 0.9
+---
+
+This topic explains how to migrate metadata when upgrading databend-query from 0.8 to 0.9
+
+Databend-query-0.9 introduces incompatible changes in metadata, these metadata has to be manually migrated.
+Databend provides a program for this job: `databend-meta-upgrade-09`, which you can find in a [release package](https://github.com/datafuselabs/databend/releases) or can be built from source.
+
+`databend-meta-upgrade-09` upgrades metadata written by databend-query before 0.9 to a databend-query-0.9 compatible format.
+It loads metadata from a `raft-dir` and upgrade `TableMeta` record to 0.9 compatible version and write them back.
+Both raft-log data and state-machine data will be converted.
+
+# Usage
+
+- Shut down all databend-meta processes.
+
+- Before proceeding, backup your data. See: [Backup and resotre metadata](../06-metasrv/30-metasrv-backup-restore.md).
+
+- Download `databend-meta-upgrade-09` from [Databend release](https://github.com/datafuselabs/databend/releases)
+  or build it with `cargo build --bin databend-meta-upgrade-09`
+
+- (Optional) To view the current `TableMeta` version, print all `TableMeta` records with the following command:
+  It should display a list of `TableMeta` record.
+  You need to upgrade only if there is a `ver` that is lower than 24.
+
+   ```text
+   databend-meta-upgrade-09 --cmd print --raft-dir "<./your/raft-dir/>"
+   # output:
+   # TableMeta { ver: 23, ..
+   ```
+
+- Upgrade metadata:
+
+  ```text
+  databend-meta-upgrade-09 --cmd upgrade --raft-dir "<./your/raft-dir/>"
+  ```
+
+- To assert upgrade has finished successfully, print all `TableMeta` records that are found in meta dir with the following command:
+  It should display a list of TableMeta record with a `ver` that is greater or equal 24.
+
+   ```text
+   databend-meta-upgrade-09 --cmd print --raft-dir "<./your/raft-dir/>"
+   # output:
+   # TableMeta { ver: 25, ..
+   # TableMeta { ver: 25, ..
+   ```

src/binaries/meta-upgrade-2023-01-15/Cargo.toml → src/binaries/meta-upgrade-09/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "databend-meta-upgrade-2022-01-15"
+name = "databend-meta-upgrade-09"
 description = "upgrade meta-service data to VER=24"
 version = { workspace = true }
 authors = { workspace = true }
@@ -28,7 +28,7 @@ tokio = { version = "1.21.1", features = ["full"] }
 tracing = "0.1.36"
 
 [[bin]]
-name = "databend-meta-upgrade-2023-01-15"
+name = "databend-meta-upgrade-09"
 path = "src/main.rs"
 doctest = false
 test = false

src/binaries/meta-upgrade-2023-01-15/src/main.rs → src/binaries/meta-upgrade-09/src/main.rs

@@ -14,18 +14,44 @@
 
 #![allow(clippy::uninlined_format_args)]
 
-//! This program load meta-service data from a `raft-dir` and upgrade TableMeta to new version and write them back.
+//! This program upgrades metadata written by databend-query before 0.9 to a databend-query-0.9 compatible format.
 //!
+//! It loads metadata from a `raft-dir` and upgrade TableMeta to new version and write them back.
 //! Both in raft-log data and in state-machine data will be converted.
 //!
 //! Usage:
 //!
-//! - Shut down databend-meta
+//! - Shut down all databend-meta processes.
+//!
 //! - Backup before proceeding: https://databend.rs/doc/deploy/metasrv/metasrv-backup-restore
-//! - Build it with `cargo build --bin databend-meta-updater-2023-01-15` and run it:
-//! ```text
-//! databend-meta-updater-2023-01-15 --cmd upgrade --raft-dir "<./your/raft-dir/>"
-//! ```
+//!
+//! - Build it with `cargo build --bin databend-meta-upgrade-09`.
+//!
+//! - To view the current TableMeta version, print all TableMeta records with the following command:
+//!   It should display a list of TableMeta record.
+//!   You need to upgrade only if there is a `ver` that is lower than 24.
+//!
+//!    ```text
+//!    databend-meta-upgrade-09 --cmd print --raft-dir "<./your/raft-dir/>"
+//!    # output:
+//!    # TableMeta { ver: 23, ..
+//!    ```
+//!
+//! - Run it:
+//!
+//!   ```text
+//!   databend-meta-upgrade-09 --cmd upgrade --raft-dir "<./your/raft-dir/>"
+//!   ```
+//!
+//! - To assert upgrade has finished successfully, print all TableMeta records that are found in meta dir with the following command:
+//!   It should display a list of TableMeta record with a `ver` that is greater or equal 24.
+//!
+//!    ```text
+//!    databend-meta-upgrade-09 --cmd print --raft-dir "<./your/raft-dir/>"
+//!    # output:
+//!    # TableMeta { ver: 25, ..
+//!    # TableMeta { ver: 25, ..
+//!    ```
 
 mod rewrite;
 
@@ -92,7 +118,7 @@ impl Default for RaftConfig {
 async fn main() -> anyhow::Result<()> {
     let config = Config::parse();
 
-    let _guards = init_logging("databend-meta-updater-2023-01-15", &LogConfig::default());
+    let _guards = init_logging("databend-meta-upgrade-09", &LogConfig::default());
 
     eprintln!();
     eprintln!("███╗   ███╗███████╗████████╗ █████╗ ");
@@ -101,20 +127,20 @@ async fn main() -> anyhow::Result<()> {
     eprintln!("██║╚██╔╝██║██╔══╝     ██║   ██╔══██║");
     eprintln!("██║ ╚═╝ ██║███████╗   ██║   ██║  ██║");
     eprintln!("╚═╝     ╚═╝╚══════╝   ╚═╝   ╚═╝  ╚═╝");
-    eprintln!("     --- Updater 2023-01-15 ---     ");
+    eprintln!("     --- Upgrade to 0.9 ---         ");
     eprintln!();
 
     eprintln!("config: {}", pretty(&config)?);
 
     match config.cmd.as_str() {
         "print" => {
-            print_all(&config)?;
+            print_table_meta(&config)?;
         }
         "upgrade" => {
             let p = GenericKVProcessor {
                 process_pb: conv_serialized_table_meta,
             };
-            rewrite::rewrite(&config, |x| p.process(x)).await?;
+            rewrite::rewrite(&config, |x| p.process(x))?;
         }
         _ => {
             return Err(anyhow::anyhow!("invalid cmd: {:?}", config.cmd));
@@ -124,7 +150,8 @@ async fn main() -> anyhow::Result<()> {
     Ok(())
 }
 
-pub fn print_all(config: &Config) -> anyhow::Result<()> {
+/// Print TableMeta in protobuf message format that are found in log or state machine.
+pub fn print_table_meta(config: &Config) -> anyhow::Result<()> {
     let p = GenericKVProcessor {
         process_pb: print_serialized_table_meta,
     };

src/binaries/meta-upgrade-2023-01-15/src/rewrite.rs → src/binaries/meta-upgrade-09/src/rewrite.rs

@@ -22,7 +22,7 @@ use crate::Config;
 ///
 /// The `convert` rewrite an entry if needed.
 /// If nothing needs to do, it should return `Ok(None)`
-pub async fn rewrite<F>(config: &Config, convert: F) -> anyhow::Result<()>
+pub fn rewrite<F>(config: &Config, convert: F) -> anyhow::Result<()>
 where F: Fn(RaftStoreEntry) -> Result<Option<RaftStoreEntry>, anyhow::Error> {
     let raft_config = &config.raft_config;
 

src/meta/proto-conv/src/util.rs

@@ -87,15 +87,18 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     //      and replace two of the variable `bytes` and `want`.
 ];
 
+/// Attribute of both a reader and a message:
 /// The version to write into a message and it is also the version of the message reader.
 pub const VER: u64 = META_CHANGE_LOG.last().unwrap().0;
 
-/// The minimal reader version that can read message of version `VER`, i.e. `message.version=VER`.
+/// Attribute of a message:
+/// The minimal reader version that can read message of version `VER`, i.e. `message.ver=VER`.
 ///
 /// This is written to every message that needs to be serialized independently.
 pub const MIN_READER_VER: u64 = 24;
 
-/// The minimal message version(`message.version`) that a reader can read.
+/// Attribute of a reader:
+/// The minimal message version(`message.ver`) that a reader can read.
 pub const MIN_MSG_VER: u64 = 1;
 
 pub fn reader_check_msg(msg_ver: u64, msg_min_reader_ver: u64) -> Result<(), Incompatible> {

tests/meta-upgrade-2023-01-15/test-meta-upgrade-to-24.sh

@@ -11,7 +11,7 @@ meta_dir="$SCRIPT_PATH/_meta_dir"
 meta_json="$SCRIPT_PATH/meta-v23.txt"
 exported="$SCRIPT_PATH/exported"
 
-chmod +x ./target/${BUILD_PROFILE}/databend-meta-upgrade-2023-01-15
+chmod +x ./target/${BUILD_PROFILE}/databend-meta-upgrade-09
 
 echo " === import into $meta_dir"
 cat $meta_json |
@@ -22,7 +22,7 @@ count_of_table_meta=$(cat "$meta_json" | grep '__fd_table_by_id/' | wc -l)
 sleep 1
 
 echo " === upgrade"
-./target/${BUILD_PROFILE}/databend-meta-upgrade-2023-01-15 --cmd upgrade --raft-dir "$meta_dir"
+./target/${BUILD_PROFILE}/databend-meta-upgrade-09 --cmd upgrade --raft-dir "$meta_dir"
 
 echo " === export from $meta_dir"
 ./target/${BUILD_PROFILE}/databend-metactl --export --raft-dir "$meta_dir" >$exported
@@ -37,7 +37,7 @@ else
 fi
 
 echo " === check ver"
-count_of_v25=$(./target/${BUILD_PROFILE}/databend-meta-upgrade-2023-01-15 --cmd print --raft-dir "$meta_dir" | grep ' ver: 25' | wc -l)
+count_of_v25=$(./target/${BUILD_PROFILE}/databend-meta-upgrade-09 --cmd print --raft-dir "$meta_dir" | grep ' ver: 25' | wc -l)
 if [ "$count_of_table_meta" == "$count_of_v25" ]; then
     echo " === count of ver=25: $count_of_v25; OK"
 else