Add incremental snapshot utils (solana-labs#18504)

This commit adds high-level functions for creating and loading-from
incremental snapshots, plus all low-level functions required to perform
those tasks.  This commit **does not** add taking incremental snapshots
as part of a running validator, nor starting up a node with an
incremental snapshot; just laying ground work.

Additionally, `snapshot_utils` and `serde_snapshot` have been
refactored to use a common code paths for the different snapshots.

Also of note, some renaming has happened:
  1. Snapshots are now either `full_` or `incremental_` throughout the
     codebase.  If not specified, the code applies to both.
  2. Bank snapshots now are called "bank snapshots"
     (before they were called "slot snapshots", "bank snapshots", or
      just "snapshots").  The one exception is within `Bank`, where they
     are still just "snapshots", because they are already "bank
     snapshots".
  3. Snapshot archives now have `_archive` in the code.  This
     should clear up an ambiguity between bank snapshots and snapshot
     archives.

core/src/accounts_hash_verifier.rs

@@ -99,6 +99,7 @@ impl AccountsHashVerifier {
         let accounts_package = solana_runtime::snapshot_utils::process_accounts_package_pre(
             accounts_package,
             thread_pool,
+            None,
         );
         Self::process_accounts_package(
             accounts_package,

core/src/snapshot_packager_service.rs

@@ -156,7 +156,7 @@ mod tests {
         }
 
         // Create a packageable snapshot
-        let output_tar_path = snapshot_utils::build_snapshot_archive_path(
+        let output_tar_path = snapshot_utils::build_full_snapshot_archive_path(
             snapshot_package_output_path,
             42,
             &Hash::default(),
@@ -177,7 +177,7 @@ mod tests {
         // Make tarball from packageable snapshot
         snapshot_utils::archive_snapshot_package(
             &snapshot_package,
-            snapshot_utils::DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            snapshot_utils::DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         )
         .unwrap();
 

core/src/test_validator.rs

@@ -13,7 +13,9 @@ use {
         genesis_utils::create_genesis_config_with_leader_ex,
         hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         snapshot_config::SnapshotConfig,
-        snapshot_utils::{ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
+        snapshot_utils::{
+            ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        },
     },
     solana_sdk::{
         account::{Account, AccountSharedData},
@@ -492,7 +494,7 @@ impl TestValidator {
                 snapshot_package_output_path: ledger_path.to_path_buf(),
                 archive_format: ArchiveFormat::Tar,
                 snapshot_version: SnapshotVersion::default(),
-                maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+                maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             }),
             enforce_ulimit_nofile: false,
             warp_slot: config.warp_slot,

core/src/validator.rs

@@ -1187,7 +1187,7 @@ fn new_banks_from_ledger(
         );
         leader_schedule_cache.set_root(&bank_forks.root_bank());
 
-        let archive_file = solana_runtime::snapshot_utils::bank_to_snapshot_archive(
+        let archive_file = solana_runtime::snapshot_utils::bank_to_full_snapshot_archive(
             ledger_path,
             &bank_forks.root_bank(),
             None,

core/tests/snapshots.rs

@@ -49,7 +49,9 @@ mod tests {
         bank_forks::BankForks,
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         snapshot_config::SnapshotConfig,
-        snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
+        snapshot_utils::{
+            self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+        },
         status_cache::MAX_CACHE_ENTRIES,
     };
     use solana_sdk::{
@@ -119,7 +121,7 @@ mod tests {
                 snapshot_path: PathBuf::from(snapshot_dir.path()),
                 archive_format: ArchiveFormat::TarBzip2,
                 snapshot_version,
-                maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+                maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
             };
             bank_forks.set_snapshot_config(Some(snapshot_config.clone()));
             SnapshotTestConfig {
@@ -148,20 +150,21 @@ mod tests {
         let old_last_bank = old_bank_forks.get(old_last_slot).unwrap();
 
         let check_hash_calculation = false;
-        let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archive(
+        let (deserialized_bank, _timing) = snapshot_utils::bank_from_snapshot_archives(
             account_paths,
             &[],
             &old_bank_forks
                 .snapshot_config
                 .as_ref()
                 .unwrap()
                 .snapshot_path,
-            snapshot_utils::build_snapshot_archive_path(
+            snapshot_utils::build_full_snapshot_archive_path(
                 snapshot_package_output_path.to_path_buf(),
                 old_last_bank.slot(),
                 &old_last_bank.get_accounts_hash(),
                 ArchiveFormat::TarBzip2,
             ),
+            None,
             ArchiveFormat::TarBzip2,
             old_genesis_config,
             None,
@@ -181,10 +184,10 @@ mod tests {
             .clone();
         assert_eq!(*bank, deserialized_bank);
 
-        let slot_snapshot_paths = snapshot_utils::get_snapshot_paths(&snapshot_path);
+        let bank_snapshot_infos = snapshot_utils::get_bank_snapshots(&snapshot_path);
 
-        for p in slot_snapshot_paths {
-            snapshot_utils::remove_snapshot(p.slot, &snapshot_path).unwrap();
+        for p in bank_snapshot_infos {
+            snapshot_utils::remove_bank_snapshot(p.slot, &snapshot_path).unwrap();
         }
     }
 
@@ -235,12 +238,11 @@ mod tests {
         let last_bank = bank_forks.get(last_slot).unwrap();
         let snapshot_config = &snapshot_test_config.snapshot_config;
         let snapshot_path = &snapshot_config.snapshot_path;
-        let last_slot_snapshot_path = snapshot_utils::get_snapshot_paths(snapshot_path)
-            .pop()
+        let last_bank_snapshot_info = snapshot_utils::get_highest_bank_snapshot_info(snapshot_path)
             .expect("no snapshots found in path");
-        let snapshot_package = snapshot_utils::package_snapshot(
+        let snapshot_package = snapshot_utils::package_full_snapshot(
             last_bank,
-            &last_slot_snapshot_path,
+            &last_bank_snapshot_info,
             snapshot_path,
             last_bank.src.slot_deltas(&last_bank.src.roots()),
             &snapshot_config.snapshot_package_output_path,
@@ -253,10 +255,11 @@ mod tests {
         let snapshot_package = snapshot_utils::process_accounts_package_pre(
             snapshot_package,
             Some(last_bank.get_thread_pool()),
+            None,
         );
         snapshot_utils::archive_snapshot_package(
             &snapshot_package,
-            DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         )
         .unwrap();
 
@@ -325,7 +328,8 @@ mod tests {
         // Take snapshot of zeroth bank
         let bank0 = bank_forks.get(0).unwrap();
         let storages = bank0.get_snapshot_storages();
-        snapshot_utils::add_snapshot(snapshot_path, bank0, &storages, snapshot_version).unwrap();
+        snapshot_utils::add_bank_snapshot(snapshot_path, bank0, &storages, snapshot_version)
+            .unwrap();
 
         // Set up snapshotting channels
         let (sender, receiver) = channel();
@@ -343,7 +347,7 @@ mod tests {
         let saved_slot = 4;
         let mut saved_archive_path = None;
 
-        for forks in 0..snapshot_utils::MAX_SNAPSHOTS + 2 {
+        for forks in 0..snapshot_utils::MAX_BANK_SNAPSHOTS + 2 {
             let bank = Bank::new_from_parent(
                 &bank_forks[forks as u64],
                 &Pubkey::default(),
@@ -420,7 +424,7 @@ mod tests {
                 let options = CopyOptions::new();
                 fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
 
-                saved_archive_path = Some(snapshot_utils::build_snapshot_archive_path(
+                saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
                     snapshot_package_output_path.to_path_buf(),
                     slot,
                     &accounts_hash,
@@ -431,11 +435,14 @@ mod tests {
 
         // Purge all the outdated snapshots, including the ones needed to generate the package
         // currently sitting in the channel
-        snapshot_utils::purge_old_snapshots(snapshot_path);
-        assert!(snapshot_utils::get_snapshot_paths(&snapshots_dir)
+        snapshot_utils::purge_old_bank_snapshots(snapshot_path);
+
+        let mut bank_snapshot_infos = snapshot_utils::get_bank_snapshots(&snapshots_dir);
+        bank_snapshot_infos.sort_unstable();
+        assert!(bank_snapshot_infos
             .into_iter()
             .map(|path| path.slot)
-            .eq(3..=snapshot_utils::MAX_SNAPSHOTS as u64 + 2));
+            .eq(3..=snapshot_utils::MAX_BANK_SNAPSHOTS as u64 + 2));
 
         // Create a SnapshotPackagerService to create tarballs from all the pending
         // SnapshotPackage's on the channel. By the time this service starts, we have already
@@ -453,7 +460,7 @@ mod tests {
             None,
             &exit,
             &cluster_info,
-            DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         );
 
         let thread_pool = accounts_db::make_min_priority_thread_pool();
@@ -471,6 +478,7 @@ mod tests {
                         solana_runtime::snapshot_utils::process_accounts_package_pre(
                             snapshot_package,
                             Some(&thread_pool),
+                            None,
                         );
                     *pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
                 }

download-utils/src/lib.rs

@@ -246,22 +246,22 @@ pub fn download_genesis_if_missing(
 
 pub fn download_snapshot<'a, 'b>(
     rpc_addr: &SocketAddr,
-    snapshot_output_dir: &Path,
+    snapshot_archives_dir: &Path,
     desired_snapshot_hash: (Slot, Hash),
     use_progress_bar: bool,
     maximum_snapshots_to_retain: usize,
     progress_notify_callback: &'a mut DownloadProgressCallbackOption<'b>,
 ) -> Result<(), String> {
-    snapshot_utils::purge_old_snapshot_archives(snapshot_output_dir, maximum_snapshots_to_retain);
+    snapshot_utils::purge_old_snapshot_archives(snapshot_archives_dir, maximum_snapshots_to_retain);
 
     for compression in &[
         ArchiveFormat::TarZstd,
         ArchiveFormat::TarGzip,
         ArchiveFormat::TarBzip2,
         ArchiveFormat::Tar, // `solana-test-validator` creates uncompressed snapshots
     ] {
-        let desired_snapshot_package = snapshot_utils::build_snapshot_archive_path(
-            snapshot_output_dir.to_path_buf(),
+        let desired_snapshot_package = snapshot_utils::build_full_snapshot_archive_path(
+            snapshot_archives_dir.to_path_buf(),
             desired_snapshot_hash.0,
             &desired_snapshot_hash.1,
             *compression,

ledger-tool/src/main.rs

@@ -30,7 +30,9 @@ use solana_runtime::{
     bank_forks::BankForks,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     snapshot_config::SnapshotConfig,
-    snapshot_utils::{self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
+    snapshot_utils::{
+        self, ArchiveFormat, SnapshotVersion, DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
+    },
 };
 use solana_sdk::{
     account::{AccountSharedData, ReadableAccount, WritableAccount},
@@ -697,7 +699,7 @@ fn load_bank_forks(
             snapshot_path,
             archive_format: ArchiveFormat::TarBzip2,
             snapshot_version: SnapshotVersion::default(),
-            maximum_snapshots_to_retain: DEFAULT_MAX_SNAPSHOTS_TO_RETAIN,
+            maximum_snapshots_to_retain: DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN,
         })
     };
     let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
@@ -904,7 +906,7 @@ fn main() {
         .default_value(SnapshotVersion::default().into())
         .help("Output snapshot version");
 
-    let default_max_snapshot_to_retain = &DEFAULT_MAX_SNAPSHOTS_TO_RETAIN.to_string();
+    let default_max_snapshot_to_retain = &DEFAULT_MAX_FULL_SNAPSHOT_ARCHIVES_TO_RETAIN.to_string();
     let maximum_snapshots_to_retain_arg = Arg::with_name("maximum_snapshots_to_retain")
         .long("maximum-snapshots-to-retain")
         .value_name("NUMBER")
@@ -2196,7 +2198,7 @@ fn main() {
                         bank.slot(),
                     );
 
-                    let archive_file = snapshot_utils::bank_to_snapshot_archive(
+                    let archive_file = snapshot_utils::bank_to_full_snapshot_archive(
                         ledger_path,
                         &bank,
                         Some(snapshot_version),

ledger/src/bank_forks_utils.rs

@@ -11,7 +11,7 @@ use solana_entry::entry::VerifyRecyclers;
 use solana_runtime::{
     bank_forks::BankForks,
     snapshot_config::SnapshotConfig,
-    snapshot_utils::{self, SnapshotArchiveInfo},
+    snapshot_utils::{self, FullSnapshotArchiveInfo},
 };
 use solana_sdk::{clock::Slot, genesis_config::GenesisConfig, hash::Hash};
 use std::{fs, path::PathBuf, process, result};
@@ -53,9 +53,11 @@ pub fn load(
         fs::create_dir_all(&snapshot_config.snapshot_path)
             .expect("Couldn't create snapshot directory");
 
-        if let Some(snapshot_archive_info) = snapshot_utils::get_highest_snapshot_archive_info(
-            &snapshot_config.snapshot_package_output_path,
-        ) {
+        if let Some(full_snapshot_archive_info) =
+            snapshot_utils::get_highest_full_snapshot_archive_info(
+                &snapshot_config.snapshot_package_output_path,
+            )
+        {
             return load_from_snapshot(
                 genesis_config,
                 blockstore,
@@ -65,7 +67,7 @@ pub fn load(
                 process_options,
                 transaction_status_sender,
                 cache_block_meta_sender,
-                &snapshot_archive_info,
+                &full_snapshot_archive_info,
             );
         } else {
             info!("No snapshot package available; will load from genesis");
@@ -113,11 +115,11 @@ fn load_from_snapshot(
     process_options: ProcessOptions,
     transaction_status_sender: Option<&TransactionStatusSender>,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
-    snapshot_archive_info: &SnapshotArchiveInfo,
+    full_snapshot_archive_info: &FullSnapshotArchiveInfo,
 ) -> LoadResult {
     info!(
         "Loading snapshot package: {:?}",
-        &snapshot_archive_info.path
+        full_snapshot_archive_info.path()
     );
 
     // Fail hard here if snapshot fails to load, don't silently continue
@@ -126,12 +128,13 @@ fn load_from_snapshot(
         process::exit(1);
     }
 
-    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archive(
+    let (deserialized_bank, timings) = snapshot_utils::bank_from_snapshot_archives(
         &account_paths,
         &process_options.frozen_accounts,
         &snapshot_config.snapshot_path,
-        &snapshot_archive_info.path,
-        snapshot_archive_info.archive_format,
+        full_snapshot_archive_info.path(),
+        None,
+        *full_snapshot_archive_info.archive_format(),
         genesis_config,
         process_options.debug_keys.clone(),
         Some(&crate::builtins::get(process_options.bpf_jit)),
@@ -152,10 +155,18 @@ fn load_from_snapshot(
         deserialized_bank.get_accounts_hash(),
     );
 
-    if deserialized_bank_slot_and_hash != (snapshot_archive_info.slot, snapshot_archive_info.hash) {
+    if deserialized_bank_slot_and_hash
+        != (
+            *full_snapshot_archive_info.slot(),
+            *full_snapshot_archive_info.hash(),
+        )
+    {
         error!(
             "Snapshot has mismatch:\narchive: {:?}\ndeserialized: {:?}",
-            (snapshot_archive_info.slot, snapshot_archive_info.hash),
+            (
+                full_snapshot_archive_info.slot(),
+                full_snapshot_archive_info.hash()
+            ),
             deserialized_bank_slot_and_hash
         );
         process::exit(1);

ledger/src/blockstore_processor.rs

@@ -567,7 +567,16 @@ fn do_process_blockstore_from_root(
         ("slot", bank_forks.root(), i64),
         ("forks", initial_forks.len(), i64),
         ("calculate_capitalization_us", time_cap.as_us(), i64),
-        ("untar_us", timings.untar_us, i64),
+        (
+            "full_snapshot_untar_us",
+            timings.full_snapshot_untar_us,
+            i64
+        ),
+        (
+            "incremental_snapshot_untar_us",
+            timings.incremental_snapshot_untar_us,
+            i64
+        ),
         (
             "rebuild_bank_from_snapshots_us",
             timings.rebuild_bank_from_snapshots_us,