Indexing dblocks

Cargo.toml

@@ -8,4 +8,5 @@ edition = "2018"
 rusqlite = "0.17.0"
 zip = "0.5.2"
 chrono = "0.4.0"
-base64 = "0.10.1"
+base64 = "0.10.1"
+pbr = "1.0.1"

src/blockid.rs

@@ -1,3 +1,5 @@
+use base64::*;
+
 pub struct BlockToFile {
     pub(self) id: i32,
     pub(self) block_id: String,
@@ -13,3 +15,7 @@ impl BlockToFile {
         }
     }
 }
+
+pub fn base64_url_to_plain(url: &str) -> String {
+    base64::encode(&base64::decode_config(url, base64::URL_SAFE).unwrap())
+}

src/database.rs

@@ -1,7 +1,11 @@
 use crate::blockid::*;
+use pbr::ProgressBar;
 use rusqlite::types::ToSql;
 use rusqlite::*;
 use std::collections::BTreeMap;
+use std::fs::File;
+use std::path::Path;
+use zip::*;
 
 pub struct DB {
     conn: Connection,
@@ -10,11 +14,18 @@ pub struct DB {
 impl DB {
     pub fn new(file: &str) -> DB {
         let conn = Connection::open(file).unwrap();
+        conn.execute("PRAGMA temp_store = memory", NO_PARAMS)
+            .unwrap();
+        conn.execute("PRAGMA page_size = 16384", NO_PARAMS).unwrap();
+        conn.execute("PRAGMA cache_size = 2048", NO_PARAMS).unwrap();
         DB { conn }
     }
 
-    pub fn create_block_id_to_filenames(self, number_to_name: &BTreeMap<usize, String>) -> Self {
-        let conn = &self.conn;
+    pub fn create_block_id_to_filenames(
+        mut self,
+        number_to_name: &BTreeMap<usize, String>,
+    ) -> Self {
+        let conn = &mut self.conn;
 
         // Create Block ID -> File Number table and empty it out if it exists
         conn.execute(
@@ -32,6 +43,29 @@ impl DB {
         conn.execute("delete from BlockIdToFile where 1", NO_PARAMS)
             .unwrap();
 
+        // Iterate through dblocks, adding them to the db
+        println!("Indexing dblocks");
+        let mut pb = ProgressBar::new(number_to_name.len() as u64);
+        for (num, path) in number_to_name.iter() {
+            let tx = conn.transaction().unwrap();
+            // Open zip file
+            let file = File::open(&Path::new(path)).unwrap();
+            let mut zip = zip::ZipArchive::new(file).unwrap();
+            // Iterate through contents and add names to database
+            for i in 0..zip.len() {
+                let inner_file = zip.by_index(i).unwrap();
+                let name = base64_url_to_plain(inner_file.name());
+                // Add name to database
+                tx.execute(
+                    "insert into BlockIdToFile (BlockId, FileNum) VALUES (?1, ?2)",
+                    &[&name, &num.to_string()],
+                )
+                .unwrap();
+            }
+            pb.inc();
+            tx.commit().unwrap();
+        }
+
         self
     }
 }

src/main.rs

@@ -4,8 +4,12 @@ mod database;
 use std::collections::BTreeMap;
 use std::fs;
 
+use database::*;
+
 fn main() {
     let backup_dir = "/home/nmccarty/tmp/config/";
+    let db_location = "/home/nmccarty/tmp/config/index.db";
+    let restore_dir = "/home/nmccarty/tmp/restore";
 
     // Get list of dblocks
     let zip_file_names = fs::read_dir(backup_dir)
@@ -20,4 +24,9 @@ fn main() {
     }
 
     println!("Found {} dblocks", number_to_name.len());
+
+    // Open dblock db connection and build db
+    println!();
+    println!("Indexing dblocks");
+    let dblock_db = DB::new(db_location).create_block_id_to_filenames(&number_to_name);
 }