chore: Upgrade to tokenizers 0.21.0 (#512)

Author: lightsofapollo

Cargo.lock

@@ -24,7 +24,7 @@ hf-hub = { version = "0.3.2", features = ["tokio", "online"], default-features =
 metrics = "0.23"
 nohash-hasher = "0.2"
 num_cpus = "1.16.0"
-tokenizers = { version = "0.19.1", default-features = false, features = ["onig", "esaxx_fast"] }
+tokenizers = { version = "0.21.0", default-features = false, features = ["onig", "esaxx_fast"] }
 tokio = { version = "1.25", features = ["rt", "rt-multi-thread", "parking_lot", "sync", "signal"] }
 tracing = "0.1"
 serde = { version = "1.0", features = ["serde_derive"] }

Cargo.toml

@@ -147,13 +147,13 @@ pub async fn run(
             .build()
             .unwrap();
         match tokenizer.get_post_processor() {
-            None => tokenizer.with_post_processor(template),
+            None => tokenizer.with_post_processor(Some(template)),
             Some(post_processor) => {
                 let post_processor = Sequence::new(vec![
                     post_processor.clone(),
                     PostProcessorWrapper::Template(template),
                 ]);
-                tokenizer.with_post_processor(post_processor)
+                tokenizer.with_post_processor(Some(post_processor))
             }
         };
     }