Remove dependence of dorado_lib on dorado_models_lib

Author: StuartAbercrombie

dorado/cli/basecaller.cpp

@@ -63,6 +63,9 @@ void setup(std::vector<std::string> args,
            const std::string& dump_stats_filter,
            const std::string& resume_from_file,
            argparse::ArgumentParser& resume_parser) {
+    auto model_config = dorado::load_crf_model_config(model_path);
+    std::string model_name = std::filesystem::canonical(model_path).filename().string();
+
     torch::set_num_threads(1);
 
     if (!DataLoader::is_read_data_present(data_path, recursive_file_loading)) {
@@ -72,7 +75,11 @@ void setup(std::vector<std::string> args,
 
     // Check sample rate of model vs data.
     auto data_sample_rate = DataLoader::get_sample_rate(data_path, recursive_file_loading);
-    auto model_sample_rate = get_model_sample_rate(model_path);
+    auto model_sample_rate = model_config.sample_rate;
+    if (model_sample_rate < 0) {
+        // If unsuccessful, find sample rate by model name.
+        model_sample_rate = utils::get_sample_rate_by_model_name(model_name);
+    }
     if (!skip_model_compatibility_check &&
         !sample_rates_compatible(data_sample_rate, model_sample_rate)) {
         std::stringstream err;
@@ -93,11 +100,9 @@ void setup(std::vector<std::string> args,
         throw std::runtime_error("Modified base models cannot be used with FASTQ output");
     }
 
-    auto model_config = dorado::load_crf_model_config(model_path);
     auto [runners, num_devices] =
             create_basecall_runners(model_config, device, num_runners, 0, batch_size, chunk_size);
 
-    std::string model_name = std::filesystem::canonical(model_path).filename().string();
     auto read_groups = DataLoader::load_read_groups(data_path, model_name, recursive_file_loading);
     auto read_list = utils::load_read_list(read_list_file_path);
 
@@ -135,10 +140,17 @@ void setup(std::vector<std::string> args,
             {read_converter}, min_qscore, default_parameters.min_sequence_length,
             std::unordered_set<std::string>{}, thread_allocations.read_filter_threads);
 
-    pipelines::create_simplex_pipeline(pipeline_desc, std::move(runners), std::move(remora_runners),
-                                       overlap, thread_allocations.scaler_node_threads,
-                                       thread_allocations.remora_threads * num_devices,
-                                       read_filter_node);
+    auto mean_qscore_start_pos = model_config.mean_qscore_start_pos;
+    if (mean_qscore_start_pos < 0) {
+        mean_qscore_start_pos = utils::get_mean_qscore_start_pos_by_model_name(model_name);
+        if (mean_qscore_start_pos < 0) {
+            throw std::runtime_error("Mean q-score start position cannot be < 0");
+        }
+    }
+    pipelines::create_simplex_pipeline(
+            pipeline_desc, std::move(runners), std::move(remora_runners), overlap,
+            mean_qscore_start_pos, thread_allocations.scaler_node_threads,
+            thread_allocations.remora_threads * num_devices, read_filter_node);
 
     // Create the Pipeline from our description.
     std::vector<dorado::stats::StatsReporter> stats_reporters;

dorado/cli/duplex.cpp

@@ -243,7 +243,12 @@ int duplex(int argc, char* argv[]) {
 
             // Check sample rate of model vs data.
             auto data_sample_rate = DataLoader::get_sample_rate(reads, recursive_file_loading);
-            auto model_sample_rate = get_model_sample_rate(model_path);
+            auto model_sample_rate = model_config.sample_rate;
+            if (model_sample_rate < 0) {
+                // If unsuccessful, find sample rate by model name.
+                model_sample_rate = utils::get_sample_rate_by_model_name(
+                        model_config.model_path.filename().string());
+            }
             auto skip_model_compatibility_check =
                     internal_parser.get<bool>("--skip-model-compatibility-check");
             if (!skip_model_compatibility_check &&
@@ -312,9 +317,18 @@ int duplex(int argc, char* argv[]) {
                 pairing_parameters = std::move(template_complement_map);
             }
 
+            auto mean_qscore_start_pos = model_config.mean_qscore_start_pos;
+            if (mean_qscore_start_pos < 0) {
+                mean_qscore_start_pos =
+                        utils::get_mean_qscore_start_pos_by_model_name(stereo_model_name);
+                if (mean_qscore_start_pos < 0) {
+                    throw std::runtime_error("Mean q-score start position cannot be < 0");
+                }
+            }
             pipelines::create_stereo_duplex_pipeline(
                     pipeline_desc, std::move(runners), std::move(stereo_runners), overlap,
-                    num_devices * 2, num_devices, std::move(pairing_parameters), read_filter_node);
+                    mean_qscore_start_pos, num_devices * 2, num_devices,
+                    std::move(pairing_parameters), read_filter_node);
 
             std::vector<dorado::stats::StatsReporter> stats_reporters;
             pipeline = Pipeline::create(std::move(pipeline_desc), &stats_reporters);

dorado/read_pipeline/Pipelines.cpp

@@ -18,6 +18,7 @@ void create_simplex_pipeline(PipelineDescriptor& pipeline_desc,
                              std::vector<dorado::Runner>&& runners,
                              std::vector<std::unique_ptr<dorado::ModBaseRunner>>&& modbase_runners,
                              size_t overlap,
+                             uint32_t mean_qscore_start_pos,
                              int scaler_node_threads,
                              int modbase_node_threads,
                              NodeHandle sink_node_handle,
@@ -43,7 +44,7 @@ void create_simplex_pipeline(PipelineDescriptor& pipeline_desc,
 
     auto basecaller_node = pipeline_desc.add_node<BasecallerNode>(
             {}, std::move(runners), overlap, kBatchTimeoutMS, model_name, 1000, "BasecallerNode",
-            false, get_model_mean_qscore_start_pos(model_config));
+            false, mean_qscore_start_pos);
 
     NodeHandle last_node_handle = PipelineDescriptor::InvalidNodeHandle;
     if (mod_base_caller_node != PipelineDescriptor::InvalidNodeHandle) {
@@ -71,6 +72,7 @@ void create_stereo_duplex_pipeline(PipelineDescriptor& pipeline_desc,
                                    std::vector<dorado::Runner>&& runners,
                                    std::vector<dorado::Runner>&& stereo_runners,
                                    size_t overlap,
+                                   uint32_t mean_qscore_start_pos,
                                    int scaler_node_threads,
                                    int splitter_node_threads,
                                    PairingParameters pairing_parameters,
@@ -89,8 +91,7 @@ void create_stereo_duplex_pipeline(PipelineDescriptor& pipeline_desc,
 
     auto stereo_basecaller_node = pipeline_desc.add_node<BasecallerNode>(
             {}, std::move(stereo_runners), adjusted_stereo_overlap, kStereoBatchTimeoutMS,
-            duplex_rg_name, 1000, "StereoBasecallerNode", true,
-            get_model_mean_qscore_start_pos(stereo_model_config));
+            duplex_rg_name, 1000, "StereoBasecallerNode", true, mean_qscore_start_pos);
 
     auto simplex_model_stride = runners.front()->model_stride();
     auto stereo_node = pipeline_desc.add_node<StereoDuplexEncoderNode>({stereo_basecaller_node},
@@ -118,8 +119,7 @@ void create_stereo_duplex_pipeline(PipelineDescriptor& pipeline_desc,
     const int kSimplexBatchTimeoutMS = 100;
     auto basecaller_node = pipeline_desc.add_node<BasecallerNode>(
             {splitter_node}, std::move(runners), adjusted_simplex_overlap, kSimplexBatchTimeoutMS,
-            model_name, 1000, "BasecallerNode", true,
-            get_model_mean_qscore_start_pos(model_config));
+            model_name, 1000, "BasecallerNode", true, mean_qscore_start_pos);
 
     auto scaler_node = pipeline_desc.add_node<ScalerNode>(
             {basecaller_node}, model_config.signal_norm_params, scaler_node_threads);

dorado/read_pipeline/Pipelines.h

@@ -2,6 +2,7 @@
 
 #include "ReadPipeline.h"
 
+#include <cstdint>
 #include <map>
 #include <memory>
 #include <string>
@@ -25,6 +26,7 @@ void create_simplex_pipeline(PipelineDescriptor& pipeline_desc,
                              std::vector<dorado::Runner>&& runners,
                              std::vector<std::unique_ptr<dorado::ModBaseRunner>>&& modbase_runners,
                              size_t overlap,
+                             uint32_t mean_qscore_start_pos,
                              int scaler_node_threads,
                              int modbase_threads,
                              NodeHandle sink_node_handle = PipelineDescriptor::InvalidNodeHandle,
@@ -38,6 +40,7 @@ void create_stereo_duplex_pipeline(
         std::vector<dorado::Runner>&& runners,
         std::vector<dorado::Runner>&& stereo_runners,
         size_t overlap,
+        uint32_t mean_qscore_start_pos,
         int scaler_node_threads,
         int splitter_node_threads,
         PairingParameters pairing_parameters,