feat: add cross process GPU locking

Cargo.toml

@@ -26,6 +26,7 @@ log = "0.4.8"
 lazy_static = "1.4.0"
 ocl = { version = "0.19.4", package = "fil-ocl", optional = true }
 itertools = { version = "0.8.0", optional = true }
+fs2 = { version = "0.4.3", optional = true }
 rand = "0.7"
 
 [dev-dependencies]
@@ -35,7 +36,7 @@ sha2 = "0.8"
 
 [features]
 default = ["groth16", "multicore"]
-gpu = ["ocl", "itertools"]
+gpu = ["ocl", "itertools", "fs2"]
 gpu-test = ["gpu"]
 groth16 = ["paired"]
 multicore = ["futures-cpupool", "crossbeam", "num_cpus"]

src/gpu/utils.rs

@@ -1,9 +1,11 @@
 use crate::gpu::error::{GPUError, GPUResult};
 use ocl::{Device, Platform};
 
+use fs2::FileExt;
 use log::info;
 use std::collections::HashMap;
-use std::env;
+use std::fs::File;
+use std::{env, io};
 
 pub const GPU_NVIDIA_PLATFORM_NAME: &str = "NVIDIA CUDA";
 // pub const CPU_INTEL_PLATFORM_NAME: &str = "Intel(R) CPU Runtime for OpenCL(TM) Applications";
@@ -62,3 +64,23 @@ pub fn get_core_count(d: Device) -> GPUResult<usize> {
         }),
     }
 }
+
+#[derive(Debug)]
+pub struct LockedFile(File);
+
+pub const LOCK_NAME: &str = "/tmp/bellman.lock";
+
+pub fn lock() -> io::Result<LockedFile> {
+    info!("Creating GPU lock file");
+    let file = File::create(LOCK_NAME)?;
+
+    file.lock_exclusive()?;
+
+    info!("GPU lock file acquired");
+    Ok(LockedFile(file))
+}
+
+pub fn unlock(lock: LockedFile) {
+    drop(lock);
+    info!("GPU lock file released");
+}

src/groth16/prover.rs

@@ -10,6 +10,8 @@ use paired::Engine;
 
 use super::{ParameterSource, Proof};
 use crate::domain::{gpu_fft_supported, EvaluationDomain, Scalar};
+#[cfg(feature = "gpu")]
+use crate::gpu;
 use crate::multicore::Worker;
 use crate::multiexp::{gpu_multiexp_supported, multiexp, DensityTracker, FullDensity};
 use crate::{Circuit, ConstraintSystem, Index, LinearCombination, SynthesisError, Variable};
@@ -181,6 +183,9 @@ where
     E: Engine,
     C: Circuit<E>,
 {
+    #[cfg(feature = "gpu")]
+    let lock = gpu::lock()?;
+
     let mut prover = ProvingAssignment {
         a_aux_density: DensityTracker::new(),
         b_input_density: DensityTracker::new(),
@@ -378,6 +383,9 @@ where
     g_c.add_assign(&h.wait()?);
     g_c.add_assign(&l.wait()?);
 
+    #[cfg(feature = "gpu")]
+    gpu::unlock(lock);
+
     Ok(Proof {
         a: g_a.into_affine(),
         b: g_b.into_affine(),