cleanup

.gitignore

@@ -1,2 +1,3 @@
 target
-Cargo.lock
+Cargo.lock
+often_used_cmds.md

README.md

@@ -2,6 +2,19 @@
 
 MLX90640 for RPi in Rust
 
+<img src="doc/prototype_setup.jpg"  width="30%">
+
+## Infos
+
+### Camera image
+
+640x480 pGAA works (SGRBG10P) -> buffer size 384000 => 10bit raw bayer packed, 5 bytes for every 4 pixels
+First run libcamera-vid
+
+### Startup
+
+Add startx /usr/bin/thermocam to .bashrc
+
 ## Cross-Compile for RPi
 
 ### Model 1 | arm (32-bit)
@@ -32,20 +45,3 @@ cargo build --target aarch64-unknown-linux-gnu
 ```
 
 Further hints here: https://chacin.dev/blog/cross-compiling-rust-for-the-raspberry-pi
-
-### OpenCV Error
-
-apt install libstdc++-12-dev
-
-
-### Startup
-
-scp target/armv7-unknown-linux-gnueabihf/release/thermocam [email protected]:/usr/bin/thermocam
-scp [email protected]:/home/dan/Bilder/test.jpg .
-scp [email protected]:/home/dan/Bilder/test.avi .
-Add startx /usr/bin/thermocam to .bashrc1^
-
-1296x972 pGAA works (SGRBG10P) 
-640x480 pGAA works (SGRBG10P) -> buffer size 384000 => 10bit raw bayer packed, 5 bytes for every 4 pixels
-YUV420
-648x486

src/lib.rs

@@ -2,9 +2,14 @@ pub mod rgb_color;
 pub mod temperature_pixel;
 pub mod thermo_image_processing;
 
+use std::fs::File;
+use std::io::Read;
+
 use image;
 use image::imageops::FilterType;
 
+use bayer;
+
 #[cfg(not(target_arch = "arm"))]
 use npyz;
 
@@ -26,7 +31,8 @@ pub fn get_thermo_image_raw_data(
     period: u64,
 ) {
     if use_simulation_data {
-        get_simulation_data(shape, mlx_sensor_data);
+        get_thermo_simulation_data(shape, mlx_sensor_data);
+        sleep(Duration::from_millis(period));
     } else {
         match sensor {
             Some(sensor) => {
@@ -39,18 +45,22 @@ pub fn get_thermo_image_raw_data(
     }
 }
 
-fn get_simulation_data(shape: &mut (u32, u32), mlx_sensor_data: &mut Vec<f32>) {
+fn get_thermo_simulation_data(shape: &mut (u32, u32), mlx_sensor_data: &mut Vec<f32>) {
     #[cfg(not(target_arch = "arm"))]
     {
         let bytes = std::fs::read("data/flir_f32.npy").unwrap();
         let reader = npyz::NpyFile::new(&bytes[..]).unwrap();
         let shape_vec = reader.shape().to_vec();
         *shape = (shape_vec[0] as u32, shape_vec[1] as u32);
         *mlx_sensor_data = reader.into_vec::<f32>().unwrap();
-        sleep(Duration::from_millis(250)); // four fps
     }
 }
 
+pub fn get_camera_simulation_data(sim_data_buffer: &mut [u8; 384000]) {
+    let mut f = File::open("data/received_image_data.bin").unwrap();
+    f.read(sim_data_buffer).unwrap();
+}
+
 pub fn process_raw_thermo_image_data(
     mlx_sensor_data: &Vec<f32>,
     shape: (u32, u32),
@@ -165,3 +175,175 @@ fn draw_cross_into_image(
         upscaled_image.put_pixel(x, y + 2, px);
     }
 }
+
+pub fn blend_images_of_different_sizes(
+    image1: image::DynamicImage,
+    image2: image::ImageBuffer<image::Rgb<u8>, Vec<u8>>,
+    foreground_alpha: f32,
+) -> image::ImageBuffer<image::Rgb<u8>, Vec<u8>> {
+    let mut blended_img = image::RgbImage::new(image1.width(), image1.height());
+    for ((x, y, pxo), pxi) in blended_img
+        .enumerate_pixels_mut()
+        .zip(image1.as_rgb8().unwrap().pixels())
+    {
+        let sample_image2_x = ((x as f32 / image1.width() as f32) * image2.width() as f32) as u32;
+        let sample_image2_y = ((y as f32 / image1.height() as f32) * image2.height() as f32) as u32;
+
+        let image2_sample = image2.get_pixel(sample_image2_x, sample_image2_y);
+
+        // luminance greyscale
+        let mut input_greyscale = 0.3 * pxi.0[0] as f32 + 0.59 * pxi.0[1] as f32 + 0.11 * pxi.0[2] as f32;
+        if input_greyscale < 0.0 {
+            input_greyscale = 255.0
+        } else if input_greyscale > 255.0 {
+            input_greyscale = 255.0
+        }
+
+        let blended_r = (image2_sample.0[0] as f32 * foreground_alpha) + (input_greyscale * (1.0 - foreground_alpha));
+        let blended_g = (image2_sample.0[1] as f32 * foreground_alpha) + (input_greyscale * (1.0 - foreground_alpha));
+        let blended_b = (image2_sample.0[2] as f32 * foreground_alpha) + (input_greyscale * (1.0 - foreground_alpha));
+
+        pxo[0] = blended_r as u8;
+        pxo[1] = blended_g as u8;
+        pxo[2] = blended_b as u8;
+    }
+    blended_img
+}
+
+pub fn sgrbg10p_to_rgb(buf: &[u8], camera_image_shape: (u32, u32), cam_rgb_raw_buf: &mut [u8]) {
+    // convert 10-bit bayer to 16 bit bayer
+    let bayer_in_buf_size = (camera_image_shape.0 * camera_image_shape.1) as f32 * 1.25;
+    let mut convert_buf = vec![0u8; bayer_in_buf_size as usize];
+    // 10-bit depth
+    let mut convert_buf_idx = 0;
+    for i in (0..bayer_in_buf_size as usize).step_by(5) {
+        let pix1 = (buf[i + 0] << 2 | ((buf[i + 4] >> 0) & 3)) as u16;
+        let pix2 = (buf[i + 1] << 2 | ((buf[i + 4] >> 2) & 3)) as u16;
+        let pix3 = (buf[i + 2] << 2 | ((buf[i + 4] >> 4) & 3)) as u16;
+        let pix4 = (buf[i + 3] << 2 | ((buf[i + 4] >> 6) & 3)) as u16;
+
+        convert_buf[convert_buf_idx + 0] = pix1 as u8;
+        convert_buf[convert_buf_idx + 1] = pix2 as u8;
+        convert_buf[convert_buf_idx + 2] = pix3 as u8;
+        convert_buf[convert_buf_idx + 3] = pix4 as u8;
+
+        convert_buf_idx += 4;
+    }
+
+    // debayer
+    let depth = bayer::RasterDepth::Depth8;
+    let mut dst = bayer::RasterMut::new(
+        camera_image_shape.0 as usize,
+        camera_image_shape.1 as usize,
+        depth,
+        cam_rgb_raw_buf,
+    );
+    let cfa = bayer::CFA::GRBG;
+    // SGRBG10P
+    let alg = bayer::Demosaic::Linear;
+
+    bayer::run_demosaic(
+        &mut convert_buf.as_slice(),
+        bayer::BayerDepth::Depth8,
+        cfa,
+        alg,
+        &mut dst,
+    )
+    .unwrap();
+}
+
+pub fn yuyv_to_rgb(yuyv_buffer: &[u8], yuyv_shape: (u32, u32), cam_rgb: &mut [u8]) {
+    // from https://gist.github.com/wlhe/fcad2999ceb4a826bd811e9fdb6fe652
+    let yuyv_buf_size: usize = yuyv_shape.0 as usize * yuyv_shape.1 as usize * 2;
+    let mut rgb_idx_offset = 0;
+
+    for yuyv_idx in (0..yuyv_buf_size).step_by(4) {
+        let y = yuyv_buffer[yuyv_idx] as i32; // y0
+        let u = yuyv_buffer[yuyv_idx + 1] as i32; // u0
+        let v = yuyv_buffer[yuyv_idx + 3] as i32; // v0
+
+        let r = y as f32 + 1.4065 * (v - 128) as f32; // r0
+        let g = y as f32 - 0.3455 * (v - 128) as f32 - 0.7169 * (v - 128) as f32; // g0
+        let b = y as f32 + 1.1790 * (u - 128) as f32; // b0
+
+        cam_rgb[0 + rgb_idx_offset] = r as u8;
+        cam_rgb[1 + rgb_idx_offset] = g as u8;
+        cam_rgb[2 + rgb_idx_offset] = b as u8;
+
+        let u = yuyv_buffer[yuyv_idx + 1] as i32; // y1
+        let y = yuyv_buffer[yuyv_idx + 2] as i32; // u1
+        let v = yuyv_buffer[yuyv_idx + 3] as i32; // v1
+
+        let mut r = y as f32 + 1.4065 * (v - 128) as f32; // r1
+        let mut g = y as f32 - 0.3455 * (v - 128) as f32 - 0.7169 * (v - 128) as f32; // g1
+        let mut b = y as f32 + 1.1790 * (u - 128) as f32; // b1
+
+        if r < 0.0 {
+            r = 0.0;
+        }
+        if g < 0.0 {
+            g = 0.0;
+        }
+        if b < 0.0 {
+            b = 0.0;
+        }
+        if r > 255.0 {
+            r = 255.0;
+        }
+        if g > 255.0 {
+            g = 255.0;
+        }
+        if b > 255.0 {
+            b = 255.0;
+        }
+
+        cam_rgb[3 + rgb_idx_offset] = r as u8;
+        cam_rgb[4 + rgb_idx_offset] = g as u8;
+        cam_rgb[5 + rgb_idx_offset] = b as u8;
+
+        rgb_idx_offset += 6;
+    }
+}
+
+pub fn yuv420_to_rgb(buf: &[u8], shape: (u32, u32)) -> image::ImageBuffer<image::Rgb<u8>, Vec<u8>> {
+    let step: u32 = shape.0;
+    let size: usize = shape.0 as usize * shape.1 as usize;
+    let mut cam_rgb = vec![0u8; size * 3];
+    for y_coo in 0..shape.1 {
+        for x_coo in 0..shape.0 {
+            let offset = (y_coo * step + x_coo) as usize;
+            let y: f32 = buf[offset] as f32;
+            let u: f32 = buf[(size as u32 + (y_coo / 2) * (step / 2) + x_coo / 2) as usize] as f32;
+            let v: f32 = buf[((size as f32 * 1.125) as u32 + (y_coo / 2) * (step / 2) + x_coo / 2) as usize] as f32;
+
+            let mut r: f32 = y + 1.402 * (v - 128.0);
+            let mut g: f32 = y - 0.344 * (u - 128.0) - 0.714 * (v - 128.0);
+            let mut b: f32 = y + 1.772 * (u - 128.0);
+
+            if r < 0.0 {
+                r = 0.0;
+            }
+            if g < 0.0 {
+                g = 0.0;
+            }
+            if b < 0.0 {
+                b = 0.0;
+            }
+            if r > 255.0 {
+                r = 255.0;
+            }
+            if g > 255.0 {
+                g = 255.0;
+            }
+            if b > 255.0 {
+                b = 255.0;
+            }
+
+            cam_rgb[(y_coo * step + x_coo) as usize] = r as u8;
+            cam_rgb[(y_coo * step + x_coo + 1) as usize] = g as u8;
+            cam_rgb[(y_coo * step + x_coo + 2) as usize] = b as u8;
+        }
+    }
+    let img = image::RgbImage::from_raw(shape.0, shape.1, cam_rgb).unwrap();
+    img
+}