Improved Crop Algorithm (HandBrake#3925)

Automatic cropping will now better handle mixed aspect ratio content.  New crop mode: "Conservative" exposes the loose cropping functionality.   Improves on HandBrake#3895

libhb/handbrake/common.h

@@ -1111,7 +1111,6 @@ struct hb_metadata_s
     hb_list_t * list_coverart;
 };
 
-// Update win/CS/HandBrake.Interop/HandBrakeInterop/HbLib/hb_title_s.cs when changing this struct
 struct hb_title_s
 {
     enum { HB_DVD_TYPE, HB_BD_TYPE, HB_STREAM_TYPE, HB_FF_STREAM_TYPE } type;
@@ -1147,6 +1146,8 @@ struct hb_title_s
     hb_content_light_metadata_t     coll;
     hb_rational_t   vrate;
     int             crop[4];
+    int             loose_crop[4];
+
     enum {HB_DVD_DEMUXER, HB_TS_DEMUXER, HB_PS_DEMUXER, HB_NULL_DEMUXER} demuxer;
     int             detected_interlacing;
     int             pcr_pid;                /* PCR PID for TS streams */

libhb/handbrake/handbrake.h

@@ -49,6 +49,11 @@ void          hb_dvd_set_dvdnav( int enable );
 void          hb_scan( hb_handle_t *, const char * path,
                        int title_index, int preview_count,
                        int store_previews, uint64_t min_duration );
+void          hb_scan2( hb_handle_t *, const char * path,
+                       int title_index, int preview_count,
+                       int store_previews, uint64_t min_duration,
+                       int crop_auto_switch_threshold, int crop_median_threshold );
+
 void          hb_scan_stop( hb_handle_t * );
 void          hb_force_rescan( hb_handle_t * );
 uint64_t      hb_first_duration( hb_handle_t * );

libhb/handbrake/internal.h

@@ -275,7 +275,8 @@ static inline hb_buffer_t * hb_video_buffer_init( int width, int height )
 hb_thread_t * hb_scan_init( hb_handle_t *, volatile int * die,
                             const char * path, int title_index,
                             hb_title_set_t * title_set, int preview_count,
-                            int store_previews, uint64_t min_duration );
+                            int store_previews, uint64_t min_duration,
+                            int crop_auto_switch_threshold, int crop_median_threshold );
 hb_thread_t * hb_work_init( hb_list_t * jobs,
                             volatile int * die, hb_error_code * error, hb_job_t ** job );
 void ReadLoop( void * _w );

libhb/hb.c

@@ -354,16 +354,27 @@ void hb_remove_previews( hb_handle_t * h )
     closedir( dir );
 }
 
+// We can remove this after we update all the UI's
+void hb_scan( hb_handle_t * h, const char * path, int title_index,
+              int preview_count, int store_previews, uint64_t min_duration )
+{
+    hb_scan2(h, path, title_index, preview_count, store_previews, min_duration, 0, 0);
+}
+
 /**
  * Initializes a scan of the by calling hb_scan_init
  * @param h Handle to hb_handle_t
  * @param path location of VIDEO_TS folder.
  * @param title_index Desired title to scan.  0 for all titles.
  * @param preview_count Number of preview images to generate.
  * @param store_previews Whether or not to write previews to disk.
+ * @param min_duration Ignore titles below a given threshold
+ * @param crop_threshold_frames The number of frames to trigger smart crop
+ * @param crop_threshold_pixels The variance in pixels detected that are allowed for.
  */
-void hb_scan( hb_handle_t * h, const char * path, int title_index,
-              int preview_count, int store_previews, uint64_t min_duration )
+void hb_scan2( hb_handle_t * h, const char * path, int title_index,
+              int preview_count, int store_previews, uint64_t min_duration,
+              int crop_threshold_frames, int crop_threshold_pixels)
 {
     hb_title_t * title;
 
@@ -435,7 +446,8 @@ void hb_scan( hb_handle_t * h, const char * path, int title_index,
     hb_log( "hb_scan: path=%s, title_index=%d", path, title_index );
     h->scan_thread = hb_scan_init( h, &h->scan_die, path, title_index,
                                    &h->title_set, preview_count,
-                                   store_previews, min_duration );
+                                   store_previews, min_duration,
+                                   crop_threshold_frames, crop_threshold_pixels);
 }
 
 void hb_force_rescan( hb_handle_t * h )

libhb/hb_json.c

@@ -235,6 +235,8 @@ static hb_dict_t* hb_title_to_dict_internal( hb_title_t *title )
         "s:{s:o, s:o, s:{s:o, s:o}},"
         // Crop[Top, Bottom, Left, Right]}
         "s:[oooo],"
+        // LooseCrop[Top, Bottom, Left, Right]}
+        "s:[oooo],"
         // Color {Format, Range, Primary, Transfer, Matrix, ChromaLocation}
         "s:{s:o, s:o, s:o, s:o, s:o, s:o},"
         // FrameRate {Num, Den}
@@ -265,6 +267,10 @@ static hb_dict_t* hb_title_to_dict_internal( hb_title_t *title )
                             hb_value_int(title->crop[1]),
                             hb_value_int(title->crop[2]),
                             hb_value_int(title->crop[3]),
+    "LooseCrop",            hb_value_int(title->loose_crop[0]),
+                            hb_value_int(title->loose_crop[1]),
+                            hb_value_int(title->loose_crop[2]),
+                            hb_value_int(title->loose_crop[3]),
     "Color",
         "Format",           hb_value_int(title->pix_fmt),
         "Range",            hb_value_int(title->color_range),

libhb/preset.c

@@ -2015,17 +2015,27 @@ int hb_preset_apply_dimensions(hb_handle_t *h, int title_index,
         int hflip = hb_dict_get_bool(rotate_settings, "hflip");
         hb_rotate_geometry(&srcGeo, &srcGeo, angle, hflip);
     }
-
-    if (!hb_value_get_bool(hb_dict_get(preset, "PictureAutoCrop")))
-    {
-        geo.crop[0] = hb_dict_get_int(preset, "PictureTopCrop");
-        geo.crop[1] = hb_dict_get_int(preset, "PictureBottomCrop");
-        geo.crop[2] = hb_dict_get_int(preset, "PictureLeftCrop");
-        geo.crop[3] = hb_dict_get_int(preset, "PictureRightCrop");
-    }
-    else
-    {
-        memcpy(geo.crop, srcGeo.crop, sizeof(geo.crop));
+
+    switch(hb_dict_get_int(preset, "PictureCropMode")) 
+    {
+        case 0: // Automatic
+          memcpy(geo.crop, srcGeo.crop, sizeof(geo.crop));
+          break;
+        case 1: // Loose
+          memcpy(geo.crop, title->loose_crop, sizeof(geo.crop));
+          break;
+        case 2: // None
+            geo.crop[0] = 0;
+            geo.crop[1] = 0;
+            geo.crop[2] = 0;
+            geo.crop[3] = 0;
+            break;
+        case 3: // Custom
+            geo.crop[0] = hb_dict_get_int(preset, "PictureTopCrop");
+            geo.crop[1] = hb_dict_get_int(preset, "PictureBottomCrop");
+            geo.crop[2] = hb_dict_get_int(preset, "PictureLeftCrop");
+            geo.crop[3] = hb_dict_get_int(preset, "PictureRightCrop");
+            break;
     }
 
     const char * pad_mode;

libhb/scan.c

@@ -28,6 +28,9 @@ typedef struct
     int            store_previews;
 
     uint64_t       min_title_duration;
+
+    int            crop_threshold_frames;
+    int            crop_threshold_pixels;
 } hb_scan_t;
 
 #define PREVIEW_READ_THRESH (200)
@@ -185,7 +188,8 @@ static int get_color_matrix(int colorspace, hb_geometry_t geometry)
 hb_thread_t * hb_scan_init( hb_handle_t * handle, volatile int * die,
                             const char * path, int title_index,
                             hb_title_set_t * title_set, int preview_count,
-                            int store_previews, uint64_t min_duration )
+                            int store_previews, uint64_t min_duration,
+                            int crop_threshold_frames, int crop_threshold_pixels)
 {
     hb_scan_t * data = calloc( sizeof( hb_scan_t ), 1 );
 
@@ -198,7 +202,10 @@ hb_thread_t * hb_scan_init( hb_handle_t * handle, volatile int * die,
     data->preview_count  = preview_count;
     data->store_previews = store_previews;
     data->min_title_duration = min_duration;
-
+
+    data->crop_threshold_frames = crop_threshold_frames;
+    data->crop_threshold_pixels = crop_threshold_pixels;
+
     // Initialize scan state
     hb_state_t state;
     hb_get_state2(handle, &state);
@@ -1213,17 +1220,81 @@ static int DecodePreviews( hb_scan_t * data, hb_title_t * title, int flush )
         if ( crops->n > 2 )
         {
             sort_crops( crops );
-            // The next line selects median cropping - at least
+
+            // Available crop modes:
+            // - Median: Selects median cropping - at least
             // 50% of the frames will have their borders removed.
-            // Other possible choices are loose cropping (i = 0) where
+            // - Loose cropping (i = 0) where
             // no non-black pixels will be cropped from any frame and a
-            // tight cropping (i = crops->n - (crops->n >> 2)) where at
+            // - Tight cropping (i = crops->n - (crops->n >> 2)) where at
             // least 75% of the frames will have their borders removed.
-            i = crops->n >> 1;
+            // - Smart: A blend between Median and Loose depending on whether 
+            // mixed AR content is found.
+
+            i = crops->n >> 1; // Median
+
+            int crop_switch_frame_count = data->crop_threshold_frames;
+            int less_than_median_crop_threshold = data->crop_threshold_pixels;
+
+            if (crop_switch_frame_count == 0) 
+            {
+                // Values seem like sensible defaults.
+                // Have observed that the optimal value does not always linearly increase with preview count.
+                 crop_switch_frame_count = 4;
+
+                if (data->preview_count >= 30){
+                    crop_switch_frame_count = 6;
+                }
+
+                if (data->preview_count > 40){
+                    crop_switch_frame_count = 8;
+                }
+            }
+
+            if (less_than_median_crop_threshold == 0) 
+            {
+                // It's not uncommon to see 2~12 px variance in cropping.
+                // Defaulting to 9 to account for that variance before switching to loose.
+                // This accounts for variance that is unlikely to be caused by mixed AR. 
+                less_than_median_crop_threshold = 9;
+            }
+
+            // Count the number of frames "substantially" less than the median.
+            int less_than_median_frame_count = 0;
+            for (int x = 0; x < crops->n; x++)
+            {
+
+                if (crops->t[x] < (crops->t[i] - less_than_median_crop_threshold) ||
+                    crops->b[x] < (crops->b[i] - less_than_median_crop_threshold) ||
+                    crops->l[x] < (crops->l[i] - less_than_median_crop_threshold) ||
+                    crops->r[x] < (crops->r[i] - less_than_median_crop_threshold)){
+                    less_than_median_frame_count = less_than_median_frame_count +1;
+                }
+
+                 hb_deep_log(2, "crop: [%d] %d/%d/%d/%d", x, crops->t[x], crops->b[x],  crops->l[x], crops->r[x]);
+            }
+
+            hb_deep_log(2, "crop: less_than_median_frame_count: %d,", less_than_median_frame_count);
+
+            // If we have a reasonable number of samples and it appears we have mixed aspect ratio, switch to loose crop.
+            if (less_than_median_frame_count >= crop_switch_frame_count) 
+            {
+                hb_deep_log(2, "crop: switching to loose crop for this source. May be mixed aspect ratio. (%d)", crop_switch_frame_count);
+                i = 0;
+            }
+
+            // Automatic "Smart" Crop.
             title->crop[0] = EVEN( crops->t[i] );
             title->crop[1] = EVEN( crops->b[i] );
             title->crop[2] = EVEN( crops->l[i] );
             title->crop[3] = EVEN( crops->r[i] );
+
+            // Loose / Conservative  (i = 0)
+            i = 0;
+            title->loose_crop[0] = EVEN( crops->t[i] );
+            title->loose_crop[1] = EVEN( crops->b[i] );
+            title->loose_crop[2] = EVEN( crops->l[i] );
+            title->loose_crop[3] = EVEN( crops->r[i] );
         }
 
         hb_log( "scan: %d previews, %dx%d, %.3f fps, autocrop = %d/%d/%d/%d, "