libhb: do not try to use the ch_layout mask when the channel order is…

… not AV_CHANNEL_ORDER_NATIVE

At the moment only the native channel order is supported in libhb, so try to convert from custom to native, or fall back to a guess based on the number of channels.

libhb/decavcodec.c

@@ -1033,7 +1033,31 @@ static int decavcodecaBSInfo( hb_work_object_t *w, const hb_buffer_t *buf,
                     }
                     else
                     {
-                        info->channel_layout = frame->ch_layout.u.mask;
+                        if (frame->ch_layout.order == AV_CHANNEL_ORDER_NATIVE)
+                        {
+                            info->channel_layout = frame->ch_layout.u.mask;
+                        }
+                        else if (frame->ch_layout.order == AV_CHANNEL_ORDER_CUSTOM)
+                        {
+                            AVChannelLayout channel_layout;
+                            av_channel_layout_copy(&channel_layout, &frame->ch_layout);
+                            int result = av_channel_layout_retype(&channel_layout,
+                                                                  AV_CHANNEL_ORDER_NATIVE,
+                                                                  0);
+                            if (result == 0)
+                            {
+                                info->channel_layout = channel_layout.u.mask;
+                            }
+                            else
+                            {
+                                hb_deep_log(2, "decavcodec: unsupported custom channel order");
+                            }
+                            av_channel_layout_uninit(&channel_layout);
+                        }
+                        else
+                        {
+                            hb_deep_log(2, "decavcodec: unsupported custom channel order");
+                        }
                     }
 
                     if (info->channel_layout == 0)