Bug 1178215 - update update.py for new libvpx r=rillian

libvpx dropped vpx_mem_set_functions,
only use it if an external libvpx
is used and still has it.

update update.py
add vpx_dsp_rtcd.h
rebase disable_pthread_on_mingw.patch
add vp9_filter_restore_aligment.patch
drop msvc2015.patch

configure.in

@@ -5591,6 +5591,9 @@ if test -n "$MOZ_VPX" -a -z "$MOZ_NATIVE_LIBVPX"; then
     else
       AC_MSG_WARN([No assembler or assembly support for libvpx. Using unoptimized C routines.])
     fi
+
+    dnl native libvpx no longer has vpx_mem_set_functions
+    AC_DEFINE(MOZ_VPX_NO_MEM_REPORTING)
 fi
 
 dnl ========================================================

layout/media/symbols.def.in

@@ -62,7 +62,6 @@ vpx_codec_get_cx_data
 vpx_codec_enc_config_default
 vpx_img_alloc
 vpx_codec_encode
-vpx_mem_set_functions
 #endif
 #endif
 #ifdef MOZ_VORBIS

media/libvpx/README_MOZILLA

@@ -6,6 +6,6 @@ Mozilla build system.
 
 The libvpx git repository is:
 
-    https://gerrit.chromium.org/gerrit/webm/libvpx
+    https://chromium.googlesource.com/webm/libvpx
 
 The git commit ID used was c74bf6d889992c3cabe017ec353ca85c323107cd

media/libvpx/disable_pthread_on_mingw.patch

@@ -3,50 +3,30 @@ Don't use pthread for libvpx in mingw builds.
 
 
 diff --git a/media/libvpx/vpx_config_x86-win32-gcc.h b/media/libvpx/vpx_config_x86-win32-gcc.h
-index c5452c0..12b5c84 100644
+index 5bc3efb..e60f84d 100644
 --- a/media/libvpx/vpx_config_x86-win32-gcc.h
 +++ b/media/libvpx/vpx_config_x86-win32-gcc.h
-@@ -30,17 +30,18 @@
- #define HAVE_SSSE3 1
- #define HAVE_SSE4_1 1
- #define HAVE_AVX 1
+@@ -32,7 +32,8 @@
  #define HAVE_AVX2 1
- #define HAVE_ALTIVEC 0
  #define HAVE_VPX_PORTS 1
  #define HAVE_STDINT_H 1
- #define HAVE_ALT_TREE_LAYOUT 0
 -#define HAVE_PTHREAD_H 1
 +#undef HAVE_PTHREAD_H
 +#define HAVE_PTHREAD_H 0
  #define HAVE_SYS_MMAN_H 1
  #define HAVE_UNISTD_H 1
  #define CONFIG_DEPENDENCY_TRACKING 1
- #define CONFIG_EXTERNAL_BUILD 0
- #define CONFIG_INSTALL_DOCS 0
- #define CONFIG_INSTALL_BINS 1
- #define CONFIG_INSTALL_LIBS 1
- #define CONFIG_INSTALL_SRCS 0
 diff --git a/media/libvpx/vpx_config_x86_64-win64-gcc.h b/media/libvpx/vpx_config_x86_64-win64-gcc.h
-index 04f382b..4d75a4b 100644
+index 4ff4339..b056a0e 100644
 --- a/media/libvpx/vpx_config_x86_64-win64-gcc.h
 +++ b/media/libvpx/vpx_config_x86_64-win64-gcc.h
-@@ -30,17 +30,18 @@
- #define HAVE_SSSE3 1
- #define HAVE_SSE4_1 1
- #define HAVE_AVX 1
+@@ -32,7 +32,8 @@
  #define HAVE_AVX2 1
- #define HAVE_ALTIVEC 0
  #define HAVE_VPX_PORTS 1
  #define HAVE_STDINT_H 1
- #define HAVE_ALT_TREE_LAYOUT 0
 -#define HAVE_PTHREAD_H 1
 +#undef HAVE_PTHREAD_H
 +#define HAVE_PTHREAD_H 0
  #define HAVE_SYS_MMAN_H 1
  #define HAVE_UNISTD_H 1
  #define CONFIG_DEPENDENCY_TRACKING 1
- #define CONFIG_EXTERNAL_BUILD 0
- #define CONFIG_INSTALL_DOCS 0
- #define CONFIG_INSTALL_BINS 1
- #define CONFIG_INSTALL_LIBS 1
- #define CONFIG_INSTALL_SRCS 0

media/libvpx/update.py

@@ -27,12 +27,13 @@
 
 mk_files = [
     'vp8/vp8_common.mk',
-    'vp8/vp8cx_arm.mk',
     'vp8/vp8cx.mk',
     'vp8/vp8dx.mk',
+    'vp8/vp8cx_arm.mk',
     'vp9/vp9_common.mk',
     'vp9/vp9cx.mk',
     'vp9/vp9dx.mk',
+    'vpx_dsp/vpx_dsp.mk',
     'vpx_mem/vpx_mem.mk',
     'vpx_ports/vpx_ports.mk',
     'vpx_scale/vpx_scale.mk',
@@ -48,6 +49,8 @@
         'API_EXPORTS',
         'API_SRCS-$(CONFIG_VP8_DECODER)',
         'API_SRCS-yes',
+        'DSP_SRCS-yes',
+        'DSP_SRCS-yes+$(CONFIG_ENCODERS)',
         'MEM_SRCS-yes',
         'PORTS_SRCS-yes',
         'SCALE_SRCS-$(CONFIG_SPATIAL_RESAMPLING)',
@@ -86,8 +89,16 @@
         'VP9_CX_SRCS-yes',
     ],
     'X86_ASM': [
+        'DSP_SRCS-$(HAVE_MMX)',
+        'DSP_SRCS-$(HAVE_MMX)+$(CONFIG_ENCODERS)',
+        'DSP_SRCS-$(HAVE_SSE2)',
+        'DSP_SRCS-$(HAVE_SSE2)+$(CONFIG_ENCODERS)',
+        'DSP_SRCS-$(HAVE_SSE3)+$(CONFIG_ENCODERS)',
+        'DSP_SRCS-$(HAVE_SSE4_1)+$(CONFIG_ENCODERS)',
+        'DSP_SRCS-$(HAVE_SSSE3)+$(CONFIG_ENCODERS)',
         'PORTS_SRCS-$(BUILD_LIBVPX)',
         'PORTS_SRCS-$(BUILD_LIBVPX)+$(ARCH_X86)$(ARCH_X86_64)',
+        'PORTS_SRCS-yes+$(ARCH_X86)$(ARCH_X86_64)',
         'VP8_COMMON_SRCS-$(ARCH_X86)$(ARCH_X86_64)',
         'VP8_COMMON_SRCS-$(HAVE_MMX)',
         'VP8_COMMON_SRCS-$(HAVE_MMX)+$(CONFIG_POSTPROC)',
@@ -127,6 +138,10 @@
         'VP9_CX_SRCS-$(HAVE_SSSE3)+$(ARCH_X86_64)',
     ],
     'ARM_ASM': [
+        'DSP_SRCS-$(HAVE_MEDIA)',
+        'DSP_SRCS-$(HAVE_MEDIA)+$(CONFIG_ENCODERS)',
+        'DSP_SRCS-$(HAVE_NEON)',
+        'DSP_SRCS-$(HAVE_NEON)+$(CONFIG_ENCODERS)',
         'PORTS_SRCS-$(ARCH_ARM)',
         'SCALE_SRCS-$(HAVE_NEON)',
         'VP8_COMMON_SRCS-$(ARCH_ARM)',
@@ -146,6 +161,8 @@
         'VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT)',
     ],
     'AVX2': [
+        'DSP_SRCS-$(HAVE_AVX2)',
+        'DSP_SRCS-$(HAVE_AVX2)+$(CONFIG_ENCODERS)',
         'VP9_COMMON_SRCS-$(HAVE_AVX2)',
         'VP9_CX_SRCS-$(HAVE_AVX2)',
     ],
@@ -170,6 +187,9 @@
     'VP9_CX_SRCS-$(CONFIG_INTERNAL_STATS)+$(CONFIG_VP9_POSTPROC)',
     'VP9_CX_SRCS-$(CONFIG_VP9_TEMPORAL_DENOISING)',
     'VP9_CX_SRCS-$(HAVE_SSE2)+$(CONFIG_VP9_TEMPORAL_DENOISING)',
+
+    # VP9_HIGHBITDEPTH
+    'DSP_SRCS-$(HAVE_SSE2)+$(CONFIG_VP9_HIGHBITDEPTH)',
     'VP9_COMMON_SRCS-$(HAVE_SSE2)+$(CONFIG_VP9_HIGHBITDEPTH)',
     'VP9_CX_SRCS-$(HAVE_SSE2)+$(CONFIG_VP9_HIGHBITDEPTH)',
 
@@ -183,6 +203,8 @@
     'VP8_COMMON_SRCS-$(HAVE_DSPR2)',
     'VP9_COMMON_SRCS-$(HAVE_DSPR2)',
     'VP8_CX_SRCS_REMOVE-$(HAVE_EDSP)',
+    'VP9_COMMON_SRCS-$(HAVE_MSA)',
+    'VP9_CX_SRCS-$(HAVE_MSA)',
 ]
 
 libvpx_files = [
@@ -221,8 +243,12 @@
 ]
 
 rename_files = {
+    #avoid clash between vpx_dsp/x86 and vp8/common/x86
+    'vp8/common/x86/variance_mmx.c': 'vp8/common/x86/vp8_variance_mmx.c',
+    'vp8/common/x86/variance_sse2.c': 'vp8/common/x86/vp8_variance_sse2.c',
+    'vp8/common/x86/variance_impl_mmx.asm': 'vp8/common/x86/vp8_variance_impl_mmx.asm',
     #avoid clash with common/arm/neon/vp9_avg_neon.c
-    'vp9/encoder/arm/neon/vp9_avg_neon.c': 'vp9/encoder/arm/neon/vp9enc_avg_neon.c'
+    'vp9/encoder/arm/neon/vp9_avg_neon.c': 'vp9/encoder/arm/neon/vp9enc_avg_neon.c',
 }
 
 files = {
@@ -250,7 +276,6 @@
     ],
     'SOURCES': [
         'vp8/common/rtcd.c',
-        'vp8/common/sad_c.c',
         'vp8/encoder/bitstream.c',
         'vp8/encoder/onyx_if.c',
         'vp8/vp8_dx_iface.c',
@@ -300,7 +325,6 @@
         'vp9/encoder/vp9_ratectrl.c',
         'vp9/encoder/vp9_rdopt.c',
         'vp9/encoder/vp9_resize.c',
-        'vp9/encoder/vp9_sad.c',
         'vp9/encoder/vp9_segmentation.c',
         'vp9/encoder/vp9_subexp.c',
         'vp9/encoder/vp9_temporal_filter.c',
@@ -348,14 +372,14 @@
     'vpx_config.asm',
     'vpx_config.h',
     'vpx_scale_rtcd.h',
+    'vpx_dsp_rtcd.h',
 ]
 
 def prepare_upstream(prefix, commit=None):
     upstream_url = 'https://chromium.googlesource.com/webm/libvpx'
     if os.path.exists(prefix):
-        print "Using existing repo in '%s'" % prefix
         os.chdir(prefix)
-        subprocess.call(['git', 'fetch', upstream_url, prefix])
+        subprocess.call(['git', 'fetch', 'origin'])
     else:
         subprocess.call(['git', 'clone', upstream_url, prefix])
         os.chdir(prefix)
@@ -368,7 +392,8 @@ def prepare_upstream(prefix, commit=None):
 
     for target in PLATFORMS:
         target_objdir = os.path.join(prefix, 'objdir', target)
-        os.makedirs(target_objdir)
+        if not os.path.exists(target_objdir):
+            os.makedirs(target_objdir)
         os.chdir(target_objdir)
         configure = ['../../configure', '--target=%s' % target,
             '--disable-examples', '--disable-install-docs',
@@ -386,10 +411,14 @@ def prepare_upstream(prefix, commit=None):
 
         if target == 'armv7-android-gcc':
             configure += ['--sdk-path=%s' % ndk_path]
-
+        print "\n" + target_objdir
+        print " ".join(configure)
+        sys.stdout.flush()
         subprocess.call(configure)
         make_targets = [f for f in platform_files if not os.path.exists(f)]
         if make_targets:
+            print " ".join(['make'] + make_targets)
+            sys.stdout.flush()
             subprocess.call(['make'] + make_targets)
         for f in make_targets:
             if not os.path.exists(f):
@@ -543,6 +572,10 @@ def copy(src, dst):
 
     # Remove unknown files from tree
     removed_files = [f for f in current_files if f not in libvpx_files and f not in rename_files.values()]
+    for f in rename_files:
+        if os.path.exists(f) and os.path.exists(rename_files[f]) and not f in removed_files:
+            removed_files.append(f)
+
     if removed_files:
         print "Remove files:"
         for f in removed_files:
@@ -552,12 +585,13 @@ def copy(src, dst):
 def apply_patches():
     # Patch to permit vpx users to specify their own <stdint.h> types.
     os.system("patch -p0 < stdint.patch")
-    # Patch to allow MSVC 2015 to compile libvpx
-    os.system("patch -p1 < msvc2015.patch")
     # Patch to fix a crash caused by MSVC 2013
     os.system("patch -p3 < bug1137614.patch")
     # Bug 1176730 - Don't use pthread for libvpx in mingw builds.
     os.system("patch -p3 < disable_pthread_on_mingw.patch")
+    # Cherry pick https://chromium-review.googlesource.com/#/c/276889/
+    # to fix crash on 32bit
+    os.system("patch -p1 < vp9_filter_restore_aligment.patch")
 
 def update_readme(commit):
     with open('README_MOZILLA') as f:

media/libvpx/vp9_filter_restore_aligment.patch

@@ -0,0 +1,27 @@
+commit 33b3953c548a20c0aee705657df0440a740c28b7
+Author: James Zern <[email protected]>
+Date:   Thu Jun 11 15:12:22 2015 -0700
+
+    vp9_filter: restore vp9_bilinear_filters alignment
+
+    the declaration containing the alignment in vp9_filter.h was removed in:
+    eb88b17 Make vp9 subpixel match vp8
+
+    fixes a crash in 32-bit builds
+
+    Change-Id: I9a97e6b4e8e94698e43ff79d0d8bb85043b73c61
+
+diff --git a/vp9/common/vp9_filter.c b/vp9/common/vp9_filter.c
+index afcdf22..b256d4a 100644
+--- a/vp9/common/vp9_filter.c
++++ b/vp9/common/vp9_filter.c
+@@ -12,7 +12,8 @@
+
+ #include "vp9/common/vp9_filter.h"
+
+-const InterpKernel vp9_bilinear_filters[SUBPEL_SHIFTS] = {
++DECLARE_ALIGNED(256, const InterpKernel,
++                vp9_bilinear_filters[SUBPEL_SHIFTS]) = {
+   { 0, 0, 0, 128,   0, 0, 0, 0 },
+   { 0, 0, 0, 120,   8, 0, 0, 0 },
+   { 0, 0, 0, 112,  16, 0, 0, 0 },

media/libvpx/vpx_dsp_rtcd.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2013 Mozilla Foundation. All Rights Reserved.
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.
+ */
+
+#if defined(_WIN64)
+/* 64 bit Windows */
+#ifdef _MSC_VER
+#include "vpx_dsp_rtcd_x86_64-win64-vs12.h"
+#else
+#include "vpx_dsp_rtcd_x86_64-win64-gcc.h"
+#endif
+
+#elif defined(_WIN32)
+/* 32 bit Windows, MSVC. */
+#ifdef _MSC_VER
+#include "vpx_dsp_rtcd_x86-win32-vs12.h"
+#else
+#include "vpx_dsp_rtcd_x86-win32-gcc.h"
+#endif
+
+#elif defined(__APPLE__) && defined(__x86_64__)
+/* 64 bit MacOS. */
+#include "vpx_dsp_rtcd_x86_64-darwin9-gcc.h"
+
+#elif defined(__APPLE__) && defined(__i386__)
+/* 32 bit MacOS. */
+#include "vpx_dsp_rtcd_x86-darwin9-gcc.h"
+
+#elif defined(__ELF__) && (defined(__i386) || defined(__i386__))
+/* 32 bit ELF platforms. */
+#include "vpx_dsp_rtcd_x86-linux-gcc.h"
+
+#elif defined(__ELF__) && (defined(__x86_64) || defined(__x86_64__))
+/* 64 bit ELF platforms. */
+#include "vpx_dsp_rtcd_x86_64-linux-gcc.h"
+
+#elif defined(VPX_ARM_ASM)
+/* Android */
+#include "vpx_dsp_rtcd_armv7-android-gcc.h"
+
+#else
+/* Assume generic GNU/GCC configuration. */
+#include "vpx_dsp_rtcd_generic-gnu.h"
+#endif