Build: refactor fetch for contrib tarballs

Fetch is now python-based and runs on the same version as does
configure. The source script is make/fetch.py. New features:

MD5 hash tracking for tarballs. Data values for all contribs added.
Upon download, the file will be verified, and only then will it be moved
into place inside downloads/ . Files that exist before the build system
does a fetch will not be md5-checked.

Multiple URLs for tarballs. Each module may specify one or more URLs and
by convention the official HandBrake should be first when possible. Each
URL is tried in sequence, and if it fails for any reason, the next URL
is tried. If no URL succeeds, a hard-error is reported.

Network fetching may be disabled via configure options. --disable-fetch
will hard-error if a fetch is attempted. --accept-fetch-url=SPEC and
--deny-fetch-url=SPEC offer an ACL-style mechanism using regex to match
against URLs. For example, --accept-fecth-url='.*/download.handbrake.fr/.*'
would skip any non-matching URLs.

Build dependencies have been lightened. wget and curl are no longer
required. TODO: GTK packaging should also be able to remove those deps.

contrib/autoconf/module.defs

@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,AUTOCONF,autoconf,M4))
 $(eval $(call import.CONTRIB.defs,AUTOCONF))
 
 AUTOCONF.FETCH.url = http://download.handbrake.fr/handbrake/contrib/autoconf-2.69.tar.bz2
+AUTOCONF.FETCH.md5 = 58e0cca25061e1b3e2bc30a5be13815b

contrib/automake/module.defs

@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,AUTOMAKE,automake,AUTOCONF))
 $(eval $(call import.CONTRIB.defs,AUTOMAKE))
 
 AUTOMAKE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/automake-1.13.1.tar.bz2
+AUTOMAKE.FETCH.md5 = c66595aafe79e4e5455b4b3491fe6b05

contrib/bzip2/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,BZIP2,bzip2))
 $(eval $(call import.CONTRIB.defs,BZIP2))
 
 BZIP2.FETCH.url = http://download.handbrake.fr/handbrake/contrib/bzip2-1.0.6.tar.gz
+BZIP2.FETCH.md5 = 6ec09fe81e3380184ed11e21ea06c215
 BZIP2.EXTRACT.tarbase = bzip2
 BZIP2.CONFIGURE = $(TOUCH.exe) $@
 

contrib/cmake/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,CMAKE,cmake))
 $(eval $(call import.CONTRIB.defs,CMAKE))
 
 CMAKE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/cmake-3.3.2.tar.gz
+CMAKE.FETCH.md5 = 5febbd11bcaac854a27eebaf4a124be2
 
 CMAKE.CONFIGURE.deps =
 CMAKE.CONFIGURE.shared =

contrib/fdk-aac/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,FDKAAC,fdkaac))
 $(eval $(call import.CONTRIB.defs,FDKAAC))
 
 FDKAAC.FETCH.url = http://download.handbrake.fr/handbrake/contrib/fdk-aac-0.1.4.tar.gz
+FDKAAC.FETCH.md5 = e274a7d7f6cd92c71ec5c78e4dc9f8b7
 
 # fdk-aac configure script fails to add compiler optimizations if the
 # CFLAGS env variable is set during configure.  Since we set it, we

contrib/ffmpeg/module.defs

@@ -7,6 +7,7 @@ $(eval $(call import.MODULE.defs,FFMPEG,ffmpeg,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,FFMPEG))
 
 FFMPEG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libav-v11.3-0-g00abc00.tar.gz
+FFMPEG.FETCH.md5 = 23648740623345f122cfd0e015063c7e
 
 FFMPEG.CONFIGURE.deps  =
 FFMPEG.CONFIGURE.env   =

contrib/fontconfig/module.defs

@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,FONTCONFIG,fontconfig,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,FONTCONFIG))
 
 FONTCONFIG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/fontconfig-2.11.94.tar.bz2
+FONTCONFIG.FETCH.md5 = c988ea12f4117330246e041109152b4a
 FONTCONFIG.EXTRACT.tarbase = fontconfig-2.11.94
 
 #

contrib/freetype/module.defs

@@ -3,5 +3,6 @@ $(eval $(call import.MODULE.defs,FREETYPE,freetype,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,FREETYPE))
 
 FREETYPE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/freetype-2.6.tar.bz2
+FREETYPE.FETCH.md5 = 5682890cb0267f6671dd3de6eabd3e69
 
 FREETYPE.CONFIGURE.extra = --with-harfbuzz=no --with-png=no

contrib/fribidi/module.defs

@@ -6,6 +6,7 @@ $(eval $(call import.MODULE.defs,FRIBIDI,fribidi))
 $(eval $(call import.CONTRIB.defs,FRIBIDI))
 
 FRIBIDI.FETCH.url = http://download.handbrake.fr/handbrake/contrib/fribidi-0.19.7.tar.bz2
+FRIBIDI.FETCH.md5 = 6c7e7cfdd39c908f7ac619351c1c5c23
 
 ifeq (1-mingw,$(BUILD.cross)-$(BUILD.system))
     FRIBIDI.CONFIGURE.extra = --with-glib=no

contrib/jansson/module.defs

@@ -2,5 +2,6 @@ $(eval $(call import.MODULE.defs,JANSSON,jansson))
 $(eval $(call import.CONTRIB.defs,JANSSON))
 
 JANSSON.FETCH.url = http://download.handbrake.fr/handbrake/contrib/jansson-2.6.tar.bz2
+JANSSON.FETCH.md5 = c70a52488db623a26f7213c7c6b7c878
 
 JANSSON.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache; mkdir m4; autoreconf -fiv;

contrib/lame/module.defs

@@ -2,4 +2,5 @@ $(eval $(call import.MODULE.defs,LAME,lame))
 $(eval $(call import.CONTRIB.defs,LAME))
 
 LAME.FETCH.url = http://download.handbrake.fr/handbrake/contrib/lame-3.98.tar.gz
+LAME.FETCH.md5 = 7036b52e792538fd665595d56b9e49a0
 LAME.EXTRACT.tarbase = lame

contrib/libass/module.defs

@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,LIBASS,libass,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,LIBASS))
 
 LIBASS.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libass-0.12.3.tar.gz
+LIBASS.FETCH.md5 = 648ee785f966c69d4b5d50948e509d93
 
 # Disable as many external dependencies as I can get away with
 # and tell configure where to find our version of freetype

contrib/libbluray/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBBLURAY,libbluray,PKGCONFIG LIBXML2 FREETYPE)
 $(eval $(call import.CONTRIB.defs,LIBBLURAY))
 
 LIBBLURAY.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libbluray-0.8.1.tar.bz2
+LIBBLURAY.FETCH.md5 = c7a89966ad091fa6075a58d31a698e70
 
 ifneq (max,$(LIBBLURAY.GCC.g))
 	LIBBLURAY.CONFIGURE.extra += --disable-debug

contrib/libdvdnav/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBDVDNAV,libdvdnav,PKGCONFIG LIBDVDREAD))
 $(eval $(call import.CONTRIB.defs,LIBDVDNAV))
 
 LIBDVDNAV.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libdvdnav-5.0.1-0-gaa3659d.tar.gz
+LIBDVDNAV.FETCH.md5 = 1fb10188c7a33f480011d3ddf32553c7
 LIBDVDNAV.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache configure; autoreconf -I m4 -fiv;
 
 ifneq (max,$(LIBDVDNAV.GCC.g))

contrib/libdvdread/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBDVDREAD,libdvdread,PKGCONFIG))
 $(eval $(call import.CONTRIB.defs,LIBDVDREAD))
 
 LIBDVDREAD.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libdvdread-5.0.0-6-gcb1ae87.tar.gz
+LIBDVDREAD.FETCH.md5 = 607a5dd41b0dd2f35433d6deac79b99e
 LIBDVDREAD.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache configure; autoreconf -I m4 -fiv;
 
 ifeq (1-mingw,$(BUILD.cross)-$(BUILD.system))

contrib/libgnurx/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBGNURX,libgnurx))
 $(eval $(call import.CONTRIB.defs,LIBGNURX))
 
 LIBGNURX.FETCH.url = http://download.handbrake.fr/handbrake/contrib/mingw-libgnurx-2.5.1-src.tar.gz
+LIBGNURX.FETCH.md5 = 35c8fed3101ca1f253e9b6b1966661f6
 LIBGNURX.EXTRACT.tarbase = mingw-libgnurx-2.5.1
 
 LIBGNURX.CONFIGURE.env += AR="$(AR.exe)"

contrib/libiconv/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBICONV,libiconv))
 $(eval $(call import.CONTRIB.defs,LIBICONV))
 
 LIBICONV.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libiconv-1.13.tar.bz2
+LIBICONV.FETCH.md5 = 9c8b41b77f508edf72e26b406e873a2f
 
 # this contrib will not build under MinGW with -std=gnu99
 ifeq (1-mingw,$(BUILD.cross)-$(BUILD.system))

contrib/libmfx/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBMFX,libmfx))
 $(eval $(call import.CONTRIB.defs,LIBMFX))
 
 LIBMFX.FETCH.url = http://download.handbrake.fr/contrib/mfx_dispatch-9f4a84d7.tar.gz
+LIBMFX.FETCH.md5 = 694058b83b43b39b7e5b5fc38dbe2b88
 
 LIBMFX.CONFIGURE.bootstrap = rm -fr aclocal.m4 autom4te.cache; autoreconf -fiv;
 

contrib/libogg/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBOGG,libogg))
 $(eval $(call import.CONTRIB.defs,LIBOGG))
 
 LIBOGG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libogg-1.3.0.tar.gz
+LIBOGG.FETCH.md5 = 0a7eb40b86ac050db3a789ab65fe21c2
 LIBOGG.EXTRACT.tarbase = libogg-1.3.0
 
 LIBOGG.CONFIGURE.extra = --disable-sdl

contrib/libsamplerate/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBSAMPLERATE,libsamplerate))
 $(eval $(call import.CONTRIB.defs,LIBSAMPLERATE))
 
 LIBSAMPLERATE.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libsamplerate-0.1.4.tar.gz
+LIBSAMPLERATE.FETCH.md5 = 69ec6c05f487458f688dda8f3e722e5d
 LIBSAMPLERATE.EXTRACT.tarbase = libsamplerate
 
 # Disable to avoid Carbon.h dependency on OSX

contrib/libtheora/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBTHEORA,libtheora,LIBOGG LIBVORBIS))
 $(eval $(call import.CONTRIB.defs,LIBTHEORA))
 
 LIBTHEORA.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libtheora-1.1.0.tar.bz2
+LIBTHEORA.FETCH.md5 = d0f83cf7f13e2b3bd068a858ca1398ad
 
 LIBTHEORA.CONFIGURE.extra = \
     --disable-examples \

contrib/libtool/module.defs

@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,LIBTOOL,libtool,AUTOCONF AUTOMAKE PKGCONFIG))
 $(eval $(call import.CONTRIB.defs,LIBTOOL))
 
 LIBTOOL.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libtool-2.4.2.tar.bz2
+LIBTOOL.FETCH.md5 = f1a2109b9cd9c03a8df1f47e672c3aeb

contrib/libvorbis/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBVORBIS,libvorbis,LIBOGG))
 $(eval $(call import.CONTRIB.defs,LIBVORBIS))
 
 LIBVORBIS.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libvorbis-aotuv_b6.03.tar.bz2
+LIBVORBIS.FETCH.md5 = 586d2ac0fa13f32cba78be5db4a16330
 LIBVORBIS.EXTRACT.tarbase = aotuv-b6.03_20110424
 
 LIBVORBIS.CONFIGURE.extra = --with-ogg=$(call fn.ABSOLUTE,$(CONTRIB.build/)) HAVE_PKG_CONFIG="no"

contrib/libvpx/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,LIBVPX,libvpx,YASM))
 $(eval $(call import.CONTRIB.defs,LIBVPX))
 
 LIBVPX.FETCH.url = http://download.handbrake.fr/contrib/libvpx-v1.3.0.tar.bz2
+LIBVPX.FETCH.md5 = 14783a148872f2d08629ff7c694eb31f
 
 LIBVPX.CONFIGURE.args.host =
 LIBVPX.CONFIGURE.deps  =

contrib/libxml2/module.defs

@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,LIBXML2,libxml2,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,LIBXML2))
 
 LIBXML2.FETCH.url = http://download.handbrake.fr/handbrake/contrib/libxml2-2.7.7.tar.gz
+LIBXML2.FETCH.md5 = 9abc9959823ca9ff904f1fbcf21df066
 LIBXML2.EXTRACT.tarbase = libxml2-2.7.7
 
 # The Python components do not build on MinGW due to the lack of a select() call

contrib/m4/module.defs

@@ -2,3 +2,4 @@ $(eval $(call import.MODULE.defs,M4,m4))
 $(eval $(call import.CONTRIB.defs,M4))
 
 M4.FETCH.url = http://download.handbrake.fr/handbrake/contrib/m4-1.4.16.tar.bz2
+M4.FETCH.md5 = 8a7cef47fecab6272eb86a6be6363b2f

contrib/pkgconfig/module.defs

@@ -2,5 +2,6 @@ $(eval $(call import.MODULE.defs,PKGCONFIG,pkgconfig))
 $(eval $(call import.CONTRIB.defs,PKGCONFIG))
 
 PKGCONFIG.FETCH.url = http://download.handbrake.fr/handbrake/contrib/pkg-config-0.28.tar.bz2
+PKGCONFIG.FETCH.md5 = 60c2d5353d501c5619d535c6d4d80f1c
 
 PKGCONFIG.CONFIGURE.extra = --with-internal-glib --disable-host-tool

contrib/pthreadw32/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,PTHREADW32,pthreadw32))
 $(eval $(call import.CONTRIB.defs,PTHREADW32))
 
 PTHREADW32.FETCH.url = http://download.handbrake.fr/handbrake/contrib/pthreads-w32-cvs20100909.tar.bz2
+PTHREADW32.FETCH.md5 = d1dea735b53176567e9841ca77388633
 
 PTHREADW32.CONFIGURE = $(TOUCH.exe) $@
 

contrib/x264/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,X264,x264,YASM PTHREADW32))
 $(eval $(call import.CONTRIB.defs,X264))
 
 X264.FETCH.url = http://download.handbrake.fr/handbrake/contrib/x264-r2665-a01e339.tar.gz
+X264.FETCH.md5 = 4ffeac9157c5a7119b5b6ff36a5d96b7
 X264.EXTRACT.tarbase = x264
 
 X264.GCC.args.c_std =

contrib/x265/module.defs

@@ -3,6 +3,7 @@ $(eval $(call import.MODULE.defs,X265,x265,$(__deps__)))
 $(eval $(call import.CONTRIB.defs,X265))
 
 X265.FETCH.url = http://download.handbrake.fr/contrib/x265_1.9.tar.gz
+X265.FETCH.md5 = f34a1c4c660ff07511365cb0983cf164
 
 X265.CONFIGURE.exe         = cmake
 X265.CONFIGURE.args.prefix = -DCMAKE_INSTALL_PREFIX="$(X265.CONFIGURE.prefix)"

contrib/yasm/module.defs

@@ -2,4 +2,5 @@ $(eval $(call import.MODULE.defs,YASM,yasm))
 $(eval $(call import.CONTRIB.defs,YASM))
 
 YASM.FETCH.url = http://download.handbrake.fr/handbrake/contrib/yasm-1.2.0.tar.gz
+YASM.FETCH.md5 = 4cfc0686cf5350dd1305c4d905eb55a6
 

contrib/zlib/module.defs

@@ -2,6 +2,7 @@ $(eval $(call import.MODULE.defs,ZLIB,zlib))
 $(eval $(call import.CONTRIB.defs,ZLIB))
 
 ZLIB.FETCH.url = http://download.handbrake.fr/handbrake/contrib/zlib-1.2.3.tar.gz
+ZLIB.FETCH.md5 = af3358a811ad3469a2e54db49f77d52a
 ZLIB.EXTRACT.tarbase = zlib
 ZLIB.CONFIGURE.args = !sete @dir !env !exe @prefix !extra
 

doc/texi/building/prerequisites.common.texi

@@ -3,10 +3,8 @@ The following general tools are used on various platforms and it is recommended
 @itemize @bullet
 @item git - 2.4.3
 @item python - Python 2.7.1
-@item curl - curl 7.21.4 (or wget)
 @item m4 - GNU M4 1.4.6
 @item make - GNU Make 3.81
 @item patch - Patch 2.5.8
 @item tar - GNU tar 1.26
-@item wget - GNU Wget 1.13.4 (or curl)
 @end itemize