Simplify patches related to downloading HTTPS

* The patch in rubygems/remote_fetcher.rb is not needed,
  as removing just goes through the rubygems/request.rb patch.
* wget is no longer needed.
* Share code to perform a https request and return a Net::HTTPResponse
  using curl.
* Only patch the https path, let other requests use the original
  implementation to minimize the patch size.

doc/user/installing-gems.md

@@ -3,12 +3,12 @@
 As mentioned in the [README](../../README.md) TruffleRuby does not yet support
 openssl and C extensions, therefore we apply a few patches internally to  make
 `rubygems` and `bundler` work out of the box. Gems with C extensions will
-install but nothing will be compiled. If the gem does not contain a pure  Ruby
-implementation of the C extension the gem will not function properly ( e.g.
+install but nothing will be compiled. If the gem does not contain a pure Ruby
+implementation of the C extension the gem will not function properly (e.g.
 `nokogiri`, Active Record drivers, etc).
 
-The patches require `wget` and `curl` to be installed, The patches will be
-eventually removed.
+The patches require `curl` to be installed.
+The patches will be eventually removed.
 
 Examples:
 

lib/patches/bundler/bundler/fetcher/downloader.rb

@@ -3,63 +3,19 @@
 if Truffle::Boot.patching_openssl_enabled?
   # TruffleRuby: Use curl in bundler downloader for https requests
 
-  module Bundler
-    class Fetcher
-      class Downloader
-        HTTP_PROXY_HEADER="HTTP/1.0 200 Connection established\r\n\r\n"
-        class CurlResponse < Net::HTTPOK
-          def body
-            @body
-          end
-        end
+  require "truffle/https_downloader"
 
-        def fetch(uri, options = {}, counter = 0)
-          raise HTTPError, "Too many redirects" if counter >= redirect_limit
-
-          # TruffleRuby: start
-          response = if uri.scheme == "https"
-                       resp           = CurlResponse.new("1.1", 200, "OK")
-                       resp_raw = if (`curl --help` rescue nil)
-                                    `curl -i -s #{uri}`
-                                  else
-                                    raise 'curl is missing'
-                                  end
-                       index_offset   = resp_raw.start_with?(HTTP_PROXY_HEADER) ? HTTP_PROXY_HEADER.size : 0
-                       blank_line_idx = resp_raw.index("\r\n\r\n", index_offset)
-                       header         = resp_raw[0, blank_line_idx]
-                       resp.body      = resp_raw[(blank_line_idx+4)..-1]
-                       if m = /ETag: (\"[[:alnum:]]*\")/.match(header)
-                         resp["ETag"] = m[1]
-                       end
-                       resp
-                     else
-                       response = request(uri, options)
-                     end
-          # TruffleRuby: end
-
-          Bundler.ui.debug("HTTP #{response.code} #{response.message}")
-
-          case response
-          when Net::HTTPSuccess, Net::HTTPNotModified
-            response
-          when Net::HTTPRedirection
-            new_uri = URI.parse(response["location"])
-            if new_uri.host == uri.host
-              new_uri.user     = uri.user
-              new_uri.password = uri.password
-            end
-            fetch(new_uri, options, counter + 1)
-          when Net::HTTPRequestEntityTooLarge
-            raise FallbackError, response.body
-          when Net::HTTPUnauthorized
-            raise AuthenticationRequiredError, uri.host
-          when Net::HTTPNotFound
-            raise FallbackError, "Net::HTTPNotFound"
-          else
-            raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
-          end
-        end
+  module Truffle::Patching::BundlerFetcherDownloaderHTTPS
+    def request(uri, options)
+      if uri.scheme == "https"
+        Truffle::HTTPSDownloader.download(uri)
+      else
+        super
       end
     end
   end
+
+  class Bundler::Fetcher::Downloader
+    prepend Truffle::Patching::BundlerFetcherDownloaderHTTPS
+  end
 end

lib/patches/stdlib/rubygems/remote_fetcher.rb

@@ -31,114 +31,3 @@ def api_endpoint(uri)
     end
   end
 end
-
-if Truffle::Boot.patching_openssl_enabled?
-  # Use wget
-
-  class Gem::RemoteFetcher
-    def download(spec, source_uri, install_dir = Gem.dir)
-      cache_dir =
-          if Dir.pwd == install_dir then # see fetch_command
-            install_dir
-          elsif File.writable? install_dir then
-            File.join install_dir, "cache"
-          else
-            File.join Gem.user_dir, "cache"
-          end
-
-      gem_file_name = File.basename spec.cache_file
-      local_gem_path = File.join cache_dir, gem_file_name
-
-      FileUtils.mkdir_p cache_dir rescue nil unless File.exist? cache_dir
-
-      # Always escape URI's to deal with potential spaces and such
-      # It should also be considered that source_uri may already be
-      # a valid URI with escaped characters. e.g. "{DESede}" is encoded
-      # as "%7BDESede%7D". If this is escaped again the percentage
-      # symbols will be escaped.
-      unless source_uri.is_a?(URI::Generic)
-        begin
-          source_uri = URI.parse(source_uri)
-        rescue
-          source_uri = URI.parse(URI.const_defined?(:DEFAULT_PARSER) ?
-                                     URI::DEFAULT_PARSER.escape(source_uri.to_s) :
-                                     URI.escape(source_uri.to_s))
-        end
-      end
-
-      scheme = source_uri.scheme
-
-      # URI.parse gets confused by MS Windows paths with forward slashes.
-      scheme = nil if scheme =~ /^[a-z]$/i
-
-      # REFACTOR: split this up and dispatch on scheme (eg download_http)
-      # REFACTOR: be sure to clean up fake fetcher when you do this... cleaner
-      case scheme
-      when 'http', 'https', 's3' then
-        unless File.exist? local_gem_path then
-          begin
-            verbose "Downloading gem #{gem_file_name}"
-
-            remote_gem_path = source_uri + "gems/#{gem_file_name}"
-
-            # TruffleRuby: start
-            if (`wget --help` rescue nil)
-              cmd = "wget -q #{remote_gem_path} -O #{local_gem_path}"
-             `#{cmd}`
-            else
-              raise 'wget is missing'
-            end
-            # self.cache_update_path remote_gem_path, local_gem_path
-            # TruffleRuby: end
-          rescue Gem::RemoteFetcher::FetchError
-            raise if spec.original_platform == spec.platform
-
-            alternate_name = "#{spec.original_name}.gem"
-
-            verbose "Failed, downloading gem #{alternate_name}"
-
-            remote_gem_path = source_uri + "gems/#{alternate_name}"
-
-            self.cache_update_path remote_gem_path, local_gem_path
-          end
-        end
-      when 'file' then
-        begin
-          path = source_uri.path
-          path = File.dirname(path) if File.extname(path) == '.gem'
-
-          remote_gem_path = correct_for_windows_path(File.join(path, 'gems', gem_file_name))
-
-          FileUtils.cp(remote_gem_path, local_gem_path)
-        rescue Errno::EACCES
-          local_gem_path = source_uri.to_s
-        end
-
-        verbose "Using local gem #{local_gem_path}"
-      when nil then # TODO test for local overriding cache
-        source_path = if Gem.win_platform? && source_uri.scheme &&
-            !source_uri.path.include?(':') then
-                        "#{source_uri.scheme}:#{source_uri.path}"
-                      else
-                        source_uri.path
-                      end
-
-        source_path = Gem::UriFormatter.new(source_path).unescape
-
-        begin
-          FileUtils.cp source_path, local_gem_path unless
-              File.identical?(source_path, local_gem_path)
-        rescue Errno::EACCES
-          local_gem_path = source_uri.to_s
-        end
-
-        verbose "Using local gem #{local_gem_path}"
-      else
-        raise ArgumentError, "unsupported URI scheme #{source_uri.scheme}"
-      end
-
-      local_gem_path
-    end
-  end
-end
-

lib/patches/stdlib/rubygems/request.rb

@@ -3,53 +3,19 @@
 if Truffle::Boot.patching_openssl_enabled?
   # TruffleRuby: Use curl when uri has https scheme
 
-  class Gem::Request
-    class CurlResponse < Net::HTTPOK
-      def body
-        @body
-      end
-    end
-
-    HTTP_PROXY_HEADER="HTTP/1.0 200 Connection established\r\n\r\n"
-
-    def fetch
-      request = @request_class.new @uri.request_uri
-
-      unless @uri.nil? || @uri.user.nil? || @uri.user.empty? then
-        request.basic_auth Gem::UriFormatter.new(@uri.user).unescape,
-                           Gem::UriFormatter.new(@uri.password).unescape
-      end
+  require "truffle/https_downloader"
 
-      request.add_field 'User-Agent', @user_agent
-      request.add_field 'Connection', 'keep-alive'
-      request.add_field 'Keep-Alive', '30'
-
-      if @last_modified then
-        request.add_field 'If-Modified-Since', @last_modified.httpdate
-      end
-
-      yield request if block_given?
-
-      # TruffleRuby: start
+  module Truffle::Patching::GemRequestPerformHTTPSRequest
+    def perform_request(request)
       if @uri.scheme == "https"
-        resp = CurlResponse.new("1.1", 200, "OK")
-        resp_raw = if (`curl --help` rescue nil)
-                     `curl -i -s #{@uri}`
-                   else
-                     raise 'curl is missing'
-                   end
-        index_offset = resp_raw.start_with?(HTTP_PROXY_HEADER) ? HTTP_PROXY_HEADER.size : 0
-        blank_line_idx = resp_raw.index("\r\n\r\n", index_offset)
-        header = resp_raw[0, blank_line_idx]
-        resp.body = resp_raw[(blank_line_idx+4)..-1]
-        if m = /ETag: (\"[[:alnum:]]*\")/.match(header)
-          resp["ETag"] = m[1]
-        end
-        resp
+        Truffle::HTTPSDownloader.download(@uri)
       else
-        perform_request request
+        super
       end
-      # TruffleRuby: end
     end
   end
+
+  class Gem::Request
+    prepend Truffle::Patching::GemRequestPerformHTTPSRequest
+  end
 end

lib/truffle/truffle/https_downloader.rb

@@ -0,0 +1,33 @@
+require 'net/http'
+
+module Truffle
+  module HTTPSDownloader
+    HTTP_PROXY_HEADER = "HTTP/1.0 200 Connection established\r\n\r\n"
+
+    class CurlResponse < Net::HTTPOK
+      def body
+        @body
+      end
+    end
+
+    def self.download(uri)
+      resp = CurlResponse.new('1.1', 200, 'OK')
+      raise 'curl is missing' unless (`curl --help` rescue nil)
+      resp_raw = `curl -i -s #{uri}`
+      index_offset = resp_raw.start_with?(HTTP_PROXY_HEADER) ? HTTP_PROXY_HEADER.size : 0
+      blank_line_idx = resp_raw.index("\r\n\r\n", index_offset)
+      header = resp_raw[0...blank_line_idx]
+      resp.body = resp_raw[(blank_line_idx+4)..-1]
+
+      if m = /ETag: (\"[[:alnum:]]*\")/.match(header)
+        resp['ETag'] = m[1]
+      end
+      resp
+    end
+
+    def self.download_to(uri, path)
+      raise 'curl is missing' unless (`curl --help` rescue nil)
+      system 'curl', '-s', uri, '-o', path
+    end
+  end
+end

tool/jt.rb

@@ -1524,7 +1524,7 @@ def install_graal
           puts "Downloading JDK8 with JVMCI"
           jvmci_releases = "https://github.com/dougxc/openjdk8-jvmci-builder/releases/download"
           filename = "openjdk1.8.0_141-#{jvmci_version}-linux-amd64.tar.gz"
-          raw_sh "wget", "#{jvmci_releases}/#{jvmci_version}/#{filename}", "-O", filename
+          raw_sh "curl", "-L", "#{jvmci_releases}/#{jvmci_version}/#{filename}", "-o", filename
           raw_sh "tar", "xf", filename
         end
         java_home = Dir[jvmci_grep].sort.first

tool/make-testing-distribution.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # This script creates a small (~12MB) distribution of TruffleRuby without Sulong
-# which only needs JRE 8, curl, wget and bash.
+# which only needs JRE 8, curl and bash.
 
 # This script should be run on a fresh checkout of truffleruby,
 # otherwise extra gems and build artifacts might be included.