Skip to content

Commit

Permalink
fix content encoding issues:
Browse files Browse the repository at this point in the history
1. correctly handle gzip/deflate encoding response body
2. make the default accept-encoding as 'identity'
3. allow setting custom headers in get_object
  • Loading branch information
rockuw committed Dec 25, 2015
1 parent 3d8e842 commit ecb7210
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 11 deletions.
14 changes: 6 additions & 8 deletions lib/aliyun/oss/bucket.rb
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,8 @@ def put_object(key, opts = {}, &block)
# * :if_unmodified_since (Time) 指定如果object从这个时间后再无修改,则下载
# * :if_match_etag (String) 指定如果object的etag等于这个值,则下载
# * :if_unmatch_etag (String) 指定如果object的etag不等于这个值,则下载
# @option opts [Hash] :headers 指定请求的HTTP Header,不区分大小
# 写。这里指定的值会覆盖通过`:range`和`:condition`设置的值。
# @option opts [Hash] :rewrite 指定下载object时Server端返回的响应头部字段的值
# * :content_type (String) 指定返回的响应中Content-Type的值
# * :content_language (String) 指定返回的响应中Content-Language的值
Expand Down Expand Up @@ -472,10 +474,7 @@ def resumable_upload(key, file, opts = {}, &block)

args[:content_type] ||= get_content_type(file)
args[:content_type] ||= get_content_type(key)

unless cpt_file = args[:cpt_file]
cpt_file = get_cpt_file(file)
end
cpt_file = args[:cpt_file] || get_cpt_file(file)

Multipart::Upload.new(
@protocol, options: args,
Expand Down Expand Up @@ -507,6 +506,8 @@ def resumable_upload(key, file, opts = {}, &block)
# 则:cpt_file会被忽略。
# @option opts [Hash] :condition 指定下载object需要满足的条件,
# 同 {#get_object}
# @option opts [Hash] :headers 指定请求的HTTP Header,不区分大小
# 写。这里指定的值会覆盖通过`:condition`设置的值。
# @option opts [Hash] :rewrite 指定下载object时Server端返回的响
# 应头部字段的值,同 {#get_object}
# @yield [Float] 如果调用的时候传递了block,则会将下载进度交由
Expand All @@ -529,10 +530,7 @@ def resumable_download(key, file, opts = {}, &block)

args[:content_type] ||= get_content_type(file)
args[:content_type] ||= get_content_type(key)

unless cpt_file = args[:cpt_file]
cpt_file = get_cpt_file(file)
end
cpt_file = args[:cpt_file] || get_cpt_file(file)

Multipart::Download.new(
@protocol, options: args,
Expand Down
3 changes: 2 additions & 1 deletion lib/aliyun/oss/download.rb
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ def download_part(p)
part_file = get_part_file(p)
File.open(part_file, 'w') do |w|
@protocol.get_object(
bucket, object, :range => p[:range]) { |chunk| w.write(chunk) }
bucket, object,
@options.merge(range: p[:range])) { |chunk| w.write(chunk) }
end

sync_update_part(p.merge(done: true, md5: get_file_md5(part_file)))
Expand Down
23 changes: 21 additions & 2 deletions lib/aliyun/oss/http.rb
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ module OSS
class HTTP

DEFAULT_CONTENT_TYPE = 'application/octet-stream'
DEFAULT_ACCEPT_ENCODING = 'identity'
STS_HEADER = 'x-oss-security-token'
OPEN_TIMEOUT = 10
READ_TIMEOUT = 120
Expand Down Expand Up @@ -156,8 +157,25 @@ def handle_response(r, &block)
r.read_body
else
# streaming read body on success
r.read_body do |chunk|
yield RestClient::Request.decode(r['content-encoding'], chunk)
encoding = r['content-encoding']
if encoding == 'gzip'
stream = StreamWriter.new { |s| r.read_body { |chunk| s << chunk } }
reader = Zlib::GzipReader.new(stream)
yield reader.read(16 * 1024) until reader.eof?
elsif encoding == 'deflate'
begin
stream = Zlib::Inflate.new
r.read_body { |chunk| stream << chunk }
stream.finish { |chunk| yield chunk }
rescue Zlib::DataError
# No luck with Zlib decompression. Let's try with raw deflate,
# like some broken web servers do.
stream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
r.read_body { |chunk| stream << chunk }
stream.finish { |chunk| yield chunk }
end
else
r.read_body { |chunk| yield chunk }
end
end
end
Expand Down Expand Up @@ -210,6 +228,7 @@ def do_request(verb, resources = {}, http_options = {}, &block)
headers['user-agent'] = get_user_agent
headers['date'] = Time.now.httpdate
headers['content-type'] ||= DEFAULT_CONTENT_TYPE
headers['accept-encoding'] ||= DEFAULT_ACCEPT_ENCODING
headers[STS_HEADER] = @config.sts_token if @config.sts_token

if body = http_options[:body]
Expand Down
4 changes: 4 additions & 0 deletions lib/aliyun/oss/protocol.rb
Original file line number Diff line number Diff line change
Expand Up @@ -718,6 +718,9 @@ def list_objects(bucket_name, opts = {})
# specified
# * :if_unmatch_etag (String) get the object if its etag
# doesn't match specified
# @option opts [Hash] :headers custom HTTP headers, case
# insensitive. Headers specified here will overwrite `:condition`
# and `:range`
# @option opts [Hash] :rewrite response headers to rewrite
# * :content_type (String) the Content-Type header
# * :content_language (String) the Content-Language header
Expand All @@ -739,6 +742,7 @@ def get_object(bucket_name, object_name, opts = {}, &block)
headers = {}
headers['range'] = get_bytes_range(range) if range
headers.merge!(get_conditions(conditions)) if conditions
headers.merge!(to_lower_case(opts[:headers])) if opts.key?(:headers)

sub_res = {}
if rewrites
Expand Down

0 comments on commit ecb7210

Please sign in to comment.