Skip to content

Commit

Permalink
Merge branch 'development'
Browse files Browse the repository at this point in the history
  • Loading branch information
@bdw429s
bdw429s committed Mar 22, 2023
2 parents a6917ed + aea5835 commit 1924461
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 34 deletions.
2 changes: 1 addition & 1 deletion box.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name":"Amazon S3 SDK",
"version":"5.5.2",
"version":"5.6.0",
"slug":"s3sdk",
"location":"https://downloads.ortussolutions.com/ortussolutions/coldbox-modules/s3sdk/@build.version@/[email protected]@.zip",
"author":"Ortus Solutions, Corp",
Expand Down
3 changes: 3 additions & 0 deletions changelog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

----
## v5.6.0 => 2023-MAR-07
### Added
* Support for overriding response headers like content type for pre-signed URLs
## v5.5.2 => 2023-FEB-07
### Fixed
* Multi-part upload concurrency fixes
Expand Down
85 changes: 53 additions & 32 deletions models/AmazonS3.cfc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,13 +645,12 @@ component accessors="true" singleton {
arguments.contentType = getFileMimeType( arguments.filepath );
}

var byteCount = getFileInfo( arguments.filepath ).size;
var byteCount = getFileInfo( arguments.filepath ).size;

if ( byteCount <= variables.multiPartByteThreshold ) {
arguments.data = fileReadBinary( arguments.filepath );
return putObject( argumentCollection = arguments );
} else {

var jFiles = createObject( "java", "java.nio.file.Files" );
var jPath = createObject( "java", "java.nio.file.Paths" ).get(
// Java is less lax on slashes than CF, so getCanonicalPath() cleans that up
Expand All @@ -668,24 +667,24 @@ component accessors="true" singleton {
for ( var i = 1; i <= numberOfUploads; i++ ) {
var remaining = byteCount - ( ( i - 1 ) * variables.multiPartByteThreshold );
parts.append( {
"uploadId" : uploadId,
"partNumber" : i,
"offset" : (i-1) * variables.multiPartByteThreshold,
"limit" : remaining <= variables.multiPartByteThreshold ? remaining : variables.multiPartByteThreshold,
"timeout" : arguments.HTTPTimeout,
"channel" : jFiles.newByteChannel( jPath, [] )
"uploadId" : uploadId,
"partNumber" : i,
"offset" : ( i - 1 ) * variables.multiPartByteThreshold,
"limit" : remaining <= variables.multiPartByteThreshold ? remaining : variables.multiPartByteThreshold,
"timeout" : arguments.HTTPTimeout,
"channel" : jFiles.newByteChannel( jPath, [] )
} );
}
try {
parts = variables.asyncManager.allApply( parts, function( part ){
var channel = part.channel.position( part.offset );
var buffer = createObject( "java", "java.nio.ByteBuffer" ).allocate( part.limit );
var buffer = createObject( "java", "java.nio.ByteBuffer" ).allocate( part.limit );
channel.read( buffer );

return {
"partNumber" : part.partNumber,
"size" : part.limit,
"channel" : part.channel,
"channel" : part.channel,
"response" : s3Request(
method = "PUT",
resource = bucketName & "/" & uri,
Expand All @@ -695,9 +694,7 @@ component accessors="true" singleton {
"uploadId" : part.uploadId,
"partNumber" : part.partNumber
},
headers = {
"content-type" : "binary/octet-stream"
}
headers = { "content-type" : "binary/octet-stream" }
)
};
} );
Expand Down Expand Up @@ -753,10 +750,11 @@ component accessors="true" singleton {
arguments.data = fileReadBinary( arguments.filepath );
return putObject( argumentCollection = arguments );
} finally {
parts.each( (p)=>{
try{
parts.each( ( p ) => {
try {
p.channel.close()
} catch( any e ) {}
} catch ( any e ) {
}
} );
}
}
Expand Down Expand Up @@ -1019,15 +1017,16 @@ component accessors="true" singleton {
/**
* Returns a query string authenticated URL to an object in S3.
*
* @bucketName The bucket the object resides in.
* @uri The uri to the object to create a link for.
* @minutesValid The minutes the link is valid for. Defaults to 60 minutes.
* @useSSL Use SSL for the returned url.
* @method HTTP method that will be used
* @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL(). If omitted, any ACL will be allowed when PUTting the file.
* @metaHeaders Additonal metadata headers to add.
* @contentType The object content type for PUT. If omitted, any content-type will be allowed when PUTting the file.
* @encryptionKey The base64 encoded AES 356 bit key for server side encryption.
* @bucketName The bucket the object resides in.
* @uri The uri to the object to create a link for.
* @minutesValid The minutes the link is valid for. Defaults to 60 minutes.
* @useSSL Use SSL for the returned url.
* @method HTTP method that will be used
* @acl The security policy to use. Specify a canned ACL like "public-read" as a string, or provide a struct in the format of the "grants" key returned by getObjectACL(). If omitted, any ACL will be allowed when PUTting the file.
* @metaHeaders Additonal metadata headers to add.
* @contentType The object content type for PUT. If omitted, any content-type will be allowed when PUTting the file.
* @encryptionKey The base64 encoded AES 356 bit key for server side encryption.
* @responseHeaders A struct of headers to be forced for the HTTP response of GET requests. Valid options are content-type, content-language, expires, cache-control, content-disposition, content-encoding
*
* @return An authenticated url to the resource.
*/
Expand All @@ -1040,7 +1039,8 @@ component accessors="true" singleton {
any acl = "",
struct metaHeaders = {},
string contentType,
string encryptionKey = variables.defaultEncryptionKey
string encryptionKey = variables.defaultEncryptionKey,
struct responseHeaders = {}
){
requireBucketName( arguments.bucketName );

Expand All @@ -1057,13 +1057,35 @@ component accessors="true" singleton {

var hostname = "#bucketName#.#variables.URLEndpointHostname#";

var requestParams = { "X-Amz-Expires" : arguments.minutesValid * 60 };
var validResponseHeaders = [
"content-type",
"content-language",
"expires",
"cache-control",
"content-disposition",
"content-encoding"
];
responseHeaders.each( ( header, value ) => {
if ( !validResponseHeaders.findNoCase( header ) ) {
throw(
message = "Invalid Reponse Header for signed URL: [#header#].",
detail = "Valid options are: [#validResponseHeaders.toList()#]"
);
}
if ( header == "content-type" && value == "auto" ) {
value = getFileMimeType( uri );
}
requestParams[ "response-" & header ] = value;
} );

var sigData = variables.signatureUtil.generateSignatureData(
requestMethod = arguments.method,
hostName = hostname,
requestURI = arguments.uri,
requestBody = "",
requestHeaders = headers,
requestParams = { "X-Amz-Expires" : arguments.minutesValid * 60 },
requestParams = requestParams,
accessKey = variables.accessKey,
secretKey = variables.secretKey,
regionName = variables.awsRegion,
Expand Down Expand Up @@ -1395,8 +1417,8 @@ component accessors="true" singleton {
"message" : "",
"responseheader" : {}
};
var param = "";
var md5 = "";
var param = "";
var md5 = "";

// Default Content Type
if ( NOT structKeyExists( arguments.headers, "content-type" ) ) {
Expand Down Expand Up @@ -1428,7 +1450,7 @@ component accessors="true" singleton {
// Let the CF engine directly save the file so it can stream large files to disk and not eat up memory
cfhttpAttributes[ "file" ] = getFileFromPath( arguments.filename );
cfhttpAttributes[ "path" ] = getDirectoryFromPath( arguments.filename );
if( !directoryExists( cfhttpAttributes[ "path" ] ) ) {
if ( !directoryExists( cfhttpAttributes[ "path" ] ) ) {
directoryCreate( cfhttpAttributes[ "path" ] );
}
if ( !isNull( server.lucee ) ) {
Expand All @@ -1437,7 +1459,6 @@ component accessors="true" singleton {
// https://luceeserver.atlassian.net/browse/LDEV-4357
cfhttpAttributes[ "result" ] = "";
}

}

cfhttp(
Expand Down Expand Up @@ -1487,7 +1508,7 @@ component accessors="true" singleton {

// Lucee behavior mentioned above regarding file download incompat with Adobe
// When Lucee direct-downnloads a file, it doesn't return ANY details from the HTTP request :/
if( isNull( local.HTTPResults ) || !isStruct( local.HTTPResults ) ) {
if ( isNull( local.HTTPResults ) || !isStruct( local.HTTPResults ) ) {
return results;
}

Expand Down
59 changes: 58 additions & 1 deletion test-harness/tests/specs/AmazonS3Spec.cfc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ component extends="coldbox.system.testing.BaseTestCase" {
expandPath( "/tests/tmp/big_file2.txt" )
);
// And confirm a hash of both file contents still matches
expect( hash( fileRead( expandPath( "/tests/tmp/big_file2.txt" ) ) ) ).toBe( hash( fileRead( expandPath( "/tests/tmp/big_file.txt" ) ) ) )
expect( hash( fileRead( expandPath( "/tests/tmp/big_file2.txt" ) ) ) ).toBe(
hash( fileRead( expandPath( "/tests/tmp/big_file.txt" ) ) )
)
} );

it(
Expand Down Expand Up @@ -600,6 +602,61 @@ component extends="coldbox.system.testing.BaseTestCase" {
};
expect( local.cfhttp.Responseheader.status_code ?: 0 ).toBe( "403", local.cfhttp.fileContent );
} );

it( "Can use presigned URL with forced response headers", function(){
s3.putObject( testBucket, "example.txt", "Hello, world!" );
var presignedURL = s3.getAuthenticatedURL(
bucketName = testBucket,
uri = "example.txt",
responseHeaders = {
"content-type" : "custom-type",
"content-language" : "custom-language",
"expires" : "custom-expires",
"cache-control" : "custom-cache",
"content-disposition" : "custom-disposition",
"content-encoding" : "custom-encoding"
}
);
cfhttp( url = "#presignedURL#", result = "local.cfhttp" );

expect( local.cfhttp.Responseheader.status_code ?: 0 ).toBe( "200", local.cfhttp.fileContent );
expect( local.cfhttp.fileContent ).toBe( "Hello, world!" );
expect( local.cfhttp.Responseheader[ "content-type" ] ).toBe( "custom-type" );
expect( local.cfhttp.Responseheader[ "content-language" ] ).toBe( "custom-language" );
expect( local.cfhttp.Responseheader[ "expires" ] ).toBe( "custom-expires" );
expect( local.cfhttp.Responseheader[ "cache-control" ] ).toBe( "custom-cache" );
expect( local.cfhttp.Responseheader[ "content-disposition" ] ).toBe( "custom-disposition" );
expect( local.cfhttp.Responseheader[ "content-encoding" ] ).toBe( "custom-encoding" );
} );

it( "Can use presigned URL with auto response content type", function(){
s3.putObject(
testBucket,
"example.txt",
"Hello, world!",
"",
"wacky-content-type"
);
var presignedURL = s3.getAuthenticatedURL(
bucketName = testBucket,
uri = "example.txt",
responseHeaders = { "content-type" : "auto" }
);
cfhttp( url = "#presignedURL#", result = "local.cfhttp" );

expect( local.cfhttp.Responseheader.status_code ?: 0 ).toBe( "200", local.cfhttp.fileContent );
expect( local.cfhttp.fileContent ).toBe( "Hello, world!" );
// Our explicit content type when storing the file is ignored and the corret type is automatically returned based on MIME type
expect( local.cfhttp.Responseheader[ "content-type" ] ).toBe( "text/plain" );
} );

it( "Creating presigned URL with invalid response header throws error", function(){
expect( () => s3.getAuthenticatedURL(
bucketName = testBucket,
uri = "example.txt",
responseHeaders = { "fake" : "" }
) ).toThrow();
} );
} );
} );

Expand Down

0 comments on commit 1924461

Please sign in to comment.