Skip to content

Commit

Permalink
generic presigned url API (minio#637)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebozduman
ebozduman authored and Krishna Srinivas committed Oct 21, 2017
1 parent fb712f4 commit ec6d0ba
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 77 deletions.
3 changes: 2 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,5 @@ logs
node_modules

/dist/
yarn.lock
yarn.lock
package-lock.json
79 changes: 62 additions & 17 deletions docs/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ var s3Client = new Minio.Client({
```
| Bucket operations | Object operations | Presigned operations | Bucket Policy & Notification operations |
| ------------- |-------------| -----| ----- |
| [`makeBucket`](#makeBucket) | [`getObject`](#getObject) | [`presignedGetObject`](#presignedGetObject) | [`getBucketNotification`](#getBucketNotification) |
| [`listBuckets`](#listBuckets) | [`getPartialObject`](#getPartialObject) | [`presignedPutObject`](#presignedPutObject) | [`setBucketNotification`](#setBucketNotification) |
| [`bucketExists`](#bucketExists) | [`fGetObject`](#fGetObject) | [`presignedPostPolicy`](#presignedPostPolicy) | [`removeAllBucketNotification`](#removeAllBucketNotification) |
| [`removeBucket`](#removeBucket) | [`putObject`](#putObject) | | [`getBucketPolicy`](#getBucketPolicy) | |
| [`makeBucket`](#makeBucket) | [`getObject`](#getObject) | [`presignedUrl`](#presignedUrl) | [`getBucketNotification`](#getBucketNotification) |
| [`listBuckets`](#listBuckets) | [`getPartialObject`](#getPartialObject) | [`presignedGetObject`](#presignedGetObject) | [`setBucketNotification`](#setBucketNotification) |
| [`bucketExists`](#bucketExists) | [`fGetObject`](#fGetObject) | [`presignedPutObject`](#presignedPutObject) | [`removeAllBucketNotification`](#removeAllBucketNotification) |
| [`removeBucket`](#removeBucket) | [`putObject`](#putObject) | [`presignedPostPolicy`](#presignedPostPolicy) | [`getBucketPolicy`](#getBucketPolicy) | |
| [`listObjects`](#listObjects) | [`fPutObject`](#fPutObject) | | [`setBucketPolicy`](#setBucketPolicy)
| [`listObjectsV2`](#listObjectsV2) | [`copyObject`](#copyObject) | | [`listenBucketNotification`](#listenBucketNotification)|
| [`listIncompleteUploads`](#listIncompleteUploads) | [`statObject`](#statObject) |
Expand Down Expand Up @@ -349,9 +349,9 @@ __Parameters__

| Param | Type | Description |
|---|---|---|
| `bucketName` | _string_ | Name of the bucket. |
| `objectName` | _string_ | Name of the object. |
| `callback(err, stream)` | _function_ | Callback is called with `err` in case of error. `stream` is the object content stream. If no callback is passed, a `Promise` is returned. |
|`bucketName` | _string_ | Name of the bucket. |
|`objectName` | _string_ | Name of the object. |
|`callback(err, stream)` | _function_ | Callback is called with `err` in case of error. `stream` is the object content stream. If no callback is passed, a `Promise` is returned. |

__Example__

Expand Down Expand Up @@ -653,10 +653,55 @@ minioClient.removeIncompleteUpload('mybucket', 'photo.jpg', function(err) {

Presigned URLs are generated for temporary download/upload access to private objects.

<a name="presignedUrl"></a>
### presignedUrl(httpMethod, bucketName, objectName, expiry[, reqParams, cb])

Generates a presigned URL for the provided HTTP method, 'httpMethod'. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which the URL is no longer valid. The default value is 7 days.


__Parameters__



| Param | Type | Description |
|---|---|---|
|`bucketName` | _string_ | Name of the bucket. |
|`objectName` | _string_ | Name of the object. |
|`expiry` | _number_ | Expiry time in seconds. Default value is 7 days. |
|`reqParams` | _object_ | request parameters. |
|`callback(err, presignedUrl)` | _function_ | Callback function is called with non `null` err value in case of error. `presignedUrl` will be the URL using which the object can be downloaded using GET request. If no callback is passed, a `Promise` is returned. |


__Example 1__


```js
// presigned url for 'getObject' method.
// expires in a day.
minioClient.presignedUrl('GET', 'mybucket', 'hello.txt', 24*60*60, function(err, presignedUrl) {
if (err) return console.log(err)
console.log(presignedUrl)
})
```


__Example 2__


```js
// presigned url for 'listObject' method.
// Lists objects in 'myBucket' with prefix 'data'.
// Lists max 1000 of them.
minioClient.presignedUrl('GET', 'mybucket', '', 1000, {'prefix': 'data', 'max-keys': 1000}, function(err, presignedUrl) {
if (err) return console.log(err)
console.log(presignedUrl)
})
```

<a name="presignedGetObject"></a>
### presignedGetObject(bucketName, objectName, expiry[, cb])

Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which the URL is no longer valid. The default expiry is set to 7 days.
Generates a presigned URL for HTTP GET operations. Browsers/Mobile clients may point to this URL to directly download objects even if the bucket is private. This presigned URL can have an associated expiration time in seconds after which the URL is no longer valid. The default value is 7 days.


__Parameters__
Expand All @@ -665,10 +710,10 @@ __Parameters__

| Param | Type | Description |
|---|---|---|
| `bucketName` | _string_ | Name of the bucket. |
|`objectName` | _string_ | Name of the object. |
| `expiry` |_number_ | Expiry in seconds. Default expiry is set to 7 days. |
| `callback(err, presignedUrl)` | _function_ | Callback function is called with non `null` err value in case of error. `presignedUrl` will be the URL using which the object can be downloaded using GET request. If no callback is passed, a `Promise` is returned. |
|`bucketName` | _string_ | Name of the bucket. |
|`objectName` | _string_ | Name of the object. |
|`expiry` | _number_ | Expiry time in seconds. Default value is 7 days. |
|`callback(err, presignedUrl)` | _function_ | Callback function is called with non `null` err value in case of error. `presignedUrl` will be the URL using which the object can be downloaded using GET request. If no callback is passed, a `Promise` is returned. |


__Example__
Expand All @@ -685,18 +730,18 @@ minioClient.presignedGetObject('mybucket', 'hello.txt', 24*60*60, function(err,
<a name="presignedPutObject"></a>
### presignedPutObject(bucketName, objectName, expiry[, callback])

Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which the URL is no longer valid. The default expiry is set to 7 days.
Generates a presigned URL for HTTP PUT operations. Browsers/Mobile clients may point to this URL to upload objects directly to a bucket even if it is private. This presigned URL can have an associated expiration time in seconds after which the URL is no longer valid. The default value is 7 days.


__Parameters__


| Param | Type | Description |
|---|---|---|
| `bucketName` | _string_ | Name of the bucket. |
| `objectName` | _string_ | Name of the object. |
| `expiry` | _number_ | Expiry in seconds. Default expiry is set to 7 days. |
| `callback(err, presignedUrl)` | _function_ | Callback function is called with non `null` err value in case of error. `presignedUrl` will be the URL using which the object can be uploaded using PUT request. If no callback is passed, a `Promise` is returned. |
|`bucketName` | _string_ | Name of the bucket. |
|`objectName` | _string_ | Name of the object. |
|`expiry` | _number_ | Expiry time in seconds. Default value is 7 days. |
|`callback(err, presignedUrl)` | _function_ | Callback function is called with non `null` err value in case of error. `presignedUrl` will be the URL using which the object can be uploaded using PUT request. If no callback is passed, a `Promise` is returned. |


__Example__
Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@
"through2": "^0.6.5",
"uuid": "^3.1.0",
"xml": "^1.0.0",
"xml2js": "^0.4.15"
"xml2js": "^0.4.15",
"querystring": "0.2.0"
},
"devDependencies": {
"browserify": "^12.0.1",
Expand Down
96 changes: 41 additions & 55 deletions src/main/minio.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import BlockStream2 from 'block-stream2'
import Xml from 'xml'
import xml2js from 'xml2js'
import async from 'async'
import querystring from 'querystring'
import mkdirp from 'mkdirp'
import path from 'path'
import _ from 'lodash'
Expand Down Expand Up @@ -1158,6 +1159,7 @@ export class Client {
if (queries.length > 0) {
query = `${queries.join('&')}`
}

var method = 'GET'
var transformer = transformers.getListObjectsTransformer()
this.makeRequest({method, bucketName, query}, '', 200, '', (e, response) => {
Expand Down Expand Up @@ -1471,31 +1473,39 @@ export class Client {
this.makeRequest({method, bucketName, query}, policyPayload, 204, '', cb)
}

// Generate a presigned URL for PUT. Using this URL, the browser can upload to S3 only with the specified object name.
// Generate a generic presigned URL which can be
// used for HTTP methods GET, PUT, HEAD and DELETE
//
// __Arguments__
// * `method` _string_: name of the HTTP method
// * `bucketName` _string_: name of the bucket
// * `objectName` _string_: name of the object
// * `expiry` _number_: expiry in seconds (optional, default 7 days)
presignedPutObject(bucketName, objectName, expires, cb) {
// * `reqParams` _object_: request parameters (optional)
presignedUrl(method, bucketName, objectName, expires, reqParams, cb) {
if (this.anonymous) {
throw new errors.AnonymousRequestError('Presigned PUT url cannot be generated for anonymous requests')
throw new errors.AnonymousRequestError('Presigned ' + method + ' url cannot be generated for anonymous requests')
}
if (isFunction(reqParams)) {
cb = reqParams
reqParams = {}
}
if (isFunction(expires)) {
cb = expires
expires = 24 * 60 * 60 * 7
}
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`)
reqParams = {}
expires = 24 * 60 * 60 * 7 // 7 days in seconds
}
if (!isNumber(expires)) {
throw new TypeError('expires should be of type "number"')
}
var method = 'PUT'
if (!isObject(reqParams)) {
throw new TypeError('reqParams should be of type "object"')
}
if (!isFunction(cb)) {
throw new TypeError('callback should be of type "function"')
}
var requestDate = new Date()
var query = querystring.stringify(reqParams)
this.getBucketRegion(bucketName, (e, region) => {
if (e) return cb(e)
// This statement is added to ensure that we send error through
Expand All @@ -1504,7 +1514,8 @@ export class Client {
var reqOptions = this.getRequestOptions({method,
region,
bucketName,
objectName})
objectName,
query})
try {
url = presignSignatureV4(reqOptions, this.accessKey, this.secretKey,
region, requestDate, expires)
Expand All @@ -1523,61 +1534,36 @@ export class Client {
// * `expiry` _number_: expiry in seconds (optional, default 7 days)
// * `respHeaders` _object_: response headers to override (optional)
presignedGetObject(bucketName, objectName, expires, respHeaders, cb) {
if (this.anonymous) {
throw new errors.AnonymousRequestError('Presigned GET url cannot be generated for anonymous requests')
}
if (isFunction(respHeaders)) {
cb = respHeaders
respHeaders = {}
}
if (isFunction(expires)) {
cb = expires
respHeaders = {}
expires = 24 * 60 * 60 * 7
}
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError(`Invalid object name: ${objectName}`)
}
if (!isNumber(expires)) {
throw new TypeError('expires should be of type "number"')
}
if (!isObject(respHeaders)) {
throw new TypeError('respHeaders should be of type "object"')
}
if (!isFunction(cb)) {
throw new TypeError('callback should be of type "function"')
}
var validRespHeaders = ['response-content-type', 'response-content-language', 'response-expires', 'response-cache-control',
'response-content-disposition', 'response-content-encoding']
validRespHeaders.forEach(header => {
if (respHeaders[header] !== undefined && !isString(respHeaders[header])) {
if (respHeaders !== undefined && respHeaders[header] !== undefined && !isString(respHeaders[header])) {
throw new TypeError(`response header ${header} should be of type "string"`)
}
})
var method = 'GET'
var requestDate = new Date()
var query = _.map(respHeaders, (value, key) => `${key}=${uriEscape(value)}`).join('&')
this.getBucketRegion(bucketName, (e, region) => {
if (e) return cb(e)
// This statement is added to ensure that we send error through
// callback on presign failure.
var url
var reqOptions = this.getRequestOptions({method,
region,
bucketName,
objectName,
query})
try {
url = presignSignatureV4(reqOptions, this.accessKey, this.secretKey,
region, requestDate, expires)
} catch (pe) {
return cb(pe)
}
cb(null, url)
})
return this.presignedUrl('GET', bucketName, objectName, expires, respHeaders, cb)
}

// Generate a presigned URL for PUT. Using this URL, the browser can upload to S3 only with the specified object name.
//
// __Arguments__
// * `bucketName` _string_: name of the bucket
// * `objectName` _string_: name of the object
// * `expiry` _number_: expiry in seconds (optional, default 7 days)
presignedPutObject(bucketName, objectName, expires, cb) {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ${bucketName}')
}
if (!isValidObjectName(objectName)) {
throw new errors.InvalidObjectNameError('Invalid object name: ${objectName}')
}
return this.presignedUrl('PUT', bucketName, objectName, expires, cb)
}

// return PostPolicy object
Expand Down
73 changes: 70 additions & 3 deletions src/test/functional/functional-tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -742,7 +742,7 @@ describe('functional tests', function() {
})

step('presignedPutObject(bucketName, objectName)__', done => {
// negative values should trigger an error
// Putting the same object should not cause any error
client.presignedPutObject(bucketName, _1byteObjectName)
.then(() => done())
.catch(done)
Expand Down Expand Up @@ -771,6 +771,29 @@ describe('functional tests', function() {
})
})

step('presignedUrl(getMethod, bucketName, objectName, expires, cb)__', done => {
client.presignedUrl('GET', bucketName, _1byteObjectName, 1000, (e, presignedUrl) => {
if (e) return done(e)
var transport = http
var options = _.pick(url.parse(presignedUrl), ['hostname', 'port', 'path', 'protocol'])
options.method = 'GET'
if (options.protocol === 'https:') transport = https
var request = transport.request(options, (response) => {
if (response.statusCode !== 200) return done(new Error(`error on put : ${response.statusCode}`))
var error = null
response.on('error', e => done(e))
response.on('end', () => done(error))
response.on('data', (data) => {
if (data.toString() !== _1byte.toString()) {
error = new Error('content mismatch')
}
})
})
request.on('error', e => done(e))
request.end()
})
})

step('presignedGetObject(bucketName, objectName, cb)__', done => {
client.presignedGetObject(bucketName, _1byteObjectName, (e, presignedUrl) => {
if (e) return done(e)
Expand Down Expand Up @@ -872,8 +895,52 @@ describe('functional tests', function() {
.catch(() => done())
})

step('removeObject(bucketName, objectName, done)__', done => {
client.removeObject(bucketName, _1byteObjectName, done)
step('presignedUrl(listObjectMethod, bucketName, \'\', expires, reqParams, cb)__', done => {
client.presignedUrl('GET', bucketName, '', 1000, {'prefix': 'data', 'max-keys': 1000}, (e, presignedUrl) => {
if (e) return done(e)
var transport = http
var options = _.pick(url.parse(presignedUrl), ['hostname', 'port', 'path', 'protocol'])
options.method = 'GET'
options.headers = {
}
var str = ''
if (options.protocol === 'https:') transport = https
var callback = function (response) {
if (response.statusCode !== 200) return done(new Error(`error on put : ${response.statusCode}`))
response.on('error', e => done(e))
response.on('end', function () {
if (!str.match(`<Key>${_1byteObjectName}</Key>`)) {
return done(new Error('Listed object does not match the object in the bucket!'))
}
done()
})
response.on('data', function (chunk) {
str += chunk
})
}
var request = transport.request(options, callback)
request.end()
})
})

step('presignedUrl(deleteMethod, bucketName, objectName, expires, cb)__', done => {
client.presignedUrl('DELETE', bucketName, _1byteObjectName, 1000, (e, presignedUrl) => {
if (e) return done(e)
var transport = http
var options = _.pick(url.parse(presignedUrl), ['hostname', 'port', 'path', 'protocol'])
options.method = 'DELETE'
options.headers = {
}
if (options.protocol === 'https:') transport = https
var request = transport.request(options, (response) => {
if (response.statusCode !== 204) return done(new Error(`error on put : ${response.statusCode}`))
response.on('error', e => done(e))
response.on('end', () => done())
response.on('data', () => {})
})
request.on('error', e => done(e))
request.end()
})
})
})

Expand Down

0 comments on commit ec6d0ba

Please sign in to comment.