Skip to content

Commit

Permalink
cleanup object-lock/bucket tagging for gateways (minio#9548)
Browse files Browse the repository at this point in the history 
This PR is to ensure that we call the relevant object
layer APIs for necessary S3 API level functionalities
allowing gateway implementations to return proper
errors as NotImplemented{}

This allows for all our tests in mint to behave
appropriately and can be handled appropriately as
well.
  • Loading branch information
@harshavardhana
harshavardhana authored May 8, 2020
1 parent 6885c72 commit a1de9ce
Show file tree
Hide file tree
Showing 49 changed files with 679 additions and 373 deletions.
4 changes: 4 additions & 0 deletions cmd/api-errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1808,6 +1808,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
apiErr = ErrNoSuchLifecycleConfiguration
case BucketSSEConfigNotFound:
apiErr = ErrNoSuchBucketSSEConfig
case BucketTaggingNotFound:
apiErr = ErrBucketTaggingNotFound
case BucketObjectLockConfigNotFound:
apiErr = ErrObjectLockConfigurationNotFound
case BucketQuotaConfigNotFound:
apiErr = ErrAdminNoSuchQuotaConfiguration
case BucketQuotaExceeded:
Expand Down
10 changes: 5 additions & 5 deletions cmd/benchmark-utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
if err != nil {
b.Fatal(err)
}
Expand Down Expand Up @@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
object := getRandomObjectName()

// create bucket.
err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
if err != nil {
b.Fatal(err)
}
Expand Down Expand Up @@ -181,7 +181,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
if err != nil {
b.Fatal(err)
}
Expand Down Expand Up @@ -278,7 +278,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
if err != nil {
b.Fatal(err)
}
Expand Down Expand Up @@ -322,7 +322,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
// obtains random bucket name.
bucket := getRandomBucketName()
// create bucket.
err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
if err != nil {
b.Fatal(err)
}
Expand Down
122 changes: 32 additions & 90 deletions cmd/bucket-handlers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,16 +292,6 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

for i := range bucketsInfo {
meta, err := loadBucketMetadata(ctx, objectAPI, bucketsInfo[i].Name)
if err == nil {
bucketsInfo[i].Created = meta.Created
}
if err != errMetaDataConverted {
logger.LogIf(ctx, err)
}
}
}

if s3Error == ErrAccessDenied {
Expand Down Expand Up @@ -409,7 +399,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
continue
}

if _, ok := globalBucketObjectLockConfig.Get(bucket); ok {
if _, ok := globalBucketObjectLockSys.Get(bucket); ok {
if err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); err != ErrNone {
dErrs[index] = err
continue
Expand Down Expand Up @@ -537,14 +527,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
if err != nil {
if err == dns.ErrNoEntriesFound {
// Proceed to creating a bucket.
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

meta := newBucketMetadata(bucket)
meta.LockEnabled = objectLockEnabled
if err := meta.save(ctx, objectAPI); err != nil {
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
Expand Down Expand Up @@ -579,24 +562,16 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
}

// Proceed to creating a bucket.
err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
err := objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

if !globalIsGateway {
meta := newBucketMetadata(bucket)
meta.LockEnabled = objectLockEnabled
if err := meta.save(ctx, objectAPI); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if objectLockEnabled {
ret := &objectlock.Retention{}
globalBucketObjectLockConfig.Set(bucket, ret)
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, ret)
}
if objectLockEnabled && !globalIsGateway {
ret := &objectlock.Retention{}
globalBucketObjectLockSys.Set(bucket, ret)
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, ret)
}

// Make sure to add Location information here only for bucket
Expand Down Expand Up @@ -933,7 +908,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
}
}

if _, ok := globalBucketObjectLockConfig.Get(bucket); ok && forceDelete {
if _, ok := globalBucketObjectLockSys.Get(bucket); ok && forceDelete {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
return
}
Expand All @@ -949,15 +924,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
if globalDNSConfig != nil {
if err := globalDNSConfig.Delete(bucket); err != nil {
// Deleting DNS entry failed, attempt to create the bucket again.
objectAPI.MakeBucketWithLocation(ctx, bucket, "")
objectAPI.MakeBucketWithLocation(ctx, bucket, "", false)
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
}

// Delete metadata, only log errors.
logger.LogIf(ctx, newBucketMetadata(bucket).delete(ctx, objectAPI))

globalNotificationSys.DeleteBucket(ctx, bucket)

// Write success response.
Expand Down Expand Up @@ -1041,6 +1013,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}

config, err := objectlock.ParseObjectLockConfig(r.Body)
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
Expand All @@ -1049,32 +1022,17 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
return
}

meta, err := loadBucketMetadata(ctx, objectAPI, bucket)
if err != nil {
if err = objectAPI.SetBucketObjectLockConfig(ctx, bucket, config); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if !meta.LockEnabled {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL, guessIsBrowserReq(r))
return
}

data, err := xml.Marshal(config)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configFile := path.Join(bucketConfigPrefix, bucket, objectLockConfig)
if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if config.Rule != nil {
retention := config.ToRetention()
globalBucketObjectLockConfig.Set(bucket, retention)
globalBucketObjectLockSys.Set(bucket, retention)
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, retention)
} else {
globalBucketObjectLockConfig.Set(bucket, &objectlock.Retention{})
globalBucketObjectLockSys.Set(bucket, &objectlock.Retention{})
globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, &objectlock.Retention{})
}

Expand Down Expand Up @@ -1106,28 +1064,16 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
return
}

meta, err := loadBucketMetadata(ctx, objectAPI, bucket)
if err != nil && err != errMetaDataConverted {
lkCfg, err := objectAPI.GetBucketObjectLockConfig(ctx, bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
if !meta.LockEnabled {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL, guessIsBrowserReq(r))
return
}

configFile := path.Join(bucketConfigPrefix, bucket, objectLockConfig)
configData, err := readConfig(ctx, objectAPI, configFile)
configData, err := xml.Marshal(lkCfg)
if err != nil {
if err != errConfigNotFound {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

// Write success response.
Expand All @@ -1154,20 +1100,16 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}

tags, err := tags.ParseBucketXML(io.LimitReader(r.Body, r.ContentLength))
if err != nil {
apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
apiErr.Description = err.Error()
writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
return
}
data, err := xml.Marshal(tags)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
configFile := path.Join(bucketConfigPrefix, bucket, bucketTaggingConfigFile)
if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {

if err = objectAPI.SetBucketTagging(ctx, bucket, tags); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
Expand All @@ -1191,21 +1133,22 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
return
}

// check if user has permissions to perform this operation
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketTaggingAction, bucket, ""); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
return
}
configFile := path.Join(bucketConfigPrefix, bucket, bucketTaggingConfigFile)
configData, err := readConfig(ctx, objectAPI, configFile)

t, err := objectAPI.GetBucketTagging(ctx, bucket)
if err != nil {
var aerr APIError
if err == errConfigNotFound {
aerr = errorCodes.ToAPIErr(ErrBucketTaggingNotFound)
} else {
aerr = toAPIError(ctx, err)
}
writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

configData, err := xml.Marshal(t)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}

Expand Down Expand Up @@ -1234,8 +1177,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
return
}

configFile := path.Join(bucketConfigPrefix, bucket, bucketTaggingConfigFile)
if err := deleteConfig(ctx, objectAPI, configFile); err != nil && err != errConfigNotFound {
if err := objectAPI.DeleteBucketTagging(ctx, bucket); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
return
}
Expand Down
14 changes: 6 additions & 8 deletions cmd/bucket-meta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,12 @@ func (b *bucketMetadata) save(ctx context.Context, o ObjectLayer) error {
return saveConfig(ctx, o, configFile, data)
}

// delete the config metadata.
// removeBucketMeta bucket metadata.
// If config does not exist no error is returned.
func (b bucketMetadata) delete(ctx context.Context, o ObjectLayer) error {
configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
err := deleteConfig(ctx, o, configFile)
if err == errConfigNotFound {
// We don't care
err = nil
func removeBucketMeta(ctx context.Context, obj ObjectLayer, bucket string) error {
configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
return err
}
return err
return nil
}
Loading

0 comments on commit a1de9ce

Please sign in to comment.