From 3d5b1c1f51c11fa5ac338794f2b2e14b0ed3214f Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 15:42:57 +0800 Subject: [PATCH 01/33] Add image resize and upload when user upload origin image --- src/cmd/gcs-server/main.go | 4 +- src/server/handler_storage_test.go | 139 ++++++++++++++++++ src/server/testutils.go | 30 ++++ .../pubsubprovider/pubsubprovider.go | 2 + 4 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 src/server/handler_storage_test.go create mode 100644 src/server/testutils.go diff --git a/src/cmd/gcs-server/main.go b/src/cmd/gcs-server/main.go index 753f722..b2aac9e 100644 --- a/src/cmd/gcs-server/main.go +++ b/src/cmd/gcs-server/main.go @@ -1,14 +1,14 @@ package main import ( - _ "context" + "context" "flag" "os" "os/signal" - "syscall" "go-gcs/src/logger" "go-gcs/src/server" + "syscall" ) func main() { diff --git a/src/server/handler_storage_test.go b/src/server/handler_storage_test.go new file mode 100644 index 0000000..6f37775 --- /dev/null +++ b/src/server/handler_storage_test.go @@ -0,0 +1,139 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + restful "github.com/emicklei/go-restful" + "github.com/stretchr/testify/suite" + "go-gcs/src/config" + "go-gcs/src/entity" + "go-gcs/src/service" + "go-gcs/src/service/googlecloud/storageprovider" +) + +type StorageSuite struct { + suite.Suite + sp *service.Container + wc *restful.Container + JWTBearer string +} + +func (suite *StorageSuite) SetupSuite() { + cf := config.MustRead("../../config/testing.json") + sp := service.NewForTesting(cf) + + // init service provider + suite.sp = sp + // init restful container + suite.wc = restful.NewContainer() + + storageService := newStorageService(suite.sp) + + suite.wc.Add(storageService) + + token, err := generateToken("myAwesomeId", sp.Config.JWTSecretKey) + suite.NotEmpty(token) + suite.NoError(err) + suite.JWTBearer = "Bearer " + token +} + +func TestStorageSuite(t *testing.T) { + suite.Run(t, new(StorageSuite)) +} + +func (suite *StorageSuite) TestCreateGCSSignedUrl() { + to := "myAwesomeBuddyId" + singlePayload := entity.SinglePayload{ + To: to, + } + bodyBytes, err := json.MarshalIndent(singlePayload, "", " ") + suite.NoError(err) + + fileName := "cat.jpg" + contentType := "image/jpeg" + tag := "single" + signedUrlRequest := entity.SignedUrlRequest{ + FileName: fileName, + ContentType: contentType, + Tag: tag, + Payload: string(bodyBytes), + } + bodyBytes, err = json.MarshalIndent(signedUrlRequest, "", " ") + suite.NoError(err) + + bodyReader := strings.NewReader(string(bodyBytes)) + httpRequest, err := http.NewRequest("POST", "http://localhost:7890/v1/storage/signurl", bodyReader) + suite.NoError(err) + + httpRequest.Header.Add("Content-Type", "application/json") + httpRequest.Header.Add("Authorization", suite.JWTBearer) + httpWriter := httptest.NewRecorder() + suite.wc.Dispatch(httpWriter, httpRequest) + assertResponseCode(suite.T(), http.StatusOK, httpWriter) + + signedUrl := entity.SignedUrl{} + err = json.Unmarshal(httpWriter.Body.Bytes(), &signedUrl) + suite.NoError(err) + suite.Equal(contentType, signedUrl.UploadHeaders.ContentType) + suite.Equal(suite.sp.GoogleCloudStorage.Config.ContentLengthRange, signedUrl.UploadHeaders.ContentLengthRange) +} + +func (suite *StorageSuite) TestResizeGCSImage() { + bucket := suite.sp.GoogleCloudStorage.Config.Bucket + path := "test/cat.jpg" + filePath := "../../test/image/cat.jpg" + err := suite.sp.GoogleCloudStorage.Upload(bucket, path, filePath) + suite.NoError(err) + + gcsPublicBaseUrl := storageprovider.GoogleCloudStoragePublicBaseUrl + url := fmt.Sprintf("%s/%s/%s", gcsPublicBaseUrl, bucket, path) + contentType := "image/jpeg" + resizeImageRequest := entity.ResizeImageRequest{ + Url: url, + ContentType: contentType, + } + bodyBytes, err := json.MarshalIndent(resizeImageRequest, "", " ") + suite.NoError(err) + + bodyReader := strings.NewReader(string(bodyBytes)) + httpRequest, err := http.NewRequest("POST", "http://localhost:7890/v1/storage/resize/image", bodyReader) + suite.NoError(err) + + httpRequest.Header.Add("Content-Type", "application/json") + httpWriter := httptest.NewRecorder() + suite.wc.Dispatch(httpWriter, httpRequest) + assertResponseCode(suite.T(), http.StatusOK, httpWriter) + + resizeImage := entity.ResizeImage{} + err = json.Unmarshal(httpWriter.Body.Bytes(), &resizeImage) + suite.NoError(err) + + imageResizeBucket := suite.sp.GoogleCloudStorage.Config.ImageResizeBucket + url = fmt.Sprintf("%s/%s/%s", gcsPublicBaseUrl, imageResizeBucket, path) + suite.Equal(url, resizeImage.Origin) + suite.Equal(url+"_100", resizeImage.ThumbWidth100) + suite.Equal(url+"_150", resizeImage.ThumbWidth150) + suite.Equal(url+"_300", resizeImage.ThumbWidth300) + suite.Equal(url+"_640", resizeImage.ThumbWidth640) + suite.Equal(url+"_1080", resizeImage.ThumbWidth1080) + + err = suite.sp.GoogleCloudStorage.Delete(bucket, path) + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path) + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_100") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_150") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_300") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_640") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_1080") + suite.NoError(err) +} diff --git a/src/server/testutils.go b/src/server/testutils.go new file mode 100644 index 0000000..0314394 --- /dev/null +++ b/src/server/testutils.go @@ -0,0 +1,30 @@ +package server + +import ( + "net/http/httptest" + "testing" + "time" + + "github.com/dgrijalva/jwt-go" +) + +// generateToken is for generating token +func generateToken(userId, secret string) (string, error) { + token := jwt.New(jwt.SigningMethodHS256) + token.Claims = jwt.MapClaims{ + "exp": time.Now().Add(time.Minute * time.Duration(30)).Unix(), + "iat": time.Now().Unix(), + "juu": userId, + } + return token.SignedString([]byte(secret)) +} + +// Assert the response code +func assertResponseCode(t *testing.T, expectedCode int, resp *httptest.ResponseRecorder) { + t.Helper() + t.Logf("code:%d", resp.Code) + if expectedCode != resp.Code { + t.Errorf("status code %d expected.", expectedCode) + t.Logf("Response:\n%s", resp.Body.String()) + } +} diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index f4f2366..bb3e1a4 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -13,6 +13,8 @@ import ( "google.golang.org/api/option" ) +const GoogleCloudStoragePublicBaseUrl = "https://storage.googleapis.com" + // PubSub is the structure for config type PubSub struct { Topic string `json:"topic"` From 6ae2d7247b364d3960d00d2a32749b618828cdfa Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 16:42:22 +0800 Subject: [PATCH 02/33] Add logger --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6c3452f..e4cf027 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# keys +/key.pem + # log **/*.log **/logs From f3341d678b442049899a4f1587284df10228b176 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 16:42:22 +0800 Subject: [PATCH 03/33] Add logger --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index e4cf027..6c3452f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# keys -/key.pem - # log **/*.log **/logs From 998356ee1433f1aa00dfb30c3c26cfed638d75b0 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 16:50:40 +0800 Subject: [PATCH 04/33] Add empty config file --- config/local.json | 32 ++++++++++++++++++++++++++++++++ config/testing.json | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 config/local.json create mode 100644 config/testing.json diff --git a/config/local.json b/config/local.json new file mode 100644 index 0000000..e40bec2 --- /dev/null +++ b/config/local.json @@ -0,0 +1,32 @@ +{ + "googlecloud":{ + "type": "service_account", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "", + "token_uri": "", + "auth_provider_x509_cert_url": "", + "client_x509_cert_url": "" + }, + "storage": { + "bucket": "", + "imageResizeBucket": "", + "contentLengthRange": "" + }, + "pubsub": { + "topic": "", + "subscription": "" + }, + "jwtSecretKey": "", + "logger": { + "dir":"./logs", + "level":"info", + "maxAge":"720h", + "suffixPattern":".%Y%m%d", + "linkName":"access_log" + }, + "version": "local" +} diff --git a/config/testing.json b/config/testing.json new file mode 100644 index 0000000..f7e98ec --- /dev/null +++ b/config/testing.json @@ -0,0 +1,32 @@ +{ + "googlecloud":{ + "type": "service_account", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "", + "token_uri": "", + "auth_provider_x509_cert_url": "", + "client_x509_cert_url": "" + }, + "storage": { + "bucket": "", + "imageResizeBucket": "", + "contentLengthRange": "" + }, + "pubsub": { + "topic": "", + "subscription": "" + }, + "jwtSecretKey": "", + "logger": { + "dir":"./logs", + "level":"info", + "maxAge":"720h", + "suffixPattern":".%Y%m%d", + "linkName":"access_log" + }, + "version": "testing" +} From 21565f48e5b68c5aa12658f8944ff9b482c549b0 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 17:00:54 +0800 Subject: [PATCH 05/33] Update read.me --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8f91331..5be87e9 100644 --- a/README.md +++ b/README.md @@ -29,27 +29,27 @@ go test ./src/... go run src/cmd/gcs-server/main.go ``` - ## Overview + + + ![](https://i.imgur.com/dalZEaf.png) Upload URL would be @@ -222,6 +247,7 @@ Example: "contentType": "image/jpeg" } ``` +Example: #### Response @@ -248,3 +274,15 @@ Example: "origin": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg" } ``` +Example: + +```json +{ + "100": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_100", + "150": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_150", + "300": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_300", + "640": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_640", + "1080": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_1080", + "origin": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg" +} +``` diff --git a/src/cmd/gcs-server/main.go b/src/cmd/gcs-server/main.go index b2aac9e..17d3043 100644 --- a/src/cmd/gcs-server/main.go +++ b/src/cmd/gcs-server/main.go @@ -1,7 +1,7 @@ package main import ( - "context" + _ "context" "flag" "os" "os/signal" diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index bb3e1a4..f4f2366 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -13,8 +13,6 @@ import ( "google.golang.org/api/option" ) -const GoogleCloudStoragePublicBaseUrl = "https://storage.googleapis.com" - // PubSub is the structure for config type PubSub struct { Topic string `json:"topic"` From fa01efb371f957bc58a6637c6feaa48a2c05c00a Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 26 Nov 2018 18:54:55 +0800 Subject: [PATCH 07/33] Add provider testing file --- src/googlecloud/storage/storage_test.go | 47 +++++++++++- src/imageresize/imageresize.go | 21 ++++++ .../pubsubprovider/pubsubprovider_test.go | 21 ++++-- .../storageprovider/storageprovider_test.go | 74 +++++++++++++++++++ 4 files changed, 153 insertions(+), 10 deletions(-) diff --git a/src/googlecloud/storage/storage_test.go b/src/googlecloud/storage/storage_test.go index a065b89..26d0ec9 100644 --- a/src/googlecloud/storage/storage_test.go +++ b/src/googlecloud/storage/storage_test.go @@ -1,11 +1,13 @@ package storage import ( + "encoding/json" "testing" "time" "github.com/stretchr/testify/suite" "go-gcs/src/config" + "go-gcs/src/entity" "go-gcs/src/service" ) @@ -25,11 +27,50 @@ func TestGoogleCloudStorageSuite(t *testing.T) { func (suite *GoogleCloudStorageSuite) TestSignURL() { path := "test/" - fileName := "test.txt" - contentType := "text/plain" + fileName := "cat.jpg" + contentType := "image/jpeg" method := "PUT" expires := time.Now().Add(time.Second * 60) - _, err := SignURL(suite.sp, path, fileName, contentType, method, expires) + url, err := SignURL(suite.sp, path, fileName, contentType, method, expires) suite.NoError(err) + suite.NotEqual("", url) +} + +func (suite *GoogleCloudStorageSuite) TestCreateGCSSingleSignedUrl() { + userId := "myAwesomeId" + fileName := "cat.jpg" + contentType := "image/jpeg" + payload := entity.SinglePayload{ + To: "myAwesomeBuddyId", + } + + p, err := json.Marshal(payload) + suite.NotNil(p) + suite.NoError(err) + + signedUrl, err := CreateGCSSingleSignedUrl(suite.sp, userId, fileName, contentType, string(p)) + suite.NotNil(signedUrl) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageSuite) TestCreateGCSGroupSignedUrl() { + userId := "myAwesomeId" + fileName := "cat.jpg" + contentType := "image/jpeg" + payload := entity.GroupPayload{ + GroupId: "myAwesomeGroupId", + } + + p, err := json.Marshal(payload) + suite.NotNil(p) + suite.NoError(err) + + signedUrl, err := CreateGCSGroupSignedUrl(suite.sp, userId, fileName, contentType, string(p)) + suite.NotNil(signedUrl) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageSuite) TestResizeGCSImage() { + } diff --git a/src/imageresize/imageresize.go b/src/imageresize/imageresize.go index c6a9aa8..fb94900 100644 --- a/src/imageresize/imageresize.go +++ b/src/imageresize/imageresize.go @@ -1,6 +1,7 @@ package imageresize import ( + "bufio" "errors" "image" "image/jpeg" @@ -33,6 +34,26 @@ func DownloadImageFromUrl(url, contentType string) (image.Image, error) { return nil, errors.New("invalid content type.") } +func ReadImageFile(contentType, path string) (image.Image, error) { + imgFp, err := os.Open(path) + if err != nil { + return nil, err + } + defer imgFp.Close() + + switch contentType { + case "image/jpg": + return jpeg.Decode(bufio.NewReader(imgFp)) + case "image/jpeg": + return jpeg.Decode(bufio.NewReader(imgFp)) + case "jpeg": + return jpeg.Decode(bufio.NewReader(imgFp)) + case "image/png": + return png.Decode(bufio.NewReader(imgFp)) + } + return nil, errors.New("invalid content type.") +} + func WriteImageFile(image image.Image, contentType, path string) error { os.MkdirAll(filepath.Dir(path), os.ModePerm) switch contentType { diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go index c3e6de0..a4f8066 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go @@ -3,13 +3,11 @@ package pubsubprovider import ( "context" "encoding/json" - "fmt" "os" "testing" "time" "github.com/stretchr/testify/suite" - "go-gcs/src/imageresize" "go-gcs/src/service/googlecloud" "go-gcs/src/service/googlecloud/storageprovider" ) @@ -74,16 +72,25 @@ func (suite *GoogleCloudPubSubProviderSuite) TestNotifyFromGCSStorage() { bucket := storageService.Config.Bucket path := "test/cat.jpg" filePath := "../../../../test/image/cat.jpg" - contentType := "image/jpeg" err = storageService.Upload(bucket, path, filePath) suite.NoError(err) // Wait for resize image - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) imageResizeBucket := storageService.Config.ImageResizeBucket - resizeUrl := fmt.Sprintf("%s/%s/%s", storageprovider.GoogleCloudStoragePublicBaseUrl, imageResizeBucket, path) - img, err := imageresize.DownloadImageFromUrl(resizeUrl, contentType) - suite.NotNil(img) + err = storageService.Delete(bucket, path) + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path) + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_100") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_150") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_300") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_640") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_1080") suite.NoError(err) } diff --git a/src/service/googlecloud/storageprovider/storageprovider_test.go b/src/service/googlecloud/storageprovider/storageprovider_test.go index d6a7ba0..e63d8f8 100644 --- a/src/service/googlecloud/storageprovider/storageprovider_test.go +++ b/src/service/googlecloud/storageprovider/storageprovider_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/suite" + "go-gcs/src/imageresize" "go-gcs/src/service/googlecloud" ) @@ -34,8 +35,81 @@ func (suite *GoogleCloudStorageProviderSuite) SetupSuite() { ctx := context.Background() service := New(ctx, cf.GoogleCloud, cf.Storage) suite.NotNil(service) + suite.service = service } func TestStorageProviderSuite(t *testing.T) { suite.Run(t, new(GoogleCloudStorageProviderSuite)) } + +func (suite *GoogleCloudStorageProviderSuite) TestUploadDelete() { + bucket := suite.service.Config.Bucket + path := "test/cat.jpg" + filePath := "../../../../test/image/cat.jpg" + err := suite.service.Upload(bucket, path, filePath) + suite.NoError(err) + + err = suite.service.Delete(bucket, path) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageProviderSuite) TestUploadImageDelete() { + contentType := "image/jpeg" + filePath := "../../../../test/image/cat.jpg" + img, err := imageresize.ReadImageFile(contentType, filePath) + suite.NotNil(img) + suite.NoError(err) + + bucket := suite.service.Config.Bucket + path := "test/cat.jpg" + err = suite.service.UploadImage(img, contentType, bucket, path) + suite.NoError(err) + + err = suite.service.Delete(bucket, path) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUploadDelete() { + contentType := "image/jpeg" + filePath := "../../../../test/image/cat.jpg" + img, err := imageresize.ReadImageFile(contentType, filePath) + suite.NotNil(img) + suite.NoError(err) + + path := "test/cat.jpg" + err = suite.service.ResizeImageAndUpload(img, 100, contentType, path) + suite.NoError(err) + + bucket := suite.service.Config.ImageResizeBucket + err = suite.service.Delete(bucket, path+"_100") + suite.NoError(err) +} + +func (suite *GoogleCloudStorageProviderSuite) TestResizeMultiImageSizeAndUpload() { + bucket := suite.service.Config.Bucket + path := "test/cat.jpg" + filePath := "../../../../test/image/cat.jpg" + err := suite.service.Upload(bucket, path, filePath) + suite.NoError(err) + + contentType := "image/jpeg" + url, err := suite.service.ResizeMultiImageSizeAndUpload(contentType, bucket, path) + suite.NoError(err) + suite.NotEqual("", url) + + imageResizeBucket := suite.service.Config.ImageResizeBucket + err = suite.service.Delete(bucket, path) + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path) + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_100") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_150") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_300") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_640") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_1080") + suite.NoError(err) +} From 2eef4130cc32f942e4922337d8879c268e72e96e Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 27 Nov 2018 18:25:00 +0800 Subject: [PATCH 08/33] Add testing coverage --- Makefile | 37 +++++++++++++++++-- src/googlecloud/storage/storage_test.go | 16 +++++++- src/server/handler_storage_test.go | 15 -------- .../pubsubprovider/pubsubprovider.go | 3 ++ .../pubsubprovider/pubsubprovider_test.go | 19 ---------- .../storageprovider/storageprovider_test.go | 34 ++--------------- 6 files changed, 54 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 044e1d4..bec6e27 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,44 @@ -EXE=file_manager +## File name for executing +FILE_NAME=file_manager +## Folder content generated files +BUILD_FOLDER = ./build +## command +GO = go +MKDIR_P = mkdir -p -all: build-linux +################################################ +.PHONY: all +all: build-linux test + +.PHONY: build-darwin build-darwin: - go build -o $(EXE) src/cmd/gcs-server/main.go + go build -o $(FILE_NAME) src/cmd/gcs-server/main.go +.PHONY: build-linux build-linux: - export GOOS=linux && export GOARCH=amd64 && go build -o $(EXE) src/cmd/gcs-server/main.go + export GOOS=linux && export GOARCH=amd64 && go build -o $(FILE_NAME) src/cmd/gcs-server/main.go + +.PHONY: test +test: build-linux + $(MAKE) src.test +.PHONY: run run: go run src/cmd/gcs-server/main.go +.PHONY: zip zip: zip -r config.zip ./config/ + +## src/ ######################################## + +.PHONY: src.test +src.test: + $(GO) test -v ./src/... + +.PHONY: src.test-coverage +src.test-coverage: + $(MKDIR_P) $(BUILD_FOLDER)/src/ + $(GO) test -v -coverprofile=$(BUILD_FOLDER)/src/coverage.txt -covermode=atomic ./src/... + $(GO) tool cover -html=$(BUILD_FOLDER)/src/coverage.txt -o $(BUILD_FOLDER)/src/coverage.html diff --git a/src/googlecloud/storage/storage_test.go b/src/googlecloud/storage/storage_test.go index 26d0ec9..59c6a2c 100644 --- a/src/googlecloud/storage/storage_test.go +++ b/src/googlecloud/storage/storage_test.go @@ -2,6 +2,7 @@ package storage import ( "encoding/json" + "fmt" "testing" "time" @@ -9,6 +10,7 @@ import ( "go-gcs/src/config" "go-gcs/src/entity" "go-gcs/src/service" + "go-gcs/src/service/googlecloud/storageprovider" ) type GoogleCloudStorageSuite struct { @@ -45,7 +47,7 @@ func (suite *GoogleCloudStorageSuite) TestCreateGCSSingleSignedUrl() { To: "myAwesomeBuddyId", } - p, err := json.Marshal(payload) + p, err := json.MarshalIndent(payload, "", " ") suite.NotNil(p) suite.NoError(err) @@ -62,7 +64,7 @@ func (suite *GoogleCloudStorageSuite) TestCreateGCSGroupSignedUrl() { GroupId: "myAwesomeGroupId", } - p, err := json.Marshal(payload) + p, err := json.MarshalIndent(payload, "", " ") suite.NotNil(p) suite.NoError(err) @@ -72,5 +74,15 @@ func (suite *GoogleCloudStorageSuite) TestCreateGCSGroupSignedUrl() { } func (suite *GoogleCloudStorageSuite) TestResizeGCSImage() { + bucket := suite.sp.GoogleCloudStorage.Config.Bucket + path := "test/cat.jpg" + filePath := "../../../test/image/cat.jpg" + err := suite.sp.GoogleCloudStorage.Upload(bucket, path, filePath) + suite.NoError(err) + url := fmt.Sprintf("%s/%s/%s", storageprovider.GoogleCloudStoragePublicBaseUrl, bucket, path) + contentType := "image/jpeg" + ri, err := ResizeGCSImage(suite.sp, url, contentType) + suite.NotNil(ri) + suite.NoError(err) } diff --git a/src/server/handler_storage_test.go b/src/server/handler_storage_test.go index 6f37775..8ba3222 100644 --- a/src/server/handler_storage_test.go +++ b/src/server/handler_storage_test.go @@ -121,19 +121,4 @@ func (suite *StorageSuite) TestResizeGCSImage() { suite.Equal(url+"_300", resizeImage.ThumbWidth300) suite.Equal(url+"_640", resizeImage.ThumbWidth640) suite.Equal(url+"_1080", resizeImage.ThumbWidth1080) - - err = suite.sp.GoogleCloudStorage.Delete(bucket, path) - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path) - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_100") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_150") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_300") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_640") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_1080") - suite.NoError(err) } diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index f4f2366..a416f45 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -34,12 +34,15 @@ type GoogleCloudStorageNotification struct { // NotifyFromGCSStorage will call if google cloud storage object update func (s *Service) NotifyFromGCSStorage(sp *storageprovider.Service) { + var sub *pubsub.Subscription sub, err := s.Client.CreateSubscription(s.Context, s.Config.Subscription, pubsub.SubscriptionConfig{ Topic: s.Client.Topic(s.Config.Topic), AckDeadline: 20 * time.Second, }) if err != nil { logger.Warnf("error while create google cloud pubsub subscription: %s", err) + logger.Info("try to use the exist subscription...") + sub = s.Client.Subscription(s.Config.Subscription) } var mu sync.Mutex diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go index a4f8066..9799522 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go @@ -74,23 +74,4 @@ func (suite *GoogleCloudPubSubProviderSuite) TestNotifyFromGCSStorage() { filePath := "../../../../test/image/cat.jpg" err = storageService.Upload(bucket, path, filePath) suite.NoError(err) - - // Wait for resize image - time.Sleep(10 * time.Second) - - imageResizeBucket := storageService.Config.ImageResizeBucket - err = storageService.Delete(bucket, path) - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path) - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_100") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_150") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_300") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_640") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_1080") - suite.NoError(err) } diff --git a/src/service/googlecloud/storageprovider/storageprovider_test.go b/src/service/googlecloud/storageprovider/storageprovider_test.go index e63d8f8..8daccf7 100644 --- a/src/service/googlecloud/storageprovider/storageprovider_test.go +++ b/src/service/googlecloud/storageprovider/storageprovider_test.go @@ -42,18 +42,15 @@ func TestStorageProviderSuite(t *testing.T) { suite.Run(t, new(GoogleCloudStorageProviderSuite)) } -func (suite *GoogleCloudStorageProviderSuite) TestUploadDelete() { +func (suite *GoogleCloudStorageProviderSuite) TestUpload() { bucket := suite.service.Config.Bucket path := "test/cat.jpg" filePath := "../../../../test/image/cat.jpg" err := suite.service.Upload(bucket, path, filePath) suite.NoError(err) - - err = suite.service.Delete(bucket, path) - suite.NoError(err) } -func (suite *GoogleCloudStorageProviderSuite) TestUploadImageDelete() { +func (suite *GoogleCloudStorageProviderSuite) TestUploadImage() { contentType := "image/jpeg" filePath := "../../../../test/image/cat.jpg" img, err := imageresize.ReadImageFile(contentType, filePath) @@ -64,12 +61,9 @@ func (suite *GoogleCloudStorageProviderSuite) TestUploadImageDelete() { path := "test/cat.jpg" err = suite.service.UploadImage(img, contentType, bucket, path) suite.NoError(err) - - err = suite.service.Delete(bucket, path) - suite.NoError(err) } -func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUploadDelete() { +func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUpload() { contentType := "image/jpeg" filePath := "../../../../test/image/cat.jpg" img, err := imageresize.ReadImageFile(contentType, filePath) @@ -79,10 +73,6 @@ func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUploadDelete() { path := "test/cat.jpg" err = suite.service.ResizeImageAndUpload(img, 100, contentType, path) suite.NoError(err) - - bucket := suite.service.Config.ImageResizeBucket - err = suite.service.Delete(bucket, path+"_100") - suite.NoError(err) } func (suite *GoogleCloudStorageProviderSuite) TestResizeMultiImageSizeAndUpload() { @@ -95,21 +85,5 @@ func (suite *GoogleCloudStorageProviderSuite) TestResizeMultiImageSizeAndUpload( contentType := "image/jpeg" url, err := suite.service.ResizeMultiImageSizeAndUpload(contentType, bucket, path) suite.NoError(err) - suite.NotEqual("", url) - - imageResizeBucket := suite.service.Config.ImageResizeBucket - err = suite.service.Delete(bucket, path) - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path) - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_100") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_150") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_300") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_640") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_1080") - suite.NoError(err) + suite.NotEmpty(url) } From 912bfbb79fe0f6bc41b6b10b63bb53260e52360e Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Nov 2018 15:05:20 +0800 Subject: [PATCH 09/33] Add circle ci --- .circleci/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..464d32a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,13 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + build: + docker: + # specify the version + - image: circleci/golang:1.11 + steps: + - checkout + # specify any bash command here prefixed with `run: ` + - run: make src.test-coverage From df53f72fa1a79a2ceefe0faebb7b67d34b35d84c Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Nov 2018 18:37:43 +0800 Subject: [PATCH 10/33] Update testing file for test circle ci --- .circleci/config.yml | 3 +++ Makefile | 5 +++++ config/testing.json | 30 +++++++++++++++--------------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 464d32a..da8b3cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,4 +10,7 @@ jobs: steps: - checkout # specify any bash command here prefixed with `run: ` + - run: make src.test-coverage-minikube + - run: cat config/testing.json - run: make src.test-coverage + diff --git a/Makefile b/Makefile index bec6e27..7aca644 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,8 @@ src.test-coverage: $(MKDIR_P) $(BUILD_FOLDER)/src/ $(GO) test -v -coverprofile=$(BUILD_FOLDER)/src/coverage.txt -covermode=atomic ./src/... $(GO) tool cover -html=$(BUILD_FOLDER)/src/coverage.txt -o $(BUILD_FOLDER)/src/coverage.html + +.PHONY: src.test-coverage-minikube +src.test-coverage-minikube: + sed -i.bak "s/{{ projectId }}/$(PROJECTID)/g; s/{{ privateKeyId }}/$(PRIVATEKEYID)/g; s#{{ privateKey }}#$(PRIVATEKEY)#g; s/{{ clientEmail }}/$(CLIENTEMAIL)/g; s/{{ clientId }}/$(CLIENTID)/g; s#{{ clientCert }}#$(CLIENTCERT)#g; s/{{ jwtSecretKey }}/$(JWTSECRETKEY)/g;" config/testing.json + echo $(sed ':a;N;$!ba;s/\n/\\n/g' $PRIVATEKEY) diff --git a/config/testing.json b/config/testing.json index f7e98ec..1948488 100644 --- a/config/testing.json +++ b/config/testing.json @@ -1,26 +1,26 @@ { "googlecloud":{ "type": "service_account", - "project_id": "", - "private_key_id": "", - "private_key": "", - "client_email": "", - "client_id": "", - "auth_uri": "", - "token_uri": "", - "auth_provider_x509_cert_url": "", - "client_x509_cert_url": "" + "project_id": "{{ projectId }}", + "private_key_id": "{{ privateKeyId }}", + "private_key": "{{ privateKey }}", + "client_email": "{{ clientEmail }}", + "client_id": "{{ clientId }}", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "{{ clientCert }}" }, "storage": { - "bucket": "", - "imageResizeBucket": "", - "contentLengthRange": "" + "bucket": "jkopay-test", + "imageResizeBucket": "jkopay-test_img_resize", + "contentLengthRange": "0,200000000" }, "pubsub": { - "topic": "", - "subscription": "" + "topic": "jello", + "subscription": "mySubscription" }, - "jwtSecretKey": "", + "jwtSecretKey": "{{ jwtSecretKey }}", "logger": { "dir":"./logs", "level":"info", From 720479df61137909d37a4b4cc59b5aebdb6be2c7 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 14:05:31 +0800 Subject: [PATCH 11/33] Update success circleci file --- .circleci/config.yml | 3 --- Makefile | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da8b3cd..568774a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,3 @@ jobs: - checkout # specify any bash command here prefixed with `run: ` - run: make src.test-coverage-minikube - - run: cat config/testing.json - - run: make src.test-coverage - diff --git a/Makefile b/Makefile index 7aca644..e843125 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,5 @@ src.test-coverage: .PHONY: src.test-coverage-minikube src.test-coverage-minikube: sed -i.bak "s/{{ projectId }}/$(PROJECTID)/g; s/{{ privateKeyId }}/$(PRIVATEKEYID)/g; s#{{ privateKey }}#$(PRIVATEKEY)#g; s/{{ clientEmail }}/$(CLIENTEMAIL)/g; s/{{ clientId }}/$(CLIENTID)/g; s#{{ clientCert }}#$(CLIENTCERT)#g; s/{{ jwtSecretKey }}/$(JWTSECRETKEY)/g;" config/testing.json - echo $(sed ':a;N;$!ba;s/\n/\\n/g' $PRIVATEKEY) + $(MAKE) src.test-coverage + mv config/testing.json.bak config/testing.json From 7248b5071b53bcd13521cf70506ca7dbd01c41fb Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 16:46:38 +0800 Subject: [PATCH 12/33] Add dockerfile license --- Dockerfile | 32 ++++ LICENSE | 201 ++++++++++++++++++++ Makefile | 18 +- src/cmd/{gcs-server => filemanager}/main.go | 0 4 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 LICENSE rename src/cmd/{gcs-server => filemanager}/main.go (100%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..249ad2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# Building stage +FROM golang:1.11-alpine3.7 + +WORKDIR /go-gcs + +RUN apk add --no-cache protobuf ca-certificates make git + +# Source code, building tools and dependences +COPY src /go-gcs +COPY Makefile /go/src/github.com/linkernetworks/vortex + +ENV CGO_ENABLED 0 +ENV GOOS linux +ENV TIMEZONE "Asia/Taipei" +RUN apk add --no-cache tzdata && \ + cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime && \ + echo $TIMEZONE > /etc/timezone && \ + apk del tzdata + +RUN make src.build +RUN mv build/src/cmd/filemanager/filemanager /go/bin + +# Production stage +FROM alpine:3.7 +RUN apk add --no-cache ca-certificates +WORKDIR /go-gcs + +# copy the go binaries from the building stage +COPY --from=0 /go/bin /go/bin + +EXPOSE 7890 +ENTRYPOINT ["/go/bin/filemanager", "-port", "7890", "-config", "/go-gcs/config/develop.json"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile index e843125..4e11b77 100644 --- a/Makefile +++ b/Makefile @@ -9,15 +9,19 @@ MKDIR_P = mkdir -p ################################################ .PHONY: all -all: build-linux test +all: build build-linux test + +.PHONY: build +build: + $(MAKE) src.build .PHONY: build-darwin build-darwin: - go build -o $(FILE_NAME) src/cmd/gcs-server/main.go + go build -o $(FILE_NAME) src/cmd/filemanger/main.go .PHONY: build-linux build-linux: - export GOOS=linux && export GOARCH=amd64 && go build -o $(FILE_NAME) src/cmd/gcs-server/main.go + export GOOS=linux && export GOARCH=amd64 && go build -o $(FILE_NAME) src/cmd/filemanager/main.go .PHONY: test test: build-linux @@ -25,7 +29,7 @@ test: build-linux .PHONY: run run: - go run src/cmd/gcs-server/main.go + go run src/cmd/filemanager/main.go .PHONY: zip zip: @@ -33,6 +37,12 @@ zip: ## src/ ######################################## +.PHONY: src.build +src.build: + $(GO) build -v ./src/... + $(MKDIR_P) $(BUILD_FOLDER)/src/cmd/filemanager/ + $(GO) build -v -o $(BUILD_FOLDER)/src/cmd/filemanager/filemanager ./src/cmd/filemanager/... + .PHONY: src.test src.test: $(GO) test -v ./src/... diff --git a/src/cmd/gcs-server/main.go b/src/cmd/filemanager/main.go similarity index 100% rename from src/cmd/gcs-server/main.go rename to src/cmd/filemanager/main.go From e090edecb3ee78061f830fc2924a11150d086330 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 17:33:54 +0800 Subject: [PATCH 13/33] Update dockerfile --- Dockerfile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 249ad2f..8c5b6e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,20 @@ FROM golang:1.11-alpine3.7 WORKDIR /go-gcs + + + + + RUN apk add --no-cache protobuf ca-certificates make git # Source code, building tools and dependences -COPY src /go-gcs -COPY Makefile /go/src/github.com/linkernetworks/vortex +COPY src /go-gcs/src +COPY Makefile /go-gcs +# We want to populate the module cache based on the go.{mod,sum} files. +COPY go.mod /go-gcs +COPY go.sum /go-gcs + ENV CGO_ENABLED 0 ENV GOOS linux @@ -16,7 +25,10 @@ RUN apk add --no-cache tzdata && \ cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime && \ echo $TIMEZONE > /etc/timezone && \ apk del tzdata +# Force the go compiler to use modules +ENV GO111MODULE=on +RUN go mod download RUN make src.build RUN mv build/src/cmd/filemanager/filemanager /go/bin From d5c7c47f05e56a03cc0f41d6af352a44a4450744 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 18:27:41 +0800 Subject: [PATCH 14/33] update circleci --- .circleci/config.yml | 18 ++++++++++++++++-- Dockerfile | 5 ----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 568774a..7a93b51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,11 +3,25 @@ # Check https://circleci.com/docs/2.0/language-go/ for more details version: 2 jobs: - build: + test: docker: # specify the version - image: circleci/golang:1.11 steps: - checkout # specify any bash command here prefixed with `run: ` - - run: make src.test-coverage-minikube + - run: + name: Test and generate code coverage + command: make src.test-coverage-minikube + - run: + name: Copy code coverage to root + command: cp build/src/coverage.txt coverage.txt + - run: + name: Copy code coverage to root + command: bash <(curl -s https://codecov.io/bash) +workflows: + version: 2 + test: + jobs: + - test + diff --git a/Dockerfile b/Dockerfile index 8c5b6e9..3eaf21f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,6 @@ FROM golang:1.11-alpine3.7 WORKDIR /go-gcs - - - - - RUN apk add --no-cache protobuf ca-certificates make git # Source code, building tools and dependences From 695f721149a67c30347947baf66d15ba8e9de151 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 30 Nov 2018 16:28:52 +0800 Subject: [PATCH 15/33] Add helm to deploy service to k8s --- .circleci/config.yml | 9 ++- Makefile | 57 +++++++++++++++++++ deployment/helm/apps/.helmignore | 21 +++++++ deployment/helm/apps/Chart.yaml | 5 ++ .../helm/apps/charts/filemanager/.helmignore | 21 +++++++ .../helm/apps/charts/filemanager/Chart.yaml | 5 ++ .../filemanager/templates/deployment.yaml | 30 ++++++++++ .../charts/filemanager/templates/service.yaml | 17 ++++++ deployment/helm/config/development.yaml | 10 ++++ deployment/helm/config/production.yaml | 10 ++++ deployment/helm/services/.helmignore | 21 +++++++ deployment/helm/services/Chart.yaml | 5 ++ .../services/charts/namespace/.helmignore | 21 +++++++ .../helm/services/charts/namespace/Chart.yaml | 5 ++ .../charts/namespace/templates/namespace.yaml | 4 ++ .../helm/services/charts/rbac/.helmignore | 21 +++++++ .../helm/services/charts/rbac/Chart.yaml | 5 ++ .../services/charts/rbac/templates/role.yaml | 30 ++++++++++ .../charts/rbac/templates/serviceaccount.yaml | 5 ++ 19 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 deployment/helm/apps/.helmignore create mode 100644 deployment/helm/apps/Chart.yaml create mode 100644 deployment/helm/apps/charts/filemanager/.helmignore create mode 100644 deployment/helm/apps/charts/filemanager/Chart.yaml create mode 100644 deployment/helm/apps/charts/filemanager/templates/deployment.yaml create mode 100644 deployment/helm/apps/charts/filemanager/templates/service.yaml create mode 100644 deployment/helm/config/development.yaml create mode 100644 deployment/helm/config/production.yaml create mode 100644 deployment/helm/services/.helmignore create mode 100644 deployment/helm/services/Chart.yaml create mode 100644 deployment/helm/services/charts/namespace/.helmignore create mode 100644 deployment/helm/services/charts/namespace/Chart.yaml create mode 100644 deployment/helm/services/charts/namespace/templates/namespace.yaml create mode 100644 deployment/helm/services/charts/rbac/.helmignore create mode 100644 deployment/helm/services/charts/rbac/Chart.yaml create mode 100644 deployment/helm/services/charts/rbac/templates/role.yaml create mode 100644 deployment/helm/services/charts/rbac/templates/serviceaccount.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a93b51..2caf9ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,8 +20,7 @@ jobs: name: Copy code coverage to root command: bash <(curl -s https://codecov.io/bash) workflows: - version: 2 - test: - jobs: - - test - + version: 2 + test: + jobs: + - test diff --git a/Makefile b/Makefile index 4e11b77..64baee4 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +## filemanager server version +SERVER_VERSION = latest ## File name for executing FILE_NAME=file_manager ## Folder content generated files @@ -58,3 +60,58 @@ src.test-coverage-minikube: sed -i.bak "s/{{ projectId }}/$(PROJECTID)/g; s/{{ privateKeyId }}/$(PRIVATEKEYID)/g; s#{{ privateKey }}#$(PRIVATEKEY)#g; s/{{ clientEmail }}/$(CLIENTEMAIL)/g; s/{{ clientId }}/$(CLIENTID)/g; s#{{ clientCert }}#$(CLIENTCERT)#g; s/{{ jwtSecretKey }}/$(JWTSECRETKEY)/g;" config/testing.json $(MAKE) src.test-coverage mv config/testing.json.bak config/testing.json + +## launch apps ############################# + +.PHONY: apps.init-helm +apps.init-helm: + helm init + kubectl create serviceaccount --namespace kube-system tiller + kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' + +.PHONY: apps.launch-dev +apps.launch-dev: + yq -y .services deployment/helm/config/development.yaml | helm install --name filemanager-services-dev --debug --wait -f - deployment/helm/services + kubectl create configmap filemanager-config --from-file=config/ -n filemanager + yq -y .apps deployment/helm/config/development.yaml | helm install --name filemanager-apps-dev --debug --wait -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.launch-prod +apps.launch-prod: + yq -y .services deployment/helm/config/production.yaml | helm install --name filemanager-services-prod --debug --wait -f - deployment/helm/services + yq -y .apps deployment/helm/config/production.yaml | helm install --name filemanager-apps-prod --debug --wait -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.upgrade-dev +apps.upgrade-dev: + yq -y .services deployment/helm/config/development.yaml | helm upgrade filemanager-services-dev --debug -f - deployment/helm/services + yq -y .apps deployment/helm/config/development.yaml | helm upgrade filemanager-apps-dev --debug -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.upgrade-prod +apps.upgrade-prod: + yq -y .services deployment/helm/config/production.yaml | helm upgrade filemanager-services-prod --debug -f - deployment/helm/services + yq -y .apps deployment/helm/config/production.yaml | helm upgrade filemanager-apps-prod --debug -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.teardown-dev +apps.teardown-dev: + helm delete --purge filemanager-services-dev + helm delete --purge filemanager-apps-dev + kubectl delete configmap -n filemanager + +.PHONY: apps.teardown-prod +apps.teardown-prod: + helm delete --purge filemanager-services-prod + helm delete --purge filemanager-apps-prod + +## dockerfiles/ ######################################## + +.PHONY: dockerfiles.build +dockerfiles.build: + docker build --tag yunchen/go-gcs:$(SERVER_VERSION) . + +## git tag version ######################################## + +.PHONY: push.tag +push.tag: + @echo "Current git tag version:"$(SERVER_VERSION) + git tag $(SERVER_VERSION) + git push --tags diff --git a/deployment/helm/apps/.helmignore b/deployment/helm/apps/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/apps/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/apps/Chart.yaml b/deployment/helm/apps/Chart.yaml new file mode 100644 index 0000000..d172ecb --- /dev/null +++ b/deployment/helm/apps/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: apps +version: 0.1.0 diff --git a/deployment/helm/apps/charts/filemanager/.helmignore b/deployment/helm/apps/charts/filemanager/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/apps/charts/filemanager/Chart.yaml b/deployment/helm/apps/charts/filemanager/Chart.yaml new file mode 100644 index 0000000..a728824 --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for filemanager-server +name: filemanager +version: 0.1.0 diff --git a/deployment/helm/apps/charts/filemanager/templates/deployment.yaml b/deployment/helm/apps/charts/filemanager/templates/deployment.yaml new file mode 100644 index 0000000..4670dab --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/templates/deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filemanager-server + namespace: filemanager + labels: + app: filemanager-server +spec: + replicas: {{ .Values.controller.replicaCount }} + selector: + matchLabels: + app: filemanager-server + template: + metadata: + labels: + app: filemanager-server + spec: + serviceAccountName: filemanager-admin + containers: + - name: filemanager-server + image: yunchen/go-gcs:{{ .Values.controller.apiserverImageTag }} + ports: + - containerPort: 7890 + volumeMounts: + - name: config-volume + mountPath: /go-gcs/config + volumes: + - name: config-volume + configMap: + name: filemanager-config diff --git a/deployment/helm/apps/charts/filemanager/templates/service.yaml b/deployment/helm/apps/charts/filemanager/templates/service.yaml new file mode 100644 index 0000000..283c572 --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: filemanager-server + namespace: filemanager + labels: + app: filemanager-server +spec: + type: NodePort + ports: + - name: apiserver + protocol: TCP + port: 7890 + targetPort: 7890 + nodePort: {{ .Values.service.ports.apiserverNodePort }} + selector: + app: filemanager-server diff --git a/deployment/helm/config/development.yaml b/deployment/helm/config/development.yaml new file mode 100644 index 0000000..fdc959f --- /dev/null +++ b/deployment/helm/config/development.yaml @@ -0,0 +1,10 @@ +# go-gcs/deploy/helm/apps +apps: + # go-gcs/deploy/helm/apps/charts/filemanager-server + filemanager: + controller: + replicaCount: 1 + apiserverImageTag: latest + service: + ports: + apiserverNodePort: 32326 diff --git a/deployment/helm/config/production.yaml b/deployment/helm/config/production.yaml new file mode 100644 index 0000000..fdc959f --- /dev/null +++ b/deployment/helm/config/production.yaml @@ -0,0 +1,10 @@ +# go-gcs/deploy/helm/apps +apps: + # go-gcs/deploy/helm/apps/charts/filemanager-server + filemanager: + controller: + replicaCount: 1 + apiserverImageTag: latest + service: + ports: + apiserverNodePort: 32326 diff --git a/deployment/helm/services/.helmignore b/deployment/helm/services/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/services/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/services/Chart.yaml b/deployment/helm/services/Chart.yaml new file mode 100644 index 0000000..ace0a01 --- /dev/null +++ b/deployment/helm/services/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: services +version: 0.1.0 diff --git a/deployment/helm/services/charts/namespace/.helmignore b/deployment/helm/services/charts/namespace/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/services/charts/namespace/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/services/charts/namespace/Chart.yaml b/deployment/helm/services/charts/namespace/Chart.yaml new file mode 100644 index 0000000..cf8f4ff --- /dev/null +++ b/deployment/helm/services/charts/namespace/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: namespace +version: 0.1.0 diff --git a/deployment/helm/services/charts/namespace/templates/namespace.yaml b/deployment/helm/services/charts/namespace/templates/namespace.yaml new file mode 100644 index 0000000..e108f69 --- /dev/null +++ b/deployment/helm/services/charts/namespace/templates/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: filemanager diff --git a/deployment/helm/services/charts/rbac/.helmignore b/deployment/helm/services/charts/rbac/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/services/charts/rbac/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/services/charts/rbac/Chart.yaml b/deployment/helm/services/charts/rbac/Chart.yaml new file mode 100644 index 0000000..e55b632 --- /dev/null +++ b/deployment/helm/services/charts/rbac/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: rbac +version: 0.1.0 diff --git a/deployment/helm/services/charts/rbac/templates/role.yaml b/deployment/helm/services/charts/rbac/templates/role.yaml new file mode 100644 index 0000000..ad5a751 --- /dev/null +++ b/deployment/helm/services/charts/rbac/templates/role.yaml @@ -0,0 +1,30 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: filemanager-admin +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + - nonResourceURLs: + - '*' + verbs: + - '*' +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: filemanager-admin + namespace: filemanager +roleRef: + kind: ClusterRole + name: filemanager-admin + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: filemanager-admin + namespace: filemanager diff --git a/deployment/helm/services/charts/rbac/templates/serviceaccount.yaml b/deployment/helm/services/charts/rbac/templates/serviceaccount.yaml new file mode 100644 index 0000000..e6213cd --- /dev/null +++ b/deployment/helm/services/charts/rbac/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filemanager-admin + namespace: filemanager From 0a0a42b20939625091ce9a198a72ce5e8c57e986 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 30 Nov 2018 17:20:45 +0800 Subject: [PATCH 16/33] Fix golint --- README.md | 5 +++++ src/entity/storage.go | 2 +- src/googlecloud/storage/storage.go | 2 +- src/imageresize/imageresize.go | 15 +++++++++++---- src/logger/logger.go | 2 +- .../googlecloud/pubsubprovider/pubsubprovider.go | 1 + .../storageprovider/storageprovider.go | 2 ++ 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1f8f7e5..2a6eea6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # go-gcs +[![CircleCI](https://circleci.com/gh/chenyunchen/go-gcs/tree/github%2Fdevelop.svg?style=svg)](https://circleci.com/gh/chenyunchen/go-gcs/tree/github%2Fdevelop) +[![Go Report Card](https://goreportcard.com/badge/github.com/chenyunchen/go-gcs)](https://goreportcard.com/report/github.com/chenyunchen/go-gcs) +[![codecov](https://codecov.io/gh/chenyunchen/go-gcs/branch/github%2Fdevelop/graph/badge.svg)](https://codecov.io/gh/chenyunchen/go-gcs) +[![Docker Build Status](https://img.shields.io/docker/build/yunchen/go-gcs.svg)](https://hub.docker.com/r/yunchen/go-gcs/) + * [1. Quck Start](#header_1) * [2. Overview](#header_2) * [3. API](#header_3) diff --git a/src/entity/storage.go b/src/entity/storage.go index bc8f60a..e0da2b8 100644 --- a/src/entity/storage.go +++ b/src/entity/storage.go @@ -18,7 +18,7 @@ type GroupPayload struct { GroupId string `json:"groupId" validate:"required"` } -// SignedUrlRequest is the structure for signed url response +// SignedUrl is the structure for signed url response type SignedUrl struct { Url string `json:"url" validate:"required"` UploadHeaders `json:"uploadHeaders" validate:"required"` diff --git a/src/googlecloud/storage/storage.go b/src/googlecloud/storage/storage.go index 6512c7e..4d770f7 100644 --- a/src/googlecloud/storage/storage.go +++ b/src/googlecloud/storage/storage.go @@ -28,7 +28,7 @@ func SignURL(sp *service.Container, path string, fileName string, contentType st rand.Seed(time.Now().UnixNano()) r := strings.NewReplacer("/", ":") - b := make([]byte, 4) //equals 8 charachters + b := make([]byte, 4) //equals 8 characters rand.Read(b) hashFileName := fmt.Sprintf("%s_%s", hex.EncodeToString(b), r.Replace(fileName)) diff --git a/src/imageresize/imageresize.go b/src/imageresize/imageresize.go index fb94900..ef11427 100644 --- a/src/imageresize/imageresize.go +++ b/src/imageresize/imageresize.go @@ -13,6 +13,7 @@ import ( "github.com/nfnt/resize" ) +// DownloadImageFromUrl will download image from url func DownloadImageFromUrl(url, contentType string) (image.Image, error) { // Fetch an image. resp, err := http.Get(url) @@ -31,9 +32,10 @@ func DownloadImageFromUrl(url, contentType string) (image.Image, error) { case "image/png": return png.Decode(resp.Body) } - return nil, errors.New("invalid content type.") + return nil, errors.New("invalid content type") } +// ReadImageFile will read image struct from file path func ReadImageFile(contentType, path string) (image.Image, error) { imgFp, err := os.Open(path) if err != nil { @@ -51,9 +53,10 @@ func ReadImageFile(contentType, path string) (image.Image, error) { case "image/png": return png.Decode(bufio.NewReader(imgFp)) } - return nil, errors.New("invalid content type.") + return nil, errors.New("invalid content type") } +// WriteImageFile will write image struct to file path func WriteImageFile(image image.Image, contentType, path string) error { os.MkdirAll(filepath.Dir(path), os.ModePerm) switch contentType { @@ -66,9 +69,10 @@ func WriteImageFile(image image.Image, contentType, path string) error { case "image/png": return WritePngImageFile(image, path) } - return errors.New("invalid content type.") + return errors.New("invalid content type") } +// WritePngImageFile will write image struct to png file path func WritePngImageFile(image image.Image, path string) error { imgFp, err := os.Create(path) if err != nil { @@ -83,6 +87,7 @@ func WritePngImageFile(image image.Image, path string) error { return nil } +// WriteJpegImageFile will write image struct to jpeg file path func WriteJpegImageFile(image image.Image, path string, quality int) error { imgFp, err := os.Create(path) if err != nil { @@ -97,16 +102,18 @@ func WriteJpegImageFile(image image.Image, path string, quality int) error { return nil } +// ResizeImage will resize image struct and return image struct func ResizeImage(image image.Image, width, height uint) image.Image { return resize.Resize(width, height, image, resize.Lanczos3) } -// Thumbnail will fail if width or height == 0 +// ThumbnailImage will fail if width or height == 0 // If maxWidth and maxHeight > image size, return the image with its original size func ThumbnailImage(image image.Image, maxWidth, maxHeight uint) image.Image { return resize.Thumbnail(maxWidth, maxHeight, image, resize.Lanczos3) } +// DeleteImage will delete image from path func DeleteImage(path string) error { return os.Remove(path) } diff --git a/src/logger/logger.go b/src/logger/logger.go index 6801c80..0b32b97 100644 --- a/src/logger/logger.go +++ b/src/logger/logger.go @@ -27,7 +27,7 @@ func init() { Logger = logrus.New() } -// Setup Logger in packge. Enable Logger after import +// Setup Logger in package. Enable Logger after import func Setup(c LoggerConfig) { if Logger == nil { Logger = logrus.New() diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index a416f45..7b61347 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -26,6 +26,7 @@ type Service struct { Context context.Context } +// GoogleCloudStorageNotification is the structure for notification type GoogleCloudStorageNotification struct { Name string `json:"name" validate:"required"` Bucket string `json:"bucket" validate:"required"` diff --git a/src/service/googlecloud/storageprovider/storageprovider.go b/src/service/googlecloud/storageprovider/storageprovider.go index 5eecf68..cfe34c7 100644 --- a/src/service/googlecloud/storageprovider/storageprovider.go +++ b/src/service/googlecloud/storageprovider/storageprovider.go @@ -16,6 +16,7 @@ import ( "google.golang.org/api/option" ) +// GoogleCloudStoragePublicBaseUrl is public url for download const GoogleCloudStoragePublicBaseUrl = "https://storage.googleapis.com" // Storage is the structure for config @@ -111,6 +112,7 @@ func (s *Service) ResizeImageAndUpload(img image.Image, width uint, contentType, return s.UploadImage(img, contentType, s.Config.ImageResizeBucket, path) } +// ResizeMultiImageSizeAndUpload will resize multiple image and upload to google cloud storage func (s *Service) ResizeMultiImageSizeAndUpload(contentType, bucket, path string) (string, error) { // Check if already resize origin image imageResizeBucket := s.Config.ImageResizeBucket From 2997a4455c5da4f40a340b0beb3b2ee0c2aaced7 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 15:42:57 +0800 Subject: [PATCH 17/33] Add image resize and upload when user upload origin image --- src/cmd/gcs-server/main.go | 4 +- src/server/handler_storage_test.go | 139 ++++++++++++++++++ src/server/testutils.go | 30 ++++ .../pubsubprovider/pubsubprovider.go | 2 + 4 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 src/server/handler_storage_test.go create mode 100644 src/server/testutils.go diff --git a/src/cmd/gcs-server/main.go b/src/cmd/gcs-server/main.go index 753f722..b2aac9e 100644 --- a/src/cmd/gcs-server/main.go +++ b/src/cmd/gcs-server/main.go @@ -1,14 +1,14 @@ package main import ( - _ "context" + "context" "flag" "os" "os/signal" - "syscall" "go-gcs/src/logger" "go-gcs/src/server" + "syscall" ) func main() { diff --git a/src/server/handler_storage_test.go b/src/server/handler_storage_test.go new file mode 100644 index 0000000..6f37775 --- /dev/null +++ b/src/server/handler_storage_test.go @@ -0,0 +1,139 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + restful "github.com/emicklei/go-restful" + "github.com/stretchr/testify/suite" + "go-gcs/src/config" + "go-gcs/src/entity" + "go-gcs/src/service" + "go-gcs/src/service/googlecloud/storageprovider" +) + +type StorageSuite struct { + suite.Suite + sp *service.Container + wc *restful.Container + JWTBearer string +} + +func (suite *StorageSuite) SetupSuite() { + cf := config.MustRead("../../config/testing.json") + sp := service.NewForTesting(cf) + + // init service provider + suite.sp = sp + // init restful container + suite.wc = restful.NewContainer() + + storageService := newStorageService(suite.sp) + + suite.wc.Add(storageService) + + token, err := generateToken("myAwesomeId", sp.Config.JWTSecretKey) + suite.NotEmpty(token) + suite.NoError(err) + suite.JWTBearer = "Bearer " + token +} + +func TestStorageSuite(t *testing.T) { + suite.Run(t, new(StorageSuite)) +} + +func (suite *StorageSuite) TestCreateGCSSignedUrl() { + to := "myAwesomeBuddyId" + singlePayload := entity.SinglePayload{ + To: to, + } + bodyBytes, err := json.MarshalIndent(singlePayload, "", " ") + suite.NoError(err) + + fileName := "cat.jpg" + contentType := "image/jpeg" + tag := "single" + signedUrlRequest := entity.SignedUrlRequest{ + FileName: fileName, + ContentType: contentType, + Tag: tag, + Payload: string(bodyBytes), + } + bodyBytes, err = json.MarshalIndent(signedUrlRequest, "", " ") + suite.NoError(err) + + bodyReader := strings.NewReader(string(bodyBytes)) + httpRequest, err := http.NewRequest("POST", "http://localhost:7890/v1/storage/signurl", bodyReader) + suite.NoError(err) + + httpRequest.Header.Add("Content-Type", "application/json") + httpRequest.Header.Add("Authorization", suite.JWTBearer) + httpWriter := httptest.NewRecorder() + suite.wc.Dispatch(httpWriter, httpRequest) + assertResponseCode(suite.T(), http.StatusOK, httpWriter) + + signedUrl := entity.SignedUrl{} + err = json.Unmarshal(httpWriter.Body.Bytes(), &signedUrl) + suite.NoError(err) + suite.Equal(contentType, signedUrl.UploadHeaders.ContentType) + suite.Equal(suite.sp.GoogleCloudStorage.Config.ContentLengthRange, signedUrl.UploadHeaders.ContentLengthRange) +} + +func (suite *StorageSuite) TestResizeGCSImage() { + bucket := suite.sp.GoogleCloudStorage.Config.Bucket + path := "test/cat.jpg" + filePath := "../../test/image/cat.jpg" + err := suite.sp.GoogleCloudStorage.Upload(bucket, path, filePath) + suite.NoError(err) + + gcsPublicBaseUrl := storageprovider.GoogleCloudStoragePublicBaseUrl + url := fmt.Sprintf("%s/%s/%s", gcsPublicBaseUrl, bucket, path) + contentType := "image/jpeg" + resizeImageRequest := entity.ResizeImageRequest{ + Url: url, + ContentType: contentType, + } + bodyBytes, err := json.MarshalIndent(resizeImageRequest, "", " ") + suite.NoError(err) + + bodyReader := strings.NewReader(string(bodyBytes)) + httpRequest, err := http.NewRequest("POST", "http://localhost:7890/v1/storage/resize/image", bodyReader) + suite.NoError(err) + + httpRequest.Header.Add("Content-Type", "application/json") + httpWriter := httptest.NewRecorder() + suite.wc.Dispatch(httpWriter, httpRequest) + assertResponseCode(suite.T(), http.StatusOK, httpWriter) + + resizeImage := entity.ResizeImage{} + err = json.Unmarshal(httpWriter.Body.Bytes(), &resizeImage) + suite.NoError(err) + + imageResizeBucket := suite.sp.GoogleCloudStorage.Config.ImageResizeBucket + url = fmt.Sprintf("%s/%s/%s", gcsPublicBaseUrl, imageResizeBucket, path) + suite.Equal(url, resizeImage.Origin) + suite.Equal(url+"_100", resizeImage.ThumbWidth100) + suite.Equal(url+"_150", resizeImage.ThumbWidth150) + suite.Equal(url+"_300", resizeImage.ThumbWidth300) + suite.Equal(url+"_640", resizeImage.ThumbWidth640) + suite.Equal(url+"_1080", resizeImage.ThumbWidth1080) + + err = suite.sp.GoogleCloudStorage.Delete(bucket, path) + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path) + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_100") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_150") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_300") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_640") + suite.NoError(err) + err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_1080") + suite.NoError(err) +} diff --git a/src/server/testutils.go b/src/server/testutils.go new file mode 100644 index 0000000..0314394 --- /dev/null +++ b/src/server/testutils.go @@ -0,0 +1,30 @@ +package server + +import ( + "net/http/httptest" + "testing" + "time" + + "github.com/dgrijalva/jwt-go" +) + +// generateToken is for generating token +func generateToken(userId, secret string) (string, error) { + token := jwt.New(jwt.SigningMethodHS256) + token.Claims = jwt.MapClaims{ + "exp": time.Now().Add(time.Minute * time.Duration(30)).Unix(), + "iat": time.Now().Unix(), + "juu": userId, + } + return token.SignedString([]byte(secret)) +} + +// Assert the response code +func assertResponseCode(t *testing.T, expectedCode int, resp *httptest.ResponseRecorder) { + t.Helper() + t.Logf("code:%d", resp.Code) + if expectedCode != resp.Code { + t.Errorf("status code %d expected.", expectedCode) + t.Logf("Response:\n%s", resp.Body.String()) + } +} diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index f4f2366..bb3e1a4 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -13,6 +13,8 @@ import ( "google.golang.org/api/option" ) +const GoogleCloudStoragePublicBaseUrl = "https://storage.googleapis.com" + // PubSub is the structure for config type PubSub struct { Topic string `json:"topic"` From 2f460a9991012ad90b16ab2796865bead5989e3a Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 16:42:22 +0800 Subject: [PATCH 18/33] Add logger --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6c3452f..e4cf027 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# keys +/key.pem + # log **/*.log **/logs From 1fb5a13c3589fae98ca6af74b07d7dbbc3e13a22 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 16:42:22 +0800 Subject: [PATCH 19/33] Add logger --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index e4cf027..6c3452f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# keys -/key.pem - # log **/*.log **/logs From d848f2a0c56c0e047fd34d378838e3f27de63c3a Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 16:50:40 +0800 Subject: [PATCH 20/33] Add empty config file --- config/local.json | 32 ++++++++++++++++++++++++++++++++ config/testing.json | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 config/local.json create mode 100644 config/testing.json diff --git a/config/local.json b/config/local.json new file mode 100644 index 0000000..e40bec2 --- /dev/null +++ b/config/local.json @@ -0,0 +1,32 @@ +{ + "googlecloud":{ + "type": "service_account", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "", + "token_uri": "", + "auth_provider_x509_cert_url": "", + "client_x509_cert_url": "" + }, + "storage": { + "bucket": "", + "imageResizeBucket": "", + "contentLengthRange": "" + }, + "pubsub": { + "topic": "", + "subscription": "" + }, + "jwtSecretKey": "", + "logger": { + "dir":"./logs", + "level":"info", + "maxAge":"720h", + "suffixPattern":".%Y%m%d", + "linkName":"access_log" + }, + "version": "local" +} diff --git a/config/testing.json b/config/testing.json new file mode 100644 index 0000000..f7e98ec --- /dev/null +++ b/config/testing.json @@ -0,0 +1,32 @@ +{ + "googlecloud":{ + "type": "service_account", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "", + "token_uri": "", + "auth_provider_x509_cert_url": "", + "client_x509_cert_url": "" + }, + "storage": { + "bucket": "", + "imageResizeBucket": "", + "contentLengthRange": "" + }, + "pubsub": { + "topic": "", + "subscription": "" + }, + "jwtSecretKey": "", + "logger": { + "dir":"./logs", + "level":"info", + "maxAge":"720h", + "suffixPattern":".%Y%m%d", + "linkName":"access_log" + }, + "version": "testing" +} From 5946ef1a36a7b728e41b1442cdf0060b29618fd3 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 22 Nov 2018 17:00:54 +0800 Subject: [PATCH 21/33] Update read.me --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8f91331..5be87e9 100644 --- a/README.md +++ b/README.md @@ -29,27 +29,27 @@ go test ./src/... go run src/cmd/gcs-server/main.go ``` - ## Overview + + + ![](https://i.imgur.com/dalZEaf.png) Upload URL would be @@ -222,6 +247,7 @@ Example: "contentType": "image/jpeg" } ``` +Example: #### Response @@ -248,3 +274,15 @@ Example: "origin": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg" } ``` +Example: + +```json +{ + "100": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_100", + "150": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_150", + "300": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_300", + "640": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_640", + "1080": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg_1080", + "origin": "https://storage.googleapis.com/jkopay-test_img_resize/test/cat.jpg" +} +``` diff --git a/src/cmd/gcs-server/main.go b/src/cmd/gcs-server/main.go index b2aac9e..17d3043 100644 --- a/src/cmd/gcs-server/main.go +++ b/src/cmd/gcs-server/main.go @@ -1,7 +1,7 @@ package main import ( - "context" + _ "context" "flag" "os" "os/signal" diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index bb3e1a4..f4f2366 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -13,8 +13,6 @@ import ( "google.golang.org/api/option" ) -const GoogleCloudStoragePublicBaseUrl = "https://storage.googleapis.com" - // PubSub is the structure for config type PubSub struct { Topic string `json:"topic"` From 738459d8a4608e4a4242e5fc5e2bd4142f591bb3 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 26 Nov 2018 18:54:55 +0800 Subject: [PATCH 23/33] Add provider testing file --- src/googlecloud/storage/storage_test.go | 47 +++++++++++- src/imageresize/imageresize.go | 21 ++++++ .../pubsubprovider/pubsubprovider_test.go | 21 ++++-- .../storageprovider/storageprovider_test.go | 74 +++++++++++++++++++ 4 files changed, 153 insertions(+), 10 deletions(-) diff --git a/src/googlecloud/storage/storage_test.go b/src/googlecloud/storage/storage_test.go index a065b89..26d0ec9 100644 --- a/src/googlecloud/storage/storage_test.go +++ b/src/googlecloud/storage/storage_test.go @@ -1,11 +1,13 @@ package storage import ( + "encoding/json" "testing" "time" "github.com/stretchr/testify/suite" "go-gcs/src/config" + "go-gcs/src/entity" "go-gcs/src/service" ) @@ -25,11 +27,50 @@ func TestGoogleCloudStorageSuite(t *testing.T) { func (suite *GoogleCloudStorageSuite) TestSignURL() { path := "test/" - fileName := "test.txt" - contentType := "text/plain" + fileName := "cat.jpg" + contentType := "image/jpeg" method := "PUT" expires := time.Now().Add(time.Second * 60) - _, err := SignURL(suite.sp, path, fileName, contentType, method, expires) + url, err := SignURL(suite.sp, path, fileName, contentType, method, expires) suite.NoError(err) + suite.NotEqual("", url) +} + +func (suite *GoogleCloudStorageSuite) TestCreateGCSSingleSignedUrl() { + userId := "myAwesomeId" + fileName := "cat.jpg" + contentType := "image/jpeg" + payload := entity.SinglePayload{ + To: "myAwesomeBuddyId", + } + + p, err := json.Marshal(payload) + suite.NotNil(p) + suite.NoError(err) + + signedUrl, err := CreateGCSSingleSignedUrl(suite.sp, userId, fileName, contentType, string(p)) + suite.NotNil(signedUrl) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageSuite) TestCreateGCSGroupSignedUrl() { + userId := "myAwesomeId" + fileName := "cat.jpg" + contentType := "image/jpeg" + payload := entity.GroupPayload{ + GroupId: "myAwesomeGroupId", + } + + p, err := json.Marshal(payload) + suite.NotNil(p) + suite.NoError(err) + + signedUrl, err := CreateGCSGroupSignedUrl(suite.sp, userId, fileName, contentType, string(p)) + suite.NotNil(signedUrl) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageSuite) TestResizeGCSImage() { + } diff --git a/src/imageresize/imageresize.go b/src/imageresize/imageresize.go index c6a9aa8..fb94900 100644 --- a/src/imageresize/imageresize.go +++ b/src/imageresize/imageresize.go @@ -1,6 +1,7 @@ package imageresize import ( + "bufio" "errors" "image" "image/jpeg" @@ -33,6 +34,26 @@ func DownloadImageFromUrl(url, contentType string) (image.Image, error) { return nil, errors.New("invalid content type.") } +func ReadImageFile(contentType, path string) (image.Image, error) { + imgFp, err := os.Open(path) + if err != nil { + return nil, err + } + defer imgFp.Close() + + switch contentType { + case "image/jpg": + return jpeg.Decode(bufio.NewReader(imgFp)) + case "image/jpeg": + return jpeg.Decode(bufio.NewReader(imgFp)) + case "jpeg": + return jpeg.Decode(bufio.NewReader(imgFp)) + case "image/png": + return png.Decode(bufio.NewReader(imgFp)) + } + return nil, errors.New("invalid content type.") +} + func WriteImageFile(image image.Image, contentType, path string) error { os.MkdirAll(filepath.Dir(path), os.ModePerm) switch contentType { diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go index c3e6de0..a4f8066 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go @@ -3,13 +3,11 @@ package pubsubprovider import ( "context" "encoding/json" - "fmt" "os" "testing" "time" "github.com/stretchr/testify/suite" - "go-gcs/src/imageresize" "go-gcs/src/service/googlecloud" "go-gcs/src/service/googlecloud/storageprovider" ) @@ -74,16 +72,25 @@ func (suite *GoogleCloudPubSubProviderSuite) TestNotifyFromGCSStorage() { bucket := storageService.Config.Bucket path := "test/cat.jpg" filePath := "../../../../test/image/cat.jpg" - contentType := "image/jpeg" err = storageService.Upload(bucket, path, filePath) suite.NoError(err) // Wait for resize image - time.Sleep(5 * time.Second) + time.Sleep(10 * time.Second) imageResizeBucket := storageService.Config.ImageResizeBucket - resizeUrl := fmt.Sprintf("%s/%s/%s", storageprovider.GoogleCloudStoragePublicBaseUrl, imageResizeBucket, path) - img, err := imageresize.DownloadImageFromUrl(resizeUrl, contentType) - suite.NotNil(img) + err = storageService.Delete(bucket, path) + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path) + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_100") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_150") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_300") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_640") + suite.NoError(err) + err = storageService.Delete(imageResizeBucket, path+"_1080") suite.NoError(err) } diff --git a/src/service/googlecloud/storageprovider/storageprovider_test.go b/src/service/googlecloud/storageprovider/storageprovider_test.go index d6a7ba0..e63d8f8 100644 --- a/src/service/googlecloud/storageprovider/storageprovider_test.go +++ b/src/service/googlecloud/storageprovider/storageprovider_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/stretchr/testify/suite" + "go-gcs/src/imageresize" "go-gcs/src/service/googlecloud" ) @@ -34,8 +35,81 @@ func (suite *GoogleCloudStorageProviderSuite) SetupSuite() { ctx := context.Background() service := New(ctx, cf.GoogleCloud, cf.Storage) suite.NotNil(service) + suite.service = service } func TestStorageProviderSuite(t *testing.T) { suite.Run(t, new(GoogleCloudStorageProviderSuite)) } + +func (suite *GoogleCloudStorageProviderSuite) TestUploadDelete() { + bucket := suite.service.Config.Bucket + path := "test/cat.jpg" + filePath := "../../../../test/image/cat.jpg" + err := suite.service.Upload(bucket, path, filePath) + suite.NoError(err) + + err = suite.service.Delete(bucket, path) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageProviderSuite) TestUploadImageDelete() { + contentType := "image/jpeg" + filePath := "../../../../test/image/cat.jpg" + img, err := imageresize.ReadImageFile(contentType, filePath) + suite.NotNil(img) + suite.NoError(err) + + bucket := suite.service.Config.Bucket + path := "test/cat.jpg" + err = suite.service.UploadImage(img, contentType, bucket, path) + suite.NoError(err) + + err = suite.service.Delete(bucket, path) + suite.NoError(err) +} + +func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUploadDelete() { + contentType := "image/jpeg" + filePath := "../../../../test/image/cat.jpg" + img, err := imageresize.ReadImageFile(contentType, filePath) + suite.NotNil(img) + suite.NoError(err) + + path := "test/cat.jpg" + err = suite.service.ResizeImageAndUpload(img, 100, contentType, path) + suite.NoError(err) + + bucket := suite.service.Config.ImageResizeBucket + err = suite.service.Delete(bucket, path+"_100") + suite.NoError(err) +} + +func (suite *GoogleCloudStorageProviderSuite) TestResizeMultiImageSizeAndUpload() { + bucket := suite.service.Config.Bucket + path := "test/cat.jpg" + filePath := "../../../../test/image/cat.jpg" + err := suite.service.Upload(bucket, path, filePath) + suite.NoError(err) + + contentType := "image/jpeg" + url, err := suite.service.ResizeMultiImageSizeAndUpload(contentType, bucket, path) + suite.NoError(err) + suite.NotEqual("", url) + + imageResizeBucket := suite.service.Config.ImageResizeBucket + err = suite.service.Delete(bucket, path) + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path) + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_100") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_150") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_300") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_640") + suite.NoError(err) + err = suite.service.Delete(imageResizeBucket, path+"_1080") + suite.NoError(err) +} From 0210ee7f49da9efad35e75b9bcb0176028039736 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 27 Nov 2018 18:25:00 +0800 Subject: [PATCH 24/33] Add testing coverage --- Makefile | 37 +++++++++++++++++-- src/googlecloud/storage/storage_test.go | 16 +++++++- src/server/handler_storage_test.go | 15 -------- .../pubsubprovider/pubsubprovider.go | 3 ++ .../pubsubprovider/pubsubprovider_test.go | 19 ---------- .../storageprovider/storageprovider_test.go | 34 ++--------------- 6 files changed, 54 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 044e1d4..bec6e27 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,44 @@ -EXE=file_manager +## File name for executing +FILE_NAME=file_manager +## Folder content generated files +BUILD_FOLDER = ./build +## command +GO = go +MKDIR_P = mkdir -p -all: build-linux +################################################ +.PHONY: all +all: build-linux test + +.PHONY: build-darwin build-darwin: - go build -o $(EXE) src/cmd/gcs-server/main.go + go build -o $(FILE_NAME) src/cmd/gcs-server/main.go +.PHONY: build-linux build-linux: - export GOOS=linux && export GOARCH=amd64 && go build -o $(EXE) src/cmd/gcs-server/main.go + export GOOS=linux && export GOARCH=amd64 && go build -o $(FILE_NAME) src/cmd/gcs-server/main.go + +.PHONY: test +test: build-linux + $(MAKE) src.test +.PHONY: run run: go run src/cmd/gcs-server/main.go +.PHONY: zip zip: zip -r config.zip ./config/ + +## src/ ######################################## + +.PHONY: src.test +src.test: + $(GO) test -v ./src/... + +.PHONY: src.test-coverage +src.test-coverage: + $(MKDIR_P) $(BUILD_FOLDER)/src/ + $(GO) test -v -coverprofile=$(BUILD_FOLDER)/src/coverage.txt -covermode=atomic ./src/... + $(GO) tool cover -html=$(BUILD_FOLDER)/src/coverage.txt -o $(BUILD_FOLDER)/src/coverage.html diff --git a/src/googlecloud/storage/storage_test.go b/src/googlecloud/storage/storage_test.go index 26d0ec9..59c6a2c 100644 --- a/src/googlecloud/storage/storage_test.go +++ b/src/googlecloud/storage/storage_test.go @@ -2,6 +2,7 @@ package storage import ( "encoding/json" + "fmt" "testing" "time" @@ -9,6 +10,7 @@ import ( "go-gcs/src/config" "go-gcs/src/entity" "go-gcs/src/service" + "go-gcs/src/service/googlecloud/storageprovider" ) type GoogleCloudStorageSuite struct { @@ -45,7 +47,7 @@ func (suite *GoogleCloudStorageSuite) TestCreateGCSSingleSignedUrl() { To: "myAwesomeBuddyId", } - p, err := json.Marshal(payload) + p, err := json.MarshalIndent(payload, "", " ") suite.NotNil(p) suite.NoError(err) @@ -62,7 +64,7 @@ func (suite *GoogleCloudStorageSuite) TestCreateGCSGroupSignedUrl() { GroupId: "myAwesomeGroupId", } - p, err := json.Marshal(payload) + p, err := json.MarshalIndent(payload, "", " ") suite.NotNil(p) suite.NoError(err) @@ -72,5 +74,15 @@ func (suite *GoogleCloudStorageSuite) TestCreateGCSGroupSignedUrl() { } func (suite *GoogleCloudStorageSuite) TestResizeGCSImage() { + bucket := suite.sp.GoogleCloudStorage.Config.Bucket + path := "test/cat.jpg" + filePath := "../../../test/image/cat.jpg" + err := suite.sp.GoogleCloudStorage.Upload(bucket, path, filePath) + suite.NoError(err) + url := fmt.Sprintf("%s/%s/%s", storageprovider.GoogleCloudStoragePublicBaseUrl, bucket, path) + contentType := "image/jpeg" + ri, err := ResizeGCSImage(suite.sp, url, contentType) + suite.NotNil(ri) + suite.NoError(err) } diff --git a/src/server/handler_storage_test.go b/src/server/handler_storage_test.go index 6f37775..8ba3222 100644 --- a/src/server/handler_storage_test.go +++ b/src/server/handler_storage_test.go @@ -121,19 +121,4 @@ func (suite *StorageSuite) TestResizeGCSImage() { suite.Equal(url+"_300", resizeImage.ThumbWidth300) suite.Equal(url+"_640", resizeImage.ThumbWidth640) suite.Equal(url+"_1080", resizeImage.ThumbWidth1080) - - err = suite.sp.GoogleCloudStorage.Delete(bucket, path) - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path) - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_100") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_150") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_300") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_640") - suite.NoError(err) - err = suite.sp.GoogleCloudStorage.Delete(imageResizeBucket, path+"_1080") - suite.NoError(err) } diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index f4f2366..a416f45 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -34,12 +34,15 @@ type GoogleCloudStorageNotification struct { // NotifyFromGCSStorage will call if google cloud storage object update func (s *Service) NotifyFromGCSStorage(sp *storageprovider.Service) { + var sub *pubsub.Subscription sub, err := s.Client.CreateSubscription(s.Context, s.Config.Subscription, pubsub.SubscriptionConfig{ Topic: s.Client.Topic(s.Config.Topic), AckDeadline: 20 * time.Second, }) if err != nil { logger.Warnf("error while create google cloud pubsub subscription: %s", err) + logger.Info("try to use the exist subscription...") + sub = s.Client.Subscription(s.Config.Subscription) } var mu sync.Mutex diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go index a4f8066..9799522 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider_test.go @@ -74,23 +74,4 @@ func (suite *GoogleCloudPubSubProviderSuite) TestNotifyFromGCSStorage() { filePath := "../../../../test/image/cat.jpg" err = storageService.Upload(bucket, path, filePath) suite.NoError(err) - - // Wait for resize image - time.Sleep(10 * time.Second) - - imageResizeBucket := storageService.Config.ImageResizeBucket - err = storageService.Delete(bucket, path) - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path) - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_100") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_150") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_300") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_640") - suite.NoError(err) - err = storageService.Delete(imageResizeBucket, path+"_1080") - suite.NoError(err) } diff --git a/src/service/googlecloud/storageprovider/storageprovider_test.go b/src/service/googlecloud/storageprovider/storageprovider_test.go index e63d8f8..8daccf7 100644 --- a/src/service/googlecloud/storageprovider/storageprovider_test.go +++ b/src/service/googlecloud/storageprovider/storageprovider_test.go @@ -42,18 +42,15 @@ func TestStorageProviderSuite(t *testing.T) { suite.Run(t, new(GoogleCloudStorageProviderSuite)) } -func (suite *GoogleCloudStorageProviderSuite) TestUploadDelete() { +func (suite *GoogleCloudStorageProviderSuite) TestUpload() { bucket := suite.service.Config.Bucket path := "test/cat.jpg" filePath := "../../../../test/image/cat.jpg" err := suite.service.Upload(bucket, path, filePath) suite.NoError(err) - - err = suite.service.Delete(bucket, path) - suite.NoError(err) } -func (suite *GoogleCloudStorageProviderSuite) TestUploadImageDelete() { +func (suite *GoogleCloudStorageProviderSuite) TestUploadImage() { contentType := "image/jpeg" filePath := "../../../../test/image/cat.jpg" img, err := imageresize.ReadImageFile(contentType, filePath) @@ -64,12 +61,9 @@ func (suite *GoogleCloudStorageProviderSuite) TestUploadImageDelete() { path := "test/cat.jpg" err = suite.service.UploadImage(img, contentType, bucket, path) suite.NoError(err) - - err = suite.service.Delete(bucket, path) - suite.NoError(err) } -func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUploadDelete() { +func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUpload() { contentType := "image/jpeg" filePath := "../../../../test/image/cat.jpg" img, err := imageresize.ReadImageFile(contentType, filePath) @@ -79,10 +73,6 @@ func (suite *GoogleCloudStorageProviderSuite) TestResizeImageAndUploadDelete() { path := "test/cat.jpg" err = suite.service.ResizeImageAndUpload(img, 100, contentType, path) suite.NoError(err) - - bucket := suite.service.Config.ImageResizeBucket - err = suite.service.Delete(bucket, path+"_100") - suite.NoError(err) } func (suite *GoogleCloudStorageProviderSuite) TestResizeMultiImageSizeAndUpload() { @@ -95,21 +85,5 @@ func (suite *GoogleCloudStorageProviderSuite) TestResizeMultiImageSizeAndUpload( contentType := "image/jpeg" url, err := suite.service.ResizeMultiImageSizeAndUpload(contentType, bucket, path) suite.NoError(err) - suite.NotEqual("", url) - - imageResizeBucket := suite.service.Config.ImageResizeBucket - err = suite.service.Delete(bucket, path) - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path) - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_100") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_150") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_300") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_640") - suite.NoError(err) - err = suite.service.Delete(imageResizeBucket, path+"_1080") - suite.NoError(err) + suite.NotEmpty(url) } From bdf08deeac7ace96d3482e660e0a9ead6430c7e7 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Nov 2018 15:05:20 +0800 Subject: [PATCH 25/33] Add circle ci --- .circleci/config.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..464d32a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,13 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + build: + docker: + # specify the version + - image: circleci/golang:1.11 + steps: + - checkout + # specify any bash command here prefixed with `run: ` + - run: make src.test-coverage From 9a205c9ad63dc2802766c6a8ad7d31ba305716fa Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Nov 2018 18:37:43 +0800 Subject: [PATCH 26/33] Update testing file for test circle ci --- .circleci/config.yml | 3 +++ Makefile | 5 +++++ config/testing.json | 30 +++++++++++++++--------------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 464d32a..da8b3cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,4 +10,7 @@ jobs: steps: - checkout # specify any bash command here prefixed with `run: ` + - run: make src.test-coverage-minikube + - run: cat config/testing.json - run: make src.test-coverage + diff --git a/Makefile b/Makefile index bec6e27..7aca644 100644 --- a/Makefile +++ b/Makefile @@ -42,3 +42,8 @@ src.test-coverage: $(MKDIR_P) $(BUILD_FOLDER)/src/ $(GO) test -v -coverprofile=$(BUILD_FOLDER)/src/coverage.txt -covermode=atomic ./src/... $(GO) tool cover -html=$(BUILD_FOLDER)/src/coverage.txt -o $(BUILD_FOLDER)/src/coverage.html + +.PHONY: src.test-coverage-minikube +src.test-coverage-minikube: + sed -i.bak "s/{{ projectId }}/$(PROJECTID)/g; s/{{ privateKeyId }}/$(PRIVATEKEYID)/g; s#{{ privateKey }}#$(PRIVATEKEY)#g; s/{{ clientEmail }}/$(CLIENTEMAIL)/g; s/{{ clientId }}/$(CLIENTID)/g; s#{{ clientCert }}#$(CLIENTCERT)#g; s/{{ jwtSecretKey }}/$(JWTSECRETKEY)/g;" config/testing.json + echo $(sed ':a;N;$!ba;s/\n/\\n/g' $PRIVATEKEY) diff --git a/config/testing.json b/config/testing.json index f7e98ec..1948488 100644 --- a/config/testing.json +++ b/config/testing.json @@ -1,26 +1,26 @@ { "googlecloud":{ "type": "service_account", - "project_id": "", - "private_key_id": "", - "private_key": "", - "client_email": "", - "client_id": "", - "auth_uri": "", - "token_uri": "", - "auth_provider_x509_cert_url": "", - "client_x509_cert_url": "" + "project_id": "{{ projectId }}", + "private_key_id": "{{ privateKeyId }}", + "private_key": "{{ privateKey }}", + "client_email": "{{ clientEmail }}", + "client_id": "{{ clientId }}", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "{{ clientCert }}" }, "storage": { - "bucket": "", - "imageResizeBucket": "", - "contentLengthRange": "" + "bucket": "jkopay-test", + "imageResizeBucket": "jkopay-test_img_resize", + "contentLengthRange": "0,200000000" }, "pubsub": { - "topic": "", - "subscription": "" + "topic": "jello", + "subscription": "mySubscription" }, - "jwtSecretKey": "", + "jwtSecretKey": "{{ jwtSecretKey }}", "logger": { "dir":"./logs", "level":"info", From 9f5effb95b559465bb69b1b13d6ffc8a604a0a8c Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 14:05:31 +0800 Subject: [PATCH 27/33] Update success circleci file --- .circleci/config.yml | 3 --- Makefile | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da8b3cd..568774a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,3 @@ jobs: - checkout # specify any bash command here prefixed with `run: ` - run: make src.test-coverage-minikube - - run: cat config/testing.json - - run: make src.test-coverage - diff --git a/Makefile b/Makefile index 7aca644..e843125 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,5 @@ src.test-coverage: .PHONY: src.test-coverage-minikube src.test-coverage-minikube: sed -i.bak "s/{{ projectId }}/$(PROJECTID)/g; s/{{ privateKeyId }}/$(PRIVATEKEYID)/g; s#{{ privateKey }}#$(PRIVATEKEY)#g; s/{{ clientEmail }}/$(CLIENTEMAIL)/g; s/{{ clientId }}/$(CLIENTID)/g; s#{{ clientCert }}#$(CLIENTCERT)#g; s/{{ jwtSecretKey }}/$(JWTSECRETKEY)/g;" config/testing.json - echo $(sed ':a;N;$!ba;s/\n/\\n/g' $PRIVATEKEY) + $(MAKE) src.test-coverage + mv config/testing.json.bak config/testing.json From dca251e0ddfce0f912e47d1c73643f9370448fb3 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 16:46:38 +0800 Subject: [PATCH 28/33] Add dockerfile license --- Dockerfile | 32 ++++ LICENSE | 201 ++++++++++++++++++++ Makefile | 18 +- src/cmd/{gcs-server => filemanager}/main.go | 0 4 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 LICENSE rename src/cmd/{gcs-server => filemanager}/main.go (100%) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..249ad2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# Building stage +FROM golang:1.11-alpine3.7 + +WORKDIR /go-gcs + +RUN apk add --no-cache protobuf ca-certificates make git + +# Source code, building tools and dependences +COPY src /go-gcs +COPY Makefile /go/src/github.com/linkernetworks/vortex + +ENV CGO_ENABLED 0 +ENV GOOS linux +ENV TIMEZONE "Asia/Taipei" +RUN apk add --no-cache tzdata && \ + cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime && \ + echo $TIMEZONE > /etc/timezone && \ + apk del tzdata + +RUN make src.build +RUN mv build/src/cmd/filemanager/filemanager /go/bin + +# Production stage +FROM alpine:3.7 +RUN apk add --no-cache ca-certificates +WORKDIR /go-gcs + +# copy the go binaries from the building stage +COPY --from=0 /go/bin /go/bin + +EXPOSE 7890 +ENTRYPOINT ["/go/bin/filemanager", "-port", "7890", "-config", "/go-gcs/config/develop.json"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile index e843125..4e11b77 100644 --- a/Makefile +++ b/Makefile @@ -9,15 +9,19 @@ MKDIR_P = mkdir -p ################################################ .PHONY: all -all: build-linux test +all: build build-linux test + +.PHONY: build +build: + $(MAKE) src.build .PHONY: build-darwin build-darwin: - go build -o $(FILE_NAME) src/cmd/gcs-server/main.go + go build -o $(FILE_NAME) src/cmd/filemanger/main.go .PHONY: build-linux build-linux: - export GOOS=linux && export GOARCH=amd64 && go build -o $(FILE_NAME) src/cmd/gcs-server/main.go + export GOOS=linux && export GOARCH=amd64 && go build -o $(FILE_NAME) src/cmd/filemanager/main.go .PHONY: test test: build-linux @@ -25,7 +29,7 @@ test: build-linux .PHONY: run run: - go run src/cmd/gcs-server/main.go + go run src/cmd/filemanager/main.go .PHONY: zip zip: @@ -33,6 +37,12 @@ zip: ## src/ ######################################## +.PHONY: src.build +src.build: + $(GO) build -v ./src/... + $(MKDIR_P) $(BUILD_FOLDER)/src/cmd/filemanager/ + $(GO) build -v -o $(BUILD_FOLDER)/src/cmd/filemanager/filemanager ./src/cmd/filemanager/... + .PHONY: src.test src.test: $(GO) test -v ./src/... diff --git a/src/cmd/gcs-server/main.go b/src/cmd/filemanager/main.go similarity index 100% rename from src/cmd/gcs-server/main.go rename to src/cmd/filemanager/main.go From 5312de36bd4dda6dffbf82c0efb0c59e9782b23b Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 17:33:54 +0800 Subject: [PATCH 29/33] Update dockerfile --- Dockerfile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 249ad2f..8c5b6e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,20 @@ FROM golang:1.11-alpine3.7 WORKDIR /go-gcs + + + + + RUN apk add --no-cache protobuf ca-certificates make git # Source code, building tools and dependences -COPY src /go-gcs -COPY Makefile /go/src/github.com/linkernetworks/vortex +COPY src /go-gcs/src +COPY Makefile /go-gcs +# We want to populate the module cache based on the go.{mod,sum} files. +COPY go.mod /go-gcs +COPY go.sum /go-gcs + ENV CGO_ENABLED 0 ENV GOOS linux @@ -16,7 +25,10 @@ RUN apk add --no-cache tzdata && \ cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime && \ echo $TIMEZONE > /etc/timezone && \ apk del tzdata +# Force the go compiler to use modules +ENV GO111MODULE=on +RUN go mod download RUN make src.build RUN mv build/src/cmd/filemanager/filemanager /go/bin From 430da7e273061060c5b748396e09df6f6b7e723b Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 29 Nov 2018 18:27:41 +0800 Subject: [PATCH 30/33] update circleci --- .circleci/config.yml | 18 ++++++++++++++++-- Dockerfile | 5 ----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 568774a..7a93b51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,11 +3,25 @@ # Check https://circleci.com/docs/2.0/language-go/ for more details version: 2 jobs: - build: + test: docker: # specify the version - image: circleci/golang:1.11 steps: - checkout # specify any bash command here prefixed with `run: ` - - run: make src.test-coverage-minikube + - run: + name: Test and generate code coverage + command: make src.test-coverage-minikube + - run: + name: Copy code coverage to root + command: cp build/src/coverage.txt coverage.txt + - run: + name: Copy code coverage to root + command: bash <(curl -s https://codecov.io/bash) +workflows: + version: 2 + test: + jobs: + - test + diff --git a/Dockerfile b/Dockerfile index 8c5b6e9..3eaf21f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,6 @@ FROM golang:1.11-alpine3.7 WORKDIR /go-gcs - - - - - RUN apk add --no-cache protobuf ca-certificates make git # Source code, building tools and dependences From e6ac5bdbb8b38a7037934f247407d10e795fb7d5 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 30 Nov 2018 16:28:52 +0800 Subject: [PATCH 31/33] Add helm to deploy service to k8s --- .circleci/config.yml | 9 ++- Makefile | 57 +++++++++++++++++++ deployment/helm/apps/.helmignore | 21 +++++++ deployment/helm/apps/Chart.yaml | 5 ++ .../helm/apps/charts/filemanager/.helmignore | 21 +++++++ .../helm/apps/charts/filemanager/Chart.yaml | 5 ++ .../filemanager/templates/deployment.yaml | 30 ++++++++++ .../charts/filemanager/templates/service.yaml | 17 ++++++ deployment/helm/config/development.yaml | 10 ++++ deployment/helm/config/production.yaml | 10 ++++ deployment/helm/services/.helmignore | 21 +++++++ deployment/helm/services/Chart.yaml | 5 ++ .../services/charts/namespace/.helmignore | 21 +++++++ .../helm/services/charts/namespace/Chart.yaml | 5 ++ .../charts/namespace/templates/namespace.yaml | 4 ++ .../helm/services/charts/rbac/.helmignore | 21 +++++++ .../helm/services/charts/rbac/Chart.yaml | 5 ++ .../services/charts/rbac/templates/role.yaml | 30 ++++++++++ .../charts/rbac/templates/serviceaccount.yaml | 5 ++ 19 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 deployment/helm/apps/.helmignore create mode 100644 deployment/helm/apps/Chart.yaml create mode 100644 deployment/helm/apps/charts/filemanager/.helmignore create mode 100644 deployment/helm/apps/charts/filemanager/Chart.yaml create mode 100644 deployment/helm/apps/charts/filemanager/templates/deployment.yaml create mode 100644 deployment/helm/apps/charts/filemanager/templates/service.yaml create mode 100644 deployment/helm/config/development.yaml create mode 100644 deployment/helm/config/production.yaml create mode 100644 deployment/helm/services/.helmignore create mode 100644 deployment/helm/services/Chart.yaml create mode 100644 deployment/helm/services/charts/namespace/.helmignore create mode 100644 deployment/helm/services/charts/namespace/Chart.yaml create mode 100644 deployment/helm/services/charts/namespace/templates/namespace.yaml create mode 100644 deployment/helm/services/charts/rbac/.helmignore create mode 100644 deployment/helm/services/charts/rbac/Chart.yaml create mode 100644 deployment/helm/services/charts/rbac/templates/role.yaml create mode 100644 deployment/helm/services/charts/rbac/templates/serviceaccount.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a93b51..2caf9ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,8 +20,7 @@ jobs: name: Copy code coverage to root command: bash <(curl -s https://codecov.io/bash) workflows: - version: 2 - test: - jobs: - - test - + version: 2 + test: + jobs: + - test diff --git a/Makefile b/Makefile index 4e11b77..64baee4 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +## filemanager server version +SERVER_VERSION = latest ## File name for executing FILE_NAME=file_manager ## Folder content generated files @@ -58,3 +60,58 @@ src.test-coverage-minikube: sed -i.bak "s/{{ projectId }}/$(PROJECTID)/g; s/{{ privateKeyId }}/$(PRIVATEKEYID)/g; s#{{ privateKey }}#$(PRIVATEKEY)#g; s/{{ clientEmail }}/$(CLIENTEMAIL)/g; s/{{ clientId }}/$(CLIENTID)/g; s#{{ clientCert }}#$(CLIENTCERT)#g; s/{{ jwtSecretKey }}/$(JWTSECRETKEY)/g;" config/testing.json $(MAKE) src.test-coverage mv config/testing.json.bak config/testing.json + +## launch apps ############################# + +.PHONY: apps.init-helm +apps.init-helm: + helm init + kubectl create serviceaccount --namespace kube-system tiller + kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}' + +.PHONY: apps.launch-dev +apps.launch-dev: + yq -y .services deployment/helm/config/development.yaml | helm install --name filemanager-services-dev --debug --wait -f - deployment/helm/services + kubectl create configmap filemanager-config --from-file=config/ -n filemanager + yq -y .apps deployment/helm/config/development.yaml | helm install --name filemanager-apps-dev --debug --wait -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.launch-prod +apps.launch-prod: + yq -y .services deployment/helm/config/production.yaml | helm install --name filemanager-services-prod --debug --wait -f - deployment/helm/services + yq -y .apps deployment/helm/config/production.yaml | helm install --name filemanager-apps-prod --debug --wait -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.upgrade-dev +apps.upgrade-dev: + yq -y .services deployment/helm/config/development.yaml | helm upgrade filemanager-services-dev --debug -f - deployment/helm/services + yq -y .apps deployment/helm/config/development.yaml | helm upgrade filemanager-apps-dev --debug -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.upgrade-prod +apps.upgrade-prod: + yq -y .services deployment/helm/config/production.yaml | helm upgrade filemanager-services-prod --debug -f - deployment/helm/services + yq -y .apps deployment/helm/config/production.yaml | helm upgrade filemanager-apps-prod --debug -f - --set filemanager-server.controller.apiserverImageTag=$(SERVER_VERSION) deployment/helm/apps + +.PHONY: apps.teardown-dev +apps.teardown-dev: + helm delete --purge filemanager-services-dev + helm delete --purge filemanager-apps-dev + kubectl delete configmap -n filemanager + +.PHONY: apps.teardown-prod +apps.teardown-prod: + helm delete --purge filemanager-services-prod + helm delete --purge filemanager-apps-prod + +## dockerfiles/ ######################################## + +.PHONY: dockerfiles.build +dockerfiles.build: + docker build --tag yunchen/go-gcs:$(SERVER_VERSION) . + +## git tag version ######################################## + +.PHONY: push.tag +push.tag: + @echo "Current git tag version:"$(SERVER_VERSION) + git tag $(SERVER_VERSION) + git push --tags diff --git a/deployment/helm/apps/.helmignore b/deployment/helm/apps/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/apps/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/apps/Chart.yaml b/deployment/helm/apps/Chart.yaml new file mode 100644 index 0000000..d172ecb --- /dev/null +++ b/deployment/helm/apps/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: apps +version: 0.1.0 diff --git a/deployment/helm/apps/charts/filemanager/.helmignore b/deployment/helm/apps/charts/filemanager/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/apps/charts/filemanager/Chart.yaml b/deployment/helm/apps/charts/filemanager/Chart.yaml new file mode 100644 index 0000000..a728824 --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for filemanager-server +name: filemanager +version: 0.1.0 diff --git a/deployment/helm/apps/charts/filemanager/templates/deployment.yaml b/deployment/helm/apps/charts/filemanager/templates/deployment.yaml new file mode 100644 index 0000000..4670dab --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/templates/deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: filemanager-server + namespace: filemanager + labels: + app: filemanager-server +spec: + replicas: {{ .Values.controller.replicaCount }} + selector: + matchLabels: + app: filemanager-server + template: + metadata: + labels: + app: filemanager-server + spec: + serviceAccountName: filemanager-admin + containers: + - name: filemanager-server + image: yunchen/go-gcs:{{ .Values.controller.apiserverImageTag }} + ports: + - containerPort: 7890 + volumeMounts: + - name: config-volume + mountPath: /go-gcs/config + volumes: + - name: config-volume + configMap: + name: filemanager-config diff --git a/deployment/helm/apps/charts/filemanager/templates/service.yaml b/deployment/helm/apps/charts/filemanager/templates/service.yaml new file mode 100644 index 0000000..283c572 --- /dev/null +++ b/deployment/helm/apps/charts/filemanager/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: filemanager-server + namespace: filemanager + labels: + app: filemanager-server +spec: + type: NodePort + ports: + - name: apiserver + protocol: TCP + port: 7890 + targetPort: 7890 + nodePort: {{ .Values.service.ports.apiserverNodePort }} + selector: + app: filemanager-server diff --git a/deployment/helm/config/development.yaml b/deployment/helm/config/development.yaml new file mode 100644 index 0000000..fdc959f --- /dev/null +++ b/deployment/helm/config/development.yaml @@ -0,0 +1,10 @@ +# go-gcs/deploy/helm/apps +apps: + # go-gcs/deploy/helm/apps/charts/filemanager-server + filemanager: + controller: + replicaCount: 1 + apiserverImageTag: latest + service: + ports: + apiserverNodePort: 32326 diff --git a/deployment/helm/config/production.yaml b/deployment/helm/config/production.yaml new file mode 100644 index 0000000..fdc959f --- /dev/null +++ b/deployment/helm/config/production.yaml @@ -0,0 +1,10 @@ +# go-gcs/deploy/helm/apps +apps: + # go-gcs/deploy/helm/apps/charts/filemanager-server + filemanager: + controller: + replicaCount: 1 + apiserverImageTag: latest + service: + ports: + apiserverNodePort: 32326 diff --git a/deployment/helm/services/.helmignore b/deployment/helm/services/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/services/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/services/Chart.yaml b/deployment/helm/services/Chart.yaml new file mode 100644 index 0000000..ace0a01 --- /dev/null +++ b/deployment/helm/services/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: services +version: 0.1.0 diff --git a/deployment/helm/services/charts/namespace/.helmignore b/deployment/helm/services/charts/namespace/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/services/charts/namespace/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/services/charts/namespace/Chart.yaml b/deployment/helm/services/charts/namespace/Chart.yaml new file mode 100644 index 0000000..cf8f4ff --- /dev/null +++ b/deployment/helm/services/charts/namespace/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: namespace +version: 0.1.0 diff --git a/deployment/helm/services/charts/namespace/templates/namespace.yaml b/deployment/helm/services/charts/namespace/templates/namespace.yaml new file mode 100644 index 0000000..e108f69 --- /dev/null +++ b/deployment/helm/services/charts/namespace/templates/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: filemanager diff --git a/deployment/helm/services/charts/rbac/.helmignore b/deployment/helm/services/charts/rbac/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/deployment/helm/services/charts/rbac/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deployment/helm/services/charts/rbac/Chart.yaml b/deployment/helm/services/charts/rbac/Chart.yaml new file mode 100644 index 0000000..e55b632 --- /dev/null +++ b/deployment/helm/services/charts/rbac/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: rbac +version: 0.1.0 diff --git a/deployment/helm/services/charts/rbac/templates/role.yaml b/deployment/helm/services/charts/rbac/templates/role.yaml new file mode 100644 index 0000000..ad5a751 --- /dev/null +++ b/deployment/helm/services/charts/rbac/templates/role.yaml @@ -0,0 +1,30 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: filemanager-admin +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + - nonResourceURLs: + - '*' + verbs: + - '*' +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: filemanager-admin + namespace: filemanager +roleRef: + kind: ClusterRole + name: filemanager-admin + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: filemanager-admin + namespace: filemanager diff --git a/deployment/helm/services/charts/rbac/templates/serviceaccount.yaml b/deployment/helm/services/charts/rbac/templates/serviceaccount.yaml new file mode 100644 index 0000000..e6213cd --- /dev/null +++ b/deployment/helm/services/charts/rbac/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filemanager-admin + namespace: filemanager From f143d9e7d853873e33d598a9a66c80fb9bf35026 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 30 Nov 2018 17:20:45 +0800 Subject: [PATCH 32/33] Fix golint --- README.md | 5 +++++ src/entity/storage.go | 2 +- src/googlecloud/storage/storage.go | 2 +- src/imageresize/imageresize.go | 15 +++++++++++---- src/logger/logger.go | 2 +- .../googlecloud/pubsubprovider/pubsubprovider.go | 1 + .../storageprovider/storageprovider.go | 2 ++ 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1f8f7e5..2a6eea6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # go-gcs +[![CircleCI](https://circleci.com/gh/chenyunchen/go-gcs/tree/github%2Fdevelop.svg?style=svg)](https://circleci.com/gh/chenyunchen/go-gcs/tree/github%2Fdevelop) +[![Go Report Card](https://goreportcard.com/badge/github.com/chenyunchen/go-gcs)](https://goreportcard.com/report/github.com/chenyunchen/go-gcs) +[![codecov](https://codecov.io/gh/chenyunchen/go-gcs/branch/github%2Fdevelop/graph/badge.svg)](https://codecov.io/gh/chenyunchen/go-gcs) +[![Docker Build Status](https://img.shields.io/docker/build/yunchen/go-gcs.svg)](https://hub.docker.com/r/yunchen/go-gcs/) + * [1. Quck Start](#header_1) * [2. Overview](#header_2) * [3. API](#header_3) diff --git a/src/entity/storage.go b/src/entity/storage.go index bc8f60a..e0da2b8 100644 --- a/src/entity/storage.go +++ b/src/entity/storage.go @@ -18,7 +18,7 @@ type GroupPayload struct { GroupId string `json:"groupId" validate:"required"` } -// SignedUrlRequest is the structure for signed url response +// SignedUrl is the structure for signed url response type SignedUrl struct { Url string `json:"url" validate:"required"` UploadHeaders `json:"uploadHeaders" validate:"required"` diff --git a/src/googlecloud/storage/storage.go b/src/googlecloud/storage/storage.go index 6512c7e..4d770f7 100644 --- a/src/googlecloud/storage/storage.go +++ b/src/googlecloud/storage/storage.go @@ -28,7 +28,7 @@ func SignURL(sp *service.Container, path string, fileName string, contentType st rand.Seed(time.Now().UnixNano()) r := strings.NewReplacer("/", ":") - b := make([]byte, 4) //equals 8 charachters + b := make([]byte, 4) //equals 8 characters rand.Read(b) hashFileName := fmt.Sprintf("%s_%s", hex.EncodeToString(b), r.Replace(fileName)) diff --git a/src/imageresize/imageresize.go b/src/imageresize/imageresize.go index fb94900..ef11427 100644 --- a/src/imageresize/imageresize.go +++ b/src/imageresize/imageresize.go @@ -13,6 +13,7 @@ import ( "github.com/nfnt/resize" ) +// DownloadImageFromUrl will download image from url func DownloadImageFromUrl(url, contentType string) (image.Image, error) { // Fetch an image. resp, err := http.Get(url) @@ -31,9 +32,10 @@ func DownloadImageFromUrl(url, contentType string) (image.Image, error) { case "image/png": return png.Decode(resp.Body) } - return nil, errors.New("invalid content type.") + return nil, errors.New("invalid content type") } +// ReadImageFile will read image struct from file path func ReadImageFile(contentType, path string) (image.Image, error) { imgFp, err := os.Open(path) if err != nil { @@ -51,9 +53,10 @@ func ReadImageFile(contentType, path string) (image.Image, error) { case "image/png": return png.Decode(bufio.NewReader(imgFp)) } - return nil, errors.New("invalid content type.") + return nil, errors.New("invalid content type") } +// WriteImageFile will write image struct to file path func WriteImageFile(image image.Image, contentType, path string) error { os.MkdirAll(filepath.Dir(path), os.ModePerm) switch contentType { @@ -66,9 +69,10 @@ func WriteImageFile(image image.Image, contentType, path string) error { case "image/png": return WritePngImageFile(image, path) } - return errors.New("invalid content type.") + return errors.New("invalid content type") } +// WritePngImageFile will write image struct to png file path func WritePngImageFile(image image.Image, path string) error { imgFp, err := os.Create(path) if err != nil { @@ -83,6 +87,7 @@ func WritePngImageFile(image image.Image, path string) error { return nil } +// WriteJpegImageFile will write image struct to jpeg file path func WriteJpegImageFile(image image.Image, path string, quality int) error { imgFp, err := os.Create(path) if err != nil { @@ -97,16 +102,18 @@ func WriteJpegImageFile(image image.Image, path string, quality int) error { return nil } +// ResizeImage will resize image struct and return image struct func ResizeImage(image image.Image, width, height uint) image.Image { return resize.Resize(width, height, image, resize.Lanczos3) } -// Thumbnail will fail if width or height == 0 +// ThumbnailImage will fail if width or height == 0 // If maxWidth and maxHeight > image size, return the image with its original size func ThumbnailImage(image image.Image, maxWidth, maxHeight uint) image.Image { return resize.Thumbnail(maxWidth, maxHeight, image, resize.Lanczos3) } +// DeleteImage will delete image from path func DeleteImage(path string) error { return os.Remove(path) } diff --git a/src/logger/logger.go b/src/logger/logger.go index 6801c80..0b32b97 100644 --- a/src/logger/logger.go +++ b/src/logger/logger.go @@ -27,7 +27,7 @@ func init() { Logger = logrus.New() } -// Setup Logger in packge. Enable Logger after import +// Setup Logger in package. Enable Logger after import func Setup(c LoggerConfig) { if Logger == nil { Logger = logrus.New() diff --git a/src/service/googlecloud/pubsubprovider/pubsubprovider.go b/src/service/googlecloud/pubsubprovider/pubsubprovider.go index a416f45..7b61347 100644 --- a/src/service/googlecloud/pubsubprovider/pubsubprovider.go +++ b/src/service/googlecloud/pubsubprovider/pubsubprovider.go @@ -26,6 +26,7 @@ type Service struct { Context context.Context } +// GoogleCloudStorageNotification is the structure for notification type GoogleCloudStorageNotification struct { Name string `json:"name" validate:"required"` Bucket string `json:"bucket" validate:"required"` diff --git a/src/service/googlecloud/storageprovider/storageprovider.go b/src/service/googlecloud/storageprovider/storageprovider.go index 5eecf68..cfe34c7 100644 --- a/src/service/googlecloud/storageprovider/storageprovider.go +++ b/src/service/googlecloud/storageprovider/storageprovider.go @@ -16,6 +16,7 @@ import ( "google.golang.org/api/option" ) +// GoogleCloudStoragePublicBaseUrl is public url for download const GoogleCloudStoragePublicBaseUrl = "https://storage.googleapis.com" // Storage is the structure for config @@ -111,6 +112,7 @@ func (s *Service) ResizeImageAndUpload(img image.Image, width uint, contentType, return s.UploadImage(img, contentType, s.Config.ImageResizeBucket, path) } +// ResizeMultiImageSizeAndUpload will resize multiple image and upload to google cloud storage func (s *Service) ResizeMultiImageSizeAndUpload(contentType, bucket, path string) (string, error) { // Check if already resize origin image imageResizeBucket := s.Config.ImageResizeBucket From 7c7cdbc8edd356637cde52fd3cdcaf4b4d8cb5e3 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 3 Dec 2018 17:52:05 +0800 Subject: [PATCH 33/33] Add preview url --- go.mod | 1 + go.sum | 2 + src/entity/preview.go | 12 ++++++ src/server/handler_preview.go | 28 ++++++++++++++ src/server/handler_preview_test.go | 60 ++++++++++++++++++++++++++++++ src/server/route.go | 8 ++++ 6 files changed, 111 insertions(+) create mode 100644 src/entity/preview.go create mode 100644 src/server/handler_preview.go create mode 100644 src/server/handler_preview_test.go diff --git a/go.mod b/go.mod index 2eaf5c2..84ac687 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ require ( cloud.google.com/go v0.32.0 github.com/alecthomas/gometalinter v2.0.11+incompatible // indirect github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect + github.com/chenyunchen/goscraper v0.0.0-20181203065544-42a7d35f86ce github.com/cosiner/argv v0.0.0-20181111161405-36fdb9e0bb2d // indirect github.com/davidrjenni/reftools v0.0.0-20180914123528-654d0ba4f96d // indirect github.com/derekparker/delve v1.1.0 // indirect diff --git a/go.sum b/go.sum index 2fb435e..3b998ef 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/alecthomas/gometalinter v2.0.11+incompatible/go.mod h1:qfIpQGGz3d+Nmg github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/chenyunchen/goscraper v0.0.0-20181203065544-42a7d35f86ce h1:ZfWmDr8+quBLU9EAv4YkZMzN/ZA+uiaSr+nJ4ym8HDc= +github.com/chenyunchen/goscraper v0.0.0-20181203065544-42a7d35f86ce/go.mod h1:qNSd/0/hSB08FlBZze7/wYioge9nWtFVlMw9o3/Ph18= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cosiner/argv v0.0.0-20181111161405-36fdb9e0bb2d h1:8x/nCVflb3lA5f+PTizN0DcF+Lt34s1dFC00Zi932rs= github.com/cosiner/argv v0.0.0-20181111161405-36fdb9e0bb2d/go.mod h1:p/NrK5tF6ICIly4qwEDsf6VDirFiWWz0FenfYBwJaKQ= diff --git a/src/entity/preview.go b/src/entity/preview.go new file mode 100644 index 0000000..99ca589 --- /dev/null +++ b/src/entity/preview.go @@ -0,0 +1,12 @@ +package entity + +// PreviewUrl is the structure for preview url response +type PreviewedUrl struct { + Type string `json:"type" validate:"required"` + Icon string `json:"icon" validate:"required"` + Name string `json:"name" validate:"required"` + Title string `json:"title" validate:"required"` + Description string `json:"desc" validate:"required"` + Image string `json:"pic" validate:"required"` + Url string `json:"url" validate:"required"` +} diff --git a/src/server/handler_preview.go b/src/server/handler_preview.go new file mode 100644 index 0000000..47a3f98 --- /dev/null +++ b/src/server/handler_preview.go @@ -0,0 +1,28 @@ +package server + +import ( + "github.com/chenyunchen/goscraper" + "go-gcs/src/entity" + "go-gcs/src/net/context" + response "go-gcs/src/net/http" +) + +func previewUrlHandler(ctx *context.Context) { + _, req, resp := ctx.ServiceProvider, ctx.Request, ctx.Response + url := req.QueryParameter("url") + s, err := goscraper.Scrape(url, 5) + if err != nil { + response.BadRequest(req.Request, resp.ResponseWriter, err) + return + } + + resp.WriteEntity(entity.PreviewedUrl{ + Type: "default", + Icon: s.Preview.Icon, + Name: s.Preview.Name, + Title: s.Preview.Title, + Description: s.Preview.Description, + Image: s.Preview.Images[0], + Url: s.Preview.Link, + }) +} diff --git a/src/server/handler_preview_test.go b/src/server/handler_preview_test.go new file mode 100644 index 0000000..a2a3e2f --- /dev/null +++ b/src/server/handler_preview_test.go @@ -0,0 +1,60 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + restful "github.com/emicklei/go-restful" + "github.com/stretchr/testify/suite" + "go-gcs/src/config" + "go-gcs/src/entity" + "go-gcs/src/service" +) + +type PreviewSuite struct { + suite.Suite + wc *restful.Container + sp *service.Container +} + +func (suite *PreviewSuite) SetupSuite() { + cf := config.MustRead("../../config/testing.json") + sp := service.NewForTesting(cf) + + // init service provider + suite.sp = sp + // init restful container + suite.wc = restful.NewContainer() + + previewService := newPreviewService(suite.sp) + + suite.wc.Add(previewService) +} + +func TestPreviewSuite(t *testing.T) { + suite.Run(t, new(PreviewSuite)) +} + +func (suite *PreviewSuite) TestPreviewUrl() { + httpRequest, err := http.NewRequest("GET", "http://localhost:7890/v1/preview", nil) + suite.NoError(err) + + url := "https://github.com/chenyunchen" + query := httpRequest.URL.Query() + query.Add("url", url) + httpRequest.URL.RawQuery = query.Encode() + httpWriter := httptest.NewRecorder() + suite.wc.Dispatch(httpWriter, httpRequest) + assertResponseCode(suite.T(), http.StatusOK, httpWriter) + + previewedUrl := entity.PreviewedUrl{} + err = json.Unmarshal(httpWriter.Body.Bytes(), &previewedUrl) + suite.NoError(err) + suite.Equal("default", previewedUrl.Type) + suite.Equal("https://assets-cdn.github.com/favicon.ico", previewedUrl.Icon) + suite.Equal("GitHub", previewedUrl.Name) + suite.Equal("chenyunchen - Overview", previewedUrl.Title) + suite.Equal(url, previewedUrl.Url) +} diff --git a/src/server/route.go b/src/server/route.go index 8943e16..05c3432 100644 --- a/src/server/route.go +++ b/src/server/route.go @@ -12,6 +12,7 @@ func (a *App) AppRoute() *mux.Router { container := restful.NewContainer() container.Filter(globalLogging) container.Add(newStorageService(a.Service)) + container.Add(newPreviewService(a.Service)) router := mux.NewRouter() router.PathPrefix("/v1/").Handler(container) @@ -26,3 +27,10 @@ func newStorageService(sp *service.Container) *restful.WebService { webService.Route(webService.POST("/resize/image").To(http.RESTfulServiceHandler(sp, resizeGCSImageHandler))) return webService } + +func newPreviewService(sp *service.Container) *restful.WebService { + webService := new(restful.WebService) + webService.Path("/v1/preview").Consumes(restful.MIME_JSON, restful.MIME_JSON).Produces(restful.MIME_JSON, restful.MIME_JSON) + webService.Route(webService.GET("/").To(http.RESTfulServiceHandler(sp, previewUrlHandler))) + return webService +}