Revert last 7 reverts (blobstore read and write in chunks)

This reverts commit 688739b.

Signed-off-by: Kam Leung <[email protected]>

.gitignore

@@ -9,5 +9,8 @@ gobin/*
 *.sublime-workspace
 *.sw*
 
+test/bosh/*.json
+test/micro_bosh
+
 /.idea
-/.vagrant
+/.vagrant

bin/test-unit

@@ -25,17 +25,10 @@ for i in `ls -1` ; do
 done
 let "result+=$?"
 
-echo -e "\n Checking with golint..."
-$bin/golint
-let "result+=$?"
-
 echo -e "\n Installing ginkgo..."
 $bin/go install ./internal/github.com/onsi/ginkgo/ginkgo
 let "result+=$?"
 
-echo -e "\n Govetting"
-$bin/govet
-let "result+=$?"
 
 echo -e "\n Checking for unhandled errors"
 $bin/test-unhandled-errors
@@ -57,4 +50,13 @@ if [ ! $QUIET ]; then
   fi
 fi
 
+
+echo -e "\n Checking with golint..."
+$bin/golint
+let "result+=$?"
+
+echo -e "\n Govetting"
+$bin/govet
+let "result+=$?"
+
 exit $result

internal/github.com/cloudfoundry/bosh-utils/blobstore/blob_manager.go

@@ -1,9 +1,13 @@
 package blobstore
 
 import (
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
-	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
+	"io"
+	"os"
 	"path/filepath"
+	"strings"
+
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
 )
 
 type BlobManager struct {
@@ -17,22 +21,36 @@ func NewBlobManager(fs boshsys.FileSystem, blobstorePath string) (manager BlobMa
 	return
 }
 
-func (manager BlobManager) Fetch(blobID string) (blobBytes []byte, err error) {
+func (manager BlobManager) Fetch(blobID string) (boshsys.File, error, int) {
 	blobPath := filepath.Join(manager.blobstorePath, blobID)
 
-	blobBytes, err = manager.fs.ReadFile(blobPath)
+	readOnlyFile, err := manager.fs.OpenFile(blobPath, os.O_RDONLY, os.ModeDir)
 	if err != nil {
-		err = bosherr.WrapError(err, "Reading blob")
+		statusCode := 500
+		if strings.Contains(err.Error(), "no such file") {
+			statusCode = 404
+		}
+		return nil, bosherr.WrapError(err, "Reading blob"), statusCode
 	}
-	return
+
+	return readOnlyFile, nil, 200
 }
 
-func (manager BlobManager) Write(blobID string, blobBytes []byte) (err error) {
+func (manager BlobManager) Write(blobID string, reader io.Reader) error {
 	blobPath := filepath.Join(manager.blobstorePath, blobID)
 
-	err = manager.fs.WriteFile(blobPath, blobBytes)
+	writeOnlyFile, err := manager.fs.OpenFile(blobPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		err = bosherr.WrapError(err, "Opening blob store file")
+		return err
+	}
+
+	defer func() {
+		_ = writeOnlyFile.Close()
+	}()
+	_, err = io.Copy(writeOnlyFile, reader)
 	if err != nil {
 		err = bosherr.WrapError(err, "Updating blob")
 	}
-	return
+	return err
 }

internal/github.com/cloudfoundry/bosh-utils/blobstore/blob_manager_test.go

@@ -1,39 +1,89 @@
 package blobstore_test
 
 import (
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
-
+	"bytes"
 	. "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore"
-	fakesys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system/fakes"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
+	boshsysfake "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system/fakes"
+	"io"
+	"os"
+	"path/filepath"
 )
 
-func createBlobManager() (blobManager BlobManager, fs *fakesys.FakeFileSystem) {
-	fs = fakesys.NewFakeFileSystem()
-	blobManager = NewBlobManager(fs, "/var/vcap/micro_bosh/data/cache")
-	return
-}
+var _ = Describe("Blob Manager", func() {
+	var (
+		fs       boshsys.FileSystem
+		logger   boshlog.Logger
+		basePath string
+		blobPath string
+		blobId   string
+		toWrite  io.Reader
+	)
+
+	BeforeEach(func() {
+		logger = boshlog.NewLogger(boshlog.LevelNone)
+		fs = boshsys.NewOsFileSystem(logger)
+		blobId = "105d33ae-655c-493d-bf9f-1df5cf3ca847"
+		basePath = "/tmp"
+		blobPath = filepath.Join(basePath, blobId)
+		toWrite = bytes.NewReader([]byte("new data"))
+	})
+
+	readFile := func(fileIO boshsys.File) []byte {
+		fileStat, _ := fileIO.Stat()
+		fileBytes := make([]byte, fileStat.Size())
+		fileIO.Read(fileBytes)
+		return fileBytes
+	}
+
+	It("fetches", func() {
+		blobManager := NewBlobManager(fs, basePath)
+		fs.WriteFileString(blobPath, "some data")
 
-var _ = Describe("Testing with Ginkgo", func() {
-	It("fetch", func() {
-		blobManager, fs := createBlobManager()
-		fs.WriteFileString("/var/vcap/micro_bosh/data/cache/105d33ae-655c-493d-bf9f-1df5cf3ca847", "some data")
+		readOnlyFile, err, _ := blobManager.Fetch(blobId)
+		defer fs.RemoveAll(readOnlyFile.Name())
 
-		blobBytes, err := blobManager.Fetch("105d33ae-655c-493d-bf9f-1df5cf3ca847")
 		Expect(err).ToNot(HaveOccurred())
-		Expect(string(blobBytes)).To(Equal("some data"))
-	})
+		fileBytes := readFile(readOnlyFile)
 
-	It("write", func() {
+		Expect(string(fileBytes)).To(Equal("some data"))
+	})
 
-		blobManager, fs := createBlobManager()
-		fs.WriteFileString("/var/vcap/micro_bosh/data/cache/105d33ae-655c-493d-bf9f-1df5cf3ca847", "some data")
+	It("writes", func() {
+		blobManager := NewBlobManager(fs, basePath)
+		fs.WriteFileString(blobPath, "some data")
+		defer fs.RemoveAll(blobPath)
 
-		err := blobManager.Write("105d33ae-655c-493d-bf9f-1df5cf3ca847", []byte("new data"))
+		err := blobManager.Write(blobId, toWrite)
 		Expect(err).ToNot(HaveOccurred())
 
-		contents, err := fs.ReadFileString("/var/vcap/micro_bosh/data/cache/105d33ae-655c-493d-bf9f-1df5cf3ca847")
+		contents, err := fs.ReadFileString(blobPath)
 		Expect(err).ToNot(HaveOccurred())
 		Expect(contents).To(Equal("new data"))
 	})
+
+	Context("when it writes", func() {
+		It("creates and closes the file", func() {
+			fs_ := boshsysfake.NewFakeFileSystem()
+			blobManager := NewBlobManager(fs_, basePath)
+			err := blobManager.Write(blobId, toWrite)
+			Expect(err).ToNot(HaveOccurred())
+			fileStats, err := fs_.FindFileStats(blobPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileStats.Open).To(BeFalse())
+		})
+		It("creates file with correct permissions", func() {
+			fs_ := boshsysfake.NewFakeFileSystem()
+			blobManager := NewBlobManager(fs_, basePath)
+			err := blobManager.Write(blobId, toWrite)
+			fileStats, err := fs_.FindFileStats(blobPath)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(fileStats.FileMode).To(Equal(os.FileMode(0666)))
+			Expect(fileStats.Flags).To(Equal(os.O_WRONLY | os.O_CREATE | os.O_TRUNC))
+		})
+	})
+
 })

internal/github.com/cloudfoundry/bosh-utils/blobstore/blobstore_suite_test.go

@@ -1,8 +1,8 @@
 package blobstore_test
 
 import (
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	"testing"
 )

internal/github.com/cloudfoundry/bosh-utils/blobstore/external_blobstore.go

@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 
 	"errors"
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
-	boshuuid "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/uuid"
+	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
 )
 
 type externalBlobstore struct {

internal/github.com/cloudfoundry/bosh-utils/blobstore/external_blobstore_test.go

@@ -5,13 +5,13 @@ import (
 	"path/filepath"
 	"strings"
 
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
-	boshassert "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/assert"
+	boshassert "github.com/cloudfoundry/bosh-utils/assert"
 	. "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore"
 	fakesys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system/fakes"
-	fakeuuid "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/uuid/fakes"
+	fakeuuid "github.com/cloudfoundry/bosh-utils/uuid/fakes"
 )
 
 var _ = Describe("externalBlobstore", func() {

internal/github.com/cloudfoundry/bosh-utils/blobstore/local_blobstore.go

@@ -4,9 +4,9 @@ import (
 	"os"
 	"path/filepath"
 
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshsys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
-	boshuuid "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/uuid"
+	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
 )
 
 const (

internal/github.com/cloudfoundry/bosh-utils/blobstore/local_blobstore_test.go

@@ -4,12 +4,12 @@ import (
 	"errors"
 	"os"
 
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	. "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore"
 	fakesys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system/fakes"
-	fakeuuid "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/uuid/fakes"
+	fakeuuid "github.com/cloudfoundry/bosh-utils/uuid/fakes"
 )
 
 var _ = Describe("localBlobstore", func() {

internal/github.com/cloudfoundry/bosh-utils/blobstore/provider.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"path/filepath"
 
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
-	boshlog "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/logger"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 	"github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system"
-	boshuuid "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/uuid"
+	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
 )
 
 const (

internal/github.com/cloudfoundry/bosh-utils/blobstore/provider_test.go

@@ -1,13 +1,13 @@
 package blobstore_test
 
 import (
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	. "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore"
-	boshlog "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/logger"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 	fakesys "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/system/fakes"
-	boshuuid "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/uuid"
+	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
 )
 
 var _ = Describe("Provider", func() {

internal/github.com/cloudfoundry/bosh-utils/blobstore/retryable_blobstore.go

@@ -1,8 +1,8 @@
 package blobstore
 
 import (
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
-	boshlog "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/logger"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 )
 
 type retryableBlobstore struct {

internal/github.com/cloudfoundry/bosh-utils/blobstore/retryable_blobstore_test.go

@@ -3,13 +3,13 @@ package blobstore_test
 import (
 	"errors"
 
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	boshblob "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore"
 	fakeblob "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore/fakes"
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
-	boshlog "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/logger"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 )
 
 var _ = Describe("retryableBlobstore", func() {

internal/github.com/cloudfoundry/bosh-utils/blobstore/sha1_verifiable_blobstore.go

@@ -6,7 +6,7 @@ import (
 	"io"
 	"os"
 
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 )
 
 type sha1VerifiableBlobstore struct {

internal/github.com/cloudfoundry/bosh-utils/blobstore/sha1_verifiable_blobstore_test.go

@@ -3,12 +3,12 @@ package blobstore_test
 import (
 	"errors"
 
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-agent/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	boshblob "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore"
 	fakeblob "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/blobstore/fakes"
-	bosherr "github.com/cloudfoundry/bosh-agent/internal/github.com/cloudfoundry/bosh-utils/errors"
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 )
 
 var _ = Describe("sha1VerifiableBlobstore", func() {

internal/github.com/cloudfoundry/bosh-utils/logger/logger_suite_test.go

@@ -1,8 +1,8 @@
 package logger_test
 
 import (
-	. "github.com/cloudfoundry/bosh-utils/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-utils/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	"testing"
 )

internal/github.com/cloudfoundry/bosh-utils/logger/logger_test.go

@@ -3,8 +3,8 @@ package logger_test
 import (
 	"fmt"
 
-	. "github.com/cloudfoundry/bosh-utils/internal/github.com/onsi/ginkgo"
-	. "github.com/cloudfoundry/bosh-utils/internal/github.com/onsi/gomega"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 
 	"bytes"
 	. "github.com/cloudfoundry/bosh-utils/logger"