Update agent to support multiple checksum algorithms

[#134619633](https://www.pivotaltracker.com/story/show/134619633)

agent/action/compile_package.go

@@ -29,11 +29,11 @@ func (a CompilePackageAction) IsLoggable() bool {
 	return true
 }
 
-func (a CompilePackageAction) Run(blobID, sha1, name, version string, deps boshcomp.Dependencies) (val map[string]interface{}, err error) {
+func (a CompilePackageAction) Run(blobID, digestString, name, version string, deps boshcomp.Dependencies) (val map[string]interface{}, err error) {
 	pkg := boshcomp.Package{
 		BlobstoreID: blobID,
 		Name:        name,
-		Sha1:        sha1,
+		Sha1:        digestString,
 		Version:     version,
 	}
 
@@ -50,15 +50,15 @@ func (a CompilePackageAction) Run(blobID, sha1, name, version string, deps boshc
 		})
 	}
 
-	uploadedBlobID, uploadedSha1, err := a.compiler.Compile(pkg, modelsDeps)
+	uploadedBlobID, uploadedDigest, err := a.compiler.Compile(pkg, modelsDeps)
 	if err != nil {
 		err = bosherr.WrapErrorf(err, "Compiling package %s", pkg.Name)
 		return
 	}
 
 	result := map[string]string{
 		"blobstore_id": uploadedBlobID,
-		"sha1":         uploadedSha1,
+		"sha1":         uploadedDigest.String(),
 	}
 
 	val = map[string]interface{}{

agent/action/compile_package_test.go

@@ -10,6 +10,7 @@ import (
 	boshmodels "github.com/cloudfoundry/bosh-agent/agent/applier/models"
 	boshcomp "github.com/cloudfoundry/bosh-agent/agent/compiler"
 	fakecomp "github.com/cloudfoundry/bosh-agent/agent/compiler/fakes"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 )
 
 func getCompileActionArguments() (blobID, sha1, name, version string, deps boshcomp.Dependencies) {
@@ -55,7 +56,7 @@ var _ = Describe("CompilePackageAction", func() {
 	Describe("Run", func() {
 		It("compile package compiles the package abd returns blob id", func() {
 			compiler.CompileBlobID = "my-blob-id"
-			compiler.CompileSha1 = "some sha1"
+			compiler.CompileDigest = boshcrypto.NewDigest("sha1", "some checksum")
 
 			expectedPkg := boshcomp.Package{
 				BlobstoreID: "fake-blobstore-id",
@@ -67,7 +68,7 @@ var _ = Describe("CompilePackageAction", func() {
 			expectedValue := map[string]interface{}{
 				"result": map[string]string{
 					"blobstore_id": "my-blob-id",
-					"sha1":         "some sha1",
+					"sha1":         "some checksum",
 				},
 			}
 

agent/action/fetch_logs.go

@@ -78,7 +78,7 @@ func (a FetchLogsAction) Run(logType string, filters []string) (value map[string
 		_ = a.compressor.CleanUp(tarball)
 	}()
 
-	blobID, _, err := a.blobstore.Create(tarball)
+	blobID, err := a.blobstore.Create(tarball)
 	if err != nil {
 		err = bosherr.WrapError(err, "Create file on blobstore")
 		return

agent/action/sync_dns.go

@@ -12,6 +12,7 @@ import (
 	boshplat "github.com/cloudfoundry/bosh-agent/platform"
 	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
 	boshblob "github.com/cloudfoundry/bosh-utils/blobstore"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 )
@@ -58,7 +59,7 @@ func (a SyncDNS) Cancel() error {
 	return errors.New("not supported")
 }
 
-func (a SyncDNS) Run(blobID, sha1 string, version uint64) (string, error) {
+func (a SyncDNS) Run(blobID, digestString string, version uint64) (string, error) {
 	requestVersionStale, err := a.isLocalStateGreaterThanOrEqual(version)
 	if err != nil {
 		return "", bosherr.WrapError(err, "reading local DNS state")
@@ -68,7 +69,12 @@ func (a SyncDNS) Run(blobID, sha1 string, version uint64) (string, error) {
 		return "synced", nil
 	}
 
-	filePath, err := a.blobstore.Get(blobID, sha1)
+	digest, err := boshcrypto.ParseDigestString(digestString)
+	if err != nil {
+		return "", bosherr.WrapError(err, "Parsing blob digest")
+	}
+
+	filePath, err := a.blobstore.Get(blobID, digest)
 	if err != nil {
 		return "", bosherr.WrapErrorf(err, "getting %s from blobstore", blobID)
 	}

agent/action/sync_dns_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/cloudfoundry/bosh-agent/agent/action/state"
 
 	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 
 	fakelogger "github.com/cloudfoundry/bosh-agent/logger/fakes"
 	fakeplatform "github.com/cloudfoundry/bosh-agent/platform/fakes"
@@ -115,7 +116,7 @@ var _ = Describe("SyncDNS", func() {
 					Expect(response).To(Equal("synced"))
 
 					Expect(fakeBlobstore.GetBlobIDs).To(ContainElement("fake-blobstore-id"))
-					Expect(fakeBlobstore.GetFingerprints).To(ContainElement("fake-fingerprint"))
+					Expect(fakeBlobstore.GetFingerprints).To(ContainElement(boshcrypto.NewDigest("sha1", "fake-fingerprint")))
 
 					Expect(fakeBlobstore.GetError).ToNot(HaveOccurred())
 					Expect(fakeBlobstore.GetFileName).ToNot(Equal(""))

agent/applier/jobs/rendered_job_applier.go

@@ -11,6 +11,7 @@ import (
 	"github.com/cloudfoundry/bosh-agent/agent/applier/packages"
 	boshjobsuper "github.com/cloudfoundry/bosh-agent/jobsupervisor"
 	boshblob "github.com/cloudfoundry/bosh-utils/blobstore"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshcmd "github.com/cloudfoundry/bosh-utils/fileutil"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
@@ -105,7 +106,12 @@ func (s *renderedJobApplier) downloadAndInstall(job models.Job, jobBundle boshbc
 		}
 	}()
 
-	file, err := s.blobstore.Get(job.Source.BlobstoreID, job.Source.Sha1)
+	digest, err := boshcrypto.ParseDigestString(job.Source.Sha1)
+	if err != nil {
+		return bosherr.WrapError(err, "Parsing job blob digest")
+	}
+
+	file, err := s.blobstore.Get(job.Source.BlobstoreID, digest)
 	if err != nil {
 		return bosherr.WrapError(err, "Getting job source from blobstore")
 	}

agent/applier/jobs/rendered_job_applier_test.go

@@ -13,6 +13,7 @@ import (
 	fakepackages "github.com/cloudfoundry/bosh-agent/agent/applier/packages/fakes"
 	fakejobsuper "github.com/cloudfoundry/bosh-agent/jobsupervisor/fakes"
 	fakeblob "github.com/cloudfoundry/bosh-utils/blobstore/fakes"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	fakecmd "github.com/cloudfoundry/bosh-utils/fileutil/fakes"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
@@ -120,7 +121,7 @@ func init() {
 					err := act()
 					Expect(err).ToNot(HaveOccurred())
 					Expect(blobstore.GetBlobIDs[0]).To(Equal("fake-blobstore-id"))
-					Expect(blobstore.GetFingerprints[0]).To(Equal("fake-blob-sha1"))
+					Expect(blobstore.GetFingerprints[0]).To(Equal(boshcrypto.NewDigest("sha1", "fake-blob-sha1")))
 
 					// downloaded file is cleaned up
 					Expect(blobstore.CleanUpFileName).To(Equal("/fake-blobstore-file-name"))
@@ -164,6 +165,35 @@ func init() {
 					Expect(err.Error()).To(ContainSubstring("fake-filesystem-tempdir-error"))
 				})
 
+				It("can process sha1 checksums in the new format", func() {
+					blobstore.GetFileName = "/fake-blobstore-file-name"
+					job.Source.Sha1 = "sha1:fake-blob-sha1"
+
+					err := act()
+					Expect(err).ToNot(HaveOccurred())
+					Expect(blobstore.GetBlobIDs[0]).To(Equal("fake-blobstore-id"))
+					Expect(blobstore.GetFingerprints[0]).To(Equal(boshcrypto.NewDigest("sha1", "fake-blob-sha1")))
+				})
+
+				It("can process sha2 checksums", func() {
+					blobstore.GetFileName = "/fake-blobstore-file-name"
+					job.Source.Sha1 = "sha256:fake-blob-sha256"
+
+					err := act()
+					Expect(err).ToNot(HaveOccurred())
+					Expect(blobstore.GetBlobIDs[0]).To(Equal("fake-blobstore-id"))
+					Expect(blobstore.GetFingerprints[0]).To(Equal(boshcrypto.NewDigest("sha256", "fake-blob-sha256")))
+				})
+
+				It("returns error when given and unsupported fingerprint", func() {
+					blobstore.GetFileName = "/fake-blobstore-file-name"
+					job.Source.Sha1 = "unsupported:checksum"
+
+					err := act()
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(ContainSubstring("Parsing job blob digest"))
+				})
+
 				It("returns error when decompressing job template fails", func() {
 					compressor.DecompressFileToDirErr = errors.New("fake-decompress-error")
 

agent/applier/packages/compiled_package_applier.go

@@ -4,6 +4,7 @@ import (
 	bc "github.com/cloudfoundry/bosh-agent/agent/applier/bundlecollection"
 	models "github.com/cloudfoundry/bosh-agent/agent/applier/models"
 	boshblob "github.com/cloudfoundry/bosh-utils/blobstore"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshcmd "github.com/cloudfoundry/bosh-utils/fileutil"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
@@ -98,7 +99,12 @@ func (s *compiledPackageApplier) downloadAndInstall(pkg models.Package, pkgBundl
 		}
 	}()
 
-	file, err := s.blobstore.Get(pkg.Source.BlobstoreID, pkg.Source.Sha1)
+	digest, err := boshcrypto.ParseDigestString(pkg.Source.Sha1)
+	if err != nil {
+		return bosherr.WrapError(err, "Parsing package blob digest")
+	}
+
+	file, err := s.blobstore.Get(pkg.Source.BlobstoreID, digest)
 	if err != nil {
 		return bosherr.WrapError(err, "Fetching package blob")
 	}

agent/applier/packages/compiled_package_applier_test.go

@@ -11,6 +11,7 @@ import (
 	models "github.com/cloudfoundry/bosh-agent/agent/applier/models"
 	. "github.com/cloudfoundry/bosh-agent/agent/applier/packages"
 	fakeblob "github.com/cloudfoundry/bosh-utils/blobstore/fakes"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	fakecmd "github.com/cloudfoundry/bosh-utils/fileutil/fakes"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
@@ -81,7 +82,7 @@ func init() {
 					err := act()
 					Expect(err).ToNot(HaveOccurred())
 					Expect(blobstore.GetBlobIDs[0]).To(Equal("fake-blobstore-id"))
-					Expect(blobstore.GetFingerprints[0]).To(Equal("fake-blob-sha1"))
+					Expect(blobstore.GetFingerprints[0]).To(Equal(boshcrypto.NewDigest("sha1", "fake-blob-sha1")))
 
 					// downloaded file is cleaned up
 					Expect(blobstore.CleanUpFileName).To(Equal("/fake-blobstore-file-name"))
@@ -128,6 +129,35 @@ func init() {
 
 				It("returns error when decompressing package blob fails", func() {
 					compressor.DecompressFileToDirErr = errors.New("fake-decompress-error")
+					pkg.Source.Sha1 = "unsupported:checksum"
+
+					err := act()
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(ContainSubstring("Parsing package blob digest"))
+				})
+
+				It("can process sha1 checksums in the new format", func() {
+					blobstore.GetFileName = "/fake-blobstore-file-name"
+					pkg.Source.Sha1 = "sha1:fake-blob-sha1"
+
+					err := act()
+					Expect(err).ToNot(HaveOccurred())
+					Expect(blobstore.GetBlobIDs[0]).To(Equal("fake-blobstore-id"))
+					Expect(blobstore.GetFingerprints[0]).To(Equal(boshcrypto.NewDigest("sha1", "fake-blob-sha1")))
+				})
+
+				It("can process sha2 checksums", func() {
+					blobstore.GetFileName = "/fake-blobstore-file-name"
+					pkg.Source.Sha1 = "sha256:fake-blob-sha256"
+
+					err := act()
+					Expect(err).ToNot(HaveOccurred())
+					Expect(blobstore.GetBlobIDs[0]).To(Equal("fake-blobstore-id"))
+					Expect(blobstore.GetFingerprints[0]).To(Equal(boshcrypto.NewDigest("sha256", "fake-blob-sha256")))
+				})
+
+				It("returns error when given and unsupported fingerprint", func() {
+					compressor.DecompressFileToDirErr = errors.New("fake-decompress-error")
 
 					err := act()
 					Expect(err).To(HaveOccurred())

agent/blobstore/cascading_blobstore.go

@@ -2,6 +2,7 @@ package blobstore
 
 import (
 	boshUtilsBlobStore "github.com/cloudfoundry/bosh-utils/blobstore"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 )
 
@@ -24,22 +25,22 @@ func NewCascadingBlobstore(
 	}
 }
 
-func (b cascadingBlobstore) Get(blobID, fingerprint string) (string, error) {
+func (b cascadingBlobstore) Get(blobID string, digest boshcrypto.Digest) (string, error) {
 	blobPath, err := b.blobManager.GetPath(blobID)
 
 	if err == nil {
 		b.logger.Debug(logTag, "Found blob with BlobManager. BlobID: %s", blobID)
 		return blobPath, nil
 	}
 
-	return b.innerBlobstore.Get(blobID, fingerprint)
+	return b.innerBlobstore.Get(blobID, digest)
 }
 
 func (b cascadingBlobstore) CleanUp(fileName string) error {
 	return b.innerBlobstore.CleanUp(fileName)
 }
 
-func (b cascadingBlobstore) Create(fileName string) (string, string, error) {
+func (b cascadingBlobstore) Create(fileName string) (string, error) {
 	return b.innerBlobstore.Create(fileName)
 }
 

agent/blobstore/cascading_blobstore_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/cloudfoundry/bosh-agent/agent/blobstore"
 	boshblob "github.com/cloudfoundry/bosh-utils/blobstore"
 	fakeblob "github.com/cloudfoundry/bosh-utils/blobstore/fakes"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 )
 
@@ -31,7 +32,7 @@ var _ = Describe("cascadingBlobstore", func() {
 			It("returns the path provided by the blobManager", func() {
 				blobManager.GetPathReturns("/path/to-copy/of-blob", nil)
 
-				filename, err := cascadingBlobstore.Get("blobID", "sha1")
+				filename, err := cascadingBlobstore.Get("blobID", boshcrypto.NewDigest("sha1", "fake-checksum"))
 
 				Expect(err).To(BeNil())
 				Expect(filename).To(Equal("/path/to-copy/of-blob"))
@@ -46,16 +47,15 @@ var _ = Describe("cascadingBlobstore", func() {
 		Describe("when blobManager returns an error", func() {
 			It("delegates the action of getting the blob to inner blobstore", func() {
 				blobID := "smurf-4"
-				sha1 := "smurf-4-sha"
+				digest := boshcrypto.NewDigest("sha1", "smurf-4-sha")
 
 				blobManager.GetPathReturns("", errors.New("broken"))
 
 				innerBlobstore.GetFileName = "/smurf-file/path"
 				innerBlobstore.GetError = nil
 				innerBlobstore.CreateBlobID = "createdBlobID"
-				innerBlobstore.CreateFingerprint = "createdSha"
 
-				filename, err := cascadingBlobstore.Get(blobID, sha1)
+				filename, err := cascadingBlobstore.Get(blobID, digest)
 
 				Expect(blobManager.GetPathCallCount()).To(Equal(1))
 				Expect(blobManager.GetPathArgsForCall(0)).To(Equal(blobID))
@@ -65,15 +65,15 @@ var _ = Describe("cascadingBlobstore", func() {
 				Expect(len(innerBlobstore.GetFingerprints)).To(Equal(1))
 
 				Expect(innerBlobstore.GetBlobIDs[0]).To(Equal(blobID))
-				Expect(innerBlobstore.GetFingerprints[0]).To(Equal(sha1))
+				Expect(innerBlobstore.GetFingerprints[0]).To(Equal(digest))
 
 				Expect(filename).To(Equal("/smurf-file/path"))
 			})
 
 			Describe("when inner blobstore returns an error", func() {
 				It("returns that error to the caller", func() {
 					blobID := "smurf-5"
-					sha1 := "smurf-5-sha"
+					sha1 := boshcrypto.NewDigest("sha1", "smurf-5-sha")
 
 					blobManager.GetPathReturns("", errors.New("broken"))
 
@@ -114,14 +114,12 @@ var _ = Describe("cascadingBlobstore", func() {
 		It("delegates the action to the inner blobstore", func() {
 			innerBlobstore.CreateErr = nil
 			innerBlobstore.CreateBlobID = "createBlobId"
-			innerBlobstore.CreateFingerprint = "createFingerprint"
 
-			createdBlobID, createdFingerprint, err := cascadingBlobstore.Create("createdFile")
+			createdBlobID, err := cascadingBlobstore.Create("createdFile")
 
 			Expect(err).To(BeNil())
 
 			Expect(createdBlobID).To(Equal("createBlobId"))
-			Expect(createdFingerprint).To(Equal("createFingerprint"))
 
 			Expect(innerBlobstore.CreateFileNames).ShouldNot(BeEmpty())
 			Expect(len(innerBlobstore.CreateFileNames)).To(Equal(1))
@@ -131,7 +129,7 @@ var _ = Describe("cascadingBlobstore", func() {
 		It("returns an error if the inner blobstore fails to create", func() {
 			innerBlobstore.CreateErr = errors.New("error creating")
 
-			_, _, err := cascadingBlobstore.Create("createdFile")
+			_, err := cascadingBlobstore.Create("createdFile")
 
 			Expect(err).ToNot(BeNil())
 			Expect(err.Error()).To(Equal("error creating"))

agent/compiler/compiler_interface.go

@@ -2,16 +2,17 @@ package compiler
 
 import (
 	boshmodels "github.com/cloudfoundry/bosh-agent/agent/applier/models"
+	boshcrypto "github.com/cloudfoundry/bosh-utils/crypto"
 )
 
 type Compiler interface {
-	Compile(pkg Package, deps []boshmodels.Package) (blobID, sha1 string, err error)
+	Compile(pkg Package, deps []boshmodels.Package) (blobID string, digest boshcrypto.Digest, err error)
 }
 
 type Package struct {
 	BlobstoreID string `json:"blobstore_id"`
 	Name        string
-	Sha1        string
+	Sha1        string `json:"sha1"`
 	Version     string
 }