Refactor checksum

auth/auth.go

@@ -16,9 +16,9 @@ import (
 const CONFIGFILE = ".dockercfg"
 
 // Only used for user auth + account creation
-const INDEXSERVER = "https://index.docker.io/v1/"
+//const INDEXSERVER = "https://index.docker.io/v1/"
 
-//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/"
+const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/"
 
 var (
 	ErrConfigFileMissing = errors.New("The Auth config file is missing")

graph.go

@@ -1,27 +1,21 @@
 package docker
 
 import (
-	"encoding/json"
 	"fmt"
-	"github.com/dotcloud/docker/registry"
 	"github.com/dotcloud/docker/utils"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 )
 
 // A Graph is a store for versioned filesystem images and the relationship between them.
 type Graph struct {
-	Root         string
-	idIndex      *utils.TruncIndex
-	checksumLock map[string]*sync.Mutex
-	lockSumFile  *sync.Mutex
-	lockSumMap   *sync.Mutex
+	Root    string
+	idIndex *utils.TruncIndex
 }
 
 // NewGraph instantiates a new graph at the given root path in the filesystem.
@@ -36,11 +30,8 @@ func NewGraph(root string) (*Graph, error) {
 		return nil, err
 	}
 	graph := &Graph{
-		Root:         abspath,
-		idIndex:      utils.NewTruncIndex(),
-		checksumLock: make(map[string]*sync.Mutex),
-		lockSumFile:  &sync.Mutex{},
-		lockSumMap:   &sync.Mutex{},
+		Root:    abspath,
+		idIndex: utils.NewTruncIndex(),
 	}
 	if err := graph.restore(); err != nil {
 		return nil, err
@@ -99,11 +90,6 @@ func (graph *Graph) Get(name string) (*Image, error) {
 			return nil, err
 		}
 	}
-	graph.lockSumMap.Lock()
-	defer graph.lockSumMap.Unlock()
-	if _, exists := graph.checksumLock[img.ID]; !exists {
-		graph.checksumLock[img.ID] = &sync.Mutex{}
-	}
 	return img, nil
 }
 
@@ -126,7 +112,6 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut
 	if err := graph.Register(layerData, layerData != nil, img); err != nil {
 		return nil, err
 	}
-	go img.Checksum()
 	return img, nil
 }
 
@@ -154,7 +139,6 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error {
 	}
 	img.graph = graph
 	graph.idIndex.Add(img.ID)
-	graph.checksumLock[img.ID] = &sync.Mutex{}
 	return nil
 }
 
@@ -311,40 +295,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) {
 func (graph *Graph) imageRoot(id string) string {
 	return path.Join(graph.Root, id)
 }
-
-func (graph *Graph) getStoredChecksums() (map[string]string, error) {
-	checksums := make(map[string]string)
-	// FIXME: Store the checksum in memory
-
-	if checksumDict, err := ioutil.ReadFile(path.Join(graph.Root, "checksums")); err == nil {
-		if err := json.Unmarshal(checksumDict, &checksums); err != nil {
-			return nil, err
-		}
-	}
-	return checksums, nil
-}
-
-func (graph *Graph) storeChecksums(checksums map[string]string) error {
-	checksumJSON, err := json.Marshal(checksums)
-	if err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(path.Join(graph.Root, "checksums"), checksumJSON, 0600); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error {
-	graph.lockSumFile.Lock()
-	defer graph.lockSumFile.Unlock()
-
-	localChecksums, err := graph.getStoredChecksums()
-	if err != nil {
-		return err
-	}
-	for id, elem := range newChecksums {
-		localChecksums[id] = elem.Checksum
-	}
-	return graph.storeChecksums(localChecksums)
-}

image.go

@@ -2,7 +2,6 @@ package docker
 
 import (
 	"crypto/rand"
-	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
@@ -72,26 +71,6 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error {
 		return err
 	}
 
-	if store {
-		layerArchive := layerArchivePath(root)
-		file, err := os.OpenFile(layerArchive, os.O_WRONLY|os.O_CREATE, 0600)
-		if err != nil {
-			return err
-		}
-		// FIXME: Retrieve the image layer size from here?
-		if _, err := io.Copy(file, layerData); err != nil {
-			return err
-		}
-		// FIXME: Don't close/open, read/write instead of Copy
-		file.Close()
-
-		file, err = os.Open(layerArchive)
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-		layerData = file
-	}
 	// If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
 		start := time.Now()
@@ -128,10 +107,6 @@ func layerPath(root string) string {
 	return path.Join(root, "layer")
 }
 
-func layerArchivePath(root string) string {
-	return path.Join(root, "layer.tar.xz")
-}
-
 func jsonPath(root string) string {
 	return path.Join(root, "json")
 }
@@ -308,80 +283,6 @@ func (img *Image) layer() (string, error) {
 	return layerPath(root), nil
 }
 
-func (img *Image) Checksum() (string, error) {
-	img.graph.checksumLock[img.ID].Lock()
-	defer img.graph.checksumLock[img.ID].Unlock()
-
-	root, err := img.root()
-	if err != nil {
-		return "", err
-	}
-
-	checksums, err := img.graph.getStoredChecksums()
-	if err != nil {
-		return "", err
-	}
-	if checksum, ok := checksums[img.ID]; ok {
-		return checksum, nil
-	}
-
-	layer, err := img.layer()
-	if err != nil {
-		return "", err
-	}
-	jsonData, err := ioutil.ReadFile(jsonPath(root))
-	if err != nil {
-		return "", err
-	}
-
-	var layerData io.Reader
-
-	if file, err := os.Open(layerArchivePath(root)); err != nil {
-		if os.IsNotExist(err) {
-			layerData, err = Tar(layer, Xz)
-			if err != nil {
-				return "", err
-			}
-		} else {
-			return "", err
-		}
-	} else {
-		defer file.Close()
-		layerData = file
-	}
-
-	h := sha256.New()
-	if _, err := h.Write(jsonData); err != nil {
-		return "", err
-	}
-	if _, err := h.Write([]byte("\n")); err != nil {
-		return "", err
-	}
-
-	if _, err := io.Copy(h, layerData); err != nil {
-		return "", err
-	}
-	hash := "sha256:" + hex.EncodeToString(h.Sum(nil))
-
-	// Reload the json file to make sure not to overwrite faster sums
-	img.graph.lockSumFile.Lock()
-	defer img.graph.lockSumFile.Unlock()
-
-	checksums, err = img.graph.getStoredChecksums()
-	if err != nil {
-		return "", err
-	}
-
-	checksums[img.ID] = hash
-
-	// Dump the checksums to disc
-	if err := img.graph.storeChecksums(checksums); err != nil {
-		return hash, err
-	}
-
-	return hash, nil
-}
-
 func (img *Image) getParentsSize(size int64) int64 {
 	parentImage, err := img.GetParent()
 	if err != nil || parentImage == nil {

registry/registry.go

@@ -330,16 +330,52 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e
 	}, nil
 }
 
+func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error {
+
+	utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum")
+
+	req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
+	req.Header.Set("X-Docker-Checksum", imgData.Checksum)
+
+	res, err := doWithCookies(r.client, req)
+	if err != nil {
+		return fmt.Errorf("Failed to upload metadata: %s", err)
+	}
+	defer res.Body.Close()
+	if len(res.Cookies()) > 0 {
+		r.client.Jar.SetCookies(req.URL, res.Cookies())
+	}
+	if res.StatusCode != 200 {
+		errBody, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err)
+		}
+		var jsonBody map[string]string
+		if err := json.Unmarshal(errBody, &jsonBody); err != nil {
+			errBody = []byte(err.Error())
+		} else if jsonBody["error"] == "Image already exists" {
+			return ErrAlreadyExists
+		}
+		return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody)
+	}
+	return nil
+}
+
 // Push a local image to the registry
 func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error {
-	// FIXME: try json with UTF8
+
+	utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json")
+
 	req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw))
 	if err != nil {
 		return err
 	}
 	req.Header.Add("Content-type", "application/json")
 	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
-	req.Header.Set("X-Docker-Checksum", imgData.Checksum)
 	r.setUserAgent(req)
 
 	utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum)
@@ -364,29 +400,33 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis
 	return nil
 }
 
-func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error {
-	req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer)
+func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) (checksum string, err error) {
+
+	utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer")
+
+	tarsumLayer := &utils.TarSum{Reader: layer}
+	req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer)
 	if err != nil {
-		return err
+		return "", err
 	}
 	req.ContentLength = -1
 	req.TransferEncoding = []string{"chunked"}
 	req.Header.Set("Authorization", "Token "+strings.Join(token, ","))
 	r.setUserAgent(req)
 	res, err := doWithCookies(r.client, req)
 	if err != nil {
-		return fmt.Errorf("Failed to upload layer: %s", err)
+		return "", fmt.Errorf("Failed to upload layer: %s", err)
 	}
 	defer res.Body.Close()
 
 	if res.StatusCode != 200 {
 		errBody, err := ioutil.ReadAll(res.Body)
 		if err != nil {
-			return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err)
+			return "", fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err)
 		}
-		return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody)
+		return "", fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody)
 	}
-	return nil
+	return tarsumLayer.Sum(), nil
 }
 
 func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) {