xl: Moved to minio/minio - fixes minio#1112

.gitignore

@@ -11,3 +11,5 @@ site/
 **/access.log
 ui-assets.go
 ui-assets.asc
+build
+isa-l

.travis.yml

@@ -1,10 +1,21 @@
-sudo: false
+sudo: required
+dist: trusty
 language: go
 
 os:
 - linux
 - osx
 
+before_install:
+  - git clone https://github.com/yasm/yasm
+  - cd yasm
+  - git checkout v1.3.0
+  - "./autogen.sh"
+  - "./configure"
+  - make
+  - export PATH=$PATH:`pwd`
+  - cd ..
+
 osx_image: xcode7.2
 
 env:
@@ -16,7 +27,6 @@ script:
 - make test GOFLAGS="-race"
 
 go:
-- 1.5.2
 - 1.5.3
 
 notifications:

Makefile

@@ -48,6 +48,15 @@ fmt:
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
 
+## Configure Intel library.
+isa-l:
+	@echo "Configuring $@:"
+	@git clone -q https://github.com/minio/isa-l.git
+	@mkdir -p build
+	@cd build; $(PWD)/isa-l/configure --prefix $(PWD)/build/lib --sysconfdir $(PWD)/build/lib --includedir $(PWD)/build/lib --libdir $(PWD)/build/lib >/dev/null
+	@make -C build >/dev/null
+	@make -C build install >/dev/null
+
 lint:
 	@echo "Running $@:"
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
@@ -58,8 +67,8 @@ cyclo:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
 
-build: getdeps verifiers $(UI_ASSETS)
-	@echo "Installing minio:"
+build: getdeps verifiers $(UI_ASSETS) isa-l
+	@GO15VENDOREXPERIMENT=1 go generate ./...
 
 deadcode:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
@@ -69,12 +78,13 @@ spelling:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
 
 test: build
-	@echo "Running all testing:"
-	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
-	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@echo "Running all minio testing:"
+	@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 gomake-all: build
-	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+	@echo "Installing minio:"
+	@CGO_CPPFLAGS="-I$(PWD)/build/lib" CGO_LDFLAGS="$(PWD)/build/lib/libisal.a" GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
@@ -100,9 +110,9 @@ release:
 
 clean:
 	@echo "Cleaning up all the generated files:"
-	@rm -fv cover.out
-	@rm -fv minio
-	@rm -fv minio.test
-	@rm -fv pkg/fs/fs.test
+	@rm -fv minio minio.test cover.out
+	@find . -name '*.test' | xargs rm -fv
 	@rm -fv ui-assets.go
 	@rm -fv ui-assets.asc
+	@rm -rf isa-l
+	@rm -rf build

accesslog-handler.go

@@ -23,7 +23,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 type accessLogHandler struct {

api-auth-utils.go

@@ -21,7 +21,7 @@ import (
 	"encoding/base64"
 	"regexp"
 
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 const (

api-signature.go

@@ -26,8 +26,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-xl/pkg/probe"
-	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
+	v4 "github.com/minio/minio/pkg/signature"
 )
 
 const (
@@ -125,7 +125,7 @@ func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) {
 }
 
 // initSignatureV4 initializing signature verification.
-func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
+func initSignatureV4(req *http.Request) (*v4.Signature, *probe.Error) {
 	// strip auth from authorization header.
 	authHeaderValue := req.Header.Get("Authorization")
 
@@ -156,7 +156,7 @@ func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
 		return nil, err.Trace(authHeaderValue)
 	}
 	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
+		signature := &v4.Signature{
 			AccessKeyID:     config.Credentials.AccessKeyID,
 			SecretAccessKey: config.Credentials.SecretAccessKey,
 			Region:          region,
@@ -203,7 +203,7 @@ func applyPolicy(formValues map[string]string) *probe.Error {
 	if e != nil {
 		return probe.NewError(e)
 	}
-	postPolicyForm, err := fs.ParsePostPolicyForm(string(policyBytes))
+	postPolicyForm, err := v4.ParsePostPolicyForm(string(policyBytes))
 	if err != nil {
 		return err.Trace()
 	}
@@ -244,7 +244,7 @@ func applyPolicy(formValues map[string]string) *probe.Error {
 }
 
 // initPostPresignedPolicyV4 initializing post policy signature verification
-func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *probe.Error) {
+func initPostPresignedPolicyV4(formValues map[string]string) (*v4.Signature, *probe.Error) {
 	credentialElements := strings.Split(strings.TrimSpace(formValues["X-Amz-Credential"]), "/")
 	if len(credentialElements) != 5 {
 		return nil, probe.NewError(errCredentialTagMalformed)
@@ -259,7 +259,7 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr
 	}
 	region := credentialElements[2]
 	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
+		signature := &v4.Signature{
 			AccessKeyID:     config.Credentials.AccessKeyID,
 			SecretAccessKey: config.Credentials.SecretAccessKey,
 			Region:          region,
@@ -272,7 +272,7 @@ func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *pr
 }
 
 // initPresignedSignatureV4 initializing presigned signature verification
-func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
+func initPresignedSignatureV4(req *http.Request) (*v4.Signature, *probe.Error) {
 	credentialElements := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-Credential")), "/")
 	if len(credentialElements) != 5 {
 		return nil, probe.NewError(errCredentialTagMalformed)
@@ -289,7 +289,7 @@ func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
 	signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";")
 	signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature"))
 	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
+		signature := &v4.Signature{
 			AccessKeyID:     config.Credentials.AccessKeyID,
 			SecretAccessKey: config.Credentials.SecretAccessKey,
 			Region:          region,

appveyor.yml

@@ -1,30 +1,46 @@
-# version format
-version: "{build}"
-
 # Operating system (build VM template)
-os: Windows Server 2012 R2
+os: Visual Studio 2015
+
+platform: x64
 
 clone_folder: c:\gopath\src\github.com\minio\minio
 
 # environment variables
 environment:
   GOPATH: c:\gopath
+  GO_EXTLINK_ENABLED: 0
   GO15VENDOREXPERIMENT: 1
   UI_ASSETS: ui-assets.go
   UI_ASSETS_ARMOR: ui-assets.asc
 
 # scripts that run after cloning repository
 install:
+  - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
+  - curl -fsSL -o c:\go\bin\yasm.exe http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
+  - curl -fsSL -o mingw-w64.7z http://downloads.sourceforge.net/project/mingw-w64-dgn/mingw-w64/mingw-w64-bin-x86_64-20151206.7z
+  - 7z x -oC:\ mingw-w64.7z > NUL
+  - set PATH=C:\mingw64\bin;%PATH%
+  - x86_64-w64-mingw32-gcc --version  
   - go version
   - go env
+  - git clone https://github.com/minio/isa-l
+  - cd isa-l
+  - make -f Makefile.unx arch=mingw
+  - mv include isa-l
+  - set CGO_CPPFLAGS=-Ic:/gopath/src/github.com/minio/minio/isa-l
+  - set CGO_LDFLAGS=c:/gopath/src/github.com/minio/minio/isa-l/isa-l.a
+  - set CC=x86_64-w64-mingw32-gcc
+  - set CXX=x86_64-w64-mingw32-g++
+  - cd %GOPATH%\src\github.com\minio\minio
 
 # to run your custom scripts instead of automatic MSBuild
 build_script:
-  - curl -o ui-assets.go -L https://dl.minio.io/assets/server/ui/%UI_ASSETS%
-  - curl -o ui-assets.asc -L https://dl.minio.io/assets/server/ui/%UI_ASSETS_ARMOR%
+  - curl -fsSL -o ui-assets.go https://dl.minio.io/assets/server/ui/%UI_ASSETS%
+  - curl -fsSL -o ui-assets.asc  https://dl.minio.io/assets/server/ui/%UI_ASSETS_ARMOR%
   - gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
   - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%
+  - go generate ./...
   - go test .
   - go test -race .
   - go test github.com/minio/minio/pkg...

bucket-handlers.go

@@ -22,9 +22,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-xl/pkg/crypto/sha256"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/crypto/sha256"
 	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
+	v4 "github.com/minio/minio/pkg/signature"
 )
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -204,7 +205,7 @@ func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Req
 		return
 	}
 
-	var signature *fs.Signature
+	var signature *v4.Signature
 	// Init signature V4 verification
 	if isRequestSignatureV4(req) {
 		var err *probe.Error
@@ -340,7 +341,7 @@ func (api CloudStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, req *h
 			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
 		case fs.BadDigest:
 			writeErrorResponse(w, req, BadDigest, req.URL.Path)
-		case fs.SignatureDoesNotMatch:
+		case v4.SigDoesNotMatch:
 			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
 		case fs.IncompleteBody:
 			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)

config-logger-main.go

@@ -20,7 +20,7 @@ import (
 	"runtime"
 
 	"github.com/minio/cli"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 // Configure logger

damerau-levenshtein.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Client (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"math"
+)
+
+// Returns the minimum value of a slice of integers
+func minimum(integers []int) (minVal int) {
+	minVal = math.MaxInt32
+	for _, v := range integers {
+		if v < minVal {
+			minVal = v
+		}
+	}
+	return
+}
+
+// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
+// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
+func DamerauLevenshteinDistance(a string, b string) int {
+	d := make([][]int, len(a)+1)
+	for i := 1; i <= len(a)+1; i++ {
+		d[i-1] = make([]int, len(b)+1)
+	}
+	for i := 0; i <= len(a); i++ {
+		d[i][0] = i
+	}
+	for j := 0; j <= len(b); j++ {
+		d[0][j] = j
+	}
+	for i := 1; i <= len(a); i++ {
+		for j := 1; j <= len(b); j++ {
+			cost := 0
+			if a[i-1] == b[j-1] {
+				cost = 0
+			} else {
+				cost = 1
+			}
+			d[i][j] = minimum([]int{
+				d[i-1][j] + 1,
+				d[i][j-1] + 1,
+				d[i-1][j-1] + cost,
+			})
+			if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
+				d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
+			}
+		}
+	}
+	return d[len(a)][len(b)]
+}

httprange.go

@@ -22,8 +22,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio-xl/pkg/probe"
 	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
 )
 
 const (

jwt.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 	"golang.org/x/crypto/bcrypt"
 )
 

logger-file-hook.go

@@ -21,7 +21,7 @@ import (
 	"os"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 type localFile struct {

logger-mongo-hook.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 )

logger-syslog-hook_nix.go

@@ -23,7 +23,7 @@ import (
 	"log/syslog"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 // syslogHook to send logs via syslog.