(COMMON/FILE) Add Zip create/extract support (thrasher-corp#411)

* Added Zip archive support * extended test coverage * extended test coverage for unzip and zip * cleaned up Zip() and moved walk to non-exported method increased test coverage * underscore unused params * remove unneeded var * check against correct error and return after * Errorf * z -> f * fixed wording on tests * handle fine handler close error
mgravitt · Jan 12, 2020 · 12159e3 · 12159e3
1 parent 44ac358
commit 12159e3
Show file tree

Hide file tree

Showing 4 changed files with 286 additions and 0 deletions.
diff --git a/common/file/archive/zip.go b/common/file/archive/zip.go
@@ -0,0 +1,181 @@
+package archive
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	log "github.com/thrasher-corp/gocryptotrader/logger"
+)
+
+const (
+	// ErrUnableToCloseFile message to display when file handler is unable to be closed normally
+	ErrUnableToCloseFile string = "Unable to close file %v %v"
+)
+
+var (
+	addFilesToZip func(z *zip.Writer, src string, isDir bool) error
+)
+
+func init() {
+	addFilesToZip = addFilesToZipWrapper
+}
+
+// UnZip extracts input zip into dest path
+func UnZip(src, dest string) (fileList []string, err error) {
+	z, err := zip.OpenReader(src)
+	if err != nil {
+		return
+	}
+
+	for x := range z.File {
+		fPath := filepath.Join(dest, z.File[x].Name) // nolint:gosec
+		// We ignore gosec linter above because the code below files the file traversal bug when extracting archives
+		if !strings.HasPrefix(fPath, filepath.Clean(dest)+string(os.PathSeparator)) {
+			err = z.Close()
+			if err != nil {
+				log.Errorf(log.Global, ErrUnableToCloseFile, z, err)
+			}
+			err = fmt.Errorf("%s: illegal file path", fPath)
+			return
+		}
+
+		if z.File[x].FileInfo().IsDir() {
+			err = os.MkdirAll(fPath, os.ModePerm)
+			if err != nil {
+				return
+			}
+			continue
+		}
+
+		err = os.MkdirAll(filepath.Dir(fPath), 0770)
+		if err != nil {
+			return
+		}
+
+		var outFile *os.File
+		outFile, err = os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, z.File[x].Mode())
+		if err != nil {
+			return
+		}
+
+		var eFile io.ReadCloser
+		eFile, err = z.File[x].Open()
+		if err != nil {
+			errCls := outFile.Close()
+			if errCls != nil {
+				log.Errorf(log.Global, ErrUnableToCloseFile, outFile, errCls)
+			}
+			return
+		}
+
+		_, errIOCopy := io.Copy(outFile, eFile)
+		if errIOCopy != nil {
+			err = z.Close()
+			if err != nil {
+				log.Errorf(log.Global, ErrUnableToCloseFile, z, err)
+			}
+			err = outFile.Close()
+			if err != nil {
+				log.Errorf(log.Global, ErrUnableToCloseFile, outFile, err)
+			}
+			err = eFile.Close()
+			if err != nil {
+				log.Errorf(log.Global, ErrUnableToCloseFile, eFile, err)
+			}
+			return fileList, errIOCopy
+		}
+		err = outFile.Close()
+		if err != nil {
+			log.Errorf(log.Global, ErrUnableToCloseFile, outFile, err)
+		}
+		err = eFile.Close()
+		if err != nil {
+			log.Errorf(log.Global, ErrUnableToCloseFile, eFile, err)
+		}
+		if err != nil {
+			return
+		}
+
+		fileList = append(fileList, fPath)
+	}
+	return fileList, z.Close()
+}
+
+// Zip archives requested file or folder
+func Zip(src, dest string) error {
+	i, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	z := zip.NewWriter(f)
+	defer z.Close()
+
+	err = addFilesToZip(z, src, i.IsDir())
+	if err != nil {
+		errCls := f.Close()
+		if errCls != nil {
+			log.Errorf(log.Global, "Failed to close file handle, manual deletion required: %v", errCls)
+			return err
+		}
+		errRemove := os.Remove(dest)
+		if errRemove != nil {
+			log.Errorf(log.Global, "Failed to remove archive, manual deletion required: %v", errRemove)
+		}
+		return err
+	}
+	return nil
+}
+
+func addFilesToZipWrapper(z *zip.Writer, src string, isDir bool) error {
+	return filepath.Walk(src, func(path string, i os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		h, err := zip.FileInfoHeader(i)
+		if err != nil {
+			return err
+		}
+
+		if isDir {
+			h.Name = filepath.Join(filepath.Base(src), strings.TrimPrefix(path, src))
+		}
+
+		if i.IsDir() {
+			h.Name += "/"
+		} else {
+			h.Method = zip.Deflate
+		}
+
+		w, err := z.CreateHeader(h)
+		if err != nil {
+			return err
+		}
+
+		if i.IsDir() {
+			return nil
+		}
+
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(w, f)
+		if err != nil {
+			log.Errorf(log.Global, "Failed to Copy data: %v", err)
+		}
+
+		return f.Close()
+	})
+}
diff --git a/common/file/archive/zip_test.go b/common/file/archive/zip_test.go
@@ -0,0 +1,105 @@
+package archive
+
+import (
+	"archive/zip"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var (
+	tempDir string
+)
+
+func TestMain(m *testing.M) {
+	var err error
+	tempDir, err = ioutil.TempDir("", "gct-temp")
+	if err != nil {
+		fmt.Printf("failed to create tempDir: %v", err)
+		os.Exit(1)
+	}
+	t := m.Run()
+	err = os.RemoveAll(tempDir)
+	if err != nil {
+		fmt.Printf("Failed to remove tempDir %v", err)
+	}
+	os.Exit(t)
+}
+
+func TestUnZip(t *testing.T) {
+	zipFile := filepath.Join("..", "..", "..", "testdata", "testdata.zip")
+	files, err := UnZip(zipFile, tempDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 2 {
+		t.Fatalf("expected 2 files to be extracted received: %v ", len(files))
+	}
+
+	zipFile = filepath.Join("..", "..", "..", "testdata", "zip-slip.zip")
+	_, err = UnZip(zipFile, tempDir)
+	if err == nil {
+		t.Fatal("Zip() expected to error due to ZipSlip detection but extracted successfully")
+	}
+
+	zipFile = filepath.Join("..", "..", "..", "testdata", "configtest.json")
+	_, err = UnZip(zipFile, tempDir)
+	if err == nil {
+		t.Fatal("Zip() expected to error due to invalid zipfile")
+	}
+}
+
+func TestZip(t *testing.T) {
+	singleFile := filepath.Join("..", "..", "..", "testdata", "configtest.json")
+	outFile := filepath.Join(tempDir, "out.zip")
+	err := Zip(singleFile, outFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	o, err := UnZip(outFile, tempDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(o) != 1 {
+		t.Fatalf("expected 1 files to be extracted received: %v ", len(o))
+	}
+
+	folder := filepath.Join("..", "..", "..", "testdata", "http_mock")
+	outFolderZip := filepath.Join(tempDir, "out_folder.zip")
+	err = Zip(folder, outFolderZip)
+	if err != nil {
+		t.Fatal(err)
+	}
+	o, err = UnZip(outFolderZip, tempDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if filepath.Base(o[0]) != "binance.json" || filepath.Base(o[4]) != "localbitcoins.json" {
+		t.Fatal("unexpected archive result received")
+	}
+	if len(o) != 6 {
+		t.Fatalf("expected 2 files to be extracted received: %v ", len(o))
+	}
+
+	folder = filepath.Join("..", "..", "..", "testdata", "invalid_file.json")
+	outFolderZip = filepath.Join(tempDir, "invalid.zip")
+	err = Zip(folder, outFolderZip)
+	if err == nil {
+		t.Fatal("expected IsNotExistError on invalid file")
+	}
+
+	addFilesToZip = addFilesToZipTestWrapper
+	folder = filepath.Join("..", "..", "..", "testdata", "http_mock")
+	outFolderZip = filepath.Join(tempDir, "error_zip.zip")
+	err = Zip(folder, outFolderZip)
+	if err == nil {
+		t.Fatal("expected Zip() to fail due to invalid addFilesToZipTestWrapper()")
+	}
+}
+
+func addFilesToZipTestWrapper(_ *zip.Writer, _ string, _ bool) error {
+	return errors.New("error")
+}
diff --git a/testdata/testdata.zip b/testdata/testdata.zip
diff --git a/testdata/zip-slip.zip b/testdata/zip-slip.zip