mirror of
https://github.com/d0zingcat/gocryptotrader.git
synced 2026-05-13 15:09:42 +00:00
(COMMON/FILE) Add Zip create/extract support (#411)
* Added Zip archive support * extended test coverage * extended test coverage for unzip and zip * cleaned up Zip() and moved walk to non-exported method increased test coverage * underscore unused params * remove unneeded var * check against correct error and return after * Errorf * z -> f * fixed wording on tests * handle fine handler close error
This commit is contained in:
Andrew
Adrian Gallagher
181
common/file/archive/zip.go
Normal file
181
common/file/archive/zip.go
Normal file
@@ -0,0 +1,181 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
log "github.com/thrasher-corp/gocryptotrader/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
// ErrUnableToCloseFile message to display when file handler is unable to be closed normally
|
||||
ErrUnableToCloseFile string = "Unable to close file %v %v"
|
||||
)
|
||||
|
||||
var (
|
||||
addFilesToZip func(z *zip.Writer, src string, isDir bool) error
|
||||
)
|
||||
|
||||
func init() {
|
||||
addFilesToZip = addFilesToZipWrapper
|
||||
}
|
||||
|
||||
// UnZip extracts input zip into dest path
|
||||
func UnZip(src, dest string) (fileList []string, err error) {
|
||||
z, err := zip.OpenReader(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for x := range z.File {
|
||||
fPath := filepath.Join(dest, z.File[x].Name) // nolint:gosec
|
||||
// We ignore gosec linter above because the code below files the file traversal bug when extracting archives
|
||||
if !strings.HasPrefix(fPath, filepath.Clean(dest)+string(os.PathSeparator)) {
|
||||
err = z.Close()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, z, err)
|
||||
}
|
||||
err = fmt.Errorf("%s: illegal file path", fPath)
|
||||
return
|
||||
}
|
||||
|
||||
if z.File[x].FileInfo().IsDir() {
|
||||
err = os.MkdirAll(fPath, os.ModePerm)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
err = os.MkdirAll(filepath.Dir(fPath), 0770)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var outFile *os.File
|
||||
outFile, err = os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, z.File[x].Mode())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var eFile io.ReadCloser
|
||||
eFile, err = z.File[x].Open()
|
||||
if err != nil {
|
||||
errCls := outFile.Close()
|
||||
if errCls != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, outFile, errCls)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
_, errIOCopy := io.Copy(outFile, eFile)
|
||||
if errIOCopy != nil {
|
||||
err = z.Close()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, z, err)
|
||||
}
|
||||
err = outFile.Close()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, outFile, err)
|
||||
}
|
||||
err = eFile.Close()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, eFile, err)
|
||||
}
|
||||
return fileList, errIOCopy
|
||||
}
|
||||
err = outFile.Close()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, outFile, err)
|
||||
}
|
||||
err = eFile.Close()
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, ErrUnableToCloseFile, eFile, err)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
fileList = append(fileList, fPath)
|
||||
}
|
||||
return fileList, z.Close()
|
||||
}
|
||||
|
||||
// Zip archives requested file or folder
|
||||
func Zip(src, dest string) error {
|
||||
i, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
z := zip.NewWriter(f)
|
||||
defer z.Close()
|
||||
|
||||
err = addFilesToZip(z, src, i.IsDir())
|
||||
if err != nil {
|
||||
errCls := f.Close()
|
||||
if errCls != nil {
|
||||
log.Errorf(log.Global, "Failed to close file handle, manual deletion required: %v", errCls)
|
||||
return err
|
||||
}
|
||||
errRemove := os.Remove(dest)
|
||||
if errRemove != nil {
|
||||
log.Errorf(log.Global, "Failed to remove archive, manual deletion required: %v", errRemove)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addFilesToZipWrapper(z *zip.Writer, src string, isDir bool) error {
|
||||
return filepath.Walk(src, func(path string, i os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
h, err := zip.FileInfoHeader(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isDir {
|
||||
h.Name = filepath.Join(filepath.Base(src), strings.TrimPrefix(path, src))
|
||||
}
|
||||
|
||||
if i.IsDir() {
|
||||
h.Name += "/"
|
||||
} else {
|
||||
h.Method = zip.Deflate
|
||||
}
|
||||
|
||||
w, err := z.CreateHeader(h)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if i.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(w, f)
|
||||
if err != nil {
|
||||
log.Errorf(log.Global, "Failed to Copy data: %v", err)
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
})
|
||||
}
|
||||
105
common/file/archive/zip_test.go
Normal file
105
common/file/archive/zip_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
tempDir string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
var err error
|
||||
tempDir, err = ioutil.TempDir("", "gct-temp")
|
||||
if err != nil {
|
||||
fmt.Printf("failed to create tempDir: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
t := m.Run()
|
||||
err = os.RemoveAll(tempDir)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to remove tempDir %v", err)
|
||||
}
|
||||
os.Exit(t)
|
||||
}
|
||||
|
||||
func TestUnZip(t *testing.T) {
|
||||
zipFile := filepath.Join("..", "..", "..", "testdata", "testdata.zip")
|
||||
files, err := UnZip(zipFile, tempDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 2 {
|
||||
t.Fatalf("expected 2 files to be extracted received: %v ", len(files))
|
||||
}
|
||||
|
||||
zipFile = filepath.Join("..", "..", "..", "testdata", "zip-slip.zip")
|
||||
_, err = UnZip(zipFile, tempDir)
|
||||
if err == nil {
|
||||
t.Fatal("Zip() expected to error due to ZipSlip detection but extracted successfully")
|
||||
}
|
||||
|
||||
zipFile = filepath.Join("..", "..", "..", "testdata", "configtest.json")
|
||||
_, err = UnZip(zipFile, tempDir)
|
||||
if err == nil {
|
||||
t.Fatal("Zip() expected to error due to invalid zipfile")
|
||||
}
|
||||
}
|
||||
|
||||
func TestZip(t *testing.T) {
|
||||
singleFile := filepath.Join("..", "..", "..", "testdata", "configtest.json")
|
||||
outFile := filepath.Join(tempDir, "out.zip")
|
||||
err := Zip(singleFile, outFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o, err := UnZip(outFile, tempDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(o) != 1 {
|
||||
t.Fatalf("expected 1 files to be extracted received: %v ", len(o))
|
||||
}
|
||||
|
||||
folder := filepath.Join("..", "..", "..", "testdata", "http_mock")
|
||||
outFolderZip := filepath.Join(tempDir, "out_folder.zip")
|
||||
err = Zip(folder, outFolderZip)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o, err = UnZip(outFolderZip, tempDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(o[0]) != "binance.json" || filepath.Base(o[4]) != "localbitcoins.json" {
|
||||
t.Fatal("unexpected archive result received")
|
||||
}
|
||||
if len(o) != 6 {
|
||||
t.Fatalf("expected 2 files to be extracted received: %v ", len(o))
|
||||
}
|
||||
|
||||
folder = filepath.Join("..", "..", "..", "testdata", "invalid_file.json")
|
||||
outFolderZip = filepath.Join(tempDir, "invalid.zip")
|
||||
err = Zip(folder, outFolderZip)
|
||||
if err == nil {
|
||||
t.Fatal("expected IsNotExistError on invalid file")
|
||||
}
|
||||
|
||||
addFilesToZip = addFilesToZipTestWrapper
|
||||
folder = filepath.Join("..", "..", "..", "testdata", "http_mock")
|
||||
outFolderZip = filepath.Join(tempDir, "error_zip.zip")
|
||||
err = Zip(folder, outFolderZip)
|
||||
if err == nil {
|
||||
t.Fatal("expected Zip() to fail due to invalid addFilesToZipTestWrapper()")
|
||||
}
|
||||
}
|
||||
|
||||
func addFilesToZipTestWrapper(_ *zip.Writer, _ string, _ bool) error {
|
||||
return errors.New("error")
|
||||
}
|
||||
Reference in New Issue
Block a user