diff --git a/common/file/archive/zip.go b/common/file/archive/zip.go new file mode 100644 index 00000000..33652c37 --- /dev/null +++ b/common/file/archive/zip.go @@ -0,0 +1,181 @@ +package archive + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + log "github.com/thrasher-corp/gocryptotrader/logger" +) + +const ( + // ErrUnableToCloseFile message to display when file handler is unable to be closed normally + ErrUnableToCloseFile string = "Unable to close file %v %v" +) + +var ( + addFilesToZip func(z *zip.Writer, src string, isDir bool) error +) + +func init() { + addFilesToZip = addFilesToZipWrapper +} + +// UnZip extracts input zip into dest path +func UnZip(src, dest string) (fileList []string, err error) { + z, err := zip.OpenReader(src) + if err != nil { + return + } + + for x := range z.File { + fPath := filepath.Join(dest, z.File[x].Name) // nolint:gosec + // We ignore gosec linter above because the code below files the file traversal bug when extracting archives + if !strings.HasPrefix(fPath, filepath.Clean(dest)+string(os.PathSeparator)) { + err = z.Close() + if err != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, z, err) + } + err = fmt.Errorf("%s: illegal file path", fPath) + return + } + + if z.File[x].FileInfo().IsDir() { + err = os.MkdirAll(fPath, os.ModePerm) + if err != nil { + return + } + continue + } + + err = os.MkdirAll(filepath.Dir(fPath), 0770) + if err != nil { + return + } + + var outFile *os.File + outFile, err = os.OpenFile(fPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, z.File[x].Mode()) + if err != nil { + return + } + + var eFile io.ReadCloser + eFile, err = z.File[x].Open() + if err != nil { + errCls := outFile.Close() + if errCls != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, outFile, errCls) + } + return + } + + _, errIOCopy := io.Copy(outFile, eFile) + if errIOCopy != nil { + err = z.Close() + if err != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, z, err) + } + err = outFile.Close() + if err != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, outFile, err) + } + err = eFile.Close() + if err != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, eFile, err) + } + return fileList, errIOCopy + } + err = outFile.Close() + if err != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, outFile, err) + } + err = eFile.Close() + if err != nil { + log.Errorf(log.Global, ErrUnableToCloseFile, eFile, err) + } + if err != nil { + return + } + + fileList = append(fileList, fPath) + } + return fileList, z.Close() +} + +// Zip archives requested file or folder +func Zip(src, dest string) error { + i, err := os.Stat(src) + if err != nil { + return err + } + + f, err := os.Create(dest) + if err != nil { + return err + } + defer f.Close() + + z := zip.NewWriter(f) + defer z.Close() + + err = addFilesToZip(z, src, i.IsDir()) + if err != nil { + errCls := f.Close() + if errCls != nil { + log.Errorf(log.Global, "Failed to close file handle, manual deletion required: %v", errCls) + return err + } + errRemove := os.Remove(dest) + if errRemove != nil { + log.Errorf(log.Global, "Failed to remove archive, manual deletion required: %v", errRemove) + } + return err + } + return nil +} + +func addFilesToZipWrapper(z *zip.Writer, src string, isDir bool) error { + return filepath.Walk(src, func(path string, i os.FileInfo, err error) error { + if err != nil { + return err + } + + h, err := zip.FileInfoHeader(i) + if err != nil { + return err + } + + if isDir { + h.Name = filepath.Join(filepath.Base(src), strings.TrimPrefix(path, src)) + } + + if i.IsDir() { + h.Name += "/" + } else { + h.Method = zip.Deflate + } + + w, err := z.CreateHeader(h) + if err != nil { + return err + } + + if i.IsDir() { + return nil + } + + f, err := os.Open(path) + if err != nil { + return err + } + _, err = io.Copy(w, f) + if err != nil { + log.Errorf(log.Global, "Failed to Copy data: %v", err) + } + + return f.Close() + }) +} diff --git a/common/file/archive/zip_test.go b/common/file/archive/zip_test.go new file mode 100644 index 00000000..b0ed10f0 --- /dev/null +++ b/common/file/archive/zip_test.go @@ -0,0 +1,105 @@ +package archive + +import ( + "archive/zip" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +var ( + tempDir string +) + +func TestMain(m *testing.M) { + var err error + tempDir, err = ioutil.TempDir("", "gct-temp") + if err != nil { + fmt.Printf("failed to create tempDir: %v", err) + os.Exit(1) + } + t := m.Run() + err = os.RemoveAll(tempDir) + if err != nil { + fmt.Printf("Failed to remove tempDir %v", err) + } + os.Exit(t) +} + +func TestUnZip(t *testing.T) { + zipFile := filepath.Join("..", "..", "..", "testdata", "testdata.zip") + files, err := UnZip(zipFile, tempDir) + if err != nil { + t.Fatal(err) + } + if len(files) != 2 { + t.Fatalf("expected 2 files to be extracted received: %v ", len(files)) + } + + zipFile = filepath.Join("..", "..", "..", "testdata", "zip-slip.zip") + _, err = UnZip(zipFile, tempDir) + if err == nil { + t.Fatal("Zip() expected to error due to ZipSlip detection but extracted successfully") + } + + zipFile = filepath.Join("..", "..", "..", "testdata", "configtest.json") + _, err = UnZip(zipFile, tempDir) + if err == nil { + t.Fatal("Zip() expected to error due to invalid zipfile") + } +} + +func TestZip(t *testing.T) { + singleFile := filepath.Join("..", "..", "..", "testdata", "configtest.json") + outFile := filepath.Join(tempDir, "out.zip") + err := Zip(singleFile, outFile) + if err != nil { + t.Fatal(err) + } + o, err := UnZip(outFile, tempDir) + if err != nil { + t.Fatal(err) + } + if len(o) != 1 { + t.Fatalf("expected 1 files to be extracted received: %v ", len(o)) + } + + folder := filepath.Join("..", "..", "..", "testdata", "http_mock") + outFolderZip := filepath.Join(tempDir, "out_folder.zip") + err = Zip(folder, outFolderZip) + if err != nil { + t.Fatal(err) + } + o, err = UnZip(outFolderZip, tempDir) + if err != nil { + t.Fatal(err) + } + if filepath.Base(o[0]) != "binance.json" || filepath.Base(o[4]) != "localbitcoins.json" { + t.Fatal("unexpected archive result received") + } + if len(o) != 6 { + t.Fatalf("expected 2 files to be extracted received: %v ", len(o)) + } + + folder = filepath.Join("..", "..", "..", "testdata", "invalid_file.json") + outFolderZip = filepath.Join(tempDir, "invalid.zip") + err = Zip(folder, outFolderZip) + if err == nil { + t.Fatal("expected IsNotExistError on invalid file") + } + + addFilesToZip = addFilesToZipTestWrapper + folder = filepath.Join("..", "..", "..", "testdata", "http_mock") + outFolderZip = filepath.Join(tempDir, "error_zip.zip") + err = Zip(folder, outFolderZip) + if err == nil { + t.Fatal("expected Zip() to fail due to invalid addFilesToZipTestWrapper()") + } +} + +func addFilesToZipTestWrapper(_ *zip.Writer, _ string, _ bool) error { + return errors.New("error") +} diff --git a/testdata/testdata.zip b/testdata/testdata.zip new file mode 100644 index 00000000..34855708 Binary files /dev/null and b/testdata/testdata.zip differ diff --git a/testdata/zip-slip.zip b/testdata/zip-slip.zip new file mode 100644 index 00000000..38b3f499 Binary files /dev/null and b/testdata/zip-slip.zip differ