From f0a03d5bd7ed6cde87945482bbe676f6ca0ba1fe Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 28 Jun 2024 01:43:50 +0200 Subject: [PATCH] add zstd compression update user agent update deps fix latent bug (leftover files) --- downloader/downloader.go | 2 +- go.mod | 13 +++-- go.sum | 28 ++++----- manager/s3_storage.go | 47 ++++++++++++++- util/archive.go | 15 ++--- util/compression.go | 85 +++++++++++++++++++++++++++ util/compression_test.go | 121 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 281 insertions(+), 30 deletions(-) create mode 100644 util/compression.go create mode 100644 util/compression_test.go diff --git a/downloader/downloader.go b/downloader/downloader.go index de42231..f823333 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -88,7 +88,7 @@ func GetVideoInformation(videoID string, stopChan stop.Chan, pool *ip_manager.IP const ( GoogleBotUA = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" - ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36" + ChromeUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" maxAttempts = 3 extractionError = "YouTube said: Unable to extract video data" throttledError = "HTTP Error 429" diff --git a/go.mod b/go.mod index f681d7b..619274e 100644 --- a/go.mod +++ b/go.mod @@ -9,22 +9,23 @@ replace gopkg.in/Graylog2/go-gelf.v2 => github.com/grafana/go-gelf/v2 v2.0.1 require ( github.com/abadojack/whatlanggo v1.0.1 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 - github.com/aws/aws-sdk-go v1.51.17 - github.com/docker/docker v26.0.0+incompatible + github.com/aws/aws-sdk-go v1.54.10 + github.com/docker/docker v27.0.2+incompatible github.com/google/gops v0.3.28 + github.com/klauspost/compress v1.17.9 github.com/lbryio/lbry.go/v2 v2.7.2-0.20230307181431-a01aa6dc0629 github.com/lbryio/reflector.go v1.1.3-0.20240409180046-de736b068d75 github.com/mitchellh/go-ps v1.0.0 github.com/prometheus-community/pro-bing v0.4.0 - github.com/prometheus/client_golang v1.19.0 - github.com/shopspring/decimal v1.3.1 + github.com/prometheus/client_golang v1.19.1 + github.com/shopspring/decimal v1.4.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/tkanos/gonfig v0.0.0-20210106201359-53e13348de2f github.com/vbauerster/mpb/v7 v7.5.3 gopkg.in/Graylog2/go-gelf.v2 v2.0.0-20191017102106-1550ee647df0 - gopkg.in/vansante/go-ffprobe.v2 v2.1.1 + gopkg.in/vansante/go-ffprobe.v2 v2.2.0 gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 1804277..491c609 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-sdk-go v1.51.17 h1:Cfa40lCdjv9OxC3X1Ks3a6O1Tu3gOANSyKHOSw/zuWU= -github.com/aws/aws-sdk-go v1.51.17/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.54.10 h1:dvkMlAttUsyacKj2L4poIQBLzOSWL2JG2ty+yWrqets= +github.com/aws/aws-sdk-go v1.54.10/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -111,15 +111,15 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v26.0.0+incompatible h1:Ng2qi+gdKADUa/VM+6b6YaY2nlZhk/lVJiKR/2bMudU= -github.com/docker/docker v26.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.0.2+incompatible h1:mNhCtgXNV1fIRns102grG7rdzIsGGCq1OlOD0KunZos= +github.com/docker/docker v27.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -331,6 +331,8 @@ github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1q github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -428,8 +430,8 @@ github.com/prometheus-community/pro-bing v0.4.0/go.mod h1:b7wRYZtCcPmt4Sz319BykU github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -462,8 +464,8 @@ github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -475,8 +477,8 @@ github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -901,8 +903,8 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64= gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/vansante/go-ffprobe.v2 v2.1.1 h1:DIh5fMn+tlBvG7pXyUZdemVmLdERnf2xX6XOFF+0BBU= -gopkg.in/vansante/go-ffprobe.v2 v2.1.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= +gopkg.in/vansante/go-ffprobe.v2 v2.2.0 h1:iuOqTsbfYuqIz4tAU9NWh22CmBGxlGHdgj4iqP+NUmY= +gopkg.in/vansante/go-ffprobe.v2 v2.2.0/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/manager/s3_storage.go b/manager/s3_storage.go index 8c898b2..c3128fa 100644 --- a/manager/s3_storage.go +++ b/manager/s3_storage.go @@ -146,11 +146,27 @@ func (s *Sync) downloadBlockchainDB() error { } blockchainDbDir := strings.Replace(defaultBDBPath, "blockchain.db", "", -1) - err = util.Untar(defaultTempBDBPath, blockchainDbDir) + extractedFileNames, err := util.Untar(defaultTempBDBPath, blockchainDbDir) if err != nil { return errors.Prefix("error extracting blockchain.db files", err) } + for _, name := range extractedFileNames { + if !strings.HasSuffix(name, ".zst") { + continue + } + pathToCompressedFile := filepath.Join(blockchainDbDir, name) + err = util.Unzstd(pathToCompressedFile, blockchainDbDir) + if err != nil { + return errors.Prefix("error unzstding blockchain.db files", err) + } + + err = os.Remove(pathToCompressedFile) + if err != nil { + return errors.Err("error removing %s: %s", pathToCompressedFile, err.Error()) + } + } + log.Printf("blockchain.db data downloaded and extracted to %s", blockchainDbDir) s.state.blockchainDbDownloaded = true return nil @@ -261,7 +277,19 @@ func (s *Sync) uploadBlockchainDB() error { return errors.Err(err) } tarPath := strings.Replace(defaultBDBDir, "blockchain.db", "", -1) + s.DbChannelData.ChannelId + ".tar" - err = util.CreateTarball(tarPath, files) + + err = util.ZstdCompressFiles(files) + if err != nil { + return err + } + + compressedFileNames := make([]string, len(files)) + + for i, file := range files { + compressedFileNames[i] = file + ".zst" + } + + err = util.CreateTarball(tarPath, compressedFileNames) if err != nil { return err } @@ -290,5 +318,18 @@ func (s *Sync) uploadBlockchainDB() error { if err != nil { return errors.Err(err) } - return os.Remove(defaultBDBDir) + + errStrings := make([]string, 0) + + filesToDelete := append(files, compressedFileNames...) + for _, f := range filesToDelete { + err = os.Remove(f) + if err != nil { + errStrings = append(errStrings, err.Error()) + } + } + if len(errStrings) > 0 { + return errors.Err(strings.Join(errStrings, "\n")) + } + return nil } diff --git a/util/archive.go b/util/archive.go index b18936c..297be30 100644 --- a/util/archive.go +++ b/util/archive.go @@ -62,37 +62,38 @@ func addFileToTarWriter(filePath string, tarWriter *tar.Writer) error { return nil } -func Untar(tarball, target string) error { +func Untar(tarball, target string) ([]string, error) { reader, err := os.Open(tarball) if err != nil { - return errors.Err(err) + return nil, errors.Err(err) } defer reader.Close() tarReader := tar.NewReader(reader) - + extractedFiles := make([]string, 0) for { header, err := tarReader.Next() if err == io.EOF { break } else if err != nil { - return errors.Err(err) + return nil, errors.Err(err) } path := filepath.Join(target, header.Name) info := header.FileInfo() if info.IsDir() { if err = os.MkdirAll(path, info.Mode()); err != nil { - return errors.Err(err) + return nil, errors.Err(err) } continue } + extractedFiles = append(extractedFiles, header.Name) err = extractFile(path, info, tarReader) if err != nil { - return err + return extractedFiles, err } } - return nil + return extractedFiles, nil } func extractFile(path string, info fs.FileInfo, tarReader *tar.Reader) error { diff --git a/util/compression.go b/util/compression.go new file mode 100644 index 0000000..f9a1b33 --- /dev/null +++ b/util/compression.go @@ -0,0 +1,85 @@ +package util + +import ( + "io" + "os" + "path/filepath" + "strings" + + "github.com/klauspost/compress/zstd" + + "github.com/lbryio/lbry.go/v2/extras/errors" +) + +// ZstdCompressFiles individually compresses each file in the filePaths slice and saves them as .zst files. +func ZstdCompressFiles(filePaths []string) error { + for _, filePath := range filePaths { + zstdFilePath := filePath + ".zst" + err := compressFile(filePath, zstdFilePath) + if err != nil { + return errors.Err("Could not compress file '%s', got error '%s'", filePath, err.Error()) + } + } + return nil +} + +// compressFile creates a zstd compressed version of the specified file. +func compressFile(sourcePath, destinationPath string) error { + inputFile, err := os.Open(sourcePath) + if err != nil { + return errors.Err("Could not open source file '%s', got error '%s'", sourcePath, err.Error()) + } + defer inputFile.Close() + + outputFile, err := os.Create(destinationPath) + if err != nil { + return errors.Err("Could not create destination file '%s', got error '%s'", destinationPath, err.Error()) + } + defer outputFile.Close() + + encoder, err := zstd.NewWriter(outputFile) + if err != nil { + return errors.Err("Could not create zstd encoder, got error '%s'", err.Error()) + } + defer encoder.Close() + + _, err = io.Copy(encoder, inputFile) + if err != nil { + return errors.Err("Could not compress the file '%s', got error '%s'", sourcePath, err.Error()) + } + + return nil +} + +// Unzstd decompresses a zstd file to a target directory. +func Unzstd(zstdFilePath, target string) error { + file, err := os.Open(zstdFilePath) + if err != nil { + return errors.Err("Could not open zstd file '%s', got error '%s'", zstdFilePath, err.Error()) + } + defer file.Close() + + decoder, err := zstd.NewReader(file) + if err != nil { + return errors.Err("Could not create zstd decoder, got error '%s'", err.Error()) + } + defer decoder.Close() + + originalFileName := strings.TrimSuffix(filepath.Base(zstdFilePath), ".zst") + err = os.MkdirAll(target, 0755) + if err != nil { + return errors.Err("Could not create target directory '%s', got error '%s'", target, err.Error()) + } + extractedFile, err := os.Create(filepath.Join(target, originalFileName)) + if err != nil { + return errors.Err("Could not create file for extraction '%s', got error '%s'", filepath.Join(target, originalFileName), err.Error()) + } + defer extractedFile.Close() + + _, err = io.Copy(extractedFile, decoder) + if err != nil { + return errors.Err("Could not extract the zstd content, got error '%s'", err.Error()) + } + + return nil +} diff --git a/util/compression_test.go b/util/compression_test.go new file mode 100644 index 0000000..d32cc1f --- /dev/null +++ b/util/compression_test.go @@ -0,0 +1,121 @@ +package util + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBlockChainDB(t *testing.T) { + blockchainDbPath := "./tmp/blockchain.db" + + if _, err := os.Stat(blockchainDbPath); os.IsNotExist(err) { + t.Fatalf("no directory found") + } + files, err := filepath.Glob(blockchainDbPath + "*") + if err != nil { + t.Fatalf(err.Error()) + } + tarPath := strings.Replace(blockchainDbPath, "blockchain.db", "", -1) + "test" + ".tar" + + err = ZstdCompressFiles(files) + if err != nil { + t.Fatalf(err.Error()) + } + + compressedFileNames := make([]string, len(files)) + for i, file := range files { + compressedFileNames[i] = file + ".zst" + } + + err = CreateTarball(tarPath, compressedFileNames) + if err != nil { + t.Fatalf(err.Error()) + } + errStrings := make([]string, 0) + filesToDelete := append(files, compressedFileNames...) + for _, f := range filesToDelete { + err = os.Remove(f) + if err != nil { + errStrings = append(errStrings, err.Error()) + } + } + if len(errStrings) > 0 { + t.Fatalf(strings.Join(errStrings, "\n")) + } + + blockchainDbDir := strings.Replace(blockchainDbPath, "blockchain.db", "", -1) + extractedFileNames, err := Untar(tarPath, blockchainDbDir) + if err != nil { + t.Fatalf(err.Error()) + } + for _, name := range extractedFileNames { + pathToCompressedFile := filepath.Join(blockchainDbDir, name) + err = Unzstd(pathToCompressedFile, blockchainDbDir) + if err != nil { + t.Fatalf(err.Error()) + } + + err = os.Remove(pathToCompressedFile) + if err != nil { + t.Fatalf(err.Error()) + } + } + err = os.Remove(tarPath) + if err != nil { + t.Fatalf(err.Error()) + } +} + +func TestIndividualFileCompression(t *testing.T) { + // Create a temporary directory for the test files + tempDir, err := os.MkdirTemp("", "test_compression") + if err != nil { + t.Fatalf("Failed to create a temporary directory: %s", err) + } + defer os.RemoveAll(tempDir) + + testFiles := []string{ + "testfile1.txt", + "testfile2.txt", + "testfile3.txt", + } + content := []string{ + "Test content 1", + "Test content 2", + "Test content 3", + } + + fullPath := make([]string, len(testFiles)) + for i, fileName := range testFiles { + filePath := filepath.Join(tempDir, fileName) + if err := os.WriteFile(filePath, []byte(content[i]), 0644); err != nil { + t.Fatalf("Failed to write to the test file: %s", err) + } + fullPath[i] = filePath + } + + if err := ZstdCompressFiles(fullPath); err != nil { + t.Fatalf("Failed to compress the file: %s", err) + } + + for i, path := range fullPath { + zstdFilePath := path + ".zst" + decompressionPath := filepath.Join(tempDir, "decompressed") + err := Unzstd(zstdFilePath, decompressionPath) + if err != nil { + t.Fatalf("Failed to decompress the file: %s", err) + } + decompressedContent, err := os.ReadFile(filepath.Join(decompressionPath, testFiles[i])) + + if err != nil { + t.Fatalf("Failed to read the decompressed file: %s", err) + } + if !assert.Equal(t, content[i], string(decompressedContent)) { + t.Fatalf("Decompressed content does not match original content.") + } + } +}