feat: sftp (#51)

* feat: sftp test: add tests for ftp and sftp * chore: ci fixes * chore: potential race fix * fix: simplify existence checks * fix: split path differently for ftp * fix: 🤷 * chore: add debug print * chore: lint * chore: idk dude * chore: ? * chore: more logs * chore: wipe mods before tests * chore: logs * chore: wat * chore: wait? * chore: no errors * chore: gh actions are 💩 * fix: always sync after copy * chore: remove some test logs * chore: remove test progress watcher * refactor: change progress to writer * chore: logs * chore: different logs * chore: whoops * chore: moar logs * chore: even moar logs * chore: what is life * chore: why are we here * chore: we are just bags of water floating through space * chore: are you real? * chore: ? * chore: if you get a single update now I call bs * chore: ok what if we just do one? * chore: ok what if we do two? * chore: this should not work * chore: wait no, this one * chore: fml * chore: remove logs * chore: what if we just wait a little * chore: retry * chore: move error * chore: verbose log * chore: remove explicit sleep * chore: remove debug * fix: linux pathing on windows * fix: clean paths properly * fix: fuck ftp * fix: send update on vanilla * feat: parallel ftp * fix: remove potential credential leak
satisfactorymodding · Dec 28, 2023 · 2672b17 · 2672b17
1 parent baacde4
commit 2672b17
Show file tree

Hide file tree

Showing 16 changed files with 675 additions and 196 deletions.
diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml
@@ -82,6 +82,10 @@ jobs:
       - name: Install Satisfactory Dedicated Server
         run: steamcmd +login anonymous +force_install_dir ${{ github.workspace }}/SatisfactoryDedicatedServer +app_update 1690800 validate +quit
 
+      - name: Change directory permissions
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: mkdir -p ${{ github.workspace }}/SatisfactoryDedicatedServer/FactoryGame/Mods && chmod -R 777 ${{ github.workspace }}/SatisfactoryDedicatedServer
+
       - name: List directory (linux)
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: ls -lR
@@ -90,6 +94,10 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' }}
         run: tree /F
 
+      - name: Boot ftp and sftp
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: docker-compose -f docker-compose-test.yml up -d
+
       - name: Download GQL schema
         run: "npx graphqurl https://api.ficsit.dev/v2/query --introspect -H 'content-type: application/json' > schema.graphql"
 

diff --git a/.gitignore b/.gitignore
@@ -128,4 +128,5 @@ dist/
 /.graphqlconfig
 schema.graphql
 *.log
-.direnv
+.direnv
+/SatisfactoryDedicatedServer
diff --git a/cfg/test_defaults.go b/cfg/test_defaults.go
@@ -1,6 +1,8 @@
 package cfg
 
 import (
+	"log/slog"
+	"os"
 	"path/filepath"
 	"runtime"
 
@@ -18,4 +20,8 @@ func SetDefaults() {
 	viper.SetDefault("api-base", "https://api.ficsit.dev")
 	viper.SetDefault("graphql-api", "/v2/query")
 	viper.SetDefault("concurrent-downloads", 5)
+
+	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	})))
 }
diff --git a/cli/cache/download.go b/cli/cache/download.go
@@ -4,11 +4,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
+	"github.com/avast/retry-go"
 	"github.com/puzpuzpuz/xsync/v3"
 	"github.com/spf13/viper"
 
@@ -84,7 +87,11 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- ut
 	outer:
 		for {
 			select {
-			case update := <-upstreamUpdates:
+			case update, ok := <-upstreamUpdates:
+				if !ok {
+					break outer
+				}
+
 				for _, u := range group.updates {
 					u <- update
 				}
@@ -94,11 +101,29 @@ func DownloadOrCache(cacheKey string, hash string, url string, updates chan<- ut
 		}
 	}()
 
-	size, err := downloadInternal(cacheKey, location, hash, url, upstreamUpdates, downloadSemaphore)
+	var size int64
+
+	err := retry.Do(func() error {
+		var err error
+		size, err = downloadInternal(cacheKey, location, hash, url, upstreamUpdates, downloadSemaphore)
+		if err != nil {
+			return fmt.Errorf("internal download error: %w", err)
+		}
+		return nil
+	},
+		retry.Attempts(5),
+		retry.Delay(time.Second),
+		retry.DelayType(retry.FixedDelay),
+		retry.OnRetry(func(n uint, err error) {
+			if n > 0 {
+				slog.Info("retrying download", slog.Uint64("n", uint64(n)), slog.String("cacheKey", cacheKey))
+			}
+		}),
+	)
 	if err != nil {
 		group.err = err
 		close(group.wait)
-		return nil, 0, err
+		return nil, 0, err // nolint
 	}
 
 	close(upstreamWaiter)
@@ -175,16 +200,17 @@ func downloadInternal(cacheKey string, location string, hash string, url string,
 	}
 
 	progresser := &utils.Progresser{
-		Reader:  resp.Body,
 		Total:   resp.ContentLength,
 		Updates: updates,
 	}
 
-	_, err = io.Copy(out, progresser)
+	_, err = io.Copy(io.MultiWriter(out, progresser), resp.Body)
 	if err != nil {
 		return 0, fmt.Errorf("failed writing file to disk: %w", err)
 	}
 
+	_ = out.Sync()
+
 	if updates != nil {
 		updates <- utils.GenericProgress{Completed: resp.ContentLength, Total: resp.ContentLength}
 	}

diff --git a/cli/context.go b/cli/context.go
@@ -2,6 +2,7 @@ package cli
 
 import (
 	"fmt"
+	"log/slog"
 
 	"github.com/Khan/genqlient/graphql"
 	"github.com/spf13/viper"
@@ -87,6 +88,8 @@ func (g *GlobalContext) ReInit() error {
 
 // Wipe will remove any trace of ficsit anywhere
 func (g *GlobalContext) Wipe() error {
+	slog.Info("wiping global context")
+
 	// Wipe all installations
 	for _, installation := range g.Installations.Installations {
 		if err := installation.Wipe(); err != nil {