Merge pull request #38 from dadav/rebased_pr

Rebased pr

Dockerfile

@@ -3,9 +3,7 @@ FROM alpine:3.21@sha256:21dc6063fd678b478f57c0e13f47560d0ea4eeba26dfc947b2a4f81f
 # Create non-root user and set up permissions in a single layer
 RUN adduser -k /dev/null -u 10001 -D gorge \
   && chgrp 0 /home/gorge \
-  && chmod -R g+rwX /home/gorge \
-  # Add additional security hardening
-  && chmod 755 /gorge
+  && chmod -R g+rwX /home/gorge
 
 # Copy application binary with explicit permissions
 COPY --chmod=755 gorge /

README.md

@@ -90,6 +90,7 @@ Flags:
       --bind string               host to listen to (default "127.0.0.1")
       --cache-max-age int         max number of seconds responses should be cached (default 86400)
       --cache-prefixes string     url prefixes to cache (default "/v3/files")
+      --cache-by-full-request-uri will cache responses by the full request URI (incl. query fragments) instead of only the request path
       --cors string               allowed cors origins separated by comma (default "*")
       --dev                       enables dev mode
       --drop-privileges           drops privileges to the given user/group
@@ -207,6 +208,7 @@ GORGE_BACKEND=filesystem
 GORGE_BIND=127.0.0.1
 GORGE_CACHE_MAX_AGE=86400
 GORGE_CACHE_PREFIXES=/v3/files
+GORGE_CACHE_BY_FULL_REQUEST_URI=false
 GORGE_CORS="*"
 GORGE_DEV=false
 GORGE_DROP_PRIVILEGES=false

cmd/serve.go

@@ -159,7 +159,13 @@ You can also enable the caching functionality to speed things up.`,
 					log.Log.Debug("Setting up cache middleware")
 					customKeyFunc := func(r *http.Request) uint64 {
 						token := r.Header.Get("Authorization")
-						return stampede.StringToHash(r.Method, r.URL.Path, strings.ToLower(token))
+						requestURI := r.URL.Path
+
+						if config.CacheByFullRequestURI {
+							requestURI = r.URL.RequestURI()
+						}
+
+						return stampede.StringToHash(r.Method, requestURI, strings.ToLower(token))
 					}
 
 					cachedMiddleware := stampede.HandlerWithKey(
@@ -209,10 +215,14 @@ You can also enable the caching functionality to speed things up.`,
 					slices.Reverse(proxies)
 
 					for _, proxy := range proxies {
-						r.Use(customMiddleware.ProxyFallback(proxy, func(status int) bool {
-							return status == http.StatusNotFound
-						},
+						r.Use(customMiddleware.ProxyFallback(
+							proxy,
+							func(status int) bool {
+								return status == http.StatusNotFound
+							},
 							func(r *http.Response) {
+								r.Header.Add("X-Proxied-To", proxy)
+
 								if config.ImportProxiedReleases && strings.HasPrefix(r.Request.URL.Path, "/v3/files/") && r.StatusCode == http.StatusOK {
 									body, err := io.ReadAll(r.Body)
 									if err != nil {
@@ -228,6 +238,7 @@ You can also enable the caching functionality to speed things up.`,
 										log.Log.Error(err)
 										return
 									}
+
 									log.Log.Infof("Imported release %s\n", release.Slug)
 								}
 							},
@@ -379,4 +390,5 @@ func init() {
 	serveCmd.Flags().Int64Var(&config.CacheMaxAge, "cache-max-age", 86400, "max number of seconds responses should be cached")
 	serveCmd.Flags().BoolVar(&config.NoCache, "no-cache", false, "disables the caching functionality")
 	serveCmd.Flags().BoolVar(&config.ImportProxiedReleases, "import-proxied-releases", false, "add every proxied modules to local store")
+	serveCmd.Flags().BoolVar(&config.CacheByFullRequestURI, "cache-by-full-request-uri", false, "will cache responses by the full request URI (incl. query fragments) instead of only the request path")
 }

internal/config/config.go

@@ -16,6 +16,7 @@ var (
 	FallbackProxyUrl      string
 	NoCache               bool
 	CachePrefixes         string
+	CacheByFullRequestURI bool
 	CacheMaxAge           int64
 	ImportProxiedReleases bool
 	JwtSecret             string

internal/middleware/stats.go

@@ -7,29 +7,33 @@ import (
 )
 
 type Statistics struct {
-	ActiveConnections       int
-	TotalConnections        int
-	TotalResponseTime       time.Duration
-	TotalCacheHits          int
-	TotalCacheMisses        int
-	ConnectionsPerEndpoint  map[string]int
-	ResponseTimePerEndpoint map[string]time.Duration
-	CacheHitsPerEndpoint    map[string]int
-	CacheMissesPerEndpoint  map[string]int
-	Mutex                   sync.Mutex
+	ActiveConnections             int
+	TotalConnections              int
+	TotalResponseTime             time.Duration
+	TotalCacheHits                int
+	TotalCacheMisses              int
+	ConnectionsPerEndpoint        map[string]int
+	ResponseTimePerEndpoint       map[string]time.Duration
+	CacheHitsPerEndpoint          map[string]int
+	CacheMissesPerEndpoint        map[string]int
+	Mutex                         sync.Mutex
+	ProxiedConnections            int
+	ProxiedConnectionsPerEndpoint map[string]int
 }
 
 func NewStatistics() *Statistics {
 	return &Statistics{
-		ActiveConnections:       0,
-		TotalConnections:        0,
-		TotalResponseTime:       0,
-		TotalCacheHits:          0,
-		TotalCacheMisses:        0,
-		ConnectionsPerEndpoint:  make(map[string]int),
-		CacheHitsPerEndpoint:    make(map[string]int),
-		CacheMissesPerEndpoint:  make(map[string]int),
-		ResponseTimePerEndpoint: make(map[string]time.Duration),
+		ActiveConnections:             0,
+		TotalConnections:              0,
+		TotalResponseTime:             0,
+		TotalCacheHits:                0,
+		TotalCacheMisses:              0,
+		ConnectionsPerEndpoint:        make(map[string]int),
+		CacheHitsPerEndpoint:          make(map[string]int),
+		CacheMissesPerEndpoint:        make(map[string]int),
+		ResponseTimePerEndpoint:       make(map[string]time.Duration),
+		ProxiedConnections:            0,
+		ProxiedConnectionsPerEndpoint: make(map[string]int),
 	}
 }
 
@@ -59,6 +63,12 @@ func StatisticsMiddleware(stats *Statistics) func(next http.Handler) http.Handle
 
 				stats.TotalResponseTime += duration
 				stats.ResponseTimePerEndpoint[r.URL.Path] += duration
+
+				if w.Header().Get("X-Proxied-To") != "" {
+					stats.ProxiedConnections++
+					stats.ProxiedConnectionsPerEndpoint[r.URL.Path]++
+				}
+
 				stats.Mutex.Unlock()
 			}()
 

internal/v3/api/release.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"github.com/dadav/gorge/internal/config"
+	"github.com/dadav/gorge/internal/log"
 	"github.com/dadav/gorge/internal/v3/backend"
 	"github.com/dadav/gorge/internal/v3/utils"
 	gen "github.com/dadav/gorge/pkg/gen/v3/openapi"
@@ -114,7 +115,6 @@ type GetFile400Response struct {
 
 // GetFile - Download module release
 func (s *ReleaseOperationsApi) GetFile(ctx context.Context, filename string) (gen.ImplResponse, error) {
-
 	if filename == "" {
 		return gen.Response(400, gen.GetFile400Response{
 			Message: "No filename provided",
@@ -261,50 +261,113 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off
 	}
 
 	results := []gen.Release{}
-	filtered := []gen.Release{}
+	filtered := []*gen.Release{}
 	allReleases, _ := backend.ConfiguredBackend.GetAllReleases()
+
+	base, _ := url.Parse("/v3/releases")
 	params := url.Values{}
+
+	filterSet := false
+
+	if module != "" {
+		filterSet = true
+		params.Add("module", module)
+	}
+
+	if owner != "" {
+		filterSet = true
+		params.Add("owner", owner)
+	}
+
 	params.Add("offset", strconv.Itoa(int(offset)))
 	params.Add("limit", strconv.Itoa(int(limit)))
 
-	if int(offset)+1 > len(allReleases) {
-		return gen.Response(404, GetRelease404Response{
-			Message: "Invalid offset",
-			Errors:  []string{"The given offset is larger than the total number of modules."},
+	base.RawQuery = params.Encode()
+	currentInf := interface{}(base.String())
+
+	// We know there's no releases and a fallback proxy, so we should return a 404 to let the proxy handle it
+	if config.FallbackProxyUrl != "" && len(allReleases) == 0 {
+		log.Log.Debugln("Could not find *any* releases in the backend, returning 404 so we can proxy if desired")
+
+		return gen.Response(http.StatusNotFound, GetRelease404Response{
+			Message: "No releases found",
+			Errors:  []string{"Did not retrieve any releases from the backend."},
 		}), nil
 	}
 
-	for _, r := range allReleases[offset:] {
-		var filterMatched, filterSet bool
-
-		if module != "" && r.Module.Slug != module {
-			filterSet = true
-			filterMatched = r.Module.Slug == module
-			params.Add("module", module)
+	if module != "" {
+		// Perform an early query to see if the module even exists in the backend, optimization for instances with _many_ modules
+		_, err := backend.ConfiguredBackend.GetModuleBySlug(module)
+		if err != nil {
+			log.Log.Debugf("Could not find module with slug '%s' in backend, returning 404 so we can proxy if desired\n", module)
+
+			if config.FallbackProxyUrl != "" {
+				return gen.Response(http.StatusNotFound, GetRelease404Response{
+					Message: "No releases found",
+					Errors:  []string{"No module(s) found for given query."},
+				}), nil
+			} else {
+				return gen.Response(http.StatusOK, gen.GetReleases200Response{
+					Pagination: gen.GetReleases200ResponsePagination{
+						Limit:    limit,
+						Offset:   offset,
+						First:    &currentInf,
+						Previous: nil,
+						Current:  &currentInf,
+						Next:     nil,
+						Total:    0,
+					},
+					Results: []gen.Release{},
+				}), nil
+			}
 		}
-		if owner != "" && r.Module.Owner.Slug != owner {
-			filterSet = true
-			filterMatched = r.Module.Owner.Slug == owner
-			params.Add("owner", owner)
+	}
+
+	if filterSet {
+		// We search through all available releases to see if they match the filter
+		for _, r := range allReleases {
+			if module != "" && r.Module.Slug != module {
+				continue
+			}
+
+			if owner != "" && r.Module.Owner.Slug != owner {
+				continue
+			}
+
+			filtered = append(filtered, r)
 		}
+	} else {
+		filtered = allReleases
+	}
+
+	if len(filtered) > int(offset) {
+		i := 1
+		for _, release := range filtered[offset:] {
+			if i > int(limit) {
+				break
+			}
 
-		if !filterSet || filterMatched {
-			filtered = append(filtered, *r)
+			results = append(results, *release)
+			i++
 		}
 	}
 
-	i := 1
-	for _, release := range filtered {
-		if i > int(limit) {
-			break
+	// If we're using a fallback-proxy, we should return a 404 so the proxy can handle the request
+	if config.FallbackProxyUrl != "" && len(results) == 0 {
+		if module != "" {
+			log.Log.Debugf("No releases for '%s' found in backend\n", module)
+		} else {
+			log.Log.Debugln("No releases found in backend")
 		}
-		results = append(results, release)
-		i++
+
+		return gen.Response(http.StatusNotFound, GetRelease404Response{
+			Message: "No releases found",
+			Errors:  []string{"No release(s) found for given query."},
+		}), nil
 	}
 
-	base, _ := url.Parse("/v3/releases")
 	base.RawQuery = params.Encode()
-	currentInf := interface{}(base.String())
+	currentInf = interface{}(base.String())
 	params.Set("offset", "0")
 	firstInf := interface{}(base.String())
 

internal/v3/ui/components/statistics.templ

@@ -10,6 +10,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) {
 	<div>
 		<h3>Statistics</h3>
 		<p>ActiveConnections: { strconv.Itoa(stats.ActiveConnections) }</p>
+		<p>ProxiedConnections: { strconv.Itoa(stats.ProxiedConnections) }</p>
 		<p>TotalConnections: { strconv.Itoa(stats.TotalConnections) }</p>
 		<p>TotalResponseTime: { stats.TotalResponseTime.String() }</p>
 		<p>TotalCacheHits: { strconv.Itoa(stats.TotalCacheHits) }</p>
@@ -19,6 +20,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) {
 				<tr>
 					<th>Path</th>
 					<th>Connections</th>
+					<th>Proxied Connections</th>
 					<th>Average ResponseTime</th>
 					<th>Total ResponseTime</th>
 					<th>Cache (Hits/Misses)</th>
@@ -29,6 +31,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) {
 					<tr>
 						<td>{ path }</td>
 						<td>{ strconv.Itoa(connections) }</td>
+						<td>{ strconv.Itoa(stats.ProxiedConnectionsPerEndpoint[path]) }</td>
 						<td>{ (stats.ResponseTimePerEndpoint[path] / time.Duration(connections)).String() }</td>
 						<td>{ stats.ResponseTimePerEndpoint[path].String() }</td>
 						if stats.CacheHitsPerEndpoint[path] > 0 || stats.CacheMissesPerEndpoint[path] > 0 {