Improve cache and fallback-proxy behaviour for /v3/release

README.md

@@ -90,6 +90,7 @@ Flags:
       --bind string               host to listen to (default "127.0.0.1")
       --cache-max-age int         max number of seconds responses should be cached (default 86400)
       --cache-prefixes string     url prefixes to cache (default "/v3/files")
+      --cache-by-full-request-uri will cache responses by the full request URI (incl. query fragments) instead of only the request path
       --cors string               allowed cors origins separated by comma (default "*")
       --dev                       enables dev mode
       --drop-privileges           drops privileges to the given user/group
@@ -207,6 +208,7 @@ GORGE_BACKEND=filesystem
 GORGE_BIND=127.0.0.1
 GORGE_CACHE_MAX_AGE=86400
 GORGE_CACHE_PREFIXES=/v3/files
+GORGE_CACHE_BY_FULL_REQUEST_URI=false
 GORGE_CORS="*"
 GORGE_DEV=false
 GORGE_DROP_PRIVILEGES=false

cmd/serve.go

@@ -138,11 +138,19 @@ You can also enable the caching functionality to speed things up.`,
 				AllowCredentials: false,
 				MaxAge:           300,
 			}))
+
 			if !config.NoCache {
 				customKeyFunc := func(r *http.Request) uint64 {
 					token := r.Header.Get("Authorization")
-					return stampede.StringToHash(r.Method, strings.ToLower(token))
+					requestURI := r.URL.Path
+
+					if config.CacheByFullRequestURI {
+						requestURI = r.URL.RequestURI()
+					}
+
+					return stampede.StringToHash(r.Method, requestURI, strings.ToLower(token))
 				}
+
 				cachedMiddleware := stampede.HandlerWithKey(512, time.Duration(config.CacheMaxAge)*time.Second, customKeyFunc, strings.Split(config.CachePrefixes, ",")...)
 				r.Use(cachedMiddleware)
 			}
@@ -165,10 +173,14 @@ You can also enable the caching functionality to speed things up.`,
 					slices.Reverse(proxies)
 
 					for _, proxy := range proxies {
-						r.Use(customMiddleware.ProxyFallback(proxy, func(status int) bool {
-							return status == http.StatusNotFound
-						},
+						r.Use(customMiddleware.ProxyFallback(
+							proxy,
+							func(status int) bool {
+								return status == http.StatusNotFound
+							},
 							func(r *http.Response) {
+								r.Header.Add("X-Proxied-To", proxy)
+
 								if config.ImportProxiedReleases && strings.HasPrefix(r.Request.URL.Path, "/v3/files/") && r.StatusCode == http.StatusOK {
 									body, err := io.ReadAll(r.Body)
 									if err != nil {
@@ -184,6 +196,7 @@ You can also enable the caching functionality to speed things up.`,
 										log.Log.Error(err)
 										return
 									}
+
 									log.Log.Infof("Imported release %s\n", release.Slug)
 								}
 							},
@@ -334,6 +347,7 @@ func init() {
 	serveCmd.Flags().Int64Var(&config.CacheMaxAge, "cache-max-age", 86400, "max number of seconds responses should be cached")
 	serveCmd.Flags().BoolVar(&config.NoCache, "no-cache", false, "disables the caching functionality")
 	serveCmd.Flags().BoolVar(&config.ImportProxiedReleases, "import-proxied-releases", false, "add every proxied modules to local store")
+	serveCmd.Flags().BoolVar(&config.CacheByFullRequestURI, "cache-by-full-request-uri", false, "will cache responses by the full request URI (incl. query fragments) instead of only the request path")
 }
 
 func checkModules(sleepSeconds int) {

internal/config/config.go

@@ -16,6 +16,7 @@ var (
 	FallbackProxyUrl      string
 	NoCache               bool
 	CachePrefixes         string
+	CacheByFullRequestURI bool
 	CacheMaxAge           int64
 	ImportProxiedReleases bool
 	JwtSecret             string

internal/middleware/stats.go

@@ -7,21 +7,25 @@ import (
 )
 
 type Statistics struct {
-	ActiveConnections       int
-	TotalConnections        int
-	TotalResponseTime       time.Duration
-	ConnectionsPerEndpoint  map[string]int
-	ResponseTimePerEndpoint map[string]time.Duration
-	Mutex                   sync.Mutex
+	ActiveConnections             int
+	TotalConnections              int
+	ProxiedConnections            int
+	TotalResponseTime             time.Duration
+	ConnectionsPerEndpoint        map[string]int
+	ProxiedConnectionsPerEndpoint map[string]int
+	ResponseTimePerEndpoint       map[string]time.Duration
+	Mutex                         sync.Mutex
 }
 
 func NewStatistics() *Statistics {
 	return &Statistics{
-		ActiveConnections:       0,
-		TotalConnections:        0,
-		TotalResponseTime:       0,
-		ConnectionsPerEndpoint:  make(map[string]int),
-		ResponseTimePerEndpoint: make(map[string]time.Duration),
+		ActiveConnections:             0,
+		TotalConnections:              0,
+		ProxiedConnections:            0,
+		TotalResponseTime:             0,
+		ConnectionsPerEndpoint:        make(map[string]int),
+		ProxiedConnectionsPerEndpoint: make(map[string]int),
+		ResponseTimePerEndpoint:       make(map[string]time.Duration),
 	}
 }
 
@@ -41,6 +45,12 @@ func StatisticsMiddleware(stats *Statistics) func(next http.Handler) http.Handle
 				stats.ActiveConnections--
 				stats.TotalResponseTime += duration
 				stats.ResponseTimePerEndpoint[r.URL.Path] += duration
+
+				if w.Header().Get("X-Proxied-To") != "" {
+					stats.ProxiedConnections++
+					stats.ProxiedConnectionsPerEndpoint[r.URL.Path]++
+				}
+
 				stats.Mutex.Unlock()
 			}()
 

internal/v3/api/release.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	"github.com/dadav/gorge/internal/config"
+	"github.com/dadav/gorge/internal/log"
 	"github.com/dadav/gorge/internal/v3/backend"
 	"github.com/dadav/gorge/internal/v3/utils"
 	gen "github.com/dadav/gorge/pkg/gen/v3/openapi"
@@ -219,48 +220,111 @@ func (s *ReleaseOperationsApi) GetReleasePlans(ctx context.Context, releaseSlug
 // GetReleases - List module releases
 func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, offset int32, sortBy string, module string, owner string, withPdk bool, operatingsystem string, operatingsystemrelease string, peRequirement string, puppetRequirement string, moduleGroups []string, showDeleted bool, hideDeprecated bool, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string, supported bool) (gen.ImplResponse, error) {
 	results := []gen.Release{}
-	filtered := []gen.Release{}
+	filtered := []*gen.Release{}
 	allReleases, _ := backend.ConfiguredBackend.GetAllReleases()
+
+	base, _ := url.Parse("/v3/releases")
 	params := url.Values{}
+
+	filterSet := false
+
+	if module != "" {
+		filterSet = true
+		params.Add("module", module)
+	}
+
+	if owner != "" {
+		filterSet = true
+		params.Add("owner", owner)
+	}
+
 	params.Add("offset", strconv.Itoa(int(offset)))
 	params.Add("limit", strconv.Itoa(int(limit)))
 
-	if int(offset)+1 > len(allReleases) {
-		return gen.Response(404, GetRelease404Response{
-			Message: "Invalid offset",
-			Errors:  []string{"The given offset is larger than the total number of modules."},
+	base.RawQuery = params.Encode()
+	currentInf := interface{}(base.String())
+
+	// We know there's no releases and a fallback proxy, so we should return a 404 to let the proxy handle it
+	if config.FallbackProxyUrl != "" && len(allReleases) == 0 {
+		log.Log.Debugln("Could not find *any* releases in the backend, returning 404 so we can proxy if desired")
+
+		return gen.Response(http.StatusNotFound, GetRelease404Response{
+			Message: "No releases found",
+			Errors:  []string{"Did not retrieve any releases from the backend."},
 		}), nil
 	}
 
-	for _, r := range allReleases[offset:] {
-		var filterMatched, filterSet bool
-
-		if module != "" && r.Module.Slug != module {
-			filterSet = true
-			filterMatched = r.Module.Slug == module
-			params.Add("module", module)
+	if module != "" {
+		// Perform an early query to see if the module even exists in the backend, optimization for instances with _many_ modules
+		_, err := backend.ConfiguredBackend.GetModuleBySlug(module)
+		if err != nil {
+			log.Log.Debugf("Could not find module with slug '%s' in backend, returning 404 so we can proxy if desired\n", module)
+
+			if config.FallbackProxyUrl != "" {
+				return gen.Response(http.StatusNotFound, GetRelease404Response{
+					Message: "No releases found",
+					Errors:  []string{"No module(s) found for given query."},
+				}), nil
+			} else {
+				return gen.Response(http.StatusOK, gen.GetReleases200Response{
+					Pagination: gen.GetReleases200ResponsePagination{
+						Limit:    limit,
+						Offset:   offset,
+						First:    &currentInf,
+						Previous: nil,
+						Current:  &currentInf,
+						Next:     nil,
+						Total:    0,
+					},
+					Results: []gen.Release{},
+				}), nil
+			}
 		}
-		if owner != "" && r.Module.Owner.Slug != owner {
-			filterSet = true
-			filterMatched = r.Module.Owner.Slug == owner
-			params.Add("owner", owner)
+	}
+
+	if filterSet {
+		// We search through all available releases to see if they match the filter
+		for _, r := range allReleases {
+			if module != "" && r.Module.Slug != module {
+				continue
+			}
+
+			if owner != "" && r.Module.Owner.Slug != owner {
+				continue
+			}
+
+			filtered = append(filtered, r)
 		}
+	} else {
+		filtered = allReleases
+	}
+
+	if len(filtered) > int(offset) {
+		i := 1
+		for _, release := range filtered[offset:] {
+			if i > int(limit) {
+				break
+			}
 
-		if !filterSet || filterMatched {
-			filtered = append(filtered, *r)
+			results = append(results, *release)
+			i++
 		}
 	}
 
-	i := 1
-	for _, release := range filtered {
-		if i > int(limit) {
-			break
+	// If we're using a fallback-proxy, we should return a 404 so the proxy can handle the request
+	if config.FallbackProxyUrl != "" && len(results) == 0 {
+		if module != "" {
+			log.Log.Debugf("No releases for '%s' found in backend\n", module)
+		} else {
+			log.Log.Debugln("No releases found in backend")
 		}
-		results = append(results, release)
-		i++
+
+		return gen.Response(http.StatusNotFound, GetRelease404Response{
+			Message: "No releases found",
+			Errors:  []string{"No release(s) found for given query."},
+		}), nil
 	}
 
-	base, _ := url.Parse("/v3/releases")
 	base.RawQuery = params.Encode()
 	currentInf := interface{}(base.String())
 	params.Set("offset", "0")

internal/v3/ui/components/statistics.templ

@@ -10,13 +10,15 @@ templ StatisticsView(stats *customMiddleware.Statistics) {
 	<div>
 		<h3>Statistics</h3>
 		<p>ActiveConnections: { strconv.Itoa(stats.ActiveConnections) }</p>
+		<p>ProxiedConnections: { strconv.Itoa(stats.ProxiedConnections) }</p>
 		<p>TotalConnections: { strconv.Itoa(stats.TotalConnections) }</p>
 		<p>TotalResponseTime: { stats.TotalResponseTime.String() }</p>
 		<table>
 			<thead>
 				<tr>
 					<th>Path</th>
 					<th>Connections</th>
+					<th>Proxied Connections</th>
 					<th>Average ResponseTime</th>
 					<th>Total ResponseTime</th>
 				</tr>
@@ -26,6 +28,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) {
 					<tr>
 						<td>{ path }</td>
 						<td>{ strconv.Itoa(connections) }</td>
+						<td>{ strconv.Itoa(stats.ProxiedConnectionsPerEndpoint[path]) }</td>
 						<td>{ (stats.ResponseTimePerEndpoint[path] / time.Duration(connections)).String() }</td>
 						<td>{ stats.ResponseTimePerEndpoint[path].String() }</td>
 					</tr>