From afb02f3224ba0f36915c7d57254ddd94a93ff93d Mon Sep 17 00:00:00 2001 From: dadav <33197631+dadav@users.noreply.github.com> Date: Sat, 2 Mar 2024 19:16:28 +0100 Subject: [PATCH] feat: Add cache expiration --- cmd/serve.go | 2 +- internal/api/v3/release.go | 146 ++++++++++++++++++++++----------- internal/backend/filesystem.go | 4 +- internal/config/config.go | 2 +- internal/middleware/cache.go | 11 ++- 5 files changed, 112 insertions(+), 53 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 269fbce..a580820 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -119,6 +119,6 @@ func init() { serveCmd.Flags().BoolVar(&config.Dev, "dev", false, "enables dev mode") serveCmd.Flags().StringVar(&config.CacheDir, "cachedir", "/var/cache/gorge", "cache directory") serveCmd.Flags().StringVar(&config.CachePrefixes, "cache-prefixes", "/v3/files", "url prefixes to cache") - serveCmd.Flags().IntVar(&config.CacheMaxAge, "cache-max-age", 86400, "max number of seconds responses should be cached") + serveCmd.Flags().Int64Var(&config.CacheMaxAge, "cache-max-age", 86400, "max number of seconds responses should be cached") serveCmd.Flags().BoolVar(&config.NoCache, "no-cache", false, "disables the caching functionality") } diff --git a/internal/api/v3/release.go b/internal/api/v3/release.go index 0c83d07..60fb1d2 100644 --- a/internal/api/v3/release.go +++ b/internal/api/v3/release.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "errors" + "fmt" "net/http" "net/url" "os" @@ -32,54 +33,67 @@ type GetRelease404Response struct { // AddRelease - Create module release func (s *ReleaseOperationsApi) AddRelease(ctx context.Context, addReleaseRequest gen.AddReleaseRequest) (gen.ImplResponse, error) { - // TODO - update AddRelease with the required logic for this service method. - // Add api_release_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - base64EncodedTarball := addReleaseRequest.File + tmpDir := "/tmp" decodedTarball, err := base64.StdEncoding.DecodeString(base64EncodedTarball) if err != nil { return gen.Response(400, gen.GetFile400Response{ - Message: "foo", - Errors: []string{"foo"}, + Message: "Could not decode provided data", + Errors: []string{err.Error()}, }), nil } - tmpTarball, err := os.CreateTemp("/tmp", "release-tarball-*") + tmpTarball, err := os.CreateTemp(tmpDir, "release-tarball-*") if err != nil { return gen.Response(400, gen.GetFile400Response{ - Message: "foo", - Errors: []string{"foo"}, + Message: "Temporary file could not be created", + Errors: []string{err.Error()}, }), nil } + tmpTarballPath := filepath.Join(tmpDir, tmpTarball.Name()) _, err = tmpTarball.Write(decodedTarball) if err != nil { return gen.Response(400, gen.GetFile400Response{ - Message: "foo", - Errors: []string{"foo"}, + Message: "Decoded data could not be written to file", + Errors: []string{err.Error()}, }), nil } // Step 2: Extract metadata of module + release, _, err := backend.ReadReleaseMetadataFromFile(tmpTarballPath) + if err != nil { + return gen.Response(400, gen.GetFile400Response{ + Message: "Metadata could not be read from given data", + Errors: []string{err.Error()}, + }), nil + } // Step 3: Check if data is valid and release does not exist yet + releaseSlug := fmt.Sprintf("%s-%s", release.Name, release.Version) + _, err = backend.ConfiguredBackend.GetReleaseBySlug(releaseSlug) + if err == nil { + return gen.Response(400, gen.GetFile400Response{ + Message: "Release already exist", + Errors: []string{"The given release already exist"}, + }), nil + } // Step 4: Moved release to correct location - // TODO: Uncomment the next line to return response Response(201, ReleaseMinimal{}) or use other options such as http.Ok ... - // return Response(201, ReleaseMinimal{}), nil - - // TODO: Uncomment the next line to return response Response(400, GetFile400Response{}) or use other options such as http.Ok ... - - // TODO: Uncomment the next line to return response Response(401, GetUserSearchFilters401Response{}) or use other options such as http.Ok ... - // return Response(401, GetUserSearchFilters401Response{}), nil - - // TODO: Uncomment the next line to return response Response(403, DeleteUserSearchFilter403Response{}) or use other options such as http.Ok ... - // return Response(403, DeleteUserSearchFilter403Response{}), nil - - // TODO: Uncomment the next line to return response Response(409, AddSearchFilter409Response{}) or use other options such as http.Ok ... - // return Response(409, AddSearchFilter409Response{}), nil + newReleasePath := filepath.Join(config.ModulesDir, release.Name, fmt.Sprintf("%s.tar.gz", releaseSlug)) + err = os.Rename(tmpTarballPath, newReleasePath) + if err != nil { + return gen.Response(400, gen.GetFile400Response{ + Message: "Could not create release", + Errors: []string{err.Error()}, + }), nil + } - return gen.Response(http.StatusNotImplemented, nil), errors.New("AddRelease method not implemented") + return gen.Response(201, gen.ReleaseMinimal{ + Uri: fmt.Sprintf("/v3/releases/%s", releaseSlug), + FileUri: fmt.Sprintf("/v3/files/%s.tar.gz", releaseSlug), + Slug: releaseSlug, + }), nil } // DeleteRelease - Delete module release @@ -131,7 +145,7 @@ type GetRelease500Response struct { // GetRelease - Fetch module release func (s *ReleaseOperationsApi) GetRelease(ctx context.Context, releaseSlug string, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string) (gen.ImplResponse, error) { - metadata, readme, err := backend.ReadReleaseMetadata(releaseSlug) + release, err := backend.ConfiguredBackend.GetReleaseBySlug(releaseSlug) if err != nil { if errors.Is(err, os.ErrNotExist) { return gen.Response(http.StatusNotFound, gen.GetFile404Response{ @@ -145,39 +159,77 @@ func (s *ReleaseOperationsApi) GetRelease(ctx context.Context, releaseSlug strin }), nil } - return gen.Response(http.StatusOK, gen.Release{ - Slug: releaseSlug, - Module: gen.ReleaseModule{Name: metadata.Name}, - Readme: readme, - }), nil + return gen.Response(http.StatusOK, release), nil +} + +func abbrReleaseToFullReleasePlan(abbrReleasePlan gen.ReleasePlanAbbreviated) gen.ReleasePlan { + planFile := fmt.Sprintf("plans/%s.pp", strings.Join(strings.Split(abbrReleasePlan.Name, "::")[1:], "/")) + return gen.ReleasePlan{ + Uri: abbrReleasePlan.Uri, + Name: abbrReleasePlan.Name, + Private: abbrReleasePlan.Private, + Filename: planFile, + PlanMetadata: gen.ReleasePlanPlanMetadata{ + Name: abbrReleasePlan.Name, + Private: abbrReleasePlan.Private, + File: planFile, + }, + } } // GetReleasePlan - Fetch module release plan func (s *ReleaseOperationsApi) GetReleasePlan(ctx context.Context, releaseSlug string, planName string) (gen.ImplResponse, error) { - // TODO - update GetReleasePlan with the required logic for this service method. - // Add api_release_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, ReleasePlan{}) or use other options such as http.Ok ... - // return Response(200, ReleasePlan{}), nil - - // TODO: Uncomment the next line to return response Response(404, GetFile404Response{}) or use other options such as http.Ok ... - // return Response(404, GetFile404Response{}), nil + release, err := backend.ConfiguredBackend.GetReleaseBySlug(releaseSlug) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return gen.Response(http.StatusNotFound, gen.GetFile404Response{ + Message: http.StatusText(http.StatusNotFound), + Errors: []string{"plan not found"}, + }), nil + } + return gen.Response(http.StatusInternalServerError, GetRelease500Response{ + Message: http.StatusText(http.StatusInternalServerError), + Errors: []string{"error while reading release metadata"}, + }), nil + } - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetReleasePlan method not implemented") + for _, plan := range release.Plans { + if plan.Name == planName { + // modulename::foo becomes plans/foo.pp + return gen.Response(200, abbrReleaseToFullReleasePlan(plan)), nil + } + } + return gen.Response(http.StatusNotFound, gen.GetFile404Response{ + Message: http.StatusText(http.StatusNotFound), + Errors: []string{"plan not found"}, + }), nil } // GetReleasePlans - List module release plans func (s *ReleaseOperationsApi) GetReleasePlans(ctx context.Context, releaseSlug string) (gen.ImplResponse, error) { - // TODO - update GetReleasePlans with the required logic for this service method. - // Add api_release_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, GetReleasePlans200Response{}) or use other options such as http.Ok ... - // return Response(200, GetReleasePlans200Response{}), nil + release, err := backend.ConfiguredBackend.GetReleaseBySlug(releaseSlug) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return gen.Response(http.StatusNotFound, gen.GetFile404Response{ + Message: http.StatusText(http.StatusNotFound), + Errors: []string{"plan not found"}, + }), nil + } + return gen.Response(http.StatusInternalServerError, GetRelease500Response{ + Message: http.StatusText(http.StatusInternalServerError), + Errors: []string{"error while reading release metadata"}, + }), nil + } - // TODO: Uncomment the next line to return response Response(404, GetFile404Response{}) or use other options such as http.Ok ... - // return Response(404, GetFile404Response{}), nil + results := []gen.ReleasePlan{} + for _, plan := range release.Plans { + results = append(results, abbrReleaseToFullReleasePlan(plan)) + } - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetReleasePlans method not implemented") + return gen.Response(200, gen.GetReleasePlans200Response{ + Pagination: gen.GetReleasePlans200ResponsePagination{}, + Results: results, + }), nil } // GetReleases - List module releases diff --git a/internal/backend/filesystem.go b/internal/backend/filesystem.go index 5bbc839..e91c916 100644 --- a/internal/backend/filesystem.go +++ b/internal/backend/filesystem.go @@ -158,7 +158,7 @@ func (s *FilesystemBackend) LoadModules() error { return nil } - releaseMetadata, releaseReadme, err := ReadReleaseMetadata(path) + releaseMetadata, releaseReadme, err := ReadReleaseMetadataFromFile(path) if err != nil { return err } @@ -255,7 +255,7 @@ func (s *FilesystemBackend) LoadModules() error { return nil } -func ReadReleaseMetadata(path string) (*model.ReleaseMetadata, string, error) { +func ReadReleaseMetadataFromFile(path string) (*model.ReleaseMetadata, string, error) { var jsonData bytes.Buffer var releaseMetadata model.ReleaseMetadata readme := new(strings.Builder) diff --git a/internal/config/config.go b/internal/config/config.go index c8766e7..4f4000b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,5 +12,5 @@ var ( NoCache bool CachePrefixes string CacheDir string - CacheMaxAge int + CacheMaxAge int64 ) diff --git a/internal/middleware/cache.go b/internal/middleware/cache.go index ba64039..c554cf3 100644 --- a/internal/middleware/cache.go +++ b/internal/middleware/cache.go @@ -9,7 +9,9 @@ import ( "os" "path/filepath" "strings" + "time" + "github.com/dadav/gorge/internal/config" "github.com/dadav/gorge/internal/log" ) @@ -44,8 +46,13 @@ func CacheMiddleware(prefixes []string, cacheDir string) func(next http.Handler) cacheControlHeader := r.Header.Get("Cache-Control") if !strings.Contains(cacheControlHeader, "no-cache") { if cacheFileInfo, err := os.Stat(cacheFilePath); err == nil { - if cacheFileInfo.ModTime().After(foo) { - // TODO: DELETE FILE + expirationTime := cacheFileInfo.ModTime().Add(time.Duration(config.CacheMaxAge) * time.Second) + if time.Now().After(expirationTime) { + log.Log.Debugf("Cached file expired: %s\n", cacheFilePath) + err := os.Remove(cacheFilePath) + if err != nil { + log.Log.Error(err) + } } else { data, err := os.ReadFile(cacheFilePath) if err == nil {