diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..aa2b211 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,32 @@ +--- +name: build + +on: + push: + branches: + - '*' + tags-ignore: + - '*' + pull_request: + types: + - opened + - reopened + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ^1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release --snapshot --clean diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ba0d488 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +--- +name: release + +on: + push: + tags: + - '*' + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: ^1.22 + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cde0123 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..d399e02 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +--- +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj + +version: 1 + +before: + hooks: + - go mod tidy + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of `uname`. + name_template: >- + {{ .ProjectName }}_ + {{- .Version }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip + +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + - '^chore:' diff --git a/README.md b/README.md new file mode 100644 index 0000000..7f2f175 --- /dev/null +++ b/README.md @@ -0,0 +1,91 @@ +# ⭐ Gorge + +Gorge is a go implementation for [forgeapi.puppet.com](https://forgeapi.puppet.com/). + +## 🌹 Installation + +Via `go install`: + +```bash +go install github.com/dadav/gorge@latest +``` + +## 💎 Usage + +```bash +Run this command to start serving your own puppet modules. +You can also enable a fallback proxy to forward the requests to +when you don't have the requested module in your local module +set yet. + +You can also enable the caching functionality to speed things up. + +Usage: + gorge serve [flags] + +Flags: + --api-version string the forge api version to use (default "v3") + --backend string backend to use (default "filesystem") + --bind string host to listen to + --cache-prefixes string url prefixes to cache (default "/v3/files") + --cachedir string cache directory (default "/var/cache/gorge") + --cors string allowed cors origins separated by comma (default "*") + --dev enables dev mode + --fallback-proxy string optional fallback upstream proxy url + -h, --help help for serve + --modulesdir string directory containing all the modules (default "/opt/gorge/modules") + --no-cache disables the caching functionality + --port int the port to listen to (default 8080) + +Global Flags: + --config string config file (default is $HOME/.gorge.yaml) +``` + +## 🐂 Examples + +```bash +# use the pupeptlabs forge as fallback +gorge serve --fallback-proxy https://forge.puppetlabs.com + +# enable cache for every request +gorge serve --fallback-proxy https://forge.puppetlabs.com --cache-prefixes /v3 +``` + +## 🍰 Configuration + +Use the `$HOME/.config/gorge.yaml` (or `./gorge.yaml`): + +```yaml +--- +api-version: v3 +backend: filesystem +bind: 127.0.0.1 +cache-prefixes: /v3/files +cachedir: /var/cache/gorge +cors: "*" +dev: false +fallback-proxy: +modulesdir: /opt/gorge/modules +no-cache: false +port: 8080 +``` + +Or the environment: + +```bash +GORGE_API_VERSION: v3 +GORGE_BACKEND: filesystem +GORGE_BIND: 127.0.0.1 +GORGE_CACHE_PREFIXES: /v3/files +GORGE_CACHEDIR: /var/cache/gorge +GORGE_CORS: "*" +GORGE_DEV: false +GORGE_FALLBACK_PROXY: +GORGE_MODULESDIR: /opt/gorge/modules +GORGE_NO_CACHE: false +GORGE_PORT: 8080 +``` + +## 🔑 License + +[Apache](./LICENSE) diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..844520f --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,3 @@ +package cmd + +var apiVersion string diff --git a/cmd/root.go b/cmd/root.go index 1c9f870..b394524 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,11 +1,11 @@ /* -Copyright © 2024 NAME HERE +Copyright © 2024 dadav Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -17,8 +17,12 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" homedir "github.com/mitchellh/go-homedir" "github.com/spf13/viper" @@ -26,19 +30,17 @@ import ( var cfgFile string +const envPrefix = "GORGE" + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "gorge", - Short: "A brief description of your application", - Long: `A longer description that spans multiple lines and likely contains -examples and usage of using your application. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, + Short: "Gorge runs a puppet forge server", + Long: `You can run this tool to provide access to your puppet modules.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + // You can bind cobra and viper in a few locations, but PersistencePreRunE on the root command works well + return initConfig(cmd) + }, } // Execute adds all child commands to the root command and sets flags appropriately. @@ -51,24 +53,16 @@ func Execute() { } func init() { - cobra.OnInitialize(initConfig) - - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.gorge.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") } // initConfig reads in config file and ENV variables if set. -func initConfig() { +func initConfig(cmd *cobra.Command) error { + v := viper.New() + if cfgFile != "" { // Use config file from the flag. - viper.SetConfigFile(cfgFile) + v.SetConfigFile(cfgFile) } else { // Find home directory. home, err := homedir.Dir() @@ -77,15 +71,37 @@ func initConfig() { os.Exit(1) } + homeConfig := filepath.Join(home, ".config") + // Search config in home directory with name ".gorge" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".gorge") + v.AddConfigPath(homeConfig) + v.AddConfigPath(".") + v.SetConfigName("gorge") + v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + v.SetEnvPrefix(envPrefix) } - viper.AutomaticEnv() // read in environment variables that match + v.AutomaticEnv() // read in environment variables that match // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Println("Using config file:", viper.ConfigFileUsed()) + if err := v.ReadInConfig(); err == nil { + fmt.Println("Using config file:", v.ConfigFileUsed()) } + + bindFlags(cmd, v) + + return nil +} + +func bindFlags(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().VisitAll(func(f *pflag.Flag) { + // Determine the naming convention of the flags when represented in the config file + configName := f.Name + + // Apply the viper config value to the flag when the flag is not set and viper has a value + if !f.Changed && v.IsSet(configName) { + val := v.Get(configName) + cmd.Flags().Set(f.Name, fmt.Sprintf("%v", val)) + } + }) } diff --git a/cmd/serve.go b/cmd/serve.go index 0530b6b..3cf77cf 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -1,5 +1,5 @@ /* -Copyright © 2024 NAME HERE +Copyright © 2024 dadav Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,42 +17,91 @@ package cmd import ( "fmt" - "log" "net/http" + "os" + "strings" v3 "github.com/dadav/gorge/internal/api/v3" + backend "github.com/dadav/gorge/internal/backend" + config "github.com/dadav/gorge/internal/config" + log "github.com/dadav/gorge/internal/log" + customMiddleware "github.com/dadav/gorge/internal/middleware" openapi "github.com/dadav/gorge/pkg/gen/v3/openapi" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/cors" "github.com/spf13/cobra" ) -var apiVersion string - // serveCmd represents the serve command var serveCmd = &cobra.Command{ Use: "serve", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: + Short: "Start the puppet forge webserver", + Long: `Run this command to start serving your own puppet modules. +You can also enable a fallback proxy to forward the requests to +when you don't have the requested module in your local module +set yet. -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, +You can also enable the caching functionality to speed things up.`, Run: func(_ *cobra.Command, _ []string) { - if apiVersion == "v3" { + log.Setup(config.Dev) + + if config.Backend == "filesystem" { + backend.ConfiguredBackend = backend.NewFilesystemBackend(config.ModulesDir) + } else { + log.Log.Fatalf("Invalid backend: %s", config.Backend) + } + + backend.ConfiguredBackend.LoadModules() + + if config.ApiVersion == "v3" { moduleService := v3.NewModuleOperationsApi() releaseService := v3.NewReleaseOperationsApi() searchFilterService := v3.NewSearchFilterOperationsApi() userService := v3.NewUserOperationsApi() - handler := openapi.NewRouter( + + r := chi.NewRouter() + + r.Use(middleware.Recoverer) + r.Use(middleware.RealIP) + r.Use(customMiddleware.RequireUserAgent) + r.Use(cors.Handler(cors.Options{ + AllowedOrigins: strings.Split(config.CORSOrigins, ","), + AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Content-Type"}, + AllowCredentials: false, + MaxAge: 300, + })) + + if !config.NoCache { + if _, err := os.Stat(config.CacheDir); err != nil { + err = os.MkdirAll(config.CacheDir, os.ModePerm) + if err != nil { + log.Log.Fatal(err) + } + } + r.Use(customMiddleware.CacheMiddleware(strings.Split(config.CachePrefixes, ","), config.CacheDir)) + } + + if config.FallbackProxyUrl != "" { + r.Use(customMiddleware.ProxyFallback(config.FallbackProxyUrl, func(status int) bool { + return status == http.StatusNotFound + })) + } + + apiRouter := openapi.NewRouter( openapi.NewModuleOperationsAPIController(moduleService), openapi.NewReleaseOperationsAPIController(releaseService), openapi.NewSearchFilterOperationsAPIController(searchFilterService), openapi.NewUserOperationsAPIController(userService), ) - fmt.Println("serve on :8080") - log.Panic(http.ListenAndServe(":8080", handler)) + + r.Mount("/", apiRouter) + + log.Log.Infof("Listen on %s:%d", config.Bind, config.Port) + log.Log.Panic(http.ListenAndServe(fmt.Sprintf("%s:%d", config.Bind, config.Port), r)) } else { - log.Panicf("%s version not supported", apiVersion) + log.Log.Panicf("%s version not supported", config.ApiVersion) } }, } @@ -60,14 +109,15 @@ to quickly create a Cobra application.`, func init() { rootCmd.AddCommand(serveCmd) - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // serveCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - serveCmd.Flags().StringVar(&apiVersion, "api-version", "v3", "the forge api version to use") + serveCmd.Flags().StringVar(&config.ApiVersion, "api-version", "v3", "the forge api version to use") + serveCmd.Flags().IntVar(&config.Port, "port", 8080, "the port to listen to") + serveCmd.Flags().StringVar(&config.Bind, "bind", "", "host to listen to") + serveCmd.Flags().StringVar(&config.ModulesDir, "modulesdir", "/opt/gorge/modules", "directory containing all the modules") + serveCmd.Flags().StringVar(&config.CacheDir, "cachedir", "/var/cache/gorge", "cache directory") + serveCmd.Flags().StringVar(&config.CachePrefixes, "cache-prefixes", "/v3/files", "url prefixes to cache") + serveCmd.Flags().StringVar(&config.Backend, "backend", "filesystem", "backend to use") + serveCmd.Flags().StringVar(&config.CORSOrigins, "cors", "*", "allowed cors origins separated by comma") + serveCmd.Flags().StringVar(&config.FallbackProxyUrl, "fallback-proxy", "", "optional fallback upstream proxy url") + serveCmd.Flags().BoolVar(&config.Dev, "dev", false, "enables dev mode") + serveCmd.Flags().BoolVar(&config.NoCache, "no-cache", false, "disables the caching functionality") } diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..da260b4 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,39 @@ +/* +Copyright © 2024 dadav + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var version string = "0.1.0-alpha" + +// versionCmd represents the version command +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print the current version, then exit.", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(version) + os.Exit(0) + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/go.mod b/go.mod index a7ed315..9d6f1fd 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,12 @@ go 1.22.0 require ( github.com/go-chi/chi/v5 v5.0.12 + github.com/go-chi/cors v1.2.1 github.com/mitchellh/go-homedir v1.1.0 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 + go.uber.org/zap v1.27.0 + golang.org/x/mod v0.12.0 ) require ( @@ -23,8 +26,7 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 4e8d45a..37d5e6e 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= +github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -52,19 +54,22 @@ github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMV github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= diff --git a/internal/api/v3/module.go b/internal/api/v3/module.go index bedcfb2..ea3de9a 100644 --- a/internal/api/v3/module.go +++ b/internal/api/v3/module.go @@ -4,7 +4,13 @@ import ( "context" "errors" "net/http" + "net/url" + "slices" + "strconv" + "strings" + "github.com/dadav/gorge/internal/backend" + "github.com/dadav/gorge/internal/log" gen "github.com/dadav/gorge/pkg/gen/v3/openapi" ) @@ -62,42 +68,135 @@ func (s *ModuleOperationsApi) DeprecateModule(ctx context.Context, moduleSlug st return gen.Response(http.StatusNotImplemented, nil), errors.New("DeprecateModule method not implemented") } -// GetModule - Fetch module -func (s *ModuleOperationsApi) GetModule(ctx context.Context, moduleSlug string, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string) (gen.ImplResponse, error) { - // TODO - update GetModule with the required logic for this service method. - // Add api_module_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, Module{}) or use other options such as http.Ok ... - // return Response(200, Module{}), nil - - // TODO: Uncomment the next line to return response Response(304, {}) or use other options such as http.Ok ... - // return Response(304, nil),nil - - // TODO: Uncomment the next line to return response Response(400, GetFile400Response{}) or use other options such as http.Ok ... - // return Response(400, GetFile400Response{}), nil +type GetModule404Response struct { + Message string `json:"message,omitempty"` + Errors []string `json:"errors,omitempty"` +} - // TODO: Uncomment the next line to return response Response(404, GetFile404Response{}) or use other options such as http.Ok ... - // return Response(404, GetFile404Response{}), nil +type GetModule500Response struct { + Message string `json:"message,omitempty"` + Errors []string `json:"errors,omitempty"` +} - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetModule method not implemented") +// GetModule - Fetch module +func (s *ModuleOperationsApi) GetModule(ctx context.Context, moduleSlug string, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string) (gen.ImplResponse, error) { + module, err := backend.ConfiguredBackend.GetModuleBySlug(moduleSlug) + if err != nil { + log.Log.Error(err) + return gen.Response( + http.StatusNotFound, + GetModule404Response{ + Message: http.StatusText(http.StatusNotFound), + Errors: []string{"Module could not be found"}, + }), nil + } + + return gen.Response(http.StatusOK, module), nil } // GetModules - List modules func (s *ModuleOperationsApi) GetModules(ctx context.Context, limit int32, offset int32, sortBy string, query string, tag string, owner string, withTasks bool, withPlans bool, withPdk bool, premium bool, excludePremium bool, endorsements []string, operatingsystem string, operatingsystemrelease string, peRequirement string, puppetRequirement string, withMinimumScore int32, moduleGroups []string, showDeleted bool, hideDeprecated bool, onlyLatest bool, slugs []string, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string, startsWith string, supported bool, withReleaseSince string) (gen.ImplResponse, error) { - // TODO - update GetModules with the required logic for this service method. - // Add api_module_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, GetModules200Response{}) or use other options such as http.Ok ... - // return Response(200, GetModules200Response{}), nil - - // TODO: Uncomment the next line to return response Response(304, {}) or use other options such as http.Ok ... - // return Response(304, nil),nil - - // TODO: Uncomment the next line to return response Response(400, GetFile400Response{}) or use other options such as http.Ok ... - // return Response(400, GetFile400Response{}), nil - - // TODO: Uncomment the next line to return response Response(404, GetFile404Response{}) or use other options such as http.Ok ... - // return Response(404, GetFile404Response{}), nil - - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetModules method not implemented") + results := []gen.Module{} + filtered := []gen.Module{} + allModules := backend.ConfiguredBackend.GetAllModules() + params := url.Values{} + params.Add("offset", strconv.Itoa(int(offset))) + params.Add("limit", strconv.Itoa(int(limit))) + + if int(offset)+1 > len(allModules) { + return gen.Response(404, GetModule404Response{ + Message: "Invalid offset", + Errors: []string{"The given offset is larger than the total number of (filtered) modules."}, + }), nil + } + + for _, m := range allModules[offset:] { + var filterMatched, filterSet bool + if query != "" { + filterSet = true + filterMatched = strings.Contains(m.Slug, query) || strings.Contains(m.Owner.Slug, query) + params.Add("query", query) + } + + if tag != "" { + filterSet = true + filterMatched = slices.Contains(m.CurrentRelease.Tags, tag) + params.Add("tag", tag) + } + + if owner != "" { + filterSet = true + filterMatched = m.Owner.Username == owner + params.Add("owner", owner) + } + + if withTasks { + filterSet = true + filterMatched = len(m.CurrentRelease.Tasks) > 0 + params.Add("with_tasks", strconv.FormatBool(withTasks)) + } + + if withPlans { + filterSet = true + filterMatched = len(m.CurrentRelease.Plans) > 0 + params.Add("with_plans", strconv.FormatBool(withPlans)) + } + + if withPdk { + filterSet = true + filterMatched = m.CurrentRelease.Pdk + params.Add("with_pdk", strconv.FormatBool(withPdk)) + } + + if premium { + filterSet = true + filterMatched = m.Premium + params.Add("premium", strconv.FormatBool(premium)) + } + + if excludePremium { + filterSet = true + filterMatched = !m.Premium + params.Add("exclude_premium", strconv.FormatBool(excludePremium)) + } + + if len(endorsements) > 0 { + filterSet = true + filterMatched = m.Endorsement != nil && slices.Contains(endorsements, *m.Endorsement) + params.Add("endorsements", "["+strings.Join(endorsements, ",")+"]") + } + + if !filterSet || filterMatched { + filtered = append(filtered, *m) + } + } + + i := 1 + for _, module := range filtered { + if i > int(limit) { + break + } + results = append(results, module) + i++ + } + + base, _ := url.Parse("/v3/modules") + base.RawQuery = params.Encode() + currentInf := interface{}(base.String()) + params.Set("offset", "0") + firstInf := interface{}(base.String()) + params.Set("offset", strconv.Itoa(int(offset)+len(results))) + nextInf := interface{}(base.String()) + + return gen.Response(http.StatusOK, gen.GetModules200Response{ + Pagination: gen.GetModules200ResponsePagination{ + Limit: limit, + Offset: offset, + First: &firstInf, + Current: ¤tInf, + Next: &nextInf, + Total: int32(len(allModules)), + }, + Results: results, + }), nil } diff --git a/internal/api/v3/release.go b/internal/api/v3/release.go index b3d7659..e18dfe1 100644 --- a/internal/api/v3/release.go +++ b/internal/api/v3/release.go @@ -4,7 +4,14 @@ import ( "context" "errors" "net/http" - + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/dadav/gorge/internal/backend" + "github.com/dadav/gorge/internal/config" gen "github.com/dadav/gorge/pkg/gen/v3/openapi" ) @@ -16,6 +23,12 @@ func NewReleaseOperationsApi() *ReleaseOperationsApi { return &ReleaseOperationsApi{} } +type GetRelease404Response struct { + Message string `json:"message,omitempty"` + + Errors []string `json:"errors,omitempty"` +} + // AddRelease - Create module release func (s *ReleaseOperationsApi) AddRelease(ctx context.Context, addReleaseRequest gen.AddReleaseRequest) (gen.ImplResponse, error) { // TODO - update AddRelease with the required logic for this service method. @@ -62,41 +75,51 @@ func (s *ReleaseOperationsApi) DeleteRelease(ctx context.Context, releaseSlug st return gen.Response(http.StatusNotImplemented, nil), errors.New("DeleteRelease method not implemented") } +func ReleaseToModule(releaseSlug string) string { + return releaseSlug[:strings.LastIndex(releaseSlug, "-")] +} + // GetFile - Download module release func (s *ReleaseOperationsApi) GetFile(ctx context.Context, filename string) (gen.ImplResponse, error) { - // TODO - update GetFile with the required logic for this service method. - // Add api_release_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, *os.File{}) or use other options such as http.Ok ... - // return Response(200, *os.File{}), nil - - // TODO: Uncomment the next line to return response Response(400, GetFile400Response{}) or use other options such as http.Ok ... - // return Response(400, GetFile400Response{}), nil - - // TODO: Uncomment the next line to return response Response(404, GetFile404Response{}) or use other options such as http.Ok ... - // return Response(404, GetFile404Response{}), nil + f, err := os.Open(filepath.Join(config.ModulesDir, ReleaseToModule(filename), filename)) + if err != nil { + if os.IsNotExist(err) { + return gen.Response(http.StatusNotFound, gen.GetFile404Response{ + Message: http.StatusText(http.StatusNotFound), + Errors: []string{"The file does not exist."}, + }), nil + } + } + + return gen.Response(http.StatusOK, f), nil +} - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetFile method not implemented") +type GetRelease500Response struct { + Message string `json:"message,omitempty"` + Errors []string `json:"errors,omitempty"` } // GetRelease - Fetch module release func (s *ReleaseOperationsApi) GetRelease(ctx context.Context, releaseSlug string, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string) (gen.ImplResponse, error) { - // TODO - update GetRelease with the required logic for this service method. - // Add api_release_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, Release{}) or use other options such as http.Ok ... - // return Response(200, Release{}), nil - - // TODO: Uncomment the next line to return response Response(304, {}) or use other options such as http.Ok ... - // return Response(304, nil),nil - - // TODO: Uncomment the next line to return response Response(400, GetFile400Response{}) or use other options such as http.Ok ... - // return Response(400, GetFile400Response{}), nil - - // TODO: Uncomment the next line to return response Response(404, GetFile404Response{}) or use other options such as http.Ok ... - // return Response(404, GetFile404Response{}), nil - - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetRelease method not implemented") + metadata, readme, err := backend.ReadReleaseMetadata(releaseSlug) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return gen.Response(http.StatusNotFound, gen.GetFile404Response{ + Message: http.StatusText(http.StatusNotFound), + Errors: []string{"release not found"}, + }), nil + } + return gen.Response(http.StatusInternalServerError, GetRelease500Response{ + Message: http.StatusText(http.StatusInternalServerError), + Errors: []string{"error while reading release metadata"}, + }), nil + } + + return gen.Response(http.StatusOK, gen.Release{ + Slug: releaseSlug, + Module: gen.ReleaseModule{Name: metadata.Name}, + Readme: readme, + }), nil } // GetReleasePlan - Fetch module release plan @@ -129,14 +152,78 @@ func (s *ReleaseOperationsApi) GetReleasePlans(ctx context.Context, releaseSlug // GetReleases - List module releases func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, offset int32, sortBy string, module string, owner string, withPdk bool, operatingsystem string, operatingsystemrelease string, peRequirement string, puppetRequirement string, moduleGroups []string, showDeleted bool, hideDeprecated bool, withHtml bool, includeFields []string, excludeFields []string, ifModifiedSince string, supported bool) (gen.ImplResponse, error) { - // TODO - update GetReleases with the required logic for this service method. - // Add api_release_operations_service.go to the .openapi-generator-ignore to avoid overwriting this service implementation when updating open api generation. - - // TODO: Uncomment the next line to return response Response(200, GetReleases200Response{}) or use other options such as http.Ok ... - // return Response(200, GetReleases200Response{}), nil - - // TODO: Uncomment the next line to return response Response(304, {}) or use other options such as http.Ok ... - // return Response(304, nil),nil - - return gen.Response(http.StatusNotImplemented, nil), errors.New("GetReleases method not implemented") + results := []gen.Release{} + filtered := []gen.Release{} + allReleases := backend.ConfiguredBackend.GetAllReleases() + params := url.Values{} + params.Add("offset", strconv.Itoa(int(offset))) + params.Add("limit", strconv.Itoa(int(limit))) + + if int(offset)+1 > len(allReleases) { + return gen.Response(404, GetRelease404Response{ + Message: "Invalid offset", + Errors: []string{"The given offset is larger than the total number of modules."}, + }), nil + } + + for _, r := range allReleases[offset:] { + var filterMatched, filterSet bool + + if module != "" && r.Module.Slug != module { + filterSet = true + filterMatched = r.Module.Slug == module + params.Add("module", module) + } + if owner != "" && r.Module.Owner.Slug != owner { + filterSet = true + filterMatched = r.Module.Owner.Slug == owner + params.Add("owner", owner) + } + + if !filterSet || filterMatched { + filtered = append(filtered, *r) + } + } + + i := 1 + for _, release := range filtered { + if i > int(limit) { + break + } + results = append(results, release) + i++ + } + + base, _ := url.Parse("/v3/releases") + base.RawQuery = params.Encode() + currentInf := interface{}(base.String()) + params.Set("offset", "0") + firstInf := interface{}(base.String()) + + var nextInf interface{} + nextOffset := int(offset) + len(results) + if nextOffset < len(filtered) { + params.Set("offset", strconv.Itoa(nextOffset)) + nextInf = interface{}(base.String()) + } + + var prevInf *string + prevOffset := int(offset) - int(limit) + if prevOffset >= 0 { + prevStr := base.String() + prevInf = &prevStr + } + + return gen.Response(http.StatusOK, gen.GetReleases200Response{ + Pagination: gen.GetReleases200ResponsePagination{ + Limit: limit, + Offset: offset, + First: &firstInf, + Previous: prevInf, + Current: ¤tInf, + Next: &nextInf, + Total: int32(len(filtered)), + }, + Results: results, + }), nil } diff --git a/internal/backend/config.go b/internal/backend/config.go new file mode 100644 index 0000000..f4ced9c --- /dev/null +++ b/internal/backend/config.go @@ -0,0 +1,3 @@ +package backend + +var ConfiguredBackend Backend diff --git a/internal/backend/filesystem.go b/internal/backend/filesystem.go new file mode 100644 index 0000000..5bbc839 --- /dev/null +++ b/internal/backend/filesystem.go @@ -0,0 +1,311 @@ +package backend + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/md5" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/dadav/gorge/internal/config" + "github.com/dadav/gorge/internal/model" + gen "github.com/dadav/gorge/pkg/gen/v3/openapi" + "golang.org/x/mod/semver" +) + +type FilesystemBackend struct { + muModules sync.RWMutex + Modules map[string]*gen.Module + ModulesDir string + muReleases sync.RWMutex + Releases map[string][]*gen.Release +} + +var _ Backend = (*FilesystemBackend)(nil) + +func NewFilesystemBackend(path string) *FilesystemBackend { + return &FilesystemBackend{ + ModulesDir: path, + } +} + +func findLatestVersion(releases []gen.ReleaseAbbreviated) string { + latest := "0.0.0" + for i, r := range releases { + if i == 0 { + latest = r.Version + continue + } + + if semver.Compare(r.Version, latest) >= 1 { + latest = r.Version + } + } + return latest +} + +func currentReleaseToAbbreviatedRelease(release *gen.ModuleCurrentRelease) *gen.ReleaseAbbreviated { + return &gen.ReleaseAbbreviated{ + Uri: release.Uri, + Slug: release.Slug, + Version: release.Version, + Supported: release.Supported, + CreatedAt: release.CreatedAt, + DeletedAt: release.DeletedAt, + FileUri: release.FileUri, + FileSize: release.FileSize, + } +} + +func (s *FilesystemBackend) GetAllReleases() []*gen.Release { + s.muReleases.Lock() + defer s.muReleases.Unlock() + result := []*gen.Release{} + + for _, v := range s.Releases { + result = append(result, v...) + } + + return result +} + +func (s *FilesystemBackend) AddRelease(name, version string, data []byte) error { + releaseFile := fmt.Sprintf("%s-%s.tar.gz", name, version) + cacheFileDir := filepath.Join(config.ModulesDir, name) + if _, err := os.Stat(cacheFileDir); err != nil { + if errors.Is(err, os.ErrNotExist) { + err = os.MkdirAll(cacheFileDir, os.ModePerm) + if err != nil { + return err + } + } else { + return err + } + } + + cacheFile := filepath.Join(cacheFileDir, releaseFile) + _, err := os.Stat(cacheFile) + if err == nil { + return nil + } + + err = os.WriteFile(cacheFile, data, 0644) + if err != nil { + return err + } + return nil +} + +func (s *FilesystemBackend) GetAllModules() []*gen.Module { + s.muModules.Lock() + defer s.muModules.Unlock() + + result := []*gen.Module{} + + for _, v := range s.Modules { + result = append(result, v) + } + + return result +} + +func (s *FilesystemBackend) GetModuleBySlug(slug string) (*gen.Module, error) { + s.muModules.Lock() + defer s.muModules.Unlock() + if module, ok := s.Modules[slug]; !ok { + return nil, errors.New("module not found") + } else { + return module, nil + } +} + +func (s *FilesystemBackend) GetReleaseBySlug(slug string) (*gen.Release, error) { + s.muReleases.Lock() + defer s.muReleases.Unlock() + for _, moduleReleases := range s.Releases { + for _, release := range moduleReleases { + if release.Slug == slug { + return release, nil + } + } + } + return nil, errors.New("release not found") +} + +func (s *FilesystemBackend) LoadModules() error { + s.muModules.Lock() + s.muReleases.Lock() + defer s.muModules.Unlock() + defer s.muReleases.Unlock() + + s.Modules = make(map[string]*gen.Module) + s.Releases = make(map[string][]*gen.Release) + + err := filepath.Walk(s.ModulesDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() || !strings.HasSuffix(info.Name(), ".tar.gz") { + return nil + } + + releaseMetadata, releaseReadme, err := ReadReleaseMetadata(path) + if err != nil { + return err + } + + moduleSlug := releaseMetadata.Name + moduleName := strings.TrimPrefix(releaseMetadata.Name, fmt.Sprintf("%s-", releaseMetadata.Author)) + releaseSlug := fmt.Sprintf("%s-%s", releaseMetadata.Name, releaseMetadata.Version) + releasePath := fmt.Sprintf("/v3/files/%s.tar.gz", releaseSlug) + + var releaseMetadataInterface map[string]interface{} + inrec, _ := json.Marshal(releaseMetadata) + json.Unmarshal(inrec, &releaseMetadataInterface) + + md5Hash := md5.New() + sha256Hash := sha256.New() + + releaseFile, err := os.Open(path) + if err != nil { + return err + } + defer releaseFile.Close() + + _, err = io.Copy(md5Hash, releaseFile) + if err != nil { + return err + } + + _, err = releaseFile.Seek(0, 0) + if err != nil { + return err + } + + _, err = io.Copy(sha256Hash, releaseFile) + if err != nil { + return err + } + + md5Sum := fmt.Sprintf("%x", md5Hash.Sum(nil)) + sha256Sum := fmt.Sprintf("%x", sha256Hash.Sum(nil)) + owner := gen.ModuleOwner{ + Uri: fmt.Sprintf("/v3/users/%s", releaseMetadata.Author), + Slug: releaseMetadata.Author, + Username: releaseMetadata.Author, + } + + newRelease := gen.Release{ + Uri: fmt.Sprintf("/%s/releases/%s", config.ApiVersion, releaseSlug), + Slug: releaseMetadata.Name, + Module: gen.ReleaseModule{ + Uri: fmt.Sprintf("/v3/modules/%s", moduleSlug), + Slug: moduleSlug, + Name: moduleName, + Owner: owner, + }, + Version: releaseMetadata.Version, + Metadata: releaseMetadataInterface, + Tags: releaseMetadata.Tags, + FileUri: releasePath, + FileSize: int32(info.Size()), + FileMd5: md5Sum, + FileSha256: sha256Sum, + Readme: releaseReadme, + License: releaseMetadata.License, + } + + currentRelease := gen.ModuleCurrentRelease(newRelease) + + if module, ok := s.Modules[moduleSlug]; !ok { + newModule := gen.Module{ + Uri: fmt.Sprintf("/%s/modules/%s", config.ApiVersion, releaseMetadata.Name), + Slug: releaseMetadata.Name, + Name: moduleName, + Owner: owner, + CurrentRelease: gen.ModuleCurrentRelease(newRelease), + Releases: []gen.ReleaseAbbreviated{*currentReleaseToAbbreviatedRelease(¤tRelease)}, + } + + s.Modules[moduleSlug] = &newModule + s.Releases[moduleSlug] = []*gen.Release{&newRelease} + } else { + s.Releases[moduleSlug] = append(s.Releases[moduleSlug], &newRelease) + module.Releases = append(module.Releases, *currentReleaseToAbbreviatedRelease(¤tRelease)) + if findLatestVersion(module.Releases) == currentRelease.Version { + module.CurrentRelease = currentRelease + } + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +func ReadReleaseMetadata(path string) (*model.ReleaseMetadata, string, error) { + var jsonData bytes.Buffer + var releaseMetadata model.ReleaseMetadata + readme := new(strings.Builder) + + f, err := os.Open(path) + if err != nil { + return nil, readme.String(), err + } + defer f.Close() + + g, err := gzip.NewReader(f) + if err != nil { + return nil, readme.String(), err + } + + tarReader := tar.NewReader(g) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, readme.String(), err + } + + if header.Typeflag != tar.TypeReg { + continue + } + + switch filepath.Base(header.Name) { + case "metadata.json": + _, err = io.Copy(&jsonData, tarReader) + if err != nil { + return nil, readme.String(), err + } + + if err := json.Unmarshal(jsonData.Bytes(), &releaseMetadata); err != nil { + return nil, readme.String(), err + } + + case "README.md": + _, err = io.Copy(readme, tarReader) + if err != nil { + return nil, readme.String(), err + } + default: + continue + } + } + return &releaseMetadata, readme.String(), nil +} diff --git a/internal/backend/interface.go b/internal/backend/interface.go new file mode 100644 index 0000000..9d146df --- /dev/null +++ b/internal/backend/interface.go @@ -0,0 +1,23 @@ +package backend + +import gen "github.com/dadav/gorge/pkg/gen/v3/openapi" + +type Backend interface { + // LoadModules loads modules into memory + LoadModules() error + + // GetAllModules returns a list of all modules + GetAllModules() []*gen.Module + + // GetModuleBySlug contains a map to modules + GetModuleBySlug(string) (*gen.Module, error) + + // GetAllReleases returns a list of all releases + GetAllReleases() []*gen.Release + + // GetReleaseBySlug contains a map to modules + GetReleaseBySlug(string) (*gen.Release, error) + + // Add a new release + AddRelease(module string, version string, data []byte) error +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..a7bb229 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,15 @@ +package config + +var ( + ApiVersion string + Port int + Bind string + Dev bool + ModulesDir string + Backend string + CORSOrigins string + FallbackProxyUrl string + NoCache bool + CachePrefixes string + CacheDir string +) diff --git a/internal/log/zap.go b/internal/log/zap.go new file mode 100644 index 0000000..8901f87 --- /dev/null +++ b/internal/log/zap.go @@ -0,0 +1,21 @@ +package log + +import ( + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var Log *zap.SugaredLogger + +func Setup(dev bool) { + var logger *zap.Logger + if dev { + config := zap.NewDevelopmentConfig() + config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + logger, _ = config.Build() + } else { + logger, _ = zap.NewProduction() + } + defer logger.Sync() + Log = logger.Sugar() +} diff --git a/internal/middleware/cache.go b/internal/middleware/cache.go new file mode 100644 index 0000000..0294722 --- /dev/null +++ b/internal/middleware/cache.go @@ -0,0 +1,88 @@ +package middleware + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/dadav/gorge/internal/log" +) + +type ContentHeaders struct { + Type string `json:"type"` + Encoding string `json:"encoding"` +} + +func CacheMiddleware(prefixes []string, cacheDir string) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + matched := false + for _, prefix := range prefixes { + if strings.HasPrefix(r.URL.Path, prefix) { + matched = true + break + } + } + + if !matched { + next.ServeHTTP(w, r) + return + } + + cacheKeyRaw := fmt.Sprintf("%s?%s", r.URL.Path, r.URL.RawQuery) + hash := sha256.New() + hash.Write([]byte(cacheKeyRaw)) + cacheKeyHash := hex.EncodeToString(hash.Sum(nil)) + cacheFilePath := filepath.Join(cacheDir, cacheKeyHash) + cacheFileHeadersPath := fmt.Sprintf("%s_headers", cacheFilePath) + + cacheControlHeader := r.Header.Get("Cache-Control") + if cacheControlHeader != "no-cache" { + if _, err := os.Stat(cacheFilePath); err == nil { + data, err := os.ReadFile(cacheFilePath) + if err == nil { + log.Log.Debugf("Send response from cache for %s\n", r.URL.Path) + headerBytes, err := os.ReadFile(cacheFileHeadersPath) + if err == nil { + var contentHeaders ContentHeaders + json.Unmarshal(headerBytes, &contentHeaders) + w.Header().Add("Content-Type", contentHeaders.Type) + w.Header().Add("Content-Encoding", contentHeaders.Encoding) + } + w.Write(data) + return + } + } + } + + capturedResponseWriter := &capturedResponseWriter{ResponseWriter: w} + next.ServeHTTP(capturedResponseWriter, r) + + if capturedResponseWriter.status == http.StatusOK && cacheControlHeader != "no-store" { + err := os.WriteFile(cacheFilePath, capturedResponseWriter.body, 0600) + if err != nil { + log.Log.Error(err) + } + + contentHeaders := ContentHeaders{ + Type: capturedResponseWriter.Header().Get("Content-Type"), + Encoding: capturedResponseWriter.Header().Get("Content-Encoding"), + } + contentHeadersBytes, err := json.Marshal(contentHeaders) + if err == nil { + err = os.WriteFile(cacheFileHeadersPath, contentHeadersBytes, 0600) + if err != nil { + log.Log.Error(err) + } + } + } + + capturedResponseWriter.sendCapturedResponse() + }) + } +} diff --git a/internal/middleware/proxy.go b/internal/middleware/proxy.go new file mode 100644 index 0000000..2c8be45 --- /dev/null +++ b/internal/middleware/proxy.go @@ -0,0 +1,111 @@ +package middleware + +import ( + "bytes" + "io" + "net/http" + "net/url" + + "github.com/dadav/gorge/internal/log" +) + +// capturedResponseWriter is a custom response writer that captures the response status +type capturedResponseWriter struct { + http.ResponseWriter + body []byte + status int +} + +func (w *capturedResponseWriter) WriteHeader(code int) { + w.status = code +} + +func (w *capturedResponseWriter) Write(body []byte) (int, error) { + w.body = body + return len(body), nil +} + +func (w *capturedResponseWriter) sendCapturedResponse() { + w.ResponseWriter.WriteHeader(w.status) + w.ResponseWriter.Write(w.body) +} + +func ProxyFallback(upstreamHost string, forwardToProxy func(int) bool) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // capture response + capturedResponseWriter := &capturedResponseWriter{ResponseWriter: w} + next.ServeHTTP(capturedResponseWriter, r) + + if forwardToProxy(capturedResponseWriter.status) { + log.Log.Infof("Forwarding request to %s\n", upstreamHost) + forwardRequest(w, r, upstreamHost) + return + } + + // If the response status is not 404, serve the original response + capturedResponseWriter.sendCapturedResponse() + }) + } +} + +func forwardRequest(w http.ResponseWriter, r *http.Request, forwardHost string) { + // Create a buffer to store the request body + var requestBodyBytes []byte + if r.Body != nil { + requestBodyBytes, _ = io.ReadAll(r.Body) + } + + // Clone the original request + forwardUrl, err := url.JoinPath(forwardHost, r.URL.Path) + if err != nil { + http.Error(w, "Failed to create forwarded request", http.StatusInternalServerError) + return + } + + forwardedRequest, err := http.NewRequest(r.Method, forwardUrl, bytes.NewBuffer(requestBodyBytes)) + if err != nil { + http.Error(w, "Failed to create forwarded request", http.StatusInternalServerError) + return + } + + // Set the parameters + forwardedRequest.URL.RawQuery = r.URL.RawQuery + + // Copy headers from the original request + forwardedRequest.Header = make(http.Header) + for key, values := range r.Header { + for _, value := range values { + forwardedRequest.Header.Add(key, value) + } + } + + // Make the request to the forward host + client := http.Client{} + resp, err := client.Do(forwardedRequest) + if err != nil { + http.Error(w, "Failed to forward request", http.StatusInternalServerError) + return + } + defer resp.Body.Close() + + // Copy the response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + + log.Log.Debugf("Response of proxied request is %d\n", resp.StatusCode) + + // Write the response status code + w.WriteHeader(resp.StatusCode) + + // Write the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, "Failed to read response body", http.StatusInternalServerError) + return + } + w.Write(body) +} diff --git a/internal/middleware/useragent.go b/internal/middleware/useragent.go new file mode 100644 index 0000000..a8dd9d2 --- /dev/null +++ b/internal/middleware/useragent.go @@ -0,0 +1,33 @@ +package middleware + +import ( + "encoding/json" + "net/http" +) + +type UserAgentNotSetResponse struct { + Message string `json:"message,omitempty"` + Errors []string `json:"errors,omitempty"` +} + +func RequireUserAgent(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent := r.Header.Get("User-Agent") + if userAgent == "" { + errorResponse := UserAgentNotSetResponse{ + Message: "User-Agent header is missing", + Errors: []string{"User-Agent must have some value"}, + } + jsonError, err := json.Marshal(errorResponse) + if err != nil { + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + w.Write(jsonError) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/internal/model/module_metadata.go b/internal/model/module_metadata.go new file mode 100644 index 0000000..ff49c38 --- /dev/null +++ b/internal/model/module_metadata.go @@ -0,0 +1,28 @@ +package model + +type SupportedOS struct { + Name string `json:"operatingsystem"` + Releases []string `json:"operatingsystemrelease,omitempty"` +} + +type ModuleDependency struct { + Name string `json:"name"` + VersionRequirement string `json:"version_requirement,omitempty"` +} + +type ModuleRequirement ModuleDependency + +type ReleaseMetadata struct { + Name string `json:"name"` + Version string `json:"version"` + Author string `json:"author"` + License string `json:"license"` + Summary string `json:"summary"` + Source string `json:"source"` + Dependencies []ModuleDependency `json:"dependencies"` + Requirements []ModuleRequirement `json:"requirements,omitempty"` + ProjectUrl string `json:"project_url,omitempty"` + IssuesUrl string `json:"issues_url,omitempty"` + OperatingsystemSupport []SupportedOS `json:"operatingsystem_support,omitempty"` + Tags []string `json:"tags,omitempty"` +} diff --git a/main.go b/main.go index 1a608bb..f1cae32 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,11 @@ /* -Copyright © 2024 NAME HERE +Copyright © 2024 dadav Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/gen/v3/openapi/model_module.go b/pkg/gen/v3/openapi/model_module.go index f6dcbb7..b56e735 100644 --- a/pkg/gen/v3/openapi/model_module.go +++ b/pkg/gen/v3/openapi/model_module.go @@ -1,7 +1,7 @@ /* * Puppet Forge v3 API * - * ## Introduction The Puppet Forge API (hereafter referred to as the Forge API) provides quick access to all the data on the Puppet Forge via a RESTful interface. Using the Forge API, you can write scripts and tools that interact with the Puppet Forge website. The Forge API's current version is `v3`. It is considered regression-stable, meaning that the returned data is guaranteed to include the fields described in the schemas on this page; however, additional data might be added in the future and clients must ignore any properties they do not recognize. ## OpenAPI Specification The Puppet Forge v3 API is described by an [OpenAPI 3.0](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md) formatted specification file. The most up-to-date version of this specification file can be accessed at [https://forgeapi.puppet.com/v3/openapi.json](/v3/openapi.json). ## Features * The API is accessed over HTTPS via either the `forgeapi.puppet.com` (IPv4 or IPv6). All data is returned in JSON format. * Blank fields are included as `null`. * Nested resources may use an abbreviated representation. A link to the full representation for the resource is always included. * All timestamps in JSON responses are returned in ISO 8601 format: `YYYY-MM-DD HH:MM:SS ±HHMM`. * The HTTP response headers include caching hints for conditional requests. ## Concepts and Terminology * **Module**: Modules are self-contained bundles of code and data with a specific directory structure. Modules are identified by a combination of the author's username and the module's name, separated by a hyphen. For example: `puppetlabs-apache` * **Release**: A single, specific version of a module is called a Release. Releases are identified by a combination of the module identifier (see above) and the Release version, separated by a hyphen. For example: `puppetlabs-apache-4.0.0` ## Errors The Puppet Forge API follows [RFC 2616](https://tools.ietf.org/html/rfc2616) and [RFC 6585](https://tools.ietf.org/html/rfc6585). Error responses are served with a `4xx` or `5xx` status code, and are sent as a JSON document with a content type of `application/json`. The error document contains the following top-level keys and values: * `message`: a string value that summarizes the problem * `errors`: a list (array) of strings containing additional details describing the underlying cause(s) of the failure An example error response is shown below: ```json { \"message\": \"400 Bad Request\", \"errors\": [ \"Cannot parse request body as JSON\" ] } ``` ## User-Agent Required All API requests must include a valid `User-Agent` header. Requests with no `User-Agent` header will be rejected. The `User-Agent` header helps identify your application or library, so we can communicate with you if necessary. If your use of the API is informal or personal, we recommend using your username as the value for the `User-Agent` header. User-Agent headers are a list of one or more product descriptions, generally taking this form: ``` / (comments) ``` For example, the following are all useful User-Agent values: ``` MyApplication/0.0.0 Her/0.6.8 Faraday/0.8.8 Ruby/1.9.3-p194 (i386-linux) My-Library-Name/1.2.4 myusername ``` ## Hostname Configuration Most tools that interact with the Forge API allow specification of the hostname to use. You can configure a few common tools to use a specified hostname as follows: For **Puppet Enterprise** users, in [r10k](https://puppet.com/docs/pe/latest/r10k_customize_config.html#r10k_configuring_forge_settings) or [Code Manager](https://puppet.com/docs/pe/latest/code_mgr_customizing.html#config_forge_settings), specify `forge_settings` in Hiera: ``` pe_r10k::forge_settings: baseurl: 'https://forgeapi.puppet.com' ``` or ``` puppet_enterprise::master::code_manager::forge_settings: baseurl: 'https://forgeapi.puppet.com' ```
If you are an **open source Puppet** user using r10k, you'll need to [edit your r10k.yaml directly](https://github.com/puppetlabs/r10k/blob/main/doc/dynamic-environments/configuration.mkd#forge): ``` forge: baseurl: 'https://forgeapi.puppet.com' ``` or set the appropriate class param for the [open source r10k module](https://forge.puppet.com/puppet/r10k#forge_settings): ``` $forge_settings = { 'baseurl' => 'https://forgeapi.puppet.com', } ```
In [**Bolt**](https://puppet.com/docs/bolt/latest/bolt_installing_modules.html#install-forge-modules-from-an-alternate-forge), set a `baseurl` for the Forge in `bolt-project.yaml`: ``` module-install: forge: baseurl: https://forgeapi.puppet.com ```
Using `puppet config`: ``` $ puppet config set module_repository https://forgeapi.puppet.com ``` + * ## Introduction The Puppet Forge API (hereafter referred to as the Forge API) provides quick access to all the data on the Puppet Forge via a RESTful interface. Using the Forge API, you can write scripts and tools that interact with the Puppet Forge website. The Forge API's current version is `v3`. It is considered regression-stable, meaning that the returned data is guaranteed to include the fields described in the schemas on this page; however, additional data might be added in the future and clients must ignore any properties they do not recognize. ## OpenAPI Specification The Puppet Forge v3 API is described by an [OpenAPI 3.0](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md) formatted specification file. The most up-to-date version of this specification file can be accessed at [https://forgeapi.puppet.com/v3/openapi.json](/v3/openapi.json). ## Features * The API is accessed over HTTPS via either the `forgeapi.puppet.com` (IPv4 or IPv6). All data is returned in JSON format. * Blank fields are included as `null`. * Nested resources may use an abbreviated representation. A link to the full representation for the resource is always included. * All timestamps in JSON responses are returned in ISO 8601 format: `YYYY-MM-DD HH:MM:SS ±HHMM`. * The HTTP response headers include caching hints for conditional requests. ## Concepts and Terminology * **Module**: Modules are self-contained bundles of code and data with a specific directory structure. Modules are identified by a combination of the author's username and the module's name, separated by a hyphen. For example: `puppetlabs-apache` * **Release**: A single, specific version of a module is called a Release. Releases are identified by a combination of the module identifier (see above) and the Release version, separated by a hyphen. For example: `puppetlabs-apache-4.0.0` ## Errors The Puppet Forge API follows [RFC 2616](https://tools.ietf.org/html/rfc2616) and [RFC 6585](https://tools.ietf.org/html/rfc6585). Error responses are served with a `4xx` or `5xx` status code, and are sent as a JSON document with a content type of `application/json`. The error document contains the following top-level keys and values: * `message`: a string value that summarizes the problem * `errors`: a list (array) of strings containing additional details describing the underlying cause(s) of the failure An example error response is shown below: ```json { \"message\": \"400 Bad Request\", \"errors\": [ \"Cannot parse request body as JSON\" ] } ``` ## User-Agent Required All API requests must include a valid `User-Agent` header. Requests with no `User-Agent` header will be rejected. The `User-Agent` header helps identify your application or library, so we can communicate with you if necessary. If your use of the API is informal or personal, we recommend using your username as the value for the `User-Agent` header. User-Agent headers are a list of one or more product descriptions, generally taking this form: ``` / (comments) ``` For example, the following are all useful User-Agent values: ``` MyApplication/0.0.0 Her/0.6.8 Faraday/0.8.8 Ruby/1.9.3-p194 (i386-linux) My-Library-Name/1.2.4 myusername ``` ## Hostname Configuration Most tools that interact with the Forge API allow specification of the hostname to use. You can configure a few common tools to use a specified hostname as follows: For **Puppet Enterprise** users, in [r10k](https://puppet.com/docs/pe/latest/r10k_customize_config.html#r10k_configuring_forge_settings) or [Code Manager](https://puppet.com/docs/pe/latest/code_mgr_customizing.html#config_forge_settings), specify `forge_settings` in Hiera: ``` pe_r10k::forge_settings: baseurl: 'https://forgeapi.puppet.com' ``` or ``` puppet_enterprise::master::code_manager::forge_settings: baseurl: 'https://forgeapi.puppet.com' ```
If you are an **open source Puppet** user using r10k, you'll need to [edit your r10k.yaml directly](https://github.com/puppetlabs/r10k/blob/main/doc/dynamic-environments/configuration.mkd#forge): ``` forge: baseurl: 'https://forgeapi.puppet.com' ``` or set the appropriate class param for the [open source r10k module](https://forge.puppet.com/puppet/r10k#forge_settings): ``` $forge_settings = { 'baseurl' => 'https://forgeapi.puppet.com', } ```
In [**Bolt**](https://puppet.com/docs/bolt/latest/bolt_installing_modules.html#install-forge-modules-from-an-alternate-forge), set a `baseurl` for the Forge in `bolt-project.yaml`: ``` module-install: forge: baseurl: https://forgeapi.puppet.com ```
Using `puppet config`: ``` $ puppet config set module_repository https://forgeapi.puppet.com ``` * * API version: 29 * Generated by: OpenAPI Generator (https://openapi-generator.tech) @@ -9,15 +9,11 @@ package gorge - import ( "errors" ) - - type Module struct { - // Relative URL for this Module resource Uri string `json:"uri,omitempty"`