misc fixes and features

- cli / admin api bug fixes - add secret resource - performance - raft updates (snapshots disabled) - add metrics
dgate-io · May 5, 2024 · e97dee0 · e97dee0
1 parent 08d7642
commit e97dee0
Show file tree

Hide file tree

Showing 75 changed files with 1,906 additions and 1,160 deletions.
diff --git a/cmd/dgate-cli/commands/collection.go → ...dgate-cli/commands/collection_commands.go b/cmd/dgate-cli/commands/collection.go → ...dgate-cli/commands/collection_commands.go
@@ -57,7 +57,13 @@ func CollectionCommand(client *dgclient.DGateClient) *cli.Command {
 				Aliases: []string{"ls"},
 				Usage:   "list collections",
 				Action: func(ctx *cli.Context) error {
-					col, err := client.ListCollection()
+					nsp, err := createMapFromArgs[dgclient.NamespacePayload](
+						ctx.Args().Slice(),
+					)
+					if err != nil {
+						return err
+					}
+					col, err := client.ListCollection(nsp.Namespace)
 					if err != nil {
 						return err
 					}

diff --git a/cmd/dgate-cli/commands/document.go → cmd/dgate-cli/commands/document_commands.go b/cmd/dgate-cli/commands/document.go → cmd/dgate-cli/commands/document_commands.go
@@ -37,18 +37,37 @@ func DocumentCommand(client *dgclient.DGateClient) *cli.Command {
 				Name:    "delete",
 				Aliases: []string{"rm"},
 				Usage:   "delete a document",
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "all",
+						Usage: "delete all documents",
+					},
+				},
 				Action: func(ctx *cli.Context) error {
-					doc, err := createMapFromArgs[spec.Document](
-						ctx.Args().Slice(), "id",
-					)
-					if err != nil {
-						return err
-					}
-					err = client.DeleteDocument(
-						doc.ID, doc.NamespaceName,
-					)
-					if err != nil {
-						return err
+					if ctx.Bool("all") {
+						doc, err := createMapFromArgs[spec.Document](
+							ctx.Args().Slice())
+						if err != nil {
+							return err
+						}
+						err = client.DeleteAllDocument(
+							doc.NamespaceName, doc.CollectionName)
+						if err != nil {
+							return err
+						}
+						return nil
+					} else {
+						doc, err := createMapFromArgs[spec.Document](
+							ctx.Args().Slice(), "id",
+						)
+						if err != nil {
+							return err
+						}
+						err = client.DeleteDocument(doc.ID,
+							doc.NamespaceName, doc.CollectionName)
+						if err != nil {
+							return err
+						}
 					}
 					return nil
 				},
@@ -58,7 +77,14 @@ func DocumentCommand(client *dgclient.DGateClient) *cli.Command {
 				Aliases: []string{"ls"},
 				Usage:   "list documents",
 				Action: func(ctx *cli.Context) error {
-					doc, err := client.ListDocument()
+					d, err := createMapFromArgs[spec.Document](
+						ctx.Args().Slice(), "collection",
+					)
+					if err != nil {
+						return err
+					}
+					doc, err := client.ListDocument(
+						d.NamespaceName, d.CollectionName)
 					if err != nil {
 						return err
 					}
@@ -75,9 +101,8 @@ func DocumentCommand(client *dgclient.DGateClient) *cli.Command {
 					if err != nil {
 						return err
 					}
-					doc, err = client.GetDocument(
-						doc.ID, doc.NamespaceName,
-					)
+					doc, err = client.GetDocument(doc.ID,
+						doc.NamespaceName, doc.CollectionName)
 					if err != nil {
 						return err
 					}

diff --git a/cmd/dgate-cli/commands/domain.go → cmd/dgate-cli/commands/domain_commands.go b/cmd/dgate-cli/commands/domain.go → cmd/dgate-cli/commands/domain_commands.go
@@ -57,7 +57,13 @@ func DomainCommand(client *dgclient.DGateClient) *cli.Command {
 				Aliases: []string{"ls"},
 				Usage:   "list domains",
 				Action: func(ctx *cli.Context) error {
-					dom, err := client.ListDomain()
+					nsp, err := createMapFromArgs[dgclient.NamespacePayload](
+						ctx.Args().Slice(),
+					)
+					if err != nil {
+						return err
+					}
+					dom, err := client.ListDomain(nsp.Namespace)
 					if err != nil {
 						return err
 					}

diff --git a/cmd/dgate-cli/commands/module.go → cmd/dgate-cli/commands/module_commands.go b/cmd/dgate-cli/commands/module.go → cmd/dgate-cli/commands/module_commands.go
@@ -57,7 +57,13 @@ func ModuleCommand(client *dgclient.DGateClient) *cli.Command {
 				Aliases: []string{"ls"},
 				Usage:   "list modules",
 				Action: func(ctx *cli.Context) error {
-					mod, err := client.ListModule()
+					nsp, err := createMapFromArgs[dgclient.NamespacePayload](
+						ctx.Args().Slice(),
+					)
+					if err != nil {
+						return err
+					}
+					mod, err := client.ListModule(nsp.Namespace)
 					if err != nil {
 						return err
 					}

diff --git a/cmd/dgate-cli/commands/namespace.go → cmd/dgate-cli/commands/namespace_commands.go b/cmd/dgate-cli/commands/namespace.go → cmd/dgate-cli/commands/namespace_commands.go
diff --git a/cmd/dgate-cli/commands/route.go → cmd/dgate-cli/commands/route_commands.go b/cmd/dgate-cli/commands/route.go → cmd/dgate-cli/commands/route_commands.go
@@ -58,7 +58,13 @@ func RouteCommand(client *dgclient.DGateClient) *cli.Command {
 				Aliases: []string{"ls"},
 				Usage:   "list routes",
 				Action: func(ctx *cli.Context) error {
-					rt, err := client.ListRoute()
+					nsp, err := createMapFromArgs[dgclient.NamespacePayload](
+						ctx.Args().Slice(),
+					)
+					if err != nil {
+						return err
+					}
+					rt, err := client.ListRoute(nsp.Namespace)
 					if err != nil {
 						return err
 					}

diff --git a/cmd/dgate-cli/commands/secret_commands.go b/cmd/dgate-cli/commands/secret_commands.go
@@ -0,0 +1,95 @@
+package commands
+
+import (
+	"github.com/dgate-io/dgate/pkg/dgclient"
+	"github.com/dgate-io/dgate/pkg/spec"
+	"github.com/urfave/cli/v2"
+)
+
+func SecretCommand(client *dgclient.DGateClient) *cli.Command {
+	return &cli.Command{
+		Name:      "secret",
+		Aliases:   []string{"sec"},
+		Args:      true,
+		ArgsUsage: "<command> <name>",
+		Usage:     "secret commands",
+		Subcommands: []*cli.Command{
+			{
+				Name:    "create",
+				Aliases: []string{"mk"},
+				Usage:   "create a secret",
+				Action: func(ctx *cli.Context) error {
+					sec, err := createMapFromArgs[spec.Secret](
+						ctx.Args().Slice(), "name", "data",
+					)
+					if err != nil {
+						return err
+					}
+					err = client.CreateSecret(sec)
+					if err != nil {
+						return err
+					}
+					// redact the data field
+					sec.Data = "**redacted**"
+					return jsonPrettyPrint(sec)
+				},
+			},
+			{
+				Name:    "delete",
+				Aliases: []string{"rm"},
+				Usage:   "delete a secret",
+				Action: func(ctx *cli.Context) error {
+					sec, err := createMapFromArgs[spec.Secret](
+						ctx.Args().Slice(), "name",
+					)
+					if err != nil {
+						return err
+					}
+					err = client.DeleteSecret(
+						sec.Name, sec.NamespaceName)
+					if err != nil {
+						return err
+					}
+					return nil
+				},
+			},
+			{
+				Name:    "list",
+				Aliases: []string{"ls"},
+				Usage:   "list services",
+				Action: func(ctx *cli.Context) error {
+					nsp, err := createMapFromArgs[dgclient.NamespacePayload](
+						ctx.Args().Slice(),
+					)
+					if err != nil {
+						return err
+					}
+					sec, err := client.ListSecret(nsp.Namespace)
+					if err != nil {
+						return err
+					}
+					return jsonPrettyPrint(sec)
+				},
+			},
+			{
+				Name:  "get",
+				Usage: "get a secret",
+				Action: func(ctx *cli.Context) error {
+					s, err := createMapFromArgs[spec.Secret](
+						ctx.Args().Slice(), "name",
+					)
+					if err != nil {
+						return err
+					}
+					sec, err := client.GetSecret(
+						s.Name, s.NamespaceName,
+					)
+					if err != nil {
+						return err
+					}
+					return jsonPrettyPrint(sec)
+				},
+			},
+		},
+	}
+}
diff --git a/cmd/dgate-cli/commands/service.go → cmd/dgate-cli/commands/service_commands.go b/cmd/dgate-cli/commands/service.go → cmd/dgate-cli/commands/service_commands.go
@@ -57,7 +57,13 @@ func ServiceCommand(client *dgclient.DGateClient) *cli.Command {
 				Aliases: []string{"ls"},
 				Usage:   "list services",
 				Action: func(ctx *cli.Context) error {
-					svc, err := client.ListService()
+					nsp, err := createMapFromArgs[dgclient.NamespacePayload](
+						ctx.Args().Slice(),
+					)
+					if err != nil {
+						return err
+					}
+					svc, err := client.ListService(nsp.Namespace)
 					if err != nil {
 						return err
 					}

diff --git a/cmd/dgate-cli/main.go b/cmd/dgate-cli/main.go
@@ -43,6 +43,19 @@ func main() {
 				EnvVars: []string{"DGATE_ADMIN_AUTH"},
 				Usage:   "basic auth username:password; or just username for password prompt",
 			},
+			&cli.BoolFlag{
+				Name:        "follow",
+				DefaultText: "false",
+				Aliases:     []string{"f"},
+				EnvVars:     []string{"DGATE_FOLLOW_REDIRECTS"},
+				Usage:       "follows redirects, useful for raft leader changes",
+			},
+			&cli.BoolFlag{
+				Name:        "verbose",
+				DefaultText: "false",
+				Aliases:     []string{"V"},
+				Usage:       "enable verbose logging",
+			},
 		},
 		Before: func(ctx *cli.Context) (err error) {
 			var authOption dgclient.Options = func(dc *dgclient.DGateClient) {}
@@ -70,11 +83,17 @@ func main() {
 			return client.Init(
 				ctx.String("admin"),
 				authOption,
+				dgclient.WithFollowRedirect(
+					ctx.Bool("follow"),
+				),
 				dgclient.WithUserAgent(
 					"DGate CLI "+version+
 						";os="+runtime.GOOS+
 						";arch="+runtime.GOARCH,
 				),
+				dgclient.WithVerboseLogging(
+					ctx.Bool("verbose"),
+				),
 			)
 		},
 		Action: func(ctx *cli.Context) error {
@@ -88,6 +107,7 @@ func main() {
 			commands.DomainCommand(client),
 			commands.CollectionCommand(client),
 			commands.DocumentCommand(client),
+			commands.SecretCommand(client),
 		},
 	}
 

diff --git a/cmd/dgate-server/main.go b/cmd/dgate-server/main.go
@@ -3,6 +3,9 @@ package main
 import (
 	"fmt"
 	"os"
+	"os/signal"
+	"runtime"
+	"syscall"
 
 	"runtime/debug"
 
@@ -34,7 +37,8 @@ func main() {
 	}
 
 	if version == "dev" {
-		version = fmt.Sprintf("dev/PID:%d", os.Getpid())
+		fmt.Printf("PID:%d\n", os.Getpid())
+		fmt.Printf("GOMAXPROCS:%d\n", runtime.GOMAXPROCS(0))
 	}
 
 	if *showVersion {
@@ -52,16 +56,26 @@ func main() {
 					"-----------------------------------\n",
 			)
 		}
-		dgateConfig, err := config.LoadConfig(*configPath)
-		if err != nil {
-			panic(err)
-		}
+		if dgateConfig, err := config.LoadConfig(*configPath); err != nil {
+			fmt.Printf("Error loading config: %s\n", err)
+			os.Exit(1)
+		} else {
+			proxyState := proxy.StartProxyGateway(version, dgateConfig)
+			admin.StartAdminAPI(dgateConfig, proxyState)
+			if err := proxyState.Start(); err != nil {
+				fmt.Printf("Error loading config: %s\n", err)
+				os.Exit(1)
+			}
 
-		proxyState, err := proxy.StartProxyGateway(dgateConfig)
-		if err != nil {
-			panic(err)
+			sigchan := make(chan os.Signal, 1)
+			signal.Notify(sigchan,
+				syscall.SIGINT,
+				syscall.SIGTERM,
+				syscall.SIGQUIT,
+			)
+			<-sigchan
+			proxyState.Stop()
+			os.Exit(1)
 		}
-
-		admin.StartAdminAPI(dgateConfig, proxyState)
 	}
 }
diff --git a/config.dgate.yaml b/config.dgate.yaml
@@ -37,24 +37,8 @@ proxy:
   tls:
     port: ${PORT_SSL:-443}
     auto_generate: true
-    cert_file: .lego/certificates/ufosoup.com.crt
-    key_file: .lego/certificates/ufosoup.com.key
-  client_transport:
-    dns_server: 8.8.8.8:53
-    dns_timeout: 10s
-    max_conns_per_host: 1000
-    max_idle_conns: 5000
-    max_idle_conns_per_host: 1000
-    idle_conn_timeout: 60s
-    tls_handshake_timeout: 10s
-    expect_continue_timeout: 5s
-    max_response_header_bytes: 4096
-    write_buffer_size: 4096
-    read_buffer_size: 4096
-    max_conns_per_client: 1000
-    disable_keep_alives: false
-    response_header_timeout: 10s
-    dial_timeout: 10s
+    cert_file: internal/proxy/testdata/server.crt
+    key_file: internal/proxy/testdata/server.key
 admin:
   port: 9080
   host: 0.0.0.0