feat(dl): implement ks3 downloader service

Signed-off-by: wuhuizuo <wuhuizuo@126.com>

dl/cmd/server/http.go

@@ -9,7 +9,9 @@ import (
 	"sync"
 	"time"
 
+	ks3svr "github.com/PingCAP-QE/ee-apps/dl/gen/http/ks3/server"
 	ocisvr "github.com/PingCAP-QE/ee-apps/dl/gen/http/oci/server"
+	ks3 "github.com/PingCAP-QE/ee-apps/dl/gen/ks3"
 	oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci"
 	goahttp "goa.design/goa/v3/http"
 	httpmdlwr "goa.design/goa/v3/http/middleware"
@@ -18,7 +20,7 @@ import (
 
 // handleHTTPServer starts configures and starts a HTTP server on the given
 // URL. It shuts down the server if any error is received in the error channel.
-func handleHTTPServer(ctx context.Context, u *url.URL, ociEndpoints *oci.Endpoints, wg *sync.WaitGroup, errc chan error, logger *log.Logger, debug bool) {
+func handleHTTPServer(ctx context.Context, u *url.URL, ociEndpoints *oci.Endpoints, ks3Endpoints *ks3.Endpoints, wg *sync.WaitGroup, errc chan error, logger *log.Logger, debug bool) {
 
 	// Setup goa log adapter.
 	var (
@@ -50,19 +52,23 @@ func handleHTTPServer(ctx context.Context, u *url.URL, ociEndpoints *oci.Endpoin
 	// responses.
 	var (
 		ociServer *ocisvr.Server
+		ks3Server *ks3svr.Server
 	)
 	{
 		eh := errorHandler(logger)
 		ociServer = ocisvr.New(ociEndpoints, mux, dec, enc, eh, nil)
+		ks3Server = ks3svr.New(ks3Endpoints, mux, dec, enc, eh, nil)
 		if debug {
 			servers := goahttp.Servers{
 				ociServer,
+				ks3Server,
 			}
 			servers.Use(httpmdlwr.Debug(mux, os.Stdout))
 		}
 	}
 	// Configure the mux.
 	ocisvr.Mount(mux, ociServer)
+	ks3svr.Mount(mux, ks3Server)
 
 	// Wrap the multiplexer with additional middlewares. Middlewares mounted
 	// here apply to all the service endpoints.
@@ -78,6 +84,9 @@ func handleHTTPServer(ctx context.Context, u *url.URL, ociEndpoints *oci.Endpoin
 	for _, m := range ociServer.Mounts {
 		logger.Printf("HTTP %q mounted on %s %s", m.Method, m.Verb, m.Pattern)
 	}
+	for _, m := range ks3Server.Mounts {
+		logger.Printf("HTTP %q mounted on %s %s", m.Method, m.Verb, m.Pattern)
+	}
 
 	(*wg).Add(1)
 	go func() {

dl/cmd/server/main.go

@@ -13,18 +13,20 @@ import (
 	"syscall"
 
 	dl "github.com/PingCAP-QE/ee-apps/dl"
+	ks3 "github.com/PingCAP-QE/ee-apps/dl/gen/ks3"
 	oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci"
 )
 
 func main() {
 	// Define command line flags, add any other flag required to configure the
 	// service.
 	var (
-		hostF     = flag.String("host", "localhost", "Server host (valid values: localhost)")
-		domainF   = flag.String("domain", "", "Host domain name (overrides host domain specified in service design)")
-		httpPortF = flag.String("http-port", "", "HTTP port (overrides host HTTP port specified in service design)")
-		secureF   = flag.Bool("secure", false, "Use secure scheme (https or grpcs)")
-		dbgF      = flag.Bool("debug", false, "Log request and response bodies")
+		hostF       = flag.String("host", "localhost", "Server host (valid values: localhost)")
+		domainF     = flag.String("domain", "", "Host domain name (overrides host domain specified in service design)")
+		httpPortF   = flag.String("http-port", "", "HTTP port (overrides host HTTP port specified in service design)")
+		secureF     = flag.Bool("secure", false, "Use secure scheme (https or grpcs)")
+		dbgF        = flag.Bool("debug", false, "Log request and response bodies")
+		ks3CfgPathF = flag.String("ks3-config", "ks3.yaml", "ks3 config yaml file path")
 	)
 	flag.Parse()
 
@@ -39,18 +41,22 @@ func main() {
 	// Initialize the services.
 	var (
 		ociSvc oci.Service
+		ks3Svc ks3.Service
 	)
 	{
 		ociSvc = dl.NewOci(logger)
+		ks3Svc = dl.NewKs3(logger, *ks3CfgPathF)
 	}
 
 	// Wrap the services in endpoints that can be invoked from other services
 	// potentially running in different processes.
 	var (
 		ociEndpoints *oci.Endpoints
+		ks3Endpoints *ks3.Endpoints
 	)
 	{
 		ociEndpoints = oci.NewEndpoints(ociSvc)
+		ks3Endpoints = ks3.NewEndpoints(ks3Svc)
 	}
 
 	// Create channel used by both the signal handler and server goroutines
@@ -92,7 +98,7 @@ func main() {
 			} else if u.Port() == "" {
 				u.Host = net.JoinHostPort(u.Host, "80")
 			}
-			handleHTTPServer(ctx, u, ociEndpoints, &wg, errc, logger, *dbgF)
+			handleHTTPServer(ctx, u, ociEndpoints, ks3Endpoints, &wg, errc, logger, *dbgF)
 		}
 
 	default:

dl/design/design.go

@@ -77,3 +77,47 @@ var _ = Service("oci", func() {
 		})
 	})
 })
+
+var _ = Service("ks3", func() {
+	Description("OCI artifacts download service")
+
+	Method("download-object", func() {
+		Payload(func() {
+			Field(1, "bucket", String, "bucket name")
+			Field(2, "key", String, "object key")
+
+			Required("bucket", "key")
+		})
+
+		// The use of Result here illustrates how HTTP headers can still be
+		// properly encoded and validated when using SkipResponseBodyEncode. It
+		// is not generally required to implement a download method.
+		Result(func() {
+			Attribute("length", Int64, "Length is the downloaded content length in bytes.", func() {
+				Example(4 * 1024 * 1024)
+			})
+			Attribute("contentDisposition", String, "Content-Disposition header for downloading", func() {
+				Example("attachment; filename*=UTF-8''tidb-v7.5.0-darwin-arm64.tar.gz")
+			})
+			Required("length", "contentDisposition")
+		})
+
+		Error("invalid_file_path", ErrorResult, "Could not locate file for download")
+		Error("internal_error", ErrorResult, "Fault while processing download.")
+
+		HTTP(func() {
+			GET("/s3-obj/{bucket}/{*key}")
+
+			// Bypass response body encoder code generation to alleviate need for
+			// loading the entire response body in memory.
+			SkipResponseBodyEncodeDecode()
+
+			Response(func() {
+				// Set the content type for binary data
+				ContentType("application/octet-stream")
+				Header("length:Content-Length")
+				Header("contentDisposition:Content-Disposition")
+			})
+		})
+	})
+})