Merge branch 'master' into amd/pre-read

Required-githooks: true

src/client/dfuse/dfuse_main.c

@@ -7,8 +7,10 @@
 #include <errno.h>
 #include <getopt.h>
 #include <dlfcn.h>
+#include <fcntl.h>
 #include <fuse3/fuse.h>
 #include <fuse3/fuse_lowlevel.h>
+#include <string.h>
 
 #include <sys/types.h>
 #include <hwloc.h>
@@ -348,6 +350,35 @@ show_help(char *name)
 	    name, DAOS_VERSION);
 }
 
+/*
+ * Checks whether a mountpoint path is a valid file descriptor.
+ *
+ * Returns the file descriptor on success, -1 on failure.
+ */
+static int
+check_fd_mountpoint(const char *mountpoint)
+{
+	int fd  = -1;
+	int len = 0;
+	int fd_flags;
+	int res;
+
+	res = sscanf(mountpoint, "/dev/fd/%u%n", &fd, &len);
+	if (res != 1) {
+		return -1;
+	}
+	if (len != strnlen(mountpoint, NAME_MAX)) {
+		return -1;
+	}
+
+	fd_flags = fcntl(fd, F_GETFD);
+	if (fd_flags == -1) {
+		return -1;
+	}
+
+	return fd;
+}
+
 int
 main(int argc, char **argv)
 {
@@ -627,8 +658,18 @@ main(int argc, char **argv)
 		duns_destroy_attr(&duns_attr);
 
 	} else if (rc == ENOENT) {
-		printf("Mount point does not exist\n");
-		D_GOTO(out_daos, rc = daos_errno2der(rc));
+		/* In order to allow FUSE daemons to run without privileges, libfuse
+		 * allows the caller to open /dev/fuse and pass the file descriptor by
+		 * specifying /dev/fd/N as the mountpoint. In some cases, realpath may
+		 * fail for these paths.
+		 */
+		int fd = check_fd_mountpoint(dfuse_info->di_mountpoint);
+		if (fd < 0) {
+			DFUSE_TRA_WARNING(dfuse_info, "Mount point is not a valid file descriptor");
+			printf("Mount point does not exist\n");
+			D_GOTO(out_daos, rc = daos_errno2der(rc));
+		}
+		DFUSE_LOG_INFO("Mounting FUSE file descriptor %d", fd);
 	} else if (rc == ENOTCONN) {
 		printf("Stale mount point, run fusermount3 and retry\n");
 		D_GOTO(out_daos, rc = daos_errno2der(rc));

src/control/cmd/daos/flags.go

@@ -99,7 +99,7 @@ func (f *ChunkSizeFlag) UnmarshalFlag(fv string) error {
 
 	size, err := humanize.ParseBytes(fv)
 	if err != nil {
-		return err
+		return errors.Errorf("invalid chunk-size %q", fv)
 	}
 	f.Size = C.uint64_t(size)
 

src/control/cmd/daos/flags_test.go

@@ -227,7 +227,7 @@ func TestFlags_ChunkSizeFlag(t *testing.T) {
 		},
 		"not a size": {
 			arg:    "snausages",
-			expErr: errors.New("ParseFloat"),
+			expErr: errors.New("invalid chunk-size \"snausages\""),
 		},
 		// TODO: More validation of allowed sizes?
 	} {

src/control/cmd/daos_agent/support.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/daos-stack/daos/src/control/common/cmdutil"
 	"github.com/daos-stack/daos/src/control/lib/support"
@@ -54,8 +55,10 @@ func (cmd *collectLogCmd) Execute(_ []string) error {
 	}
 
 	if cmd.TargetFolder == "" {
-		cmd.TargetFolder = filepath.Join(os.TempDir(), "daos_support_client_logs")
+		folderName := fmt.Sprintf("daos_support_client_logs_%s", time.Now().Format(time.RFC3339))
+		cmd.TargetFolder = filepath.Join(os.TempDir(), folderName)
 	}
+
 	cmd.Infof("Support Logs will be copied to %s", cmd.TargetFolder)
 
 	progress.Steps = 100 / progress.Total

src/control/cmd/daos_server/start.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2019-2022 Intel Corporation.
+// (C) Copyright 2019-2023 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -27,13 +27,14 @@ type startCmd struct {
 	Port                uint16  `short:"p" long:"port" description:"Port for the gRPC management interfect to listen on"`
 	MountPath           string  `short:"s" long:"storage" description:"Storage path"`
 	Modules             *string `short:"m" long:"modules" description:"List of server modules to load"`
-	Targets             uint16  `short:"t" long:"targets" description:"number of targets to use (default use all cores)"`
-	NrXsHelpers         *uint16 `short:"x" long:"xshelpernr" description:"number of helper XS per VOS target"`
-	FirstCore           uint16  `short:"f" long:"firstcore" default:"0" description:"index of first core for service thread"`
+	Targets             uint16  `short:"t" long:"targets" description:"Number of targets to use (default use all cores)"`
+	NrXsHelpers         *uint16 `short:"x" long:"xshelpernr" description:"Number of helper XS per VOS target"`
+	FirstCore           uint16  `short:"f" long:"firstcore" default:"0" description:"Index of first core for service thread"`
 	Group               string  `short:"g" long:"group" description:"Server group name"`
 	SocketDir           string  `short:"d" long:"socket_dir" description:"Location for all daos_server & daos_engine sockets"`
-	Insecure            bool    `short:"i" long:"insecure" description:"allow for insecure connections"`
-	RecreateSuperblocks bool    `long:"recreate-superblocks" description:"recreate missing superblocks rather than failing"`
+	Insecure            bool    `short:"i" long:"insecure" description:"Allow for insecure connections"`
+	RecreateSuperblocks bool    `long:"recreate-superblocks" description:"Recreate missing superblocks rather than failing"`
+	AutoFormat          bool    `long:"auto-format" description:"Automatically format storage on server start to bring-up engines without requiring dmg storage format command"`
 }
 
 func (cmd *startCmd) setCLIOverrides() error {
@@ -161,5 +162,7 @@ func (cmd *startCmd) Execute(args []string) error {
 		return err
 	}
 
+	cmd.config.AutoFormat = cmd.AutoFormat
+
 	return cmd.start(cmd.Logger, cmd.config)
 }

src/control/cmd/daos_server/support.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/daos-stack/daos/src/control/common/cmdutil"
 	"github.com/daos-stack/daos/src/control/lib/support"
@@ -52,7 +53,8 @@ func (cmd *collectLogCmd) Execute(_ []string) error {
 	}
 
 	if cmd.TargetFolder == "" {
-		cmd.TargetFolder = filepath.Join(os.TempDir(), "daos_support_server_logs")
+		folderName := fmt.Sprintf("daos_support_server_logs_%s", time.Now().Format(time.RFC3339))
+		cmd.TargetFolder = filepath.Join(os.TempDir(), folderName)
 	}
 	cmd.Infof("Support logs will be copied to %s", cmd.TargetFolder)
 

src/control/cmd/dmg/pool.go

@@ -92,7 +92,7 @@ func (trf *tierRatioFlag) UnmarshalFlag(fv string) error {
 	for _, trStr := range strings.Split(fv, ",") {
 		tr, err := strconv.ParseFloat(strings.TrimSpace(strings.Trim(trStr, "%")), 64)
 		if err != nil {
-			return errors.Wrapf(err, "invalid tier ratio %s", trStr)
+			return errors.Errorf("invalid tier ratio %q", trStr)
 		}
 		trf.ratios = append(trf.ratios, roundFloatTo(tr, 2)/100)
 	}
@@ -137,7 +137,7 @@ func (sf *sizeFlag) UnmarshalFlag(fv string) (err error) {
 
 	sf.bytes, err = humanize.ParseBytes(fv)
 	if err != nil {
-		return errors.Wrapf(err, "invalid size %q", fv)
+		return errors.Errorf("invalid size %q", fv)
 	}
 
 	return nil

src/control/cmd/dmg/pool_test.go

@@ -42,6 +42,10 @@ func Test_Dmg_PoolTierRatioFlag(t *testing.T) {
 		"empty": {
 			expErr: errors.New("no tier ratio specified"),
 		},
+		"invalid": {
+			input:  "ABCD",
+			expErr: errors.New("invalid tier ratio \"ABCD\""),
+		},
 		"less than 100%": {
 			input:  "10,80",
 			expErr: errors.New("must add up to"),

src/control/cmd/dmg/pretty/printers.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2020-2022 Intel Corporation.
+// (C) Copyright 2020-2023 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -14,6 +14,7 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/daos-stack/daos/src/control/common"
 	"github.com/daos-stack/daos/src/control/fault"
 	"github.com/daos-stack/daos/src/control/lib/control"
 	"github.com/daos-stack/daos/src/control/lib/txtfmt"
@@ -155,3 +156,41 @@ func PrintResponseErrors(resp hostErrorsGetter, out io.Writer, opts ...PrintConf
 
 	return nil
 }
+
+// PrintErrorsSummary generates a human-readable representation of the supplied
+// HostErrorsMap summary struct and writes it to the supplied io.Writer.
+func UpdateErrorSummary(resp hostErrorsGetter, cmd string, out io.Writer, opts ...PrintConfigOption) error {
+	if common.InterfaceIsNil(resp) {
+		return errors.Errorf("nil %T", resp)
+	}
+
+	if len(resp.GetHostErrors()) > 0 {
+		setTitle := "Hosts"
+		cmdTitle := "Command"
+		errTitle := "Error"
+
+		tablePrint := txtfmt.NewTableFormatter(setTitle, cmdTitle, errTitle)
+		tablePrint.InitWriter(out)
+		table := []txtfmt.TableRow{}
+
+		for _, errStr := range resp.GetHostErrors().Keys() {
+			errHosts := getPrintHosts(resp.GetHostErrors()[errStr].HostSet.RangedString(), opts...)
+			row := txtfmt.TableRow{setTitle: errHosts}
+
+			// Unpack the root cause error. If it's a fault,
+			// just print the description.
+			hostErr := errors.Cause(resp.GetHostErrors()[errStr].HostError)
+			row[cmdTitle] = cmd
+			row[errTitle] = hostErr.Error()
+			if f, ok := hostErr.(*fault.Fault); ok {
+				row[errTitle] = f.Description
+			}
+
+			table = append(table, row)
+		}
+
+		tablePrint.Format(table)
+	}
+
+	return nil
+}

src/control/cmd/dmg/pretty/printers_test.go

@@ -1,5 +1,5 @@
 //
-// (C) Copyright 2020-2021 Intel Corporation.
+// (C) Copyright 2020-2023 Intel Corporation.
 //
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 //
@@ -13,6 +13,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 
+	"github.com/daos-stack/daos/src/control/common/test"
 	"github.com/daos-stack/daos/src/control/lib/control"
 )
 
@@ -107,3 +108,98 @@ host1:1,host2:2 whoops
 		})
 	}
 }
+
+func TestControl_UpdateErrorSummary(t *testing.T) {
+	for name, tc := range map[string]struct {
+		resp      *control.CollectLogResp
+		cmd       string
+		expStdout string
+		expErr    error
+	}{
+		"nil response": {
+			resp:      nil,
+			cmd:       "empty",
+			expStdout: ``,
+			expErr:    errors.New("nil *control.CollectLogResp"),
+		},
+		"empty response": {
+			resp:      new(control.CollectLogResp),
+			cmd:       "empty",
+			expStdout: ``,
+			expErr:    nil,
+		},
+		"one host error": {
+			cmd: "hostname",
+			resp: &control.CollectLogResp{
+				HostErrorsResp: control.MockHostErrorsResp(t,
+					&control.MockHostError{
+						Hosts: "host1",
+						Error: "command not found",
+					}),
+			},
+			expStdout: `
+Hosts Command  Error             
+----- -------  -----             
+host1 hostname command not found 
+`,
+			expErr: nil,
+		},
+		"Two host, same error": {
+			cmd: "hostname",
+			resp: &control.CollectLogResp{
+				HostErrorsResp: control.MockHostErrorsResp(t,
+					&control.MockHostError{
+						Hosts: "host1",
+						Error: "command not found",
+					},
+					&control.MockHostError{
+						Hosts: "host2",
+						Error: "command not found",
+					}),
+			},
+			expStdout: `
+Hosts     Command  Error             
+-----     -------  -----             
+host[1-2] hostname command not found 
+`,
+			expErr: nil,
+		},
+		"Two host, different error": {
+			cmd: "hostname",
+			resp: &control.CollectLogResp{
+				HostErrorsResp: control.MockHostErrorsResp(t,
+					&control.MockHostError{
+						Hosts: "host1",
+						Error: "command not found",
+					},
+					&control.MockHostError{
+						Hosts: "host2",
+						Error: "command not available",
+					}),
+			},
+			expStdout: `
+Hosts Command  Error                 
+----- -------  -----                 
+host1 hostname command not found     
+host2 hostname command not available 
+`,
+			expErr: nil,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			var out strings.Builder
+
+			err := UpdateErrorSummary(tc.resp, tc.cmd, &out)
+
+			test.CmpErr(t, tc.expErr, err)
+			if tc.expErr != nil {
+				return
+			}
+
+			if diff := cmp.Diff(strings.TrimLeft(tc.expStdout, "\n"), out.String()); diff != "" {
+				t.Fatalf("unexpected print output (-want, +got):\n%s\n", diff)
+			}
+
+		})
+	}
+}