From 933c7a4c0af4ffe908bb0ee5d26fe584df71d51f Mon Sep 17 00:00:00 2001 From: Samir Raval Date: Wed, 28 Jun 2023 17:54:49 +0000 Subject: [PATCH] DAOS-13763 control: Fix daos_metrics collection for support collect-log. - Fixed the daos_metrics collection so it will collect for any running engine. - Removed the --target-host option and instead AdminNode will be used for doing RSync. Test-tag: pr support Required-githooks: true Signed-off-by: Samir Raval --- src/control/cmd/dmg/support.go | 54 ++++------------------ src/control/common/proto/ctl/support.pb.go | 38 +++++++-------- src/control/lib/control/support.go | 4 +- src/control/lib/support/log.go | 9 ++-- src/control/lib/support/log_test.go | 13 ++---- src/control/server/ctl_support_rpc.go | 2 +- src/proto/ctl/support.proto | 2 +- 7 files changed, 40 insertions(+), 82 deletions(-) diff --git a/src/control/cmd/dmg/support.go b/src/control/cmd/dmg/support.go index 965290905bb..e2e95bfbd40 100644 --- a/src/control/cmd/dmg/support.go +++ b/src/control/cmd/dmg/support.go @@ -7,16 +7,12 @@ package main import ( - "bytes" "context" "fmt" "os" - "os/exec" "path/filepath" "strings" - "github.com/pkg/errors" - "github.com/daos-stack/daos/src/control/cmd/dmg/pretty" "github.com/daos-stack/daos/src/control/lib/control" "github.com/daos-stack/daos/src/control/lib/support" @@ -37,56 +33,19 @@ type collectLogCmd struct { support.CollectLogSubCmd } -// Copy Admin logs to TargetHost (central location) if option is provided. -func (cmd *collectLogCmd) rsyncAdminLog() error { - - runcmd := strings.Join([]string{ - "rsync", - "-av", - "--blocking-io", - cmd.TargetFolder + "/", - cmd.TargetHost + ":" + cmd.TargetFolder}, " ") - - cmd.Debugf("Rsync Admin Log command = %s", runcmd) - rsyncCmd := exec.Command("sh", "-c", runcmd) - var stdout, stderr bytes.Buffer - rsyncCmd.Stdout = &stdout - rsyncCmd.Stderr = &stderr - err := rsyncCmd.Run() - outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes()) - if err != nil { - return errors.Wrapf(err, "Error running command %s with %s", rsyncCmd, err) - } - cmd.Debugf("rsyncCmd:= %s stdout:\n%s\nstderr:\n%s\n", rsyncCmd, outStr, errStr) - - return nil -} - -// gRPC call to initiate the rsync and copy the logs to Admin/TargetHost (central location). +// gRPC call to initiate the rsync and copy the logs to Admin (central location). func (cmd *collectLogCmd) rsyncLog() error { hostName, err := support.GetHostName() if err != nil { return err } - // Admin host will be the central host to collect all the logs from servers. - // Logs will be rsync to TargetHost is provided. - if cmd.TargetHost == "" { - cmd.TargetHost = hostName - } else { - // initiate the rsync and copy the Admin logs to targetHost - err = cmd.rsyncAdminLog() - if err != nil && cmd.Stop { - return err - } - } - req := &control.CollectLogReq{ TargetFolder: cmd.TargetFolder, - TargetHost: cmd.TargetHost, + AdminNode: hostName, LogFunction: support.RsyncLogEnum, } - cmd.Debugf("Rsync logs from servers to %s:%s ", cmd.TargetHost, cmd.TargetFolder) + cmd.Debugf("Rsync logs from servers to %s:%s ", hostName, cmd.TargetFolder) resp, err := control.CollectLog(context.Background(), cmd.ctlInvoker, req) if err != nil && cmd.Stop { return err @@ -105,9 +64,14 @@ func (cmd *collectLogCmd) rsyncLog() error { // gRPC call to Archive the logs on individual servers. func (cmd *collectLogCmd) archLogsOnServer() error { + hostName, err := support.GetHostName() + if err != nil { + return err + } + req := &control.CollectLogReq{ TargetFolder: cmd.TargetFolder, - TargetHost: cmd.TargetHost, + AdminNode: hostName, LogFunction: support.ArchiveLogsEnum, } cmd.Debugf("Archiving the Log Folder %s", cmd.TargetFolder) diff --git a/src/control/common/proto/ctl/support.pb.go b/src/control/common/proto/ctl/support.pb.go index 74f1655107f..70ed52d5ea7 100644 --- a/src/control/common/proto/ctl/support.pb.go +++ b/src/control/common/proto/ctl/support.pb.go @@ -33,7 +33,7 @@ type CollectLogReq struct { TargetFolder string `protobuf:"bytes,1,opt,name=TargetFolder,proto3" json:"TargetFolder,omitempty"` ExtraLogsDir string `protobuf:"bytes,2,opt,name=ExtraLogsDir,proto3" json:"ExtraLogsDir,omitempty"` - TargetHost string `protobuf:"bytes,3,opt,name=TargetHost,proto3" json:"TargetHost,omitempty"` + AdminNode string `protobuf:"bytes,3,opt,name=AdminNode,proto3" json:"AdminNode,omitempty"` JsonOutput bool `protobuf:"varint,4,opt,name=JsonOutput,proto3" json:"JsonOutput,omitempty"` LogFunction int32 `protobuf:"varint,5,opt,name=LogFunction,proto3" json:"LogFunction,omitempty"` LogCmd string `protobuf:"bytes,6,opt,name=LogCmd,proto3" json:"LogCmd,omitempty"` @@ -85,9 +85,9 @@ func (x *CollectLogReq) GetExtraLogsDir() string { return "" } -func (x *CollectLogReq) GetTargetHost() string { +func (x *CollectLogReq) GetAdminNode() string { if x != nil { - return x.TargetHost + return x.AdminNode } return "" } @@ -164,27 +164,27 @@ var File_ctl_support_proto protoreflect.FileDescriptor var file_ctl_support_proto_rawDesc = []byte{ 0x0a, 0x11, 0x63, 0x74, 0x6c, 0x2f, 0x73, 0x75, 0x70, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x63, 0x74, 0x6c, 0x22, 0xd1, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6c, + 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x63, 0x74, 0x6c, 0x22, 0xcf, 0x01, 0x0a, 0x0d, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x71, 0x12, 0x22, 0x0a, 0x0c, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x45, 0x78, 0x74, 0x72, 0x61, 0x4c, 0x6f, 0x67, 0x73, 0x44, 0x69, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x45, 0x78, 0x74, 0x72, 0x61, 0x4c, 0x6f, 0x67, 0x73, 0x44, - 0x69, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x48, 0x6f, 0x73, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x48, 0x6f, - 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x4a, 0x73, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4a, 0x73, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, - 0x75, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x4c, 0x6f, 0x67, 0x46, 0x75, 0x6e, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x4c, 0x6f, 0x67, 0x43, 0x6d, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x4c, 0x6f, 0x67, 0x43, 0x6d, 0x64, 0x22, 0x28, 0x0a, 0x0e, - 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, - 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, - 0x64, 0x61, 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x74, - 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x4e, 0x6f, 0x64, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x4e, 0x6f, 0x64, 0x65, + 0x12, 0x1e, 0x0a, 0x0a, 0x4a, 0x73, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x4a, 0x73, 0x6f, 0x6e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, + 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x4c, 0x6f, 0x67, 0x46, 0x75, 0x6e, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x4c, 0x6f, 0x67, 0x43, 0x6d, 0x64, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x4c, 0x6f, 0x67, 0x43, 0x6d, 0x64, 0x22, 0x28, 0x0a, 0x0e, 0x43, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x67, 0x52, 0x65, 0x73, 0x70, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x64, 0x61, 0x6f, 0x73, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, 0x61, + 0x6f, 0x73, 0x2f, 0x73, 0x72, 0x63, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x63, + 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x74, 0x6c, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/src/control/lib/control/support.go b/src/control/lib/control/support.go index 75d3107b247..3ca9323aecd 100644 --- a/src/control/lib/control/support.go +++ b/src/control/lib/control/support.go @@ -19,7 +19,7 @@ type ( CollectLogReq struct { unaryRequest TargetFolder string - TargetHost string + AdminNode string ExtraLogsDir string JsonOutput bool LogFunction int32 @@ -41,7 +41,7 @@ func CollectLog(ctx context.Context, rpcClient UnaryInvoker, req *CollectLogReq) req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) { return ctlpb.NewCtlSvcClient(conn).CollectLog(ctx, &ctlpb.CollectLogReq{ TargetFolder: req.TargetFolder, - TargetHost: req.TargetHost, + AdminNode: req.AdminNode, ExtraLogsDir: req.ExtraLogsDir, JsonOutput: req.JsonOutput, LogFunction: req.LogFunction, diff --git a/src/control/lib/support/log.go b/src/control/lib/support/log.go index 32b24bfb8d3..e8d10dbd94e 100644 --- a/src/control/lib/support/log.go +++ b/src/control/lib/support/log.go @@ -48,7 +48,6 @@ type CollectLogSubCmd struct { TargetFolder string `short:"t" long:"target-folder" description:"Target Folder location where log will be copied"` Archive bool `short:"z" long:"archive" description:"Archive the log/config files"` ExtraLogsDir string `short:"c" long:"extra-logs-dir" description:"Collect the Logs from given directory"` - TargetHost string `short:"m" long:"target-host" description:"Rsync all the logs to central system"` } // Folder names to copy logs and configs @@ -120,7 +119,7 @@ type CollectLogsParams struct { Config string Hostlist string TargetFolder string - TargetHost string + AdminNode string ExtraLogsDir string JsonOutput bool LogFunction int32 @@ -341,7 +340,7 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error return hostNames, nil } -// R sync logs from individual servers to Admin/TargetHost node +// R sync logs from individual servers to Admin node func rsyncLog(log logging.Logger, opts ...CollectLogsParams) error { targetLocation, err := createHostFolder(opts[0].TargetFolder, log) if err != nil { @@ -353,7 +352,7 @@ func rsyncLog(log logging.Logger, opts ...CollectLogsParams) error { "-av", "--blocking-io", targetLocation, - opts[0].TargetHost + ":" + opts[0].TargetFolder}, + opts[0].AdminNode + ":" + opts[0].TargetFolder}, " ") out, err := exec.Command("sh", "-c", cmd).Output() @@ -650,7 +649,7 @@ func collectDaosMetrics(daosNodeLocation string, log logging.Logger, opts ...Col _, err := cpOutputToFile(daosNodeLocation, log, daos) if err != nil { - return err + log.Errorf("Failed to run %s: %v", daos.cmd, err) } } } else { diff --git a/src/control/lib/support/log_test.go b/src/control/lib/support/log_test.go index 1923db178de..43ea3651f62 100644 --- a/src/control/lib/support/log_test.go +++ b/src/control/lib/support/log_test.go @@ -410,28 +410,23 @@ func TestSupport_rsyncLog(t *testing.T) { for name, tc := range map[string]struct { targetFolder string - TargetHost string + AdminNode string expErr error }{ "rsync to invalid Target directory": { targetFolder: targetTestDir + "/foo/bar", - TargetHost: hostName + ":/tmp/foo/bar/", - expErr: errors.New("Error running command"), - }, - "rsync to invalid Target Host": { - targetFolder: targetTestDir + "/foo/bar", - TargetHost: "invalid-host", + AdminNode: hostName + ":/tmp/foo/bar/", expErr: errors.New("Error running command"), }, "rsync invalid log directory": { targetFolder: srcPath + "/file1", - TargetHost: hostName, + AdminNode: hostName, expErr: errors.New("not a directory"), }, } { t.Run(name, func(t *testing.T) { rsLog.TargetFolder = tc.targetFolder - rsLog.TargetHost = tc.TargetHost + rsLog.AdminNode = tc.AdminNode gotErr := rsyncLog(log, rsLog) test.CmpErr(t, tc.expErr, gotErr) }) diff --git a/src/control/server/ctl_support_rpc.go b/src/control/server/ctl_support_rpc.go index 42b79bd33f6..8a4fec99d16 100644 --- a/src/control/server/ctl_support_rpc.go +++ b/src/control/server/ctl_support_rpc.go @@ -20,7 +20,7 @@ func (c *ControlService) CollectLog(ctx context.Context, req *ctlpb.CollectLogRe params := support.CollectLogsParams{} params.TargetFolder = req.TargetFolder params.ExtraLogsDir = req.ExtraLogsDir - params.TargetHost = req.TargetHost + params.AdminNode = req.AdminNode params.JsonOutput = req.JsonOutput params.LogFunction = req.LogFunction params.LogCmd = req.LogCmd diff --git a/src/proto/ctl/support.proto b/src/proto/ctl/support.proto index a4b421aab1a..7adab464cd9 100644 --- a/src/proto/ctl/support.proto +++ b/src/proto/ctl/support.proto @@ -14,7 +14,7 @@ option go_package = "github.com/daos-stack/daos/src/control/common/proto/ctl"; message CollectLogReq { string TargetFolder = 1; string ExtraLogsDir = 2; - string TargetHost = 3; + string AdminNode = 3; bool JsonOutput = 4; int32 LogFunction = 5; string LogCmd = 6;