DAOS-13763 control: Fix daos_metrics collection for support collect-l…

…og. (#12555)

- Fixed the daos_metrics collection so it will collect for any running engine.
- Removed the --target-host option and instead AdminNode will be used for doing RSync.

Signed-off-by: Samir Raval <samir.raval@intel.com>

src/control/cmd/dmg/support.go

@@ -7,16 +7,12 @@
 package main
 
 import (
- "bytes"
  "context"
  "fmt"
  "os"
- "os/exec"
  "path/filepath"
  "strings"
 
- "github.com/pkg/errors"
-
  "github.com/daos-stack/daos/src/control/cmd/dmg/pretty"
  "github.com/daos-stack/daos/src/control/common/cmdutil"
  "github.com/daos-stack/daos/src/control/lib/control"
@@ -38,56 +34,19 @@ type collectLogCmd struct {
  support.CollectLogSubCmd
 }
 
-// Copy Admin logs to TargetHost (central location) if option is provided.
-func (cmd *collectLogCmd) rsyncAdminLog() error {
-
- runcmd := strings.Join([]string{
- "rsync",
- "-av",
- "--blocking-io",
- cmd.TargetFolder + "/",
- cmd.TargetHost + ":" + cmd.TargetFolder}, " ")
-
- cmd.Debugf("Rsync Admin Log command = %s", runcmd)
- rsyncCmd := exec.Command("sh", "-c", runcmd)
- var stdout, stderr bytes.Buffer
- rsyncCmd.Stdout = &stdout
- rsyncCmd.Stderr = &stderr
- err := rsyncCmd.Run()
- outStr, errStr := string(stdout.Bytes()), string(stderr.Bytes())
- if err != nil {
- return errors.Wrapf(err, "Error running command %s with %s", rsyncCmd, err)
- }
- cmd.Debugf("rsyncCmd:= %s stdout:\n%s\nstderr:\n%s\n", rsyncCmd, outStr, errStr)
-
- return nil
-}
-
-// gRPC call to initiate the rsync and copy the logs to Admin/TargetHost (central location).
+// gRPC call to initiate the rsync and copy the logs to Admin (central location).
 func (cmd *collectLogCmd) rsyncLog() error {
  hostName, err := support.GetHostName()
  if err != nil {
  return err
  }
 
- // Admin host will be the central host to collect all the logs from servers.
- // Logs will be rsync to TargetHost is provided.
- if cmd.TargetHost == "" {
- cmd.TargetHost = hostName
- } else {
- // initiate the rsync and copy the Admin logs to targetHost
- err = cmd.rsyncAdminLog()
- if err != nil && cmd.Stop {
- return err
- }
- }
-
  req := &control.CollectLogReq{
  TargetFolder: cmd.TargetFolder,
- TargetHost: cmd.TargetHost,
+ AdminNode:  hostName,
  LogFunction: support.RsyncLogEnum,
  }
- cmd.Debugf("Rsync logs from servers to %s:%s ", cmd.TargetHost, cmd.TargetFolder)
+ cmd.Debugf("Rsync logs from servers to %s:%s ", hostName, cmd.TargetFolder)
  resp, err := control.CollectLog(context.Background(), cmd.ctlInvoker, req)
  if err != nil && cmd.Stop {
  return err
@@ -106,9 +65,14 @@ func (cmd *collectLogCmd) rsyncLog() error {
 
 // gRPC call to Archive the logs on individual servers.
 func (cmd *collectLogCmd) archLogsOnServer() error {
+ hostName, err := support.GetHostName()
+ if err != nil {
+ return err
+ }
+
  req := &control.CollectLogReq{
  TargetFolder: cmd.TargetFolder,
- TargetHost: cmd.TargetHost,
+ AdminNode:  hostName,
  LogFunction: support.ArchiveLogsEnum,
  }
  cmd.Debugf("Archiving the Log Folder %s", cmd.TargetFolder)

src/control/lib/control/support.go

@@ -19,7 +19,7 @@ type (
  CollectLogReq struct {
  unaryRequest
  TargetFolder string
- TargetHost string
+ AdminNode  string
  ExtraLogsDir string
  JsonOutput bool
  LogFunction int32
@@ -41,7 +41,7 @@ func CollectLog(ctx context.Context, rpcClient UnaryInvoker, req *CollectLogReq)
  req.setRPC(func(ctx context.Context, conn *grpc.ClientConn) (proto.Message, error) {
  return ctlpb.NewCtlSvcClient(conn).CollectLog(ctx, &ctlpb.CollectLogReq{
  TargetFolder: req.TargetFolder,
- TargetHost: req.TargetHost,
+ AdminNode:  req.AdminNode,
  ExtraLogsDir: req.ExtraLogsDir,
  JsonOutput: req.JsonOutput,
  LogFunction: req.LogFunction,

src/control/lib/support/log.go

@@ -48,7 +48,6 @@ type CollectLogSubCmd struct {
  TargetFolder string `short:"t" long:"target-folder" description:"Target Folder location where log will be copied"`
  Archive bool `short:"z" long:"archive" description:"Archive the log/config files"`
  ExtraLogsDir string `short:"c" long:"extra-logs-dir" description:"Collect the Logs from given directory"`
- TargetHost string `short:"m" long:"target-host" description:"Rsync all the logs to central system"`
 }
 
 // Folder names to copy logs and configs
@@ -120,7 +119,7 @@ type CollectLogsParams struct {
  Config string
  Hostlist string
  TargetFolder string
- TargetHost string
+ AdminNode  string
  ExtraLogsDir string
  JsonOutput bool
  LogFunction int32
@@ -341,7 +340,7 @@ func getSysNameFromQuery(configPath string, log logging.Logger) ([]string, error
  return hostNames, nil
 }
 
-// R sync logs from individual servers to Admin/TargetHost node
+// R sync logs from individual servers to Admin node
 func rsyncLog(log logging.Logger, opts ...CollectLogsParams) error {
  targetLocation, err := createHostFolder(opts[0].TargetFolder, log)
  if err != nil {
@@ -353,7 +352,7 @@ func rsyncLog(log logging.Logger, opts ...CollectLogsParams) error {
  "-av",
  "--blocking-io",
  targetLocation,
- opts[0].TargetHost + ":" + opts[0].TargetFolder},
+ opts[0].AdminNode + ":" + opts[0].TargetFolder},
  " ")
 
  out, err := exec.Command("sh", "-c", cmd).Output()
@@ -650,7 +649,7 @@ func collectDaosMetrics(daosNodeLocation string, log logging.Logger, opts ...Col
 
  _, err := cpOutputToFile(daosNodeLocation, log, daos)
  if err != nil {
- return err
+ log.Errorf("Failed to run %s: %v", daos.cmd, err)
  }
  }
  } else {

src/control/lib/support/log_test.go

@@ -344,28 +344,23 @@ func TestSupport_rsyncLog(t *testing.T) {
 
  for name, tc := range map[string]struct {
  targetFolder string
- TargetHost string
+ AdminNode  string
  expErr error
  }{
  "rsync to invalid Target directory": {
  targetFolder: targetTestDir + "/foo/bar",
- TargetHost: hostName + ":/tmp/foo/bar/",
- expErr: errors.New("Error running command"),
- },
- "rsync to invalid Target Host": {
- targetFolder: targetTestDir + "/foo/bar",
- TargetHost: "invalid-host",
+ AdminNode: hostName + ":/tmp/foo/bar/",
  expErr: errors.New("Error running command"),
  },
  "rsync invalid log directory": {
  targetFolder: srcPath + "/file1",
- TargetHost: hostName,
+ AdminNode:  hostName,
  expErr: errors.New("not a directory"),
  },
  } {
  t.Run(name, func(t *testing.T) {
  rsLog.TargetFolder = tc.targetFolder
- rsLog.TargetHost = tc.TargetHost
+ rsLog.AdminNode = tc.AdminNode
  gotErr := rsyncLog(log, rsLog)
  test.CmpErr(t, tc.expErr, gotErr)
  })

src/control/server/ctl_support_rpc.go

@@ -20,7 +20,7 @@ func (c *ControlService) CollectLog(ctx context.Context, req *ctlpb.CollectLogRe
  params := support.CollectLogsParams{}
  params.TargetFolder = req.TargetFolder
  params.ExtraLogsDir = req.ExtraLogsDir
- params.TargetHost = req.TargetHost
+ params.AdminNode = req.AdminNode
  params.JsonOutput = req.JsonOutput
  params.LogFunction = req.LogFunction
  params.LogCmd = req.LogCmd

src/proto/ctl/support.proto

@@ -14,7 +14,7 @@ option go_package = "github.com/daos-stack/daos/src/control/common/proto/ctl";
 message CollectLogReq {
  string TargetFolder = 1;
  string ExtraLogsDir = 2;
- string TargetHost = 3;
+ string AdminNode = 3;
  bool JsonOutput = 4;
  int32 LogFunction = 5;
  string LogCmd = 6;