Merge pull request #210 from mimiro-io/feat/stats

add stats command

internal/command/stats.go

@@ -0,0 +1,23 @@
+package command
+
+import (
+	"github.com/mimiro-io/datahub-cli/internal/stats"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+)
+
+var StatsCmd = &cobra.Command{
+	Use:   "stats",
+	Short: "retrieve storage statistics from datahub.",
+	Long: `retrieve storage statistics from datahub. statistics are compiled in a long running task in datahub and may not be available before the task is done.
+Examples:
+mim stats
+`,
+	Run: func(cmd *cobra.Command, args []string) {
+		pterm.Success.Println("Fetching storage statistics")
+		stats.Fetch(cmd)
+	},
+}
+
+func init() {
+}

internal/stats/stats.go

@@ -0,0 +1,184 @@
+package stats
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/mimiro-io/datahub-cli/internal/login"
+	"github.com/mimiro-io/datahub-cli/internal/utils"
+	"github.com/mimiro-io/datahub-cli/internal/web"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+	"github.com/tidwall/pretty"
+	"slices"
+	"strings"
+)
+
+func Fetch(cmd *cobra.Command) {
+	format := utils.ResolveFormat(cmd)
+	if format != "term" { // turn of pterm output
+		pterm.DisableOutput()
+	}
+	server, token, err := login.ResolveCredentials()
+
+	utils.HandleError(err)
+
+	pterm.EnableDebugMessages()
+
+	pterm.DefaultSection.Println("Fetching storage statistics for " + server)
+
+	stats, err := web.GetRequest(server, token, "/statistics")
+	utils.HandleError(err)
+
+	output(stats, format)
+
+	pterm.Println()
+}
+func output(statsBytes []byte, format string) {
+	switch format {
+	case "pretty":
+		f := pretty.Pretty(statsBytes)
+		result := pretty.Color(f, nil)
+		fmt.Println(string(result))
+	case "json":
+		fmt.Println(string(statsBytes))
+	default:
+		result := make(map[string]any)
+		err := json.Unmarshal(statsBytes, &result)
+		utils.HandleError(err)
+
+		ents, _ := result["entity"].(map[string]any)
+		out := make([][]string, 0)
+		out = append(out, []string{"Dataset", "Changes", "Entities", "Refs", "Size Entity Keys", "Size Entity Values", "Size Ref Keys"})
+		rows := map[string][6]int64{}
+		for ix, sub := range ents {
+			datasets := sub.(map[string]any)
+			for k, v := range datasets {
+				ds := v.(map[string]any)
+
+				keys := int64(ds["keys"].(float64))
+				sizeKeys := int64(ds["size-keys"].(float64))
+				sizeValues := int64(ds["size-values"].(float64))
+				row, _ := rows[k]
+				if ix == "DATASET_ENTITY_CHANGE_LOG" {
+					row[0] = keys + row[0]
+					row[3] = sizeKeys + row[3]
+				} else if ix == "DATASET_LATEST_ENTITIES" {
+					row[1] = keys + row[1]
+					row[3] = sizeKeys + row[3]
+				} else {
+					row[3] = sizeKeys + row[3]
+					row[4] = sizeValues + row[4]
+				}
+				rows[k] = row
+			}
+		}
+		refs, _ := result["refs"].(map[string]any)
+		for _, sub := range refs {
+			datasets := sub.(map[string]any)
+			for k, v := range datasets {
+				ds := v.(map[string]any)
+
+				keys := int64(ds["keys"].(float64))
+				sizeKeys := int64(ds["size-keys"].(float64))
+				//sizeValues := int64(ds["size-values"].(float64))
+				k = strings.ReplaceAll(k, " (deleted)", "")
+				row, _ := rows[k]
+				row[2] = keys + row[2]
+				row[5] = sizeKeys + row[5]
+				rows[k] = row
+			}
+		}
+
+		for k, v := range rows {
+			out = append(out, []string{k,
+				fmt.Sprintf("%10d", v[0]),
+				fmt.Sprintf("%10d", v[1]),
+				fmt.Sprintf("%10d", v[2]),
+				ByteCountIEC(v[3]),
+				ByteCountIEC(v[4]),
+				ByteCountIEC(v[5]),
+			})
+		}
+
+		slices.SortStableFunc(out, func(i []string, j []string) int {
+			val1 := i[0]
+			val2 := j[0]
+			if val1 < val2 {
+				return -1
+			} else if val1 > val2 {
+				return 1
+			} else {
+				return 0
+			}
+		})
+		pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Print("Entities")
+		pterm.DefaultTable.WithHasHeader().WithData(out).Render()
+		pterm.DefaultHeader.WithFullWidth().WithBackgroundStyle(pterm.NewStyle(pterm.BgLightBlue)).WithMargin(10).Print("Other")
+
+		sys, _ := result["sys"].(map[string]any)
+		unknown, _ := result["unknown"].(map[string]any)
+		uris, _ := result["urimap"].(map[string]any)
+
+		var keyCount int64
+		var keySize int64
+		var valueSize int64
+		for _, sub := range sys {
+			datasets := sub.(map[string]any)
+			for _, v := range datasets {
+				ds := v.(map[string]any)
+				keys := int64(ds["keys"].(float64))
+				sizeKeys := int64(ds["size-keys"].(float64))
+				sizeValues := int64(ds["size-values"].(float64))
+
+				keyCount += keys
+				keySize += sizeKeys
+				valueSize += sizeValues
+			}
+		}
+		for _, sub := range unknown {
+			datasets := sub.(map[string]any)
+			for _, v := range datasets {
+				ds := v.(map[string]any)
+				keys := int64(ds["keys"].(float64))
+				sizeKeys := int64(ds["size-keys"].(float64))
+				sizeValues := int64(ds["size-values"].(float64))
+
+				keyCount += keys
+				keySize += sizeKeys
+				valueSize += sizeValues
+			}
+		}
+		for _, sub := range uris {
+			datasets := sub.(map[string]any)
+			for _, v := range datasets {
+				ds := v.(map[string]any)
+				keys := int64(ds["keys"].(float64))
+				sizeKeys := int64(ds["size-keys"].(float64))
+				sizeValues := int64(ds["size-values"].(float64))
+
+				keyCount += keys
+				keySize += sizeKeys
+				valueSize += sizeValues
+			}
+		}
+
+		pterm.DefaultPanel.WithPanels(pterm.Panels{[]pterm.Panel{
+			{Data: pterm.DefaultHeader.Sprintf(fmt.Sprintf("Keys: %10d", keyCount))},
+			{Data: pterm.DefaultHeader.Sprintf("Key Size: " + ByteCountIEC(keySize))},
+			{Data: pterm.DefaultHeader.Sprintf("Value Size: " + ByteCountIEC(valueSize))}}}).WithPadding(5).Render()
+	}
+
+}
+func ByteCountIEC(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %ciB",
+		float64(b)/float64(div), "KMGTPE"[exp])
+}

root.go

@@ -33,7 +33,7 @@ import (
 
 var cfgFile string
 
-// rootCmd represents the base command when called without any subcommands
+// RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "mim",
 	Short: "MIMIRO Data Hub CLI",
@@ -78,6 +78,7 @@ func addCommands() {
 	RootCmd.AddCommand(command.ProviderCmd)
 	RootCmd.AddCommand(command.VersionCmd)
 	RootCmd.AddCommand(command.GatewayCmd)
+	RootCmd.AddCommand(command.StatsCmd)
 }
 
 // initConfig reads in config file and ENV variables if set.