Merge pull request #4008 from MaxMcAdam/anax-3996

Issue #3996 - Add agent APIs/CLIs for deleting eventlog entry from ag…

agreementbot/secrets/vault/vault.go

@@ -122,11 +122,11 @@ func (vs *AgbotVaultSecrets) listSecret(user, token, org, name, url string) erro
 // List all secrets in the specified org in vault.
 func (vs *AgbotVaultSecrets) ListAllSecrets(user, token, org, path string) ([]string, error) {
 
-        glog.V(3).Infof(vaultPluginLogString(fmt.Sprintf("list all secrets in %v", org)))
+	glog.V(3).Infof(vaultPluginLogString(fmt.Sprintf("list all secrets in %v", org)))
 
-        url := fmt.Sprintf("%s/v1/openhorizon/metadata/%s"+cliutils.AddSlash(path), vs.cfg.GetAgbotVaultURL(), org)
-        glog.V(3).Infof(vaultPluginLogString(fmt.Sprintf("listing all secrets in org %s", org)))
-        return vs.listSecrets(user, token, org, url, path, true)
+	url := fmt.Sprintf("%s/v1/openhorizon/metadata/%s"+cliutils.AddSlash(path), vs.cfg.GetAgbotVaultURL(), org)
+	glog.V(3).Infof(vaultPluginLogString(fmt.Sprintf("listing all secrets in org %s", org)))
+	return vs.listSecrets(user, token, org, url, path, true)
 }
 
 // List all org-level secrets at a specified path in vault.

agreementbot/secure_api.go

@@ -1189,7 +1189,7 @@ type SecretRequestInfo struct {
 	user            string // if applicable, the user whose resources are being accessed
 	node            string // if applicable, the node whose resources are being accessed
 	vaultSecretName string // the name of the secret being accessed
-	listAll		bool   // true to indicate listing all secrets in the org including node and user-level
+	listAll         bool   // true to indicate listing all secrets in the org including node and user-level
 
 	msgPrinter *message.Printer
 }
@@ -1448,54 +1448,54 @@ func (a *SecureAPI) orgSecrets(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *SecureAPI) allSecrets(w http.ResponseWriter, r *http.Request) {
-        info := a.secretsSetup(w, r)
-        if info == nil {
-                return
-        }
+	info := a.secretsSetup(w, r)
+	if info == nil {
+		return
+	}
 
 	info.listAll = true
 
-        // handle API options
-        switch r.Method {
-        // swagger:operation LIST /org/{org}/allsecrets allSecrets
-        //
-        // List all secrets belonging to the org including node and user-level secrets.
-        //
-        // ---
-        // consumes:
-        //   - application/json
-        // produces:
-        //   - application/json
-        // responses:
-        //  '200':
-        //    description: "Success or no secrets found."
-        //    type: array
-        //    items: string
-        //  '401':
-        //    description: "Unauthenticated user."
-        //    type: string
-        //  '403':
-        //    description: "Secrets permission denied to user."
-        //    type: string
-        //  '503':
-        //    description: "Secret provider unavailable"
-        //    type: string
-        //  '500':
-        //    description: "Invalid vault response"
-        //    type: string
-        case "LIST":
-                if payload, err, httpCode := a.listVaultSecret(info); err != nil {
-                        glog.Errorf(APIlogString(err.Error()))
-                        writeResponse(w, err.Error(), httpCode)
-                } else {
-                        writeResponse(w, payload, http.StatusOK)
-                }
-        case "OPTIONS":
-                w.Header().Set("Allow", "LIST, OPTIONS")
-                w.WriteHeader(http.StatusOK)
-        default:
-                w.WriteHeader(http.StatusMethodNotAllowed)
-        }
+	// handle API options
+	switch r.Method {
+	// swagger:operation LIST /org/{org}/allsecrets allSecrets
+	//
+	// List all secrets belonging to the org including node and user-level secrets.
+	//
+	// ---
+	// consumes:
+	//   - application/json
+	// produces:
+	//   - application/json
+	// responses:
+	//  '200':
+	//    description: "Success or no secrets found."
+	//    type: array
+	//    items: string
+	//  '401':
+	//    description: "Unauthenticated user."
+	//    type: string
+	//  '403':
+	//    description: "Secrets permission denied to user."
+	//    type: string
+	//  '503':
+	//    description: "Secret provider unavailable"
+	//    type: string
+	//  '500':
+	//    description: "Invalid vault response"
+	//    type: string
+	case "LIST":
+		if payload, err, httpCode := a.listVaultSecret(info); err != nil {
+			glog.Errorf(APIlogString(err.Error()))
+			writeResponse(w, err.Error(), httpCode)
+		} else {
+			writeResponse(w, payload, http.StatusOK)
+		}
+	case "OPTIONS":
+		w.Header().Set("Allow", "LIST, OPTIONS")
+		w.WriteHeader(http.StatusOK)
+	default:
+		w.WriteHeader(http.StatusMethodNotAllowed)
+	}
 }
 
 // handler for /org/<org>/secrets/<secret> - GET, LIST, PUT, POST, DELETE, OPTIONS
@@ -2228,8 +2228,8 @@ func (a *SecureAPI) listVaultSecret(info *SecretRequestInfo) (interface{}, error
 		secretNames := make([]string, 0)
 		var err error
 		if info.listAll {
-			// listing all secrets in the org including node and user-level 
-                        secretNames, err = a.secretProvider.ListAllSecrets(info.ec.GetExchangeId(), info.ec.GetExchangeToken(), info.org, "")
+			// listing all secrets in the org including node and user-level
+			secretNames, err = a.secretProvider.ListAllSecrets(info.ec.GetExchangeId(), info.ec.GetExchangeToken(), info.org, "")
 		} else if info.user != "" && info.node != "" {
 			// listing node user level secrets
 			secretNames, err = a.secretProvider.ListUserNodeSecrets(info.ec.GetExchangeId(), info.ec.GetExchangeToken(), info.org, info.node, userNodePath)

api/api.go

@@ -97,9 +97,12 @@ func (a *API) router(includeStaticRedirects bool) *mux.Router {
 
 	// Used to get the event logs on this node.
 	// get the eventlogs for current registration.
-	router.HandleFunc("/eventlog", a.eventlog).Methods("GET", "OPTIONS")
+	// delete eventlogs with selectors
+	router.HandleFunc("/eventlog", a.eventlog).Methods("GET", "DELETE", "OPTIONS")
 	// get the eventlogs for all registrations.
 	router.HandleFunc("/eventlog/all", a.eventlog).Methods("GET", "OPTIONS")
+	// delete all eventlogs from previous registrations
+	router.HandleFunc("/eventlog/prune", a.eventlog).Methods("DELETE", "OPTIONS")
 	//get the active surface errors for this node
 	router.HandleFunc("/eventlog/surface", a.surface).Methods("GET", "OPTIONS")
 

api/api_eventlog.go

@@ -41,7 +41,35 @@ func (a *API) eventlog(w http.ResponseWriter, r *http.Request) {
 		} else {
 			writeResponse(w, out, http.StatusOK)
 		}
+	case "DELETE":
+		// get message printer with the language passed in from the header
+		lan := r.Header.Get("Accept-Language")
+		if lan == "" {
+			lan = i18n.DEFAULT_LANGUAGE
+		}
+		msgPrinter := i18n.GetMessagePrinterWithLocale(lan)
+
+		prune := false
+		if r.URL != nil && strings.Contains(r.URL.Path, "prune") {
+			prune = true
+		}
 
+		if err := r.ParseForm(); err != nil {
+			errorHandler(NewAPIUserInputError(msgPrinter.Sprintf("Error parsing the selections %v. %v", r.Form, err), "selection"))
+			return
+		}
+
+		glog.V(5).Infof(apiLogString(fmt.Sprintf("Handling %v on resource %v with selection %v. Language: %v", r.Method, resource, r.Form, lan)))
+
+		if err := DeleteEventLogs(a.db, prune, r.Form, msgPrinter); err != nil {
+			errorHandler(NewSystemError(msgPrinter.Sprintf("Error deleting %v, error %v", resource, err)))
+		} else {
+			if prune {
+				writeResponse(w, "Successfully pruned event logs.", http.StatusOK)
+			} else {
+				writeResponse(w, "Successfully deleted event logs.", http.StatusOK)
+			}
+		}
 	case "OPTIONS":
 		w.Header().Set("Allow", "GET, OPTIONS")
 		w.WriteHeader(http.StatusOK)

api/path_eventlog.go

@@ -32,6 +32,35 @@ func FindEventLogsForOutput(db *bolt.DB, all_logs bool, selections map[string][]
 	}
 }
 
+// This API deletes the selected event logs saved on the db.
+func DeleteEventLogs(db *bolt.DB, prune bool, selections map[string][]string, msgPrinter *message.Printer) error {
+	s := map[string][]persistence.Selector{}
+	if prune {
+		lastUnreg, err := persistence.GetLastUnregistrationTime(db)
+		if err != nil {
+			return fmt.Errorf("Failed to get the last unregistration time stamp from db. %v", err)
+		}
+
+		s["timestamp"] = []persistence.Selector{persistence.Selector{Op: "<", MatchValue: lastUnreg}}
+
+		glog.V(5).Infof(apiLogString(fmt.Sprintf("Selectors for pruning are: %v.", s)))
+	} else {
+		glog.V(5).Infof(apiLogString(fmt.Sprintf("Deleting event logs from the db. The selectors are: %v.", selections)))
+
+		//convert to selectors
+		var err error
+		s, err = persistence.ConvertToSelectors(selections)
+		if err != nil {
+			return fmt.Errorf(msgPrinter.Sprintf("Error converting the selections into Selectors: %v", err))
+		} else {
+			glog.V(5).Infof(apiLogString(fmt.Sprintf("Converted selections into a map of persistence.Selector arrays: %v.", s)))
+		}
+	}
+
+	err := eventlog.DeleteEventLogs(db, s, msgPrinter)
+	return err
+}
+
 func FindSurfaceLogsForOutput(db *bolt.DB, msgPrinter *message.Printer) ([]persistence.SurfaceError, error) {
 	outputLogs := make([]persistence.SurfaceError, 0)
 	surfaceLogs, err := persistence.FindSurfaceErrors(db)

cli/eventlog/eventlog.go

@@ -63,6 +63,44 @@ func getSelectionString(selections []string) (string, error) {
 	return strings.Join(sels, "&"), nil
 }
 
+func Delete(selections []string, force bool) {
+	// format the eventlog api string
+	url_s := "eventlog"
+
+	if len(selections) > 0 {
+		if s, err := getSelectionString(selections); err != nil {
+			cliutils.Fatal(cliutils.CLI_INPUT_ERROR, "%v", err)
+		} else {
+			url_s = fmt.Sprintf("%v?%v", url_s, s)
+		}
+		if !force {
+			cliutils.ConfirmRemove(i18n.GetMessagePrinter().Sprintf("Are you sure you want to remove all event logs matching the given selectors?"))
+		}
+	} else if !force {
+		cliutils.ConfirmRemove(i18n.GetMessagePrinter().Sprintf("Are you sure you want to remove all event logs?"))
+	}
+
+	cliutils.HorizonDelete(url_s, []int{200, 204}, []int{}, false)
+
+	if len(selections) > 0 {
+		fmt.Println(i18n.GetMessagePrinter().Sprintf("Successfully deleted the selected eventlogs."))
+	} else {
+		fmt.Println(i18n.GetMessagePrinter().Sprintf("Successfully deleted the eventlogs."))
+	}
+}
+
+func Prune(force bool) {
+	url := "eventlog/prune"
+
+	if !force {
+		cliutils.ConfirmRemove(i18n.GetMessagePrinter().Sprintf("Are you sure you want to remove all event logs from previous registrations?"))
+	}
+
+	cliutils.HorizonDelete(url, []int{200, 204}, []int{}, false)
+
+	fmt.Println(i18n.GetMessagePrinter().Sprintf("Successfully pruned the eventlogs."))
+}
+
 func List(all bool, detail bool, selections []string, tailing bool) {
 
 	// format the eventlog api string

cli/hzn.go

@@ -292,6 +292,11 @@ Environment Variables:
 	listAllEventlogs := eventlogListCmd.Flag("all", msgPrinter.Sprintf("List all the event logs including the previous registrations.")).Short('a').Bool()
 	listDetailedEventlogs := eventlogListCmd.Flag("long", msgPrinter.Sprintf("List event logs with details.")).Short('l').Bool()
 	listSelectedEventlogs := eventlogListCmd.Flag("select", msgPrinter.Sprintf("Selection string. This flag can be repeated which means 'AND'. Each flag should be in the format of attribute=value, attribute~value, \"attribute>value\" or \"attribute<value\", where '~' means contains. The common attribute names are timestamp, severity, message, event_code, source_type, agreement_id, service_url etc. Use the '-l' flag to see all the attribute names.")).Short('s').Strings()
+	eventlogDeleteCmd := eventlogCmd.Command("delete | del", msgPrinter.Sprintf("Delete the all event logs or those matching the provided selectors.")).Alias("del").Alias("delete")
+	deleteSelectedEventlogs := eventlogDeleteCmd.Flag("select", msgPrinter.Sprintf("Selection string. This flag can be repeated which means 'AND'. Each flag should be in the format of attribute=value, attribute~value, \"attribute>value\" or\"attribute<value\", where '~' means contains. The common attribute names are timestamp, severity, message, event_code, source_type, agreement_id, service_url etc. Use the '-l' flag to see all the attribute names.")).Short('s').Strings()
+	deleteEventLogsForce := eventlogDeleteCmd.Flag("force", msgPrinter.Sprintf("Skip the 'are you sure?' prompt.")).Short('f').Bool()
+	eventlogPruneCmd := eventlogCmd.Command("prune | pr", msgPrinter.Sprintf("Delete the all event logs from previous registrations.")).Alias("pr").Alias("prune")
+	pruneEventLogsForce := eventlogPruneCmd.Flag("force", msgPrinter.Sprintf("Skip the 'are you sure?' prompt.")).Short('f').Bool()
 	surfaceErrorsEventlogs := eventlogCmd.Command("surface | sf", msgPrinter.Sprintf("List all the active errors that will be shared with the Exchange if the node is online.")).Alias("sf").Alias("surface")
 	surfaceErrorsEventlogsLong := surfaceErrorsEventlogs.Flag("long", msgPrinter.Sprintf("List the full event logs of the surface errors.")).Short('l').Bool()
 
@@ -1403,6 +1408,10 @@ Environment Variables:
 		status.DisplayStatus(*statusLong, false)
 	case eventlogListCmd.FullCommand():
 		eventlog.List(*listAllEventlogs, *listDetailedEventlogs, *listSelectedEventlogs, *listTail)
+	case eventlogDeleteCmd.FullCommand():
+		eventlog.Delete(*deleteSelectedEventlogs, *deleteEventLogsForce)
+	case eventlogPruneCmd.FullCommand():
+		eventlog.Prune(*pruneEventLogsForce)
 	case surfaceErrorsEventlogs.FullCommand():
 		eventlog.ListSurfaced(*surfaceErrorsEventlogsLong)
 	case devServiceNewCmd.FullCommand():

cli/secrets_manager/secrets_manager.go

@@ -103,7 +103,7 @@ func SecretList(org, credToUse, secretName, secretNodeId string, listAll bool) {
 		//            org/secrets/user/{userId}/node/{nodeId}/test-password
 		if listAll {
 			return cliutils.AgbotList("org"+cliutils.AddSlash(org)+"/allsecrets", cliutils.OrgAndCreds(org, credToUse),
-                                []int{200, 400, 401, 403, 404, 503, 504}, &resp)
+				[]int{200, 400, 401, 403, 404, 503, 504}, &resp)
 		} else {
 			return cliutils.AgbotList("org"+cliutils.AddSlash(org)+"/secrets"+cliutils.AddSlash(secretName), cliutils.OrgAndCreds(org, credToUse),
 				[]int{200, 400, 401, 403, 404, 503, 504}, &resp)

eventlog/eventlog_manager.go

@@ -96,6 +96,10 @@ func GetEventLogs(db *bolt.DB, all_logs bool, selectors map[string][]persistence
 	return persistence.FindEventLogsWithSelectors(db, all_logs, selectors, msgPrinter)
 }
 
+func DeleteEventLogs(db *bolt.DB, selectors map[string][]persistence.Selector, msgPrinter *message.Printer) error {
+	return persistence.DeleteEventLogsWithSelectors(db, selectors, msgPrinter)
+}
+
 type EventLogByTimestamp []persistence.EventLog
 
 func (s EventLogByTimestamp) Len() int {

persistence/eventlogs.go

@@ -409,6 +409,38 @@ func FindEventLogs(db *bolt.DB, filters []EventLogFilter) ([]EventLog, error) {
 	}
 }
 
+// delete event logs from the db that match the given selectors
+func DeleteEventLogsWithSelectors(db *bolt.DB, selectors map[string][]Selector, msgPrinter *message.Printer) error {
+	// separate base selectors from the source selectors
+	base_selectors, source_selectors := GroupSelectors(selectors)
+
+	if msgPrinter == nil {
+		msgPrinter = i18n.GetMessagePrinter()
+	}
+
+	return db.Update(func(tx *bolt.Tx) error {
+		if b := tx.Bucket([]byte(EVENT_LOGS)); b != nil {
+			b.ForEach(func(k, v []byte) error {
+				var el EventLogRaw
+
+				if err := json.Unmarshal(v, &el); err != nil {
+					glog.Errorf("Unable to deserialize event log db record: %v. Error: %v", v, err)
+				} else {
+					if el.EventLogBase.Matches(base_selectors) {
+						if esrc, err := GetRealEventSource(el.SourceType, el.Source); err != nil {
+							glog.Errorf("Unable to convert event source: %v. Error: %v", el.Source, err)
+						} else if (*esrc).Matches(source_selectors) {
+							b.Delete(k)
+						}
+					}
+				}
+				return nil
+			})
+		}
+		return nil
+	})
+}
+
 // find event logs from the db for the given given selectors.
 // If all_logs is false, only the event logs for the current registration is returned.
 func FindEventLogsWithSelectors(db *bolt.DB, all_logs bool, selectors map[string][]Selector, msgPrinter *message.Printer) ([]EventLog, error) {