From e51a4c089306709ce8d8e0ea117c499151f8f1ed Mon Sep 17 00:00:00 2001 From: Clark Andrianasolo Date: Fri, 26 Apr 2024 15:00:59 +0200 Subject: [PATCH] Fixes #24779: Groups compliance summary need API pagination --- .../rudder/apidata/JsonQueryObjects.scala | 12 ++ .../repository/NodeGroupRepository.scala | 13 +- .../ldap/LDAPNodeGroupRepository.scala | 63 +++++-- .../com/normation/rudder/MockServices.scala | 13 +- .../rudder/rest/data/Compliance.scala | 7 + .../rudder/rest/lift/ComplianceApi.scala | 155 ++++++++++++------ .../com/normation/rudder/MockServices.scala | 6 +- 7 files changed, 194 insertions(+), 75 deletions(-) diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/apidata/JsonQueryObjects.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/apidata/JsonQueryObjects.scala index 3c11f58f071..9e1d5d62dbf 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/apidata/JsonQueryObjects.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/apidata/JsonQueryObjects.scala @@ -37,6 +37,7 @@ package com.normation.rudder.apidata +import cats.syntax.traverse.* import com.normation.GitVersion import com.normation.GitVersion.ParseRev import com.normation.GitVersion.Revision @@ -59,6 +60,7 @@ import com.normation.rudder.domain.properties.PropertyProvider import com.normation.rudder.domain.queries.NodeReturnType import com.normation.rudder.domain.queries.Query import com.normation.rudder.domain.queries.QueryReturnType +import com.normation.rudder.domain.reports.CompliancePrecision import com.normation.rudder.rule.category.RuleCategory import com.normation.rudder.rule.category.RuleCategoryId import com.normation.rudder.services.queries.CmdbQueryParser @@ -622,4 +624,14 @@ class ZioJsonExtractor(queryParser: CmdbQueryParser with JsonQueryLexer) { } } + def extractCompliancePrecisionFromParams(params: Map[String, List[String]]): PureResult[Option[CompliancePrecision]] = { + for { + precision <- + params.parseString("precision", s => s.toIntOption.toRight(s"percent precison must be an integer, was: '${s}'")) + res <- precision.traverse(CompliancePrecision.fromPrecision(_).toPureResult) + } yield { + res + } + } + } diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/NodeGroupRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/NodeGroupRepository.scala index fc9bb83e57e..a38cd8f66ee 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/NodeGroupRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/NodeGroupRepository.scala @@ -244,11 +244,6 @@ trait RoNodeGroupRepository { // TODO: add QC def getAll(): IOResult[Seq[NodeGroup]] - /** - * Get all node groups by ids - */ - def getAllByIds(ids: Seq[NodeGroupId]): IOResult[Seq[NodeGroup]] - /** * Get all the node group id and the set of ndoes within * Goal is to be more efficient @@ -277,6 +272,14 @@ trait RoNodeGroupRepository { qc: QueryContext ): IOResult[SortedMap[List[NodeGroupCategoryId], CategoryAndNodeGroup]] + /** + * Get all node groups grouped by their direct parent category. + * Group ids can be filtered, by default return all groups with an empty filter. + */ + def getGroupsByCategoryByIds(ids: Seq[NodeGroupId] = Seq.empty, includeSystem: Boolean = false)(implicit + qc: QueryContext + ): IOResult[Map[NodeGroupCategory, Seq[NodeGroup]]] + /** * Retrieve all groups that have at least one of the given * node ID in there member list. diff --git a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala index 9b88bd4c7fc..8aa2525ad60 100644 --- a/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala +++ b/webapp/sources/rudder/rudder-core/src/main/scala/com/normation/rudder/repository/ldap/LDAPNodeGroupRepository.scala @@ -210,6 +210,52 @@ class RoLDAPNodeGroupRepository( } } + def getGroupsByCategoryByIds(ids: Seq[NodeGroupId], includeSystem: Boolean = false)(implicit + qc: QueryContext + ): IOResult[Map[NodeGroupCategory, Seq[NodeGroup]]] = { + groupLibMutex.readLock { + for { + con <- ldap + entries <- if (ids.isEmpty) con.searchSub(rudderDit.GROUP.dn, IS(OC_RUDDER_NODE_GROUP)) + else con.searchSub(rudderDit.GROUP.dn, OR(ids.map(id => EQ(A_NODE_GROUP_UUID, id.serialize))*)) + groups <- ZIO.foreach(entries)(groupEntry => { + for { + g <- mapper + .entry2NodeGroup(groupEntry) + .toIO + .chainError(s"Error when mapping server group entry into a Group instance. Entry: ${groupEntry}") + allNodes <- nodeFactRepo.getAll() + nodeIds = g.serverList.intersect(allNodes.keySet.toSet) + y = g.copy(serverList = nodeIds) + } yield (groupEntry, y) + }) + cats <- + ZIO.foreach(groups) { + case (groupEntry, g) => { + for { + parentCategoryEntry <- + con + .get(groupEntry.dn.getParent) + .notOptional(s"Parent category of entry with ID '${g.id.serialize}' was not found") + parentCategory <- + mapper + .entry2NodeGroupCategory(parentCategoryEntry) + .toIO + .chainError( + "Error when transforming LDAP entry %s into an active technique category".format(parentCategoryEntry) + ) + } yield { + parentCategory + } + } + } + result = cats.zip(groups).groupBy(_._1).map { case (cat, pairs) => (cat, pairs.map(_._2._2)) } + } yield { + result + } + } + } + def getNodeGroupOpt(id: NodeGroupId)(implicit qc: QueryContext): IOResult[Option[(NodeGroup, NodeGroupCategoryId)]] = { groupLibMutex.readLock(for { con <- ldap @@ -446,23 +492,6 @@ class RoLDAPNodeGroupRepository( } } - def getAllByIds(ids: Seq[NodeGroupId]): IOResult[Seq[NodeGroup]] = { - for { - con <- ldap - // for each directive entry, map it. if one fails, all fails - entries <- - groupLibMutex.readLock(con.searchSub(rudderDit.GROUP.dn, OR(ids.map(id => EQ(A_NODE_GROUP_UUID, id.serialize))*))) - groups <- ZIO.foreach(entries)(groupEntry => { - mapper - .entry2NodeGroup(groupEntry) - .toIO - .chainError(s"Error when transforming LDAP entry into a Group instance. Entry: ${groupEntry}") - }) - } yield { - groups - } - } - def getAllNodeIds(): IOResult[Map[NodeGroupId, Set[NodeId]]] = { for { con <- ldap diff --git a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/MockServices.scala b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/MockServices.scala index 7751aa9aaf5..d8d6717526e 100644 --- a/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/MockServices.scala +++ b/webapp/sources/rudder/rudder-core/src/test/scala/com/normation/rudder/MockServices.scala @@ -2356,8 +2356,17 @@ class MockNodeGroups(nodesRepo: MockNodes) { } yield cat } override def getAll(): IOResult[Seq[NodeGroup]] = categories.get.map(_.allGroups.values.map(_.nodeGroup).toSeq) - override def getAllByIds(ids: Seq[NodeGroupId]): IOResult[Seq[NodeGroup]] = { - categories.get.map(_.allGroups.values.map(_.nodeGroup).filter(g => ids.contains(g.id)).toSeq) + override def getGroupsByCategoryByIds(ids: Seq[NodeGroupId], includeSystem: Boolean = false)(implicit + qc: QueryContext + ): IOResult[Map[NodeGroupCategory, Seq[NodeGroup]]] = { + categories.get.map { root => + val groups = root.allGroups.values.map(_.nodeGroup).filter(g => ids.contains(g.id)).toSeq + val categories = groups.map(g => root.categoryByGroupId(g.id)).distinct + categories.map { c => + val cat = root.allCategories(c).toNodeGroupCategory + (cat, groups.filter(g => root.categoryByGroupId(g.id) == c)) + }.toMap + } } override def getAllNodeIds(): IOResult[Map[NodeGroupId, Set[NodeId]]] = diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala index 4a950112726..0b282f68aaf 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/data/Compliance.scala @@ -129,6 +129,13 @@ final case class ByDirectiveByNodeRuleCompliance( components: Seq[ByRuleByNodeByDirectiveByComponentCompliance] ) +final case class ByNodeGroupFullCompliance( + id: String, + name: String, + category: String, + targeted: ByNodeGroupCompliance, + global: ByNodeGroupCompliance +) final case class ByNodeGroupCompliance( id: String, name: String, diff --git a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala index da748ec4bb3..3a611a94c65 100644 --- a/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala +++ b/webapp/sources/rudder/rudder-rest/src/main/scala/com/normation/rudder/rest/lift/ComplianceApi.scala @@ -41,6 +41,7 @@ import com.normation.box.* import com.normation.errors.* import com.normation.inventory.domain.NodeId import com.normation.rudder.api.ApiVersion +import com.normation.rudder.apidata.ZioJsonExtractor import com.normation.rudder.domain.logger.TimingDebugLogger import com.normation.rudder.domain.logger.TimingDebugLoggerPure import com.normation.rudder.domain.nodes.NodeGroupCategoryId @@ -61,6 +62,7 @@ import com.normation.rudder.domain.policies.RuleTarget import com.normation.rudder.domain.policies.SimpleTarget import com.normation.rudder.domain.reports.BlockStatusReport import com.normation.rudder.domain.reports.ComplianceLevel +import com.normation.rudder.domain.reports.CompliancePercent import com.normation.rudder.domain.reports.CompliancePrecision import com.normation.rudder.domain.reports.ComponentStatusReport import com.normation.rudder.domain.reports.DirectiveStatusReport @@ -79,6 +81,7 @@ import com.normation.rudder.rest.ComplianceApi as API import com.normation.rudder.rest.RestExtractorService import com.normation.rudder.rest.RestUtils.* import com.normation.rudder.rest.data.* +import com.normation.rudder.rest.implicits.* import com.normation.rudder.services.reports.ReportingService import com.normation.rudder.services.reports.ReportingServiceUtils import com.normation.rudder.web.services.ComputePolicyMode @@ -101,6 +104,7 @@ class ComplianceApi( readDirective: RoDirectiveRepository ) extends LiftApiModuleProvider[API] { + import ComplianceAPIService.* import CsvCompliance.* import JsonCompliance.* @@ -319,7 +323,7 @@ class ComplianceApi( object GetNodeGroupSummary extends LiftApiModule0 { val schema: API.GetNodeGroupComplianceSummary.type = API.GetNodeGroupComplianceSummary - val restExtractor = restExtractorService + val restExtractor = zioJsonExtractor def process0( version: ApiVersion, path: ApiPath, @@ -327,32 +331,15 @@ class ComplianceApi( params: DefaultParams, authzToken: AuthzToken ): LiftResponse = { - implicit val action = schema.name - implicit val prettify = params.prettify implicit val qc: QueryContext = authzToken.qc (for { - precision <- restExtractor.extractPercentPrecision(req.params) - targets = req.params.getOrElse("groups", List.empty).flatMap { nodeGroups => - nodeGroups.split(",").toList.flatMap(parseSimpleTargetOrNodeGroupId(_).toOption) - } - - group <- complianceService.getNodeGroupComplianceSummary(targets, precision) + precision <- restExtractor.extractCompliancePrecisionFromParams(req.params).toIO + filters = QueryFilter(req) + group <- complianceService.getNodeGroupComplianceSummary(filters, precision) } yield { - JArray(group.toList.map { - case (id, (global, targeted)) => - (("id" -> id) ~ - ("targeted" -> targeted.toJson(1, precision.getOrElse(CompliancePrecision.Level2))) ~ - ("global" -> global.toJson(1, precision.getOrElse(CompliancePrecision.Level2)))) - }) - }) match { - case Full(groups) => - toJsonResponse(None, ("nodeGroups" -> groups)) - - case eb: EmptyBox => - val message = (eb ?~ (s"Could not get compliance summary")).messageChain - toJsonError(None, JString(message)) - } + group + }).chainError("Could not get compliance summary").toLiftResponseOne(params, schema, _ => None) } } @@ -545,13 +532,6 @@ class ComplianceApi( } } - private[this] def parseSimpleTargetOrNodeGroupId(str: String): PureResult[SimpleTarget] = { - // attempt to parse a "target" first because format is more specific - RuleTarget.unserOne(str) match { - case None => NodeGroupId.parse(str).map(GroupTarget(_)).left.map(Inconsistency(_)) - case Some(value) => Right(value) - } - } } /** @@ -1194,9 +1174,9 @@ class ComplianceAPIService( * Get global and targeted compliance at level 1 (without any details) with global compliance at left and targeted at right */ def getNodeGroupComplianceSummary( - targets: Seq[SimpleTarget], + filter: ComplianceAPIService.QueryFilter, precision: Option[CompliancePrecision] - )(implicit qc: QueryContext): Box[Map[String, (ByNodeGroupCompliance, ByNodeGroupCompliance)]] = { + )(implicit qc: QueryContext): IOResult[List[ByNodeGroupFullCompliance]] = { for { t1 <- currentTimeMillis nodeFacts <- nodeFactRepos.getAll() @@ -1208,29 +1188,37 @@ class ComplianceAPIService( t3 <- currentTimeMillis _ <- TimingDebugLoggerPure.trace(s"getByNodeGroupCompliance - getFullDirectiveLibrary in ${t3 - t2} ms") - (nonGroupTargets, nodeGroupIds) = targets.partitionMap { + (nonGroupTargets, nodeGroupIds) = filter.groups.partitionMap { case GroupTarget(groupId) => Right(groupId) case t: NonGroupRuleTarget => Left(t) } + nodeGroupsByCat <- nodeGroupRepo.getGroupsByCategoryByIds(nodeGroupIds, includeSystem = true) + nodeGroupsGroupInfos = nodeGroupsByCat.toList.flatMap { case (cat, groups) => groups.map(cat -> _) }.map { + case (cat, g) => g.id.serialize -> (cat, g.name, g.serverList, GroupTarget(g.id)) + }.toMap + // all group info including ones for non-group targets (within SystemGroups category) nodeGroupsInfo <- { for { - nodeGroups <- nodeGroupRepo.getAllByIds(nodeGroupIds) - nodeGroupInfos = nodeGroups.map(g => g.id.serialize -> (g.name, g.serverList, GroupTarget(g.id))).toMap - - systemCategory <- nodeGroupRepo.getGroupCategory(NodeGroupCategoryId("SystemGroups")) + systemCategory <- + ZIO + .fromOption(nodeGroupsByCat.keys.find(_.id == NodeGroupCategoryId("SystemGroups"))) + // system groups could not be included in the groups of the filter + .catchAll(_ => nodeGroupRepo.getGroupCategory(NodeGroupCategoryId("SystemGroups"))) targetsInfos = { nonGroupTargets .flatMap(t => { systemCategory.items .find(_.target == t) - .map(i => i.target.target -> (i.name, targetServerList(t)(nodeSettings), t)) + .map(i => i.target.target -> (systemCategory, i.name, targetServerList(t)(nodeSettings), t)) }) .toMap } - } yield nodeGroupInfos ++ targetsInfos + } yield nodeGroupsGroupInfos ++ targetsInfos } t4 <- currentTimeMillis - _ <- TimingDebugLoggerPure.trace(s"getByNodeGroupCompliance - nodeGroupRepo.getAllByIds in ${t4 - t3} ms") + _ <- TimingDebugLoggerPure.trace( + s"getByNodeGroupCompliance - nodeGroupRepo.getGroupsByCategoryByIds and transformations in ${t4 - t3} ms" + ) compliance <- getGlobalComplianceMode t5 <- currentTimeMillis @@ -1268,7 +1256,7 @@ class ComplianceAPIService( // global compliance : filter our rules that are applicable to any node in this group globalRulesByGroup = nodeGroupsInfo.map { - case (g, (_, serverList, _)) => + case (g, (_, _, serverList, _)) => ( g, rules.filter(rule => { @@ -1280,14 +1268,14 @@ class ComplianceAPIService( // targeted compliance : filter rules that only include this group in its targets targetedRulesByGroup = globalRulesByGroup.map { case (g, rules) => - (g, rules.filter(rule => RuleTarget.merge(rule.targets).includes(nodeGroupsInfo(g)._3))) + (g, rules.filter(rule => RuleTarget.merge(rule.targets).includes(nodeGroupsInfo(g)._4))) } level = Some(1) - bothGlobalTargeted <- + fullCompliance <- ZIO.foreach(nodeGroupsInfo.toList) { - case (id, (name, serverList, _)) => + case (id, (cat, name, serverList, _)) => (getByNodeGroupCompliance( id, name, @@ -1310,14 +1298,12 @@ class ComplianceAPIService( allRuleInfos, level, false - )).map( - (id, _) - ) + )).map { case (global, targeted) => ByNodeGroupFullCompliance.apply(id, name, cat.name, global, targeted) } } } yield { - bothGlobalTargeted.toMap + filter.apply(fullCompliance) } - }.toBox + } def getDirectivesCompliance(level: Option[Int])(implicit qc: QueryContext): Box[Seq[ByDirectiveCompliance]] = { for { @@ -1524,3 +1510,74 @@ class ComplianceAPIService( } } } + +object ComplianceAPIService { + + /** + * Query params supported by the compliance summary endpoint. + * Emptiness always means that there is no filter on the field + */ + case class QueryFilter( + groups: List[SimpleTarget], + limit: Option[Int], + offset: Option[Int], + order: Option[Ordering[ByNodeGroupFullCompliance]] + ) { + + def apply(compliances: List[ByNodeGroupFullCompliance]): List[ByNodeGroupFullCompliance] = { + val sorted = order.map(compliances.sorted(_)).getOrElse(compliances) + val dropped = offset.map(sorted.drop).getOrElse(sorted) + limit.map(dropped.take).getOrElse(dropped) + } + } + + object QueryFilter { + + def apply(req: Req): QueryFilter = { + val groups = req.params.getOrElse("groups", List.empty).flatMap { nodeGroups => + nodeGroups.split(",").toList.flatMap(parseSimpleTargetOrNodeGroupId(_).toOption) + } + val limit = req.params.get("limit").flatMap(_.headOption).flatMap(_.toIntOption) + val offset = req.params.get("offset").flatMap(_.headOption).flatMap(_.toIntOption) + val sort = req.params.get("sort").flatMap(_.headOption) + val order = req.params + .get("order") + .flatMap(_.headOption) + .flatMap(v => { + if (v == "desc") { + Some(()) + } else { + None + } + }) + val ordering: Option[Ordering[ByNodeGroupFullCompliance]] = { + implicit val complianceOrdering: Ordering[ComplianceLevel] = + Ordering.by(CompliancePercent.fromLevels(_, ComplianceLevel.PERCENT_PRECISION).compliance) + val asc: Option[Ordering[ByNodeGroupFullCompliance]] = sort match { + case Some("id") => + Some(Ordering.by(_.id)) + case Some("name") => + Some(Ordering.by(_.name)) + case Some("category") => + Some(Ordering.by(_.category)) + case Some("targeted") => + Some(Ordering.by(_.targeted.compliance)) + case Some("global") => + Some(Ordering.by(_.global.compliance)) + case _ => + None + } + order.map(_ => asc.map(_.reverse)).getOrElse(asc) + } + QueryFilter(groups, limit, offset, ordering) + } + } + + def parseSimpleTargetOrNodeGroupId(str: String): PureResult[SimpleTarget] = { + // attempt to parse a "target" first because format is more specific + RuleTarget.unserOne(str) match { + case None => NodeGroupId.parse(str).map(GroupTarget(_)).left.map(Inconsistency(_)) + case Some(value) => Right(value) + } + } +} diff --git a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala index a54e9a1ed90..d1d70a128ac 100644 --- a/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala +++ b/webapp/sources/rudder/rudder-rest/src/test/scala/com/normation/rudder/MockServices.scala @@ -225,8 +225,10 @@ class MockCompliance(mockDirectives: MockDirectives) { def categoryExists(id: NodeGroupCategoryId): IOResult[Boolean] = ??? def getNodeGroupCategory(id: NodeGroupId): IOResult[NodeGroupCategory] = ??? - def getAll(): IOResult[Seq[NodeGroup]] = ??? - def getAllByIds(ids: Seq[NodeGroupId]): IOResult[Seq[NodeGroup]] = ??? + def getAll(): IOResult[Seq[NodeGroup]] = ??? + def getGroupsByCategoryByIds(ids: Seq[NodeGroupId], includeSystem: Boolean = false)(implicit + qc: QueryContext + ): IOResult[Map[NodeGroupCategory, Seq[NodeGroup]]] = ??? def getAllNodeIds(): IOResult[Map[NodeGroupId, Set[NodeId]]] = ??? def getGroupsByCategory(includeSystem: Boolean)(implicit qc: QueryContext