Preserve the framework dependencies in the reporting structure (#636)

There are major changes here:

First, compliance frameworks were not connected to the root reporting
job. On the backend, we iterate from the root reporting job to figure
out all reporting jobs that get saved. Without this, we don't save the
compliance frameworks or controls

Second change is the structure of the framework dependencies is
preserved in the reporting jobs. For example, asset frameworks point to
the space framework which point to the global framework which point to
some actual frameworks. This keeps it consistent with the way handle
policies.

One issue I ran into was that we have space/asset frameworks and
space/asset policies. I chose to not create separate reporting jobs for
those because having multiple reporting jobs with the same query is
likely to break something. Instead, the policies would just get attached
to the existing policy jobs, but the impact is set to unscored


![reporting_structure](https://github.com/mondoohq/cnspec/assets/27443/3a8ade88-86b7-4b26-bd5e-5df42c26bf26)

policy/framework.go

@@ -412,12 +412,15 @@ func ResolveFramework(mrn string, frameworks map[string]*Framework) *ResolvedFra
  for i := range framework.FrameworkMaps {
  fmap := framework.FrameworkMaps[i]
 
- for j := range fmap.Controls {
- ctl := fmap.Controls[j]
+ for _, ctl := range fmap.Controls {
  res.addReportLink(framework.Mrn, ctl.Mrn)
  res.addControl(ctl)
  }
  }
+ // FIXME: why do these not show up in the framework map
+ for _, depFramework := range framework.Dependencies {
+ res.addReportLink(framework.Mrn, depFramework.Mrn)
+ }
  }
 
  return res

policy/resolver.go

@@ -567,15 +567,27 @@ func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetF
  Str("policy", bundleMrn).
  Msg("resolver> phase 4: aggregate queries and jobs [ok]")
 
- // phase 5: add controls
+ // phase 5: add frameworks and controls
  resolvedFramework := ResolveFramework(bundleMrn, bundleMap.Frameworks)
+ cacheFrameworkJobs := &frameworkResolverCache{
+ resolverCache: cache,
+ frameworkJobsByMrn: make(map[string]*ReportingJob),
+ }
+ if err := s.jobsToFrameworks(cacheFrameworkJobs, resolvedFramework, collectorJob, bundleMrn, reportingJob); err != nil {
+ logCtx.Error().Err(err).
+ Str("bundle", bundleMrn).
+ Msg("resolver> phase 5: internal error, trying to attach framework to resolved policy")
+ return nil, err
+ }
+
  queries := bundleMap.QueryMap()
- if err := s.jobsToControls(cache, resolvedFramework, collectorJob, queries); err != nil {
+ if err := s.jobsToControls(cacheFrameworkJobs, resolvedFramework, collectorJob, queries); err != nil {
  logCtx.Error().
  Err(err).
  Str("bundle", bundleMrn).
  Msg("resolver> phase 5: internal error, trying to attach controls to resolved policy [ok]")
  }
+
  logCtx.Debug().
  Str("bundle", bundleMrn).
  Msg("resolver> phase 5: resolve controls [ok]")
@@ -1366,7 +1378,7 @@ func mquery2executionQuery(query queryLike, props map[string]*llx.Primitive, pro
  return &res, dataChecksum, nil
 }
 
-func ensureControlJob(cache *resolverCache, jobs map[string]*ReportingJob, controlMrn string, framework *ResolvedFramework) *ReportingJob {
+func ensureControlJob(cache *frameworkResolverCache, jobs map[string]*ReportingJob, controlMrn string, framework *ResolvedFramework) *ReportingJob {
  uuid := cache.relativeChecksum(controlMrn)
 
  if found, ok := jobs[uuid]; ok {
@@ -1386,16 +1398,9 @@ func ensureControlJob(cache *resolverCache, jobs map[string]*ReportingJob, contr
  for _, parentMrn := range parents {
  parentUuid := cache.relativeChecksum(parentMrn)
 
- frameworkJob, ok := jobs[parentUuid]
+ frameworkJob, ok := cache.frameworkJobsByMrn[parentMrn]
  if !ok {
- frameworkJob = &ReportingJob{
- Uuid: parentUuid,
- QrId: parentMrn,
- ChildJobs: map[string]*explorer.Impact{},
- ScoringSystem: explorer.ScoringSystem_WORST,
- Type: ReportingJob_FRAMEWORK,
- }
- jobs[parentUuid] = frameworkJob
+ continue
  }
 
  // FIXME: we need to take user exceptions into account!
@@ -1406,7 +1411,65 @@ func ensureControlJob(cache *resolverCache, jobs map[string]*ReportingJob, contr
  return controlJob
 }
 
-func (s *LocalServices) jobsToControls(cache *resolverCache, framework *ResolvedFramework, job *CollectorJob, querymap map[string]*explorer.Mquery) error {
+type frameworkResolverCache struct {
+ *resolverCache
+ frameworkJobsByMrn map[string]*ReportingJob
+}
+
+func (s *LocalServices) jobsToFrameworks(cache *frameworkResolverCache, resolvedFramework *ResolvedFramework, job *CollectorJob, frameworkMrn string, parent *ReportingJob) error {
+ for k, rj := range job.ReportingJobs {
+ if frameworkJob := cache.bundleMap.Frameworks[rj.QrId]; frameworkJob != nil {
+ cache.frameworkJobsByMrn[rj.QrId] = job.ReportingJobs[k]
+ }
+ }
+ return s.jobsToFrameworksInner(cache, resolvedFramework, job, frameworkMrn, parent)
+}
+
+func (s *LocalServices) jobsToFrameworksInner(cache *frameworkResolverCache, resolvedFramework *ResolvedFramework, job *CollectorJob, frameworkMrn string, parent *ReportingJob) error {
+ for _, source := range resolvedFramework.ReportSources[frameworkMrn] {
+ if childFramework, ok := cache.bundleMap.Frameworks[source]; ok {
+ var reportingJob *ReportingJob
+ if found, ok := cache.frameworkJobsByMrn[childFramework.Mrn]; ok {
+ // Look for an existing job. This will happen for asset and space frameworks.
+ // Creating a new job for these would likely cause confusion, as then we'd
+ // end up with multiple jobs with the same QrId
+ reportingJob = found
+ } else {
+ uuid := cache.relativeChecksum(childFramework.Mrn)
+ reportingJob = &ReportingJob{
+ Uuid: uuid,
+ QrId: childFramework.Mrn,
+ ChildJobs: map[string]*explorer.Impact{},
+ ScoringSystem: explorer.ScoringSystem_WORST,
+ Type: ReportingJob_FRAMEWORK,
+ }
+ }
+
+ if _, exist := parent.ChildJobs[reportingJob.Uuid]; !exist {
+ // If we already have a child job, we don't need to do anything
+ // In the case that its a space or asset, we defiently don't want to
+ // overwrite it as that would change the scoring system
+ impact := &explorer.Impact{}
+ if parent.Type == ReportingJob_FRAMEWORK {
+ impact.Scoring = explorer.ScoringSystem_WORST
+ } else {
+ impact.Scoring = explorer.ScoringSystem_IGNORE_SCORE
+ }
+ parent.ChildJobs[reportingJob.Uuid] = impact
+ }
+ reportingJob.Notify = append(reportingJob.Notify, parent.Uuid)
+ job.ReportingJobs[reportingJob.Uuid] = reportingJob
+ cache.frameworkJobsByMrn[childFramework.Mrn] = reportingJob
+ if err := s.jobsToFrameworksInner(cache, resolvedFramework, job, source, reportingJob); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func (s *LocalServices) jobsToControls(cache *frameworkResolverCache, framework *ResolvedFramework, job *CollectorJob, querymap map[string]*explorer.Mquery) error {
  nuJobs := map[string]*ReportingJob{}
 
  for _, rj := range job.ReportingJobs {

policy/resolver_test.go

@@ -2,6 +2,7 @@ package policy_test
 
 import (
  "context"
+ "strings"
  "testing"
  "time"
 
@@ -14,8 +15,9 @@ import (
 )
 
 type testAsset struct {
- asset string
- policies []string
+ asset string
+ policies []string
+ frameworks []string
 }
 
 func parseBundle(t *testing.T, data string) *policy.Bundle {
@@ -37,8 +39,9 @@ func initResolver(t *testing.T, assets []*testAsset, bundles []*policy.Bundle) *
  for i := range assets {
  asset := assets[i]
  _, err := srv.Assign(context.Background(), &policy.PolicyAssignment{
- AssetMrn: asset.asset,
- PolicyMrns: asset.policies,
+ AssetMrn: asset.asset,
+ PolicyMrns: asset.policies,
+ FrameworkMrns: asset.frameworks,
  })
  require.NoError(t, err)
  }
@@ -50,6 +53,18 @@ func policyMrn(uid string) string {
  return "//test.sth/policies/" + uid
 }
 
+func frameworkMrn(uid string) string {
+ return "//test.sth/frameworks/" + uid
+}
+
+func controlMrn(uid string) string {
+ return "//test.sth/controls/" + uid
+}
+
+func queryMrn(uid string) string {
+ return "//test.sth/queries/" + uid
+}
+
 func TestResolve_EmptyPolicy(t *testing.T) {
  b := parseBundle(t, `
 owner_mrn: //test.sth
@@ -58,7 +73,7 @@ policies:
 `)
 
  srv := initResolver(t, []*testAsset{
- {"asset1", []string{policyMrn("policy1")}},
+ {asset: "asset1", policies: []string{policyMrn("policy1")}},
  }, []*policy.Bundle{b})
 
  t.Run("resolve w/o filters", func(t *testing.T) {
@@ -108,7 +123,7 @@ policies:
 `)
 
  srv := initResolver(t, []*testAsset{
- {"asset1", []string{policyMrn("policy1")}},
+ {asset: "asset1", policies: []string{policyMrn("policy1")}},
  }, []*policy.Bundle{b})
 
  checkResolvedPolicy := func(t *testing.T, rp *policy.ResolvedPolicy) {
@@ -191,7 +206,7 @@ policies:
 `)
 
  srv := initResolver(t, []*testAsset{
- {"asset1", []string{policyMrn("policy-active"), policyMrn("policy-ignored")}},
+ {asset: "asset1", policies: []string{policyMrn("policy-active"), policyMrn("policy-ignored")}},
  }, []*policy.Bundle{b})
 
  t.Run("resolve with ignored policy", func(t *testing.T) {
@@ -310,3 +325,151 @@ policies:
  require.Len(t, rp.ExecutionJob.Queries, 2)
  })
 }
+
+func TestResolve_Frameworks(t *testing.T) {
+ b := parseBundle(t, `
+owner_mrn: //test.sth
+policies:
+- uid: policy1
+ groups:
+ - filters: "true"
+ checks:
+ - uid: check-fail
+ mql: 1 == 2
+ - uid: check-pass-1
+ mql: 1 == 1
+ - uid: check-pass-2
+ mql: 2 == 2
+
+frameworks:
+- uid: framework1
+ name: framework1
+ groups:
+ - title: group1
+ controls:
+ - uid: control1
+ title: control1
+ - uid: control2
+ title: control2
+ - uid: control3
+ title: control3
+- uid: parent-framework
+ dependencies:
+ - mrn: `+frameworkMrn("framework1")+`
+
+framework_maps:
+- uid: framework-map1
+ framework_owner: framework1
+ policy_dependencies:
+ - uid: policy1
+ controls:
+ - uid: control1
+ checks:
+ - uid: check-pass-1
+ - uid: control2
+ checks:
+ - uid: check-pass-2
+ - uid: check-fail
+`)
+
+ srv := initResolver(t, []*testAsset{
+ {asset: "asset1", policies: []string{policyMrn("policy1")}, frameworks: []string{frameworkMrn("parent-framework")}},
+ }, []*policy.Bundle{b})
+
+ t.Run("resolve with correct filters", func(t *testing.T) {
+ bundle, err := srv.GetBundle(context.Background(), &policy.Mrn{Mrn: "asset1"})
+ require.NoError(t, err)
+
+ bundleMap, err := bundle.Compile(context.Background(), nil)
+ require.NoError(t, err)
+
+ mrnToQueryId := map[string]string{}
+ for _, q := range bundleMap.Queries {
+ mrnToQueryId[q.Mrn] = q.CodeId
+ }
+
+ rp, err := srv.Resolve(context.Background(), &policy.ResolveReq{
+ PolicyMrn: "asset1",
+ AssetFilters: []*explorer.Mquery{{Mql: "true"}},
+ })
+ require.NoError(t, err)
+ require.NotNil(t, rp)
+
+ require.Len(t, rp.ExecutionJob.Queries, 3)
+
+ rjTester := frameworkReportingJobTester{
+ t: t,
+ queryIdToReportingJob: map[string]*policy.ReportingJob{},
+ rjIdToReportingJob: rp.CollectorJob.ReportingJobs,
+ }
+
+ for _, rj := range rjTester.rjIdToReportingJob {
+ rjTester.queryIdToReportingJob[rj.QrId] = rj
+ }
+
+ // control3 had no checks, so it should not have a reporting job.
+ // TODO: is that the desired behavior?
+ require.Nil(t, rjTester.queryIdToReportingJob[controlMrn("control3")])
+ rjTester.requireReportsTo(mrnToQueryId[queryMrn("check-pass-1")], controlMrn("control1"))
+ rjTester.requireReportsTo(mrnToQueryId[queryMrn("check-pass-2")], controlMrn("control2"))
+ rjTester.requireReportsTo(mrnToQueryId[queryMrn("check-fail")], controlMrn("control2"))
+ rjTester.requireReportsTo(controlMrn("control1"), frameworkMrn("framework1"))
+ rjTester.requireReportsTo(controlMrn("control2"), frameworkMrn("framework1"))
+ rjTester.requireReportsTo(frameworkMrn("framework1"), frameworkMrn("parent-framework"))
+ rjTester.requireReportsTo(frameworkMrn("parent-framework"), "root")
+ })
+}
+
+type frameworkReportingJobTester struct {
+ t *testing.T
+ queryIdToReportingJob map[string]*policy.ReportingJob
+ rjIdToReportingJob map[string]*policy.ReportingJob
+}
+
+func isFramework(queryId string) bool {
+ return strings.Contains(queryId, "/frameworks/")
+}
+
+func isControl(queryId string) bool {
+ return strings.Contains(queryId, "/controls/")
+}
+
+func isPolicy(queryId string) bool {
+ return strings.Contains(queryId, "/policies/")
+}
+
+func (tester *frameworkReportingJobTester) requireReportsTo(childQueryId string, parentQueryId string) {
+ tester.t.Helper()
+
+ childRj, ok := tester.queryIdToReportingJob[childQueryId]
+ require.True(tester.t, ok)
+
+ parentRj, ok := tester.queryIdToReportingJob[parentQueryId]
+ require.True(tester.t, ok)
+
+ require.Contains(tester.t, parentRj.ChildJobs, childRj.Uuid)
+ require.Contains(tester.t, childRj.Notify, parentRj.Uuid)
+
+ if isFramework(parentQueryId) {
+ require.Equal(tester.t, policy.ReportingJob_FRAMEWORK, parentRj.Type)
+ } else if isControl(parentQueryId) {
+ require.Equal(tester.t, policy.ReportingJob_CONTROL, parentRj.Type)
+ } else if isPolicy(parentQueryId) || parentQueryId == "root" {
+ require.Equal(tester.t, policy.ReportingJob_POLICY, parentRj.Type)
+ // The root/asset reporting job is not a framework, but a policy
+ childImpact := parentRj.ChildJobs[childRj.Uuid]
+ require.Equal(tester.t, explorer.ScoringSystem_IGNORE_SCORE, childImpact.Scoring)
+ } else {
+ require.Equal(tester.t, policy.ReportingJob_CHECK, parentRj.Type)
+ }
+
+ if isControl(childQueryId) {
+ require.Equal(tester.t, policy.ReportingJob_CONTROL, childRj.Type)
+ } else if isFramework(childQueryId) {
+ require.Equal(tester.t, policy.ReportingJob_FRAMEWORK, childRj.Type)
+ } else if isPolicy(childQueryId) {
+ require.Equal(tester.t, policy.ReportingJob_POLICY, childRj.Type)
+ } else {
+ require.Equal(tester.t, policy.ReportingJob_CHECK, childRj.Type)
+ }
+}