Skip to content

Commit

Permalink
Added a Windows.Sigma.Base.CaptureTestSet artifact (#41)
Browse files Browse the repository at this point in the history
This artifact helps to capture test sets that can be used in development
of Sigma rules.
  • Loading branch information
scudette authored Nov 18, 2024
1 parent 342b135 commit 3481f21
Show file tree
Hide file tree
Showing 10 changed files with 193 additions and 29 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/gh-pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,13 @@ jobs:
run: |
make linux
./velosigmac compile --config ./config/windows_base.yaml --output ./output/Windows-Sigma-Base.zip --yaml ./output/Windows.Sigma.Base.yaml
./velosigmac compile --config ./config/windows_base_test.yaml --yaml ./output/Windows.Sigma.Base.CaptureTestSet.yaml
./velosigmac compile --config ./config/windows_hayabusa_rules.yaml --output ./output/Velociraptor-Hayabusa-Rules.zip --rejects ./rejected/windows_hayabusa_rejects.json --ignore_previous_rejects
./velosigmac compile --config ./config/windows_base_events.yaml --output ./output/Windows-Sigma-BaseEvents.zip --yaml ./output/Windows.Sigma.BaseEvents.yaml
./velosigmac compile --config ./config/windows_hayabusa_event_monitoring.yaml --output ./output/Velociraptor-Hayabusa-Monitoring.zip --rejects rejected/windows_hayabusa_rejects.json --ignore_previous_rejects
./velosigmac compile --config ./config/ChopChopGo_rules.yaml --output ./output/Velociraptor-ChopChopGo-Rules.zip --rejects rejected/ChopChopGo_rules_rejects.json --ignore_previous_rejects
cp output/*.zip docs/static/
./velosigmac gen_profiles --output ./output/profiles.json ./config/windows_base.yaml ./config/windows_base_events.yaml
cp output/*.{zip,json} docs/static/
- name: Build
run: cd docs/ && make build_indexes && hugo --minify
Expand Down
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ compileThirdParty: compileHayabusa compileHayabusaMonitoring compileChopChopGo

compileWindowsBase:
./velosigmac compile --config ./config/windows_base.yaml --output ./output/Windows-Sigma-Base.zip --yaml ./output/Windows.Sigma.Base.yaml
./velosigmac compile --config ./config/windows_base_test.yaml --yaml ./output/Windows.Sigma.Base.CaptureTestSet.yaml

compileWindowsBaseEvents:
./velosigmac compile --config ./config/windows_base_events.yaml --output ./output/Windows-Sigma-BaseEvents.zip --yaml ./output/Windows.Sigma.BaseEvents.yaml
Expand Down
18 changes: 12 additions & 6 deletions config/windows_base.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
Name: Windows.Sigma.Base
Description: |
Standard Velociraptor Sigma Log sources for Windows.
This profile includes many Windows EVTX log sources as well as
multiple artifact based log sources.
# The preamble include the artifact parameters, name, description and
# type - basically everything that goes at the top of the artifact
Expand Down Expand Up @@ -73,24 +78,26 @@ Preamble: |
# provide. This help in building dependent artifacts.
ExportTemplate: |
export: |
LET DateAfterTime <= DateAfter || timestamp(epoch="1600-01-01")
LET DateBeforeTime <= DateBefore || timestamp(epoch="2200-01-01")
LET X = scope()
LET ExcludeRegex <= if(condition=RuleExclusions,
LET DateAfterTime <= X.DateAfter || timestamp(epoch="1600-01-01")
LET DateBeforeTime <= X.DateBefore || timestamp(epoch="2200-01-01")
LET ExcludeRegex <= if(condition=X.RuleExclusions,
then=join(array=RuleExclusions.RuleTitleRegex, sep="|"),
else="XXXXXXX")
LET RuleStatusRegex <= get(item=dict(
`Stable`="stable",
`Stable and Experimental`="stable|experimental",
`Stable and Test`="stable|test",
`All Rules`="."), member=RuleStatus)
`All Rules`="."), member=X.RuleStatus || "All Rules")
LET RuleLevelRegex <= get(item=dict(
`Critical`="critical",
`Critical and High`="critical|high",
`Critical, High, and Medium`="critical|high|medium|default",
`All`="."), member=RuleLevel)
`All`="."), member=X.RuleLevel || "All")
LET MandatoryLabelLookup <= dict(
`S-1-16-0`= 'UNTRUSTED',
Expand Down Expand Up @@ -134,7 +141,6 @@ ExportTemplate: |
LET DefaultDetailsLambda = '''{{.Base64DefaultDetailsQuery}}'''
LET RuleFilterLambda = '''x=>x.Level =~ RuleLevelRegex AND x.Status =~ RuleStatusRegex AND x.Title =~ RuleTitleFilter AND NOT x.Title =~ ExcludeRegex'''
LET X = scope()
# FieldMapping are global (i.e. shared between all log sources)
# mappings between a field name and its corresponding VQL getter.
Expand Down
5 changes: 5 additions & 0 deletions config/windows_base_events.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
Name: Windows.Sigma.BaseEvents
Description: |
Standard Velociraptor Sigma Log sources for Windows.
This is a real time monitoring profile which allows live monitoring
of Windows systems using Sigma rules.
Preamble: |
name: Windows.Sigma.BaseEvents
Expand Down
69 changes: 69 additions & 0 deletions config/windows_base_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
Name: Windows.Sigma.Base.CaptureTestSet
ImportConfigs:
- config/windows_base.yaml

Preamble: |
name: Windows.Sigma.Base.CaptureTestSet
description: |
This artifact captures a test set of the log sources defined by
Windows.Sigma.Base. It is used to acquire a dataset for the
`SigmaStudio` notebook.
type: CLIENT
parameters:
- name: ROOT
description: The Event Log Directory we use to read all logs
default: C:/Windows/System32/WinEvt/Logs/
- name: LogSourceFilter
description: Only capture log sources that match this regex.
type: regex
default: .
- name: SelectedLogSources
description: Set to capture only those log sources.
type: multichoice
choices:
{{- range .ImportedLogSources }}
- "{{ .Name }}"
{{- end }}
- name: DateAfter
description: "search for events after this date. YYYY-MM-DDTmm:hh:ss Z"
type: timestamp
- name: DateBefore
description: "search for events before this date. YYYY-MM-DDTmm:hh:ss Z"
type: timestamp
- name: EventRegex
description: Only capture events that match this regex (the event is converted to JSON first).
type: regex
default: .
imports:
- Windows.Sigma.Base
QueryTemplate: |
sources:
- name: MatchingSources
query: |
SELECT _key AS SourceName
FROM items(item=LogSources)
WHERE SourceName =~ LogSourceFilter
AND if(condition=SelectedLogSources, then=SourceName in SelectedLogSources, else=TRUE)
- query: |
SELECT * FROM foreach(row={
SELECT _key AS SourceName, _value AS Query
FROM items(item=LogSources)
WHERE SourceName =~ LogSourceFilter
AND if(condition=SelectedLogSources, then=SourceName in SelectedLogSources, else=TRUE)
}, query={
SELECT * FROM foreach(row={
SELECT * FROM items(item={
SELECT * FROM query(query=Query, copy_env=TRUE)
})
WHERE _value =~ EventRegex
}, column="_value")
})
1 change: 1 addition & 0 deletions config/windows_hayabusa_rules.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ ImportConfigs:

IncludeArtifacts:
- output/Windows.Sigma.Base.yaml
- output/Windows.Sigma.Base.CaptureTestSet.yaml

Preamble: |
name: Windows.Hayabusa.Rules
Expand Down
5 changes: 5 additions & 0 deletions src/artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ func (self *CompilerContext) GetArtifact() (string, error) {
LogSources: BuildLogSource(self.config_obj),
}

for _, imp := range self.imported_configs {
params.ImportedLogSources = append(params.ImportedLogSources,
BuildLogSource(imp)...)
}

// Allow the artifact to export functions to other artifacts.
export_templ, err := calculateTemplate(self.config_obj.ExportTemplate, params)
if err != nil {
Expand Down
45 changes: 23 additions & 22 deletions src/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@ package main

type DefaultDetails struct {
// A lambda that will be used to get the default description
Query string `json:"Query"`
Lookup map[string]string `json:"Lookup"`
Query string `json:"Query,omitempty"`
Lookup map[string]string `json:"Lookup,omitempty"`
}

type Query struct {
Query string `json:"query"`
Channel []string `json:"channel"`
Fields []string `json:"fields"`
Description string `json:"description"`
Name string `json:"name"`
Query string `json:"query,omitempty"`
Channel []string `json:"channel,omitempty"`
Fields []string `json:"fields,omitempty"`
Description string `json:"description,omitempty"`
Name string `json:"name,omitempty"`
}

// Specify source transformations.
Expand All @@ -25,36 +25,37 @@ type SourceRemapping struct {
}

type Config struct {
Name string `json:"Name"`
Preamble string `json:"Preamble"`
FieldMappings map[string]string `json:"FieldMappings"`
DefaultDetails DefaultDetails `json:"DefaultDetails"`
Sources map[string]Query `json:"Sources"`
ExportTemplate string `json:"ExportTemplate"`
QueryTemplate string `json:"QueryTemplate"`
Postscript string `json:"Postscript"`
Name string `json:"Name,omitempty"`
Description string `json:"Description,omitempty"`
Preamble string `json:"Preamble,omitempty"`
FieldMappings map[string]string `json:"FieldMappings,omitempty"`
DefaultDetails DefaultDetails `json:"DefaultDetails,omitempty"`
Sources map[string]Query `json:"Sources,omitempty"`
ExportTemplate string `json:"ExportTemplate,omitempty"`
QueryTemplate string `json:"QueryTemplate,omitempty"`
Postscript string `json:"Postscript,omitempty"`

// If this is set then we generate a reference URL for each rule.
BaseReferenceURL string `json:"BaseReferenceURL"`
RuleDirectories []string `json:"RuleDirectories"`
BaseReferenceURL string `json:"BaseReferenceURL,omitempty"`
RuleDirectories []string `json:"RuleDirectories,omitempty"`

// Many rules are broken and have bad field mappings or log
// sources. The following list suppresses these warnings (but the
// rules are still rejected)
BadFieldMappings []string `json:"BadFieldMappings"`
BadSources []string `json:"BadSources"`
BadFieldMappings []string `json:"BadFieldMappings,omitempty"`
BadSources []string `json:"BadSources,omitempty"`

EventResolver string `json:"EventResolver"`
EventResolver string `json:"EventResolver,omitempty"`

// Include these artifacts into the zip bundle as well. There are
// relative paths to the included files. These are usually used to
// include dependent artifacts.
IncludeArtifacts []string `json:"IncludeArtifacts"`
IncludeArtifacts []string `json:"IncludeArtifacts,omitempty"`

// Read these configs also. Many attributes are merged with
// included configs (for example FieldMappings, and Sources). This
// allows to build derived artifacts based on other artifacts.
ImportConfigs []string `json:"ImportConfigs"`
ImportConfigs []string `json:"ImportConfigs,omitempty"`

// Merged results from imported configs
sources map[string]Query
Expand Down
69 changes: 69 additions & 0 deletions src/profiles.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package main

import (
"encoding/json"
"io/ioutil"
"os"

"github.com/Velocidex/yaml/v2"
kingpin "github.com/alecthomas/kingpin/v2"
)

var (
profile_cmd = app.Command("gen_profiles", "Generate profile JSON")
profile_cmd_output = profile_cmd.Flag("output", "File to write the profile").Required().String()
profile_cmd_args = profile_cmd.Arg("configs", "Config files to read").Required().Strings()
)

func doProfile() error {
profiles := make(map[string]*Config)

for _, config_file := range *profile_cmd_args {
fd, err := os.Open(config_file)
if err != nil {
return err
}

data, err := ioutil.ReadAll(fd)
if err != nil {
return err
}

conf := &Config{}
err = yaml.Unmarshal(data, conf)
if err != nil {
return err
}

profiles[conf.Name] = conf
}

serialized, err := json.Marshal(profiles)
if err != nil {
return err
}

outfd, err := os.OpenFile(*profile_cmd_output,
os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer outfd.Close()

_, err = outfd.Write(serialized)
return err
}

func init() {
command_handlers = append(command_handlers, func(command string) bool {
switch command {
case profile_cmd.FullCommand():
err := doProfile()
kingpin.FatalIfError(err, "Compiling profiles")

default:
return false
}
return true
})
}
5 changes: 5 additions & 0 deletions src/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ func BuildLogSource(config_obj *Config) []Query {
return nil
}

if config_obj.sources == nil {
config_obj.mergeConfig(config_obj)
}

for k, v := range config_obj.sources {
query := strings.TrimSpace(v.Query)
if len(query) > 0 {
Expand All @@ -41,6 +45,7 @@ type ArtifactContent struct {
Base64DefaultDetailsLookup string
Base64DefaultDetailsQuery string
LogSources []Query
ImportedLogSources []Query
}

func indentTemplate(args ...interface{}) interface{} {
Expand Down

0 comments on commit 3481f21

Please sign in to comment.