diff --git a/README.md b/README.md index 6ad217f..e9b55f4 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Yapscan is a **YA**ra based **P**rocess **SCAN**ner, aimed at giving more control about what to scan and giving detailed reports on matches. +**The report format is now versioned and a stable version 1.0.0 is released with compatibility guarantees, see the [report format documentation](report/v1.0.0/README.md).** + ## Features You can use yapscan to selectively scan the memory of running processes as well as files in local hard drives and/or mounted shares. diff --git a/acceptanceTests/reports_test.go b/acceptanceTests/reports_test.go index 2760c82..5071266 100644 --- a/acceptanceTests/reports_test.go +++ b/acceptanceTests/reports_test.go @@ -4,7 +4,6 @@ import ( "archive/tar" "bytes" "context" - "encoding/json" "fmt" "io" "io/ioutil" @@ -15,13 +14,15 @@ import ( "testing" "testing/quick" + "github.com/fkie-cad/yapscan/testutil" + + "github.com/fkie-cad/yapscan/report" + "github.com/fkie-cad/yapscan/procio" "github.com/fkie-cad/yapscan/system" "golang.org/x/crypto/openpgp" - "github.com/fkie-cad/yapscan/output" - "github.com/klauspost/compress/zstd" "github.com/fkie-cad/yapscan/app" @@ -169,7 +170,7 @@ func TestFullReportIsWritten_Unencrypted(t *testing.T) { cleanupCapture() conveyMatchWasSuccessful(c, addressOfData, err, stdout, stderr) - conveyReportIsReadable(c, openReportCleartext(), pid, addressOfData, reportDir) + conveyReportIsValidAndHasMatch(c, openReportCleartext(), pid, addressOfData, reportDir) }) } @@ -195,8 +196,8 @@ func TestFullReportIsWritten_Unencrypted_WhenScanningTwoProcesses(t *testing.T) cleanupCapture() conveyMatchWasSuccessful(c, addressOfMatchingData, err, stdout, stderr) - conveyReportIsReadable(c, openReportCleartext(), matchingPID, addressOfMatchingData, reportDir) - conveyReportIsReadableButDoesNotHaveMatch(c, openReportCleartext(), nonMatchingPID, addressOfNonMatchingData, reportDir) + conveyReportIsValidAndHasMatch(c, openReportCleartext(), matchingPID, addressOfMatchingData, reportDir) + conveyReportIsValidButDoesNotHaveMatch(c, openReportCleartext(), nonMatchingPID, addressOfNonMatchingData, reportDir) }) Convey("Scanning two prepared processes (first benign, then matching) with full-report", t, func(c C) { @@ -220,8 +221,8 @@ func TestFullReportIsWritten_Unencrypted_WhenScanningTwoProcesses(t *testing.T) cleanupCapture() conveyMatchWasSuccessful(c, addressOfMatchingData, err, stdout, stderr) - conveyReportIsReadable(c, openReportCleartext(), matchingPID, addressOfMatchingData, reportDir) - conveyReportIsReadableButDoesNotHaveMatch(c, openReportCleartext(), nonMatchingPID, addressOfNonMatchingData, reportDir) + conveyReportIsValidAndHasMatch(c, openReportCleartext(), matchingPID, addressOfMatchingData, reportDir) + conveyReportIsValidButDoesNotHaveMatch(c, openReportCleartext(), nonMatchingPID, addressOfNonMatchingData, reportDir) }) } @@ -250,7 +251,7 @@ func TestPasswordProtectedFullReport(t *testing.T) { conveyMatchWasSuccessful(c, addressOfData, err, stdout, stderr) conveyReportIsNotReadable(c, openReportCleartext(), reportDir) - conveyReportIsReadable(c, openReportWithPassword(password), pid, addressOfData, reportDir) + conveyReportIsValidAndHasMatch(c, openReportWithPassword(password), pid, addressOfData, reportDir) }) } @@ -279,7 +280,7 @@ func TestPGPProtectedFullReport(t *testing.T) { conveyMatchWasSuccessful(c, addressOfData, err, stdout, stderr) conveyReportIsNotReadable(c, openReportCleartext(), reportDir) - conveyReportIsReadable(c, openReportPGP(keyring), pid, addressOfData, reportDir) + conveyReportIsValidAndHasMatch(c, openReportPGP(keyring), pid, addressOfData, reportDir) }) } @@ -334,56 +335,31 @@ func (r *readerWithCloser) Close() error { return r.closer.Close() } -type reportOpenFunc func(reportPath string) (io.ReadCloser, error) +type reportOpenFunc func(reportPath string) (report.Reader, error) func openReportCleartext() reportOpenFunc { - return func(reportPath string) (io.ReadCloser, error) { - return os.Open(reportPath) + return func(reportPath string) (report.Reader, error) { + return report.NewFileReader(reportPath), nil } } func openReportWithPassword(password string) reportOpenFunc { - return func(reportPath string) (io.ReadCloser, error) { - f, err := os.Open(reportPath) - if err != nil { - return nil, err - } - - prompt := func(keys []openpgp.Key, symmetric bool) ([]byte, error) { - return []byte(password), nil - } - msg, err := openpgp.ReadMessage(f, nil, prompt, nil) - if err != nil { - f.Close() - return nil, err - } - return &readerWithCloser{ - rdr: msg.UnverifiedBody, - closer: f, - }, nil + return func(reportPath string) (report.Reader, error) { + rdr := report.NewFileReader(reportPath) + rdr.SetPassword(password) + return rdr, nil } } func openReportPGP(keyring openpgp.EntityList) reportOpenFunc { - return func(reportPath string) (io.ReadCloser, error) { - f, err := os.Open(reportPath) - if err != nil { - return nil, err - } - - msg, err := openpgp.ReadMessage(f, keyring, nil, nil) - if err != nil { - f.Close() - return nil, err - } - return &readerWithCloser{ - rdr: msg.UnverifiedBody, - closer: f, - }, nil + return func(reportPath string) (report.Reader, error) { + rdr := report.NewFileReader(reportPath) + rdr.SetKeyring(keyring) + return rdr, nil } } -func conveyReportIsReadable(c C, openReport reportOpenFunc, pid int, addressOfData uintptr, reportDir string) { +func conveyReportIsValidAndHasMatch(c C, openReport reportOpenFunc, pid int, addressOfData uintptr, reportDir string) { c.Convey("should yield a valid report", func(c C) { reportPath, exists := findReportPath(reportDir) @@ -392,36 +368,22 @@ func conveyReportIsReadable(c C, openReport reportOpenFunc, pid int, addressOfDa return } - report, err := openReport(reportPath) + reportRdr, err := openReport(reportPath) So(err, ShouldBeNil) - defer report.Close() + defer reportRdr.Close() - reportFiles, err := readReport(report) + projectRoot, err := testutil.GetProjectRoot() + So(err, ShouldBeNil) - c.So(reportFiles, ShouldNotBeEmpty) - c.So(err, ShouldBeNil) + validator := report.NewOfflineValidator(projectRoot + "/report") + err = validator.ValidateReport(reportRdr) + So(err, ShouldBeNil) - var memoryScansJSON *file - filenames := make([]string, len(reportFiles)) - for i, file := range reportFiles { - filenames[i] = file.Name - if file.Name == "memory-scans.json" { - memoryScansJSON = file - } - } - c.Convey("which contains the expected files", func(c C) { - c.So(filenames, ShouldContain, "systeminfo.json") - c.So(filenames, ShouldContain, "processes.json") - c.So(filenames, ShouldContain, "memory-scans.json") - c.So(filenames, ShouldContain, "stats.json") - c.So(memoryScansJSON, ShouldNotBeNil) - - conveyReportHasMatch(c, pid, addressOfData, memoryScansJSON) - }) + conveyReportHasMatch(c, pid, addressOfData, reportRdr) }) } -func conveyReportIsReadableButDoesNotHaveMatch(c C, openReport reportOpenFunc, pid int, addressOfData uintptr, reportDir string) { +func conveyReportIsValidButDoesNotHaveMatch(c C, openReport reportOpenFunc, pid int, addressOfData uintptr, reportDir string) { c.Convey("should yield a readable report", func(c C) { reportPath, exists := findReportPath(reportDir) @@ -430,32 +392,18 @@ func conveyReportIsReadableButDoesNotHaveMatch(c C, openReport reportOpenFunc, p return } - report, err := openReport(reportPath) + reportRdr, err := openReport(reportPath) So(err, ShouldBeNil) - defer report.Close() + defer reportRdr.Close() - reportFiles, err := readReport(report) + projectRoot, err := testutil.GetProjectRoot() + So(err, ShouldBeNil) - c.So(reportFiles, ShouldNotBeEmpty) - c.So(err, ShouldBeNil) + validator := report.NewOfflineValidator(projectRoot + "/report") + err = validator.ValidateReport(reportRdr) + So(err, ShouldBeNil) - var memoryScansJSON *file - filenames := make([]string, len(reportFiles)) - for i, file := range reportFiles { - filenames[i] = file.Name - if file.Name == "memory-scans.json" { - memoryScansJSON = file - } - } - c.Convey("which contains the expected files", func(c C) { - c.So(filenames, ShouldContain, "systeminfo.json") - c.So(filenames, ShouldContain, "processes.json") - c.So(filenames, ShouldContain, "memory-scans.json") - c.So(filenames, ShouldContain, "stats.json") - c.So(memoryScansJSON, ShouldNotBeNil) - - conveyReportDoesNotHaveMatch(c, pid, addressOfData, memoryScansJSON) - }) + conveyReportDoesNotHaveMatch(c, pid, addressOfData, reportRdr) }) } @@ -468,14 +416,16 @@ func conveyReportIsAnonymized(c C, openReport reportOpenFunc, reportDir string) return } - report, err := openReport(reportPath) + reportRdr, err := openReport(reportPath) So(err, ShouldBeNil) - defer report.Close() + defer reportRdr.Close() - reportFiles, err := readReport(report) + projectRoot, err := testutil.GetProjectRoot() + So(err, ShouldBeNil) - c.So(reportFiles, ShouldNotBeEmpty) - c.So(err, ShouldBeNil) + validator := report.NewOfflineValidator(projectRoot + "/report") + err = validator.ValidateReport(reportRdr) + So(err, ShouldBeNil) c.Convey("which does not contain the hostname, username or any IPs.", func(c C) { info, err := system.GetInfo() @@ -487,15 +437,45 @@ func conveyReportIsAnonymized(c C, openReport reportOpenFunc, reportDir string) selfInfo, err := self.Info() So(err, ShouldBeNil) - allJSONBuilder := &strings.Builder{} - for _, file := range reportFiles { - if strings.Contains(file.Name, ".json") { - allJSONBuilder.Write(file.Data) - } - } - allJSON := allJSONBuilder.String() + buffer := &bytes.Buffer{} - fmt.Println(info.Hostname) + r, err := reportRdr.OpenMeta() + So(err, ShouldBeNil) + _, err = io.Copy(buffer, r) + So(err, ShouldBeNil) + r.Close() + + r, err = reportRdr.OpenStatistics() + So(err, ShouldBeNil) + _, err = io.Copy(buffer, r) + So(err, ShouldBeNil) + r.Close() + + r, err = reportRdr.OpenSystemInformation() + So(err, ShouldBeNil) + _, err = io.Copy(buffer, r) + So(err, ShouldBeNil) + r.Close() + + r, err = reportRdr.OpenProcesses() + So(err, ShouldBeNil) + _, err = io.Copy(buffer, r) + So(err, ShouldBeNil) + r.Close() + + r, err = reportRdr.OpenMemoryScans() + So(err, ShouldBeNil) + _, err = io.Copy(buffer, r) + So(err, ShouldBeNil) + r.Close() + + r, err = reportRdr.OpenFileScans() + So(err, ShouldBeNil) + _, err = io.Copy(buffer, r) + So(err, ShouldBeNil) + r.Close() + + allJSON := buffer.String() So(allJSON, ShouldNotBeEmpty) So(allJSON, ShouldNotContainSubstring, info.Hostname) @@ -516,60 +496,62 @@ func conveyReportIsNotReadable(c C, openReport reportOpenFunc, reportDir string) return } - report, err := openReport(reportPath) + reportRdr, err := openReport(reportPath) if err != nil { So(err, ShouldNotBeNil) return } - defer report.Close() - - _, err = readReport(report) - c.So(err, ShouldNotBeNil) + defer reportRdr.Close() + + _, errMeta := reportRdr.OpenMeta() + _, errStatistics := reportRdr.OpenStatistics() + _, errSystemInformation := reportRdr.OpenSystemInformation() + _, errProcesses := reportRdr.OpenProcesses() + _, errMemoryScans := reportRdr.OpenMemoryScans() + _, errFileScans := reportRdr.OpenFileScans() + So(errMeta, ShouldNotBeNil) + So(errStatistics, ShouldNotBeNil) + So(errSystemInformation, ShouldNotBeNil) + So(errProcesses, ShouldNotBeNil) + So(errMemoryScans, ShouldNotBeNil) + So(errFileScans, ShouldNotBeNil) }) } -func conveyReportHasMatch(c C, pid int, addressOfData uintptr, memoryScansJSON *file) { +func conveyReportHasMatch(c C, pid int, addressOfData uintptr, reportRdr report.Reader) { c.Convey("with the memory-scans.json containing the correct match.", func() { - dec := json.NewDecoder(bytes.NewReader(memoryScansJSON.Data)) - foundCorrectMatch := false - var err error - for { - report := new(output.MemoryScanProgressReport) - err = dec.Decode(report) - if err != nil { - break - } + parser := report.NewParser() + rprt, err := parser.Parse(reportRdr) + So(err, ShouldBeNil) - if report.PID == pid && report.MemorySegment == addressOfData && len(report.Matches) > 0 { + foundCorrectMatch := false + for _, scan := range rprt.MemoryScans { + if scan.PID == pid && scan.MemorySegment == addressOfData && len(scan.Matches) > 0 { foundCorrectMatch = true + break } } - c.So(err, ShouldResemble, io.EOF) c.So(foundCorrectMatch, ShouldBeTrue) }) } -func conveyReportDoesNotHaveMatch(c C, pid int, addressOfData uintptr, memoryScansJSON *file) { +func conveyReportDoesNotHaveMatch(c C, pid int, addressOfData uintptr, reportRdr report.Reader) { c.Convey("with the memory-scans.json not containing a false positive.", func() { - dec := json.NewDecoder(bytes.NewReader(memoryScansJSON.Data)) + parser := report.NewParser() + rprt, err := parser.Parse(reportRdr) + So(err, ShouldBeNil) + foundMatchForPID := false foundMatchForAddressInPID := false - var err error - for { - report := new(output.MemoryScanProgressReport) - err = dec.Decode(report) - if err != nil { - break - } - - if report.PID == pid && len(report.Matches) > 0 { + for _, scan := range rprt.MemoryScans { + if scan.PID == pid && len(scan.Matches) > 0 { foundMatchForPID = true - if report.MemorySegment == addressOfData { + if scan.MemorySegment == addressOfData { foundMatchForAddressInPID = true + break } } } - c.So(err, ShouldResemble, io.EOF) c.So(foundMatchForPID, ShouldBeFalse) c.So(foundMatchForAddressInPID, ShouldBeFalse) }) diff --git a/app/app.go b/app/app.go index 82dab4e..cb573da 100644 --- a/app/app.go +++ b/app/app.go @@ -6,6 +6,8 @@ import ( "runtime" "strings" + "github.com/fkie-cad/yapscan/version" + "github.com/fkie-cad/yapscan" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -207,7 +209,7 @@ func MakeApp(args []string) *cli.App { Name: "yapscan", HelpName: "yapscan", Description: "A yara based scanner for files and process memory with some extras.", - Version: "0.12.0", + Version: version.YapscanVersion.String(), Writer: os.Stdout, ErrWriter: os.Stderr, Authors: []*cli.Author{ diff --git a/app/filter.go b/app/filter.go index 304e464..032eadd 100644 --- a/app/filter.go +++ b/app/filter.go @@ -60,7 +60,7 @@ func BuildFilterType(fStr []string) (yapscan.MemorySegmentFilter, error) { if s == "" { continue } - types[i], err = procio.ParseSegmentType(strings.ToUpper(s[0:1]) + strings.ToLower(s[1:])) + types[i], err = procio.ParseSegmentType(s) if err != nil { return nil, fmt.Errorf("could not parse type \"%s\", reason: %w", s, err) } @@ -81,7 +81,7 @@ func BuildFilterState(fStr []string) (yapscan.MemorySegmentFilter, error) { if s == "" { continue } - states[i], err = procio.ParseState(strings.ToUpper(s[0:1]) + strings.ToLower(s[1:])) + states[i], err = procio.ParseState(s) if err != nil { return nil, fmt.Errorf("could not parse state \"%s\", reason: %w", s, err) } diff --git a/app/scan.go b/app/scan.go index d4dc39e..e28c39b 100644 --- a/app/scan.go +++ b/app/scan.go @@ -12,6 +12,8 @@ import ( "strconv" "time" + "github.com/fatih/color" + "github.com/fkie-cad/yapscan/system" "github.com/fkie-cad/yapscan" @@ -218,11 +220,17 @@ func scan(c *cli.Context) error { fmt.Printf("Dumps will be written to \"%s\".\n", dumpArchivePath) } + + tmpReporter, err := repFac.Build() + if err != nil { + return err + } + reporter = &output.MultiReporter{ Reporters: []output.Reporter{ reporter, &output.FilteringReporter{ - Reporter: repFac.Build(), + Reporter: tmpReporter, Filter: analysisFilter, }, }, @@ -270,12 +278,14 @@ func scan(c *cli.Context) error { for _, pid := range pids { func() { if pid == os.Getpid() { + color.Yellow("\nWARN: PID %d is the yapscan process, skipping!", pid) // Don't scan yourself as that will cause unwanted matches. return } proc, err := procio.OpenProcess(pid) if err != nil { + color.Red("\nERROR: Could not open process %d for scanning, reason: %v!", pid, err) logrus.WithError(err).Errorf("could not open process %d for scanning", pid) return } diff --git a/arch/bitness.go b/arch/bitness.go index 50880e5..0cb4666 100644 --- a/arch/bitness.go +++ b/arch/bitness.go @@ -4,7 +4,7 @@ package arch // Bitness describes the bitness of an architecture. /* ENUM( -Invalid +invalid 32Bit = 32 64Bit = 64 ) diff --git a/arch/bitness_enum.go b/arch/bitness_enum.go index 1cd7355..7d7a2bb 100644 --- a/arch/bitness_enum.go +++ b/arch/bitness_enum.go @@ -20,7 +20,7 @@ const ( Bitness64Bit Bitness = iota + 62 ) -const _BitnessName = "Invalid32Bit64Bit" +const _BitnessName = "invalid32Bit64Bit" var _BitnessNames = []string{ _BitnessName[0:7], diff --git a/fileio/file.go b/fileio/file.go index 0f0cc35..fa58c2a 100644 --- a/fileio/file.go +++ b/fileio/file.go @@ -14,8 +14,8 @@ type File interface { type OSFile struct { FilePath string `json:"path"` - MD5Sum string `json:"MD5,omitempty"` - SHA256Sum string `json:"SHA256,omitempty"` + MD5Sum string `json:"md5,omitempty"` + SHA256Sum string `json:"sha256,omitempty"` } func NewFile(path string) File { diff --git a/generate.sh b/generate.sh index 0040b3d..d9d10f3 100755 --- a/generate.sh +++ b/generate.sh @@ -2,19 +2,11 @@ cd $(dirname "$0") || exit 1 -update="" if [[ "$1" == "-u" ]]; then - update="-u" + go install github.com/abice/go-enum@latest + go install github.com/vektra/mockery/v2@latest fi -go mod tidy -go mod vendor - -go get -v $update github.com/abice/go-enum -go get -v $update github.com/vektra/mockery/v2/.../ -go mod tidy - find . -name 'mock_*_test.go' -type f -delete go generate ./... - diff --git a/go.mod b/go.mod index 0d43bfa..1588c1d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,9 @@ require ( github.com/hillu/go-yara/v4 v4.1.0 github.com/klauspost/compress v1.13.6 github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-colorable v0.1.11 // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect + github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 github.com/sirupsen/logrus v1.8.1 github.com/smartystreets/assertions v1.2.0 // indirect github.com/smartystreets/goconvey v1.6.4 @@ -20,8 +22,8 @@ require ( github.com/targodan/go-errors v1.0.0 github.com/urfave/cli/v2 v2.3.0 github.com/yeka/zip v0.0.0-20180914125537-d046722c6feb - golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8 - golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c + golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index e5682d6..8b4cdce 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= @@ -172,6 +172,10 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= +github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= +github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE= +github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= @@ -225,8 +229,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8 h1:5QRxNnVsaJP6NAse0UdkRgL3zHMvCRRkrDVLNdNpdy4= -golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU= +golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -292,11 +296,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c h1:DHcbWVXeY+0Y8HHKR+rbLwnoh2F4tNCY7rTiHJ30RmA= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/mock_MemoryScanner_test.go b/mock_MemoryScanner_test.go index e34d4e8..1eb4ba5 100644 --- a/mock_MemoryScanner_test.go +++ b/mock_MemoryScanner_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_MemorySegmentFilterFunc_test.go b/mock_MemorySegmentFilterFunc_test.go index 2d730cb..30a67d9 100644 --- a/mock_MemorySegmentFilterFunc_test.go +++ b/mock_MemorySegmentFilterFunc_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_MemorySegmentFilter_test.go b/mock_MemorySegmentFilter_test.go index 2df0b4b..ae2aeda 100644 --- a/mock_MemorySegmentFilter_test.go +++ b/mock_MemorySegmentFilter_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_Rules_test.go b/mock_Rules_test.go index a9a6965..20a910a 100644 --- a/mock_Rules_test.go +++ b/mock_Rules_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_memoryReaderFactory_test.go b/mock_memoryReaderFactory_test.go index 9713969..d7ec90c 100644 --- a/mock_memoryReaderFactory_test.go +++ b/mock_memoryReaderFactory_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_memoryReader_test.go b/mock_memoryReader_test.go index 0c65706..0d51c3a 100644 --- a/mock_memoryReader_test.go +++ b/mock_memoryReader_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_process_test.go b/mock_process_test.go index b544114..ee27a9d 100644 --- a/mock_process_test.go +++ b/mock_process_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/mock_segmentScanner_test.go b/mock_segmentScanner_test.go index 66963fd..076513a 100644 --- a/mock_segmentScanner_test.go +++ b/mock_segmentScanner_test.go @@ -1,4 +1,4 @@ -// Code generated by mockery v0.0.0-dev. DO NOT EDIT. +// Code generated by mockery v2.9.4. DO NOT EDIT. package yapscan diff --git a/output/analysisReporter.go b/output/analysisReporter.go index 49a8b9c..b9534bf 100644 --- a/output/analysisReporter.go +++ b/output/analysisReporter.go @@ -8,29 +8,19 @@ import ( "github.com/fkie-cad/yapscan" "github.com/fkie-cad/yapscan/fileio" "github.com/fkie-cad/yapscan/procio" + "github.com/fkie-cad/yapscan/report" "github.com/fkie-cad/yapscan/system" "github.com/hillu/go-yara/v4" "github.com/sirupsen/logrus" "github.com/targodan/go-errors" ) -// SystemInfoFileName is the name of the file, where system info is stored. -const SystemInfoFileName = "systeminfo.json" - -// RulesFileName is the name of the file, where the used rules will be stored. -const RulesFileName = "rules.yarc" - -// ProcessFileName is the name of the file used to report information about processes. -const ProcessFileName = "processes.json" - -// MemoryProgressFileName is the name of the file used to report information about memory scans. -const MemoryProgressFileName = "memory-scans.json" - -// FSProgressFileName is the name of the file used to report information about file scans. -const FSProgressFileName = "file-scans.json" - -// ScanningStatisticsFileName is the name of the file used to report scanning. -const ScanningStatisticsFileName = "stats.json" +// FileScan represents all matches on a file. +type FileScan struct { + File fileio.File `json:"file"` + Matches []*report.Match `json:"match"` + Error interface{} `json:"error"` +} // AnalysisReporter implements a Reporter, which is // specifically intended for later analysis of the report @@ -45,11 +35,25 @@ type AnalysisReporter struct { processInfos map[int]*procio.ProcessInfo } +func (r *AnalysisReporter) reportMeta() error { + w, err := r.archiver.Create(r.filenamePrefix + report.MetaFileName) + if err != nil { + return err + } + + err = json.NewEncoder(w).Encode(report.GetMetaInformation()) + if err != nil { + return errors.NewMultiError(err, w.Close()) + } + + return w.Close() +} + // ReportSystemInfo reports info about the running system. // This function may only called once, otherwise the behaviour depends on the // used Archiver. func (r *AnalysisReporter) ReportSystemInfo(info *system.Info) error { - w, err := r.archiver.Create(r.filenamePrefix + SystemInfoFileName) + w, err := r.archiver.Create(r.filenamePrefix + report.SystemInfoFileName) if err != nil { return err } @@ -66,7 +70,7 @@ func (r *AnalysisReporter) ReportSystemInfo(info *system.Info) error { // This function may only called once, otherwise the behaviour depends on the // used Archiver. func (r *AnalysisReporter) ReportScanningStatistics(stats *yapscan.ScanningStatistics) error { - w, err := r.archiver.Create(r.filenamePrefix + ScanningStatisticsFileName) + w, err := r.archiver.Create(r.filenamePrefix + report.ScanningStatisticsFileName) if err != nil { return err } @@ -83,7 +87,7 @@ func (r *AnalysisReporter) ReportScanningStatistics(stats *yapscan.ScanningStati // This function may only called once, otherwise the behaviour depends on the // used Archiver. func (r *AnalysisReporter) ReportRules(rules *yara.Rules) error { - w, err := r.archiver.Create(r.filenamePrefix + RulesFileName) + w, err := r.archiver.Create(r.filenamePrefix + report.RulesFileName) if err != nil { return err } @@ -96,8 +100,20 @@ func (r *AnalysisReporter) ReportRules(rules *yara.Rules) error { return w.Close() } +func (r *AnalysisReporter) flattenSubsegments(segments []*procio.MemorySegmentInfo) []*procio.MemorySegmentInfo { + newSegments := make([]*procio.MemorySegmentInfo, 0, len(segments)) + for _, seg := range segments { + newSegments = append(newSegments, seg) + if len(seg.SubSegments) > 0 { + subSegments := r.flattenSubsegments(seg.SubSegments) + newSegments = append(newSegments, subSegments...) + } + } + return newSegments +} + func (r *AnalysisReporter) reportProcessInfos() error { - w, err := r.archiver.Create(r.filenamePrefix + ProcessFileName) + w, err := r.archiver.Create(r.filenamePrefix + report.ProcessesFileName) if err != nil { return err } @@ -109,6 +125,8 @@ func (r *AnalysisReporter) reportProcessInfos() error { encoder := json.NewEncoder(w) for _, info := range r.processInfos { + info.MemorySegments = r.flattenSubsegments(info.MemorySegments) + err = encoder.Encode(info) if err != nil { logrus.WithError(err).Error("Could not report process info.") @@ -123,7 +141,7 @@ func (r *AnalysisReporter) reportProcessInfos() error { // This function may only called once, otherwise the behaviour depends on the // used Archiver. func (r *AnalysisReporter) ConsumeMemoryScanProgress(progress <-chan *yapscan.MemoryScanProgress) error { - w, err := r.archiver.Create(r.filenamePrefix + MemoryProgressFileName) + w, err := r.archiver.Create(r.filenamePrefix + report.MemoryScansFileName) if err != nil { return err } @@ -159,7 +177,7 @@ func (r *AnalysisReporter) ConsumeMemoryScanProgress(progress <-chan *yapscan.Me if prog.Error != nil { jsonErr = prog.Error.Error() } - err = encoder.Encode(&MemoryScanProgressReport{ + err = encoder.Encode(&report.MemoryScan{ PID: info.PID, MemorySegment: prog.MemorySegment.BaseAddress, Matches: ConvertYaraMatchRules(prog.Matches), @@ -187,7 +205,7 @@ func (r *AnalysisReporter) ConsumeMemoryScanProgress(progress <-chan *yapscan.Me // This function may only called once, otherwise the behaviour depends on the // used Archiver. func (r *AnalysisReporter) ConsumeFSScanProgress(progress <-chan *fileio.FSScanProgress) error { - w, err := r.archiver.Create(r.filenamePrefix + FSProgressFileName) + w, err := r.archiver.Create(r.filenamePrefix + report.FileScansFileName) if err != nil { return err } @@ -209,7 +227,7 @@ func (r *AnalysisReporter) ConsumeFSScanProgress(progress <-chan *fileio.FSScanP } } - err = encoder.Encode(&FSScanProgressReport{ + err = encoder.Encode(&FileScan{ File: prog.File, Matches: ConvertYaraMatchRules(prog.Matches), Error: jsonErr, diff --git a/output/factory.go b/output/factory.go index 904a1b0..c64b0c7 100644 --- a/output/factory.go +++ b/output/factory.go @@ -1,5 +1,7 @@ package output +import "github.com/targodan/go-errors" + type AnalysisReporterFactory struct { reporter *AnalysisReporter } @@ -27,6 +29,10 @@ func (f *AnalysisReporterFactory) WithFilenamePrefix(prefix string) *AnalysisRep return f } -func (f *AnalysisReporterFactory) Build() *AnalysisReporter { - return f.reporter +func (f *AnalysisReporterFactory) Build() (*AnalysisReporter, error) { + err := f.reporter.reportMeta() + if err != nil { + return nil, errors.NewMultiError(err, f.reporter.Close()) + } + return f.reporter, nil } diff --git a/output/filtering.go b/output/filtering.go index 9647a62..df7d02a 100644 --- a/output/filtering.go +++ b/output/filtering.go @@ -348,8 +348,8 @@ func (f *AnonymizingFilter) FilterFSScanProgress(scan *fileio.FSScanProgress) *f type AnonymizedFile struct { FilePath string `json:"path"` - MD5Sum string `json:"MD5,omitempty"` - SHA256Sum string `json:"SHA256,omitempty"` + MD5Sum string `json:"md5,omitempty"` + SHA256Sum string `json:"sha256,omitempty"` origFile fileio.File } diff --git a/output/output.go b/output/output.go index ef743fb..483b0c2 100644 --- a/output/output.go +++ b/output/output.go @@ -3,6 +3,8 @@ package output import ( "io" + "github.com/fkie-cad/yapscan/report" + "github.com/fkie-cad/yapscan" "github.com/fkie-cad/yapscan/fileio" "github.com/fkie-cad/yapscan/system" @@ -19,32 +21,18 @@ type Reporter interface { io.Closer } -// Match represents the match of a yara Rule. -type Match struct { - Rule string `json:"rule"` - Namespace string `json:"namespace"` - Strings []*MatchString `json:"strings"` -} - -// A MatchString represents a string declared and matched in a rule. -type MatchString struct { - Name string `json:"name"` - Base uint64 `json:"base"` - Offset uint64 `json:"offset"` -} - // ConvertYaraMatchRules converts the given slice of yara.MatchRule to // a slice of *Match. -func ConvertYaraMatchRules(mr []yara.MatchRule) []*Match { - ret := make([]*Match, len(mr)) +func ConvertYaraMatchRules(mr []yara.MatchRule) []*report.Match { + ret := make([]*report.Match, len(mr)) for i, match := range mr { - ret[i] = &Match{ + ret[i] = &report.Match{ Rule: match.Rule, Namespace: match.Namespace, - Strings: make([]*MatchString, len(match.Strings)), + Strings: make([]*report.MatchString, len(match.Strings)), } for j, s := range match.Strings { - ret[i].Strings[j] = &MatchString{ + ret[i].Strings[j] = &report.MatchString{ Name: s.Name, Base: s.Base, Offset: s.Offset, @@ -53,19 +41,3 @@ func ConvertYaraMatchRules(mr []yara.MatchRule) []*Match { } return ret } - -// MemoryScanProgressReport represents all matches on a single memory -// segment of a process. -type MemoryScanProgressReport struct { - PID int `json:"pid"` - MemorySegment uintptr `json:"memorySegment"` - Matches []*Match `json:"match"` - Error interface{} `json:"error"` -} - -// FSScanProgressReport represents all matches on a file. -type FSScanProgressReport struct { - File fileio.File `json:"file"` - Matches []*Match `json:"match"` - Error interface{} `json:"error"` -} diff --git a/procio/crash.go b/procio/crash.go index a682202..09a2aa5 100644 --- a/procio/crash.go +++ b/procio/crash.go @@ -4,7 +4,7 @@ package procio // CrashMethod selects a method to crash a process. /* ENUM( -CreateThreadOnNull +createThreadOnNull ) */ type CrashMethod int diff --git a/procio/crash_enum.go b/procio/crash_enum.go index 88a64ba..8a43c15 100644 --- a/procio/crash_enum.go +++ b/procio/crash_enum.go @@ -16,7 +16,7 @@ const ( CrashMethodCreateThreadOnNull CrashMethod = iota ) -const _CrashMethodName = "CreateThreadOnNull" +const _CrashMethodName = "createThreadOnNull" var _CrashMethodNames = []string{ _CrashMethodName[0:18], diff --git a/procio/memory.go b/procio/memory.go index 3072345..5689041 100644 --- a/procio/memory.go +++ b/procio/memory.go @@ -54,7 +54,7 @@ type MemorySegmentInfo struct { // SubSegments contains sub-segments, i.e. segment where their ParentBaseAddress // is equal to this segments BaseAddress. // If no such segments exist, this will be a slice of length 0. - SubSegments []*MemorySegmentInfo `json:"subSegments"` + SubSegments []*MemorySegmentInfo `json:"-"` } // EstimateRAMIncreaseByScanning estimates the increase in RAM usage when @@ -90,13 +90,13 @@ func (s *MemorySegmentInfo) CopyWithoutSubSegments() *MemorySegmentInfo { // Permissions describes the permissions of a memory segment. type Permissions struct { // Is read-only access allowed - Read bool `yaml:"read"` + Read bool `json:"read"` // Is write access allowed (also true if COW is enabled) - Write bool `yaml:"write"` + Write bool `json:"write"` // Is copy-on-write access allowed (if this is true, then so is Write) - COW bool `yaml:"cow"` + COW bool `json:"COW"` // Is execute access allowed - Execute bool `yaml:"execute"` + Execute bool `json:"execute"` } // PermR is readonly Permissions. @@ -223,9 +223,9 @@ func (p Permissions) String() string { // State represents the state of a memory segment. /* ENUM( -Commit -Free -Reserve +commit +free +reserve ) */ type State int @@ -233,10 +233,10 @@ type State int // SegmentType represents the type of a memory segment. /* ENUM( -Image -Mapped -Private -PrivateMapped +image +mapped +private +privateMapped ) */ type SegmentType int diff --git a/procio/memory_enum.go b/procio/memory_enum.go index a0a2b0e..aedceef 100644 --- a/procio/memory_enum.go +++ b/procio/memory_enum.go @@ -22,7 +22,7 @@ const ( SegmentTypePrivateMapped ) -const _SegmentTypeName = "ImageMappedPrivatePrivateMapped" +const _SegmentTypeName = "imagemappedprivateprivateMapped" var _SegmentTypeNames = []string{ _SegmentTypeName[0:5], @@ -97,7 +97,7 @@ const ( StateReserve ) -const _StateName = "CommitFreeReserve" +const _StateName = "commitfreereserve" var _StateNames = []string{ _StateName[0:6], diff --git a/procio/process_linux.go b/procio/process_linux.go index 42c2dd2..680ca28 100644 --- a/procio/process_linux.go +++ b/procio/process_linux.go @@ -60,6 +60,16 @@ func GetRunningPIDs() ([]int, error) { } func open(pid int) (Process, error) { + _, err := os.Stat(fmt.Sprintf("/proc/%d", pid)) + if os.IsNotExist(err) { + return nil, fmt.Errorf("process does not exist") + } + if os.IsPermission(err) { + return nil, fmt.Errorf("insufficient permissions") + } + if err != nil { + return nil, fmt.Errorf("unexpected error: %w", err) + } return &processLinux{pid: pid}, nil } diff --git a/report/latest b/report/latest new file mode 120000 index 0000000..60453e6 --- /dev/null +++ b/report/latest @@ -0,0 +1 @@ +v1.0.0 \ No newline at end of file diff --git a/report/meta.go b/report/meta.go new file mode 100644 index 0000000..b42197e --- /dev/null +++ b/report/meta.go @@ -0,0 +1,72 @@ +package report + +import ( + "fmt" + "strings" + + "github.com/fkie-cad/yapscan/version" +) + +// SystemInfoFileName is the name of the file, where system info is stored. +const SystemInfoFileName = "systeminfo.json" + +// RulesFileName is the name of the file, where the used rules will be stored. +const RulesFileName = "rules.yarc" + +// ProcessesFileName is the name of the file used to report information about processes. +const ProcessesFileName = "processes.json" + +// MemoryScansFileName is the name of the file used to report information about memory scans. +const MemoryScansFileName = "memory-scans.json" + +// FileScansFileName is the name of the file used to report information about file scans. +const FileScansFileName = "file-scans.json" + +// ScanningStatisticsFileName is the name of the file used to report scanning. +const ScanningStatisticsFileName = "stats.json" + +// MetaFileName is the name of the file containing meta information about the report format. +const MetaFileName = "meta.json" + +var FormatVersion = version.Version{ + Major: 1, + Minor: 0, + Bugfix: 0, +} + +const schemaURLBase = "https://yapscan.targodan.de/reportFormat" + +var schemaURLFormat = schemaURLBase + "/v%s/%s" + +var MetaV1Schema = fmt.Sprintf(schemaURLFormat, "1.0.0", "meta.schema.json") + +type MetaInformation struct { + YapscanVersion version.Version `json:"yapscanVersion"` + FormatVersion version.Version `json:"formatVersion"` + SchemaURLs map[string]string `json:"schemaURLs"` +} + +func generateSchemaURLs(files []string) map[string]string { + ret := make(map[string]string) + for _, file := range files { + fileParts := strings.Split(file, ".") + schemaFile := strings.Join(fileParts[0:len(fileParts)-1], ".") + ".schema." + fileParts[len(fileParts)-1] + ret[file] = fmt.Sprintf(schemaURLFormat, FormatVersion, schemaFile) + } + return ret +} + +func GetMetaInformation() *MetaInformation { + return &MetaInformation{ + YapscanVersion: version.YapscanVersion, + FormatVersion: FormatVersion, + SchemaURLs: generateSchemaURLs([]string{ + SystemInfoFileName, + ProcessesFileName, + MemoryScansFileName, + FileScansFileName, + ScanningStatisticsFileName, + MetaFileName, + }), + } +} diff --git a/report/parser.go b/report/parser.go new file mode 100644 index 0000000..2ab500b --- /dev/null +++ b/report/parser.go @@ -0,0 +1,160 @@ +package report + +import ( + "encoding/json" + "fmt" + "io" +) + +type Parser struct{} + +func NewParser() *Parser { + return &Parser{} +} + +func (p *Parser) Parse(rdr Reader) (*Report, error) { + meta, err := p.parseMeta(rdr) + if err != nil { + return nil, err + } + + if meta.FormatVersion.String() != "1.0.0" { + return nil, fmt.Errorf("unsupported report version \"%v\", expected \"1.0.0\"", meta.FormatVersion) + } + + stats, err := p.parseStatistics(rdr) + if err != nil { + return nil, err + } + + sysInfo, err := p.parseSystemInformation(rdr) + if err != nil { + return nil, err + } + + processes, err := p.parseProcesses(rdr) + if err != nil { + return nil, err + } + + memScans, err := p.parseMemoryScans(rdr) + if err != nil { + return nil, err + } + + fileScans, err := p.parseFileScans(rdr) + if err != nil { + return nil, err + } + + return &Report{ + Meta: meta, + Stats: stats, + SystemInfo: sysInfo, + Processes: processes, + MemoryScans: memScans, + FileScans: fileScans, + }, nil +} + +func (p *Parser) parseMeta(rdr Reader) (*MetaInformation, error) { + r, err := rdr.OpenMeta() + if err != nil { + return nil, err + } + var data MetaInformation + err = json.NewDecoder(r).Decode(&data) + return &data, err +} + +func (p *Parser) parseStatistics(rdr Reader) (*ScanningStatistics, error) { + r, err := rdr.OpenStatistics() + if err != nil { + return nil, err + } + var data ScanningStatistics + err = json.NewDecoder(r).Decode(&data) + return &data, err +} + +func (p *Parser) parseSystemInformation(rdr Reader) (*SystemInfo, error) { + r, err := rdr.OpenSystemInformation() + if err != nil { + return nil, err + } + var data SystemInfo + err = json.NewDecoder(r).Decode(&data) + return &data, err +} + +func (p *Parser) parseProcesses(rdr Reader) ([]*ProcessInfo, error) { + r, err := rdr.OpenProcesses() + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(r) + + data := make([]*ProcessInfo, 0) + for { + var obj ProcessInfo + err = decoder.Decode(&obj) + if err != nil { + break + } + data = append(data, &obj) + } + if err != io.EOF { + return nil, err + } + + return data, nil +} + +func (p *Parser) parseMemoryScans(rdr Reader) ([]*MemoryScan, error) { + r, err := rdr.OpenMemoryScans() + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(r) + + data := make([]*MemoryScan, 0) + for { + var obj MemoryScan + err = decoder.Decode(&obj) + if err != nil { + break + } + data = append(data, &obj) + } + if err != io.EOF { + return nil, err + } + + return data, nil +} + +func (p *Parser) parseFileScans(rdr Reader) ([]*FileScan, error) { + r, err := rdr.OpenFileScans() + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(r) + + data := make([]*FileScan, 0) + for { + var obj FileScan + err = decoder.Decode(&obj) + if err != nil { + break + } + data = append(data, &obj) + } + if err != io.EOF { + return nil, err + } + + return data, nil +} diff --git a/report/reader.go b/report/reader.go new file mode 100644 index 0000000..91f28f8 --- /dev/null +++ b/report/reader.go @@ -0,0 +1,190 @@ +package report + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + + "golang.org/x/crypto/openpgp" + + "github.com/klauspost/compress/zstd" +) + +type Reader interface { + SetPassword(password string) + SetKeyring(keyring openpgp.KeyRing) + OpenMeta() (io.ReadCloser, error) + OpenSystemInformation() (io.ReadCloser, error) + OpenStatistics() (io.ReadCloser, error) + OpenProcesses() (io.ReadCloser, error) + OpenMemoryScans() (io.ReadCloser, error) + OpenFileScans() (io.ReadCloser, error) + io.Closer +} + +type FileReader struct { + path string + password string + keyring openpgp.KeyRing + + hasRead bool + lastError error + + metaBuffer []byte + statsBuffer []byte + systemInfoBuffer []byte + processesBuffer []byte + memoryScansBuffer []byte + fileScansBuffer []byte +} + +func NewFileReader(path string) Reader { + return &FileReader{ + path: path, + } +} + +func (rdr *FileReader) decryptIfNecessary(in io.Reader) (io.Reader, error) { + if rdr.password == "" && rdr.keyring == nil { + return in, nil + } + + var prompt openpgp.PromptFunction + + if rdr.password != "" { + prompt = func(keys []openpgp.Key, symmetric bool) ([]byte, error) { + return []byte(rdr.password), nil + } + } + + msg, err := openpgp.ReadMessage(in, rdr.keyring, prompt, nil) + if err != nil { + return nil, err + } + + return msg.UnverifiedBody, nil +} + +func (rdr *FileReader) readAll() { + if rdr.hasRead { + return + } + defer func() { + rdr.hasRead = true + }() + + file, err := os.Open(rdr.path) + if err != nil { + rdr.lastError = err + return + } + defer file.Close() + + fileRdr, err := rdr.decryptIfNecessary(file) + if err != nil { + rdr.lastError = err + return + } + + zstdRdr, err := zstd.NewReader(fileRdr) + if err != nil { + rdr.lastError = err + return + } + defer zstdRdr.Close() + + tarRdr := tar.NewReader(zstdRdr) + for { + var hdr *tar.Header + hdr, err = tarRdr.Next() + if err != nil { + break + } + if hdr.Typeflag == tar.TypeReg { + buf := &bytes.Buffer{} + if _, err = io.Copy(buf, tarRdr); err != nil { + break + } + + switch filepath.Base(hdr.Name) { + case MetaFileName: + rdr.metaBuffer = buf.Bytes() + case ScanningStatisticsFileName: + rdr.statsBuffer = buf.Bytes() + case SystemInfoFileName: + rdr.systemInfoBuffer = buf.Bytes() + case ProcessesFileName: + rdr.processesBuffer = buf.Bytes() + case MemoryScansFileName: + rdr.memoryScansBuffer = buf.Bytes() + case FileScansFileName: + rdr.fileScansBuffer = buf.Bytes() + } + } + } + + if err == io.EOF { + err = nil + } + + rdr.lastError = err +} + +func (rdr *FileReader) SetPassword(password string) { + rdr.password = password +} + +func (rdr *FileReader) SetKeyring(keyring openpgp.KeyRing) { + rdr.keyring = keyring +} + +func (rdr *FileReader) OpenMeta() (io.ReadCloser, error) { + rdr.readAll() + return io.NopCloser(bytes.NewReader(rdr.metaBuffer)), rdr.lastError +} + +func (rdr *FileReader) OpenSystemInformation() (io.ReadCloser, error) { + rdr.readAll() + return io.NopCloser(bytes.NewReader(rdr.systemInfoBuffer)), rdr.lastError +} + +func (rdr *FileReader) OpenStatistics() (io.ReadCloser, error) { + rdr.readAll() + return io.NopCloser(bytes.NewReader(rdr.statsBuffer)), rdr.lastError +} + +func (rdr *FileReader) OpenProcesses() (io.ReadCloser, error) { + rdr.readAll() + return io.NopCloser(bytes.NewReader(rdr.processesBuffer)), rdr.lastError +} + +func (rdr *FileReader) OpenMemoryScans() (io.ReadCloser, error) { + rdr.readAll() + return io.NopCloser(bytes.NewReader(rdr.memoryScansBuffer)), rdr.lastError +} + +func (rdr *FileReader) OpenFileScans() (io.ReadCloser, error) { + rdr.readAll() + return io.NopCloser(bytes.NewReader(rdr.fileScansBuffer)), rdr.lastError +} + +func (rdr *FileReader) Close() error { + return nil +} + +func ReadArmoredKeyring(path string) (openpgp.KeyRing, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("could not open keyring, reason: %w", err) + } + defer f.Close() + + keyring, err := openpgp.ReadArmoredKeyRing(f) + if err != nil { + return nil, fmt.Errorf("could not read keyring, reason: %w", err) + } + return keyring, nil +} diff --git a/report/report.go b/report/report.go new file mode 100644 index 0000000..9bedf0c --- /dev/null +++ b/report/report.go @@ -0,0 +1,140 @@ +package report + +import ( + "github.com/fkie-cad/yapscan/arch" + "github.com/fkie-cad/yapscan/procio" +) + +type Report struct { + Meta *MetaInformation + Stats *ScanningStatistics + SystemInfo *SystemInfo + Processes []*ProcessInfo + MemoryScans []*MemoryScan + FileScans []*FileScan +} + +type ProfilingInformation struct { + Time Time `json:"time"` + FreeRAM uintptr `json:"freeRAM"` + FreeSwap uintptr `json:"freeSwap"` + LoadAvgOneMinute float64 `json:"loadAvgOneMinute"` + LoadAvgFiveMinutes float64 `json:"loadAvgFiveMinutes"` + LoadAvgFifteenMinutes float64 `json:"loadAvgFifteenMinutes"` +} + +// ScanningStatistics holds statistic information about a scan. +type ScanningStatistics struct { + Start Time `json:"start"` + End Time `json:"end"` + NumberOfProcessesScanned uint64 `json:"numberOfProcessesScanned"` + NumberOfSegmentsScanned uint64 `json:"numberOfSegmentsScanned"` + NumberOfMemoryBytesScanned uint64 `json:"numberOfMemoryBytesScanned"` + NumberOfFileBytesScanned uint64 `json:"numberOfFileBytesScanned"` + NumberOfFilesScanned uint64 `json:"numberOfFilesScanned"` + ProfilingInformation []*ProfilingInformation `json:"profilingInformation"` +} + +// ProcessInfo represents information about a Process. +type ProcessInfo struct { + PID int `json:"pid"` + Bitness arch.Bitness `json:"bitness"` + ExecutablePath string `json:"executablePath"` + ExecutableMD5 string `json:"executableMD5"` + ExecutableSHA256 string `json:"executableSHA256"` + Username string `json:"username"` + MemorySegments []*MemorySegmentInfo `json:"memorySegments"` +} + +// MemorySegmentInfo contains information about a memory segment. +type MemorySegmentInfo struct { + // ParentBaseAddress is the base address of the parent segment. + // If no parent segment exists, this is equal to the BaseAddress. + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->AllocationBase + ParentBaseAddress uintptr `json:"parentBaseAddress"` + + // BaseAddress is the base address of the current memory segment. + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->BaseAddress + BaseAddress uintptr `json:"baseAddress"` + + // AllocatedPermissions is the Permissions that were used to initially + // allocate this segment. + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->AllocationProtect + AllocatedPermissions procio.Permissions `json:"allocatedPermissions"` + + // CurrentPermissions is the Permissions that the segment currently has. + // This may differ from AllocatedPermissions if the permissions where changed + // at some point (e.g. via VirtualProtect). + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->Protect + CurrentPermissions procio.Permissions `json:"currentPermissions"` + + // Size contains the size of the segment in bytes. + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->RegionSize + Size uintptr `json:"size"` + + // RSS contains the ResidentSetSize as reported on linux, i.e. + // the amount of RAM this segment actually uses right now. + // Equivalence on windows: No equivalence, this is currently always equal to Size. + RSS uintptr `json:"rss"` + + // State contains the current State of the segment. + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->State + State procio.State `json:"state"` + + // Type contains the Type of the segment. + // Equivalence on windows: _MEMORY_BASIC_INFORMATION->SegmentType + Type procio.SegmentType `json:"type"` + + // File contains the path to the mapped file, or empty string if + // no file mapping is associated with this memory segment. + MappedFile *File `json:"mappedFile"` +} + +// SystemInfo contains information about the running system. +type SystemInfo struct { + OSName string `json:"osName"` + OSVersion string `json:"osVersion"` + OSFlavour string `json:"osFlavour"` + OSArch arch.T `json:"osArch"` + Hostname string `json:"hostname"` + IPs []string `json:"ips"` + NumCPUs int `json:"numCPUs"` + TotalRAM uintptr `json:"totalRAM"` + TotalSwap uintptr `json:"totalSwap"` +} + +// MemoryScan represents all matches on a single memory +// segment of a process. +type MemoryScan struct { + PID int `json:"pid"` + MemorySegment uintptr `json:"memorySegment"` + Matches []*Match `json:"match"` + Error interface{} `json:"error"` +} + +// FileScan represents all matches on a file. +type FileScan struct { + File *File `json:"file"` + Matches []*Match `json:"match"` + Error interface{} `json:"error"` +} + +// Match represents the match of a yara Rule. +type Match struct { + Rule string `json:"rule"` + Namespace string `json:"namespace"` + Strings []*MatchString `json:"strings"` +} + +// A MatchString represents a string declared and matched in a rule. +type MatchString struct { + Name string `json:"name"` + Base uint64 `json:"base"` + Offset uint64 `json:"offset"` +} + +type File struct { + FilePath string `json:"path"` + MD5Sum string `json:"md5,omitempty"` + SHA256Sum string `json:"sha256,omitempty"` +} diff --git a/report/time.go b/report/time.go new file mode 100644 index 0000000..07a125a --- /dev/null +++ b/report/time.go @@ -0,0 +1,37 @@ +package report + +import ( + "encoding/json" + "fmt" + "time" +) + +const Format = "2006-01-02T15:04:05.000000Z-07:00" + +type Time struct { + time.Time +} + +func Now() Time { + return Time{time.Now()} +} + +func (t Time) MarshalJSON() ([]byte, error) { + b := make([]byte, 0, len(Format)+2) + b = append(b, '"') + b = t.AppendFormat(b, Format) + b = append(b, '"') + return b, nil +} + +func (t *Time) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return fmt.Errorf("expected a JSON-string as Time, %w", err) + } + + tmp, err := time.Parse(Format, s) + t.Time = tmp + return err +} diff --git a/report/v1.0.0/README.md b/report/v1.0.0/README.md new file mode 100644 index 0000000..20b1c49 --- /dev/null +++ b/report/v1.0.0/README.md @@ -0,0 +1,79 @@ +# Yapscan Report Format + +The Yapscan report format is versioned independently of the Yapscan executable. +Its versioning is inspired by semantic versioning of the form `MAJOR.MINOR.BUGFIX`. +Changes to the different parts of the versioning promise different compatibility. + +- **MAJOR-Update:** + These updates would not promise any backwards or forwards compatibility. + Parsers might require close to a complete rewrite. + Switching from JSON to e.g. YAML would change the major version. +- **MINOR-Update:** + These updates promise backwards compatibility, with only small efforts on the parser implementation. + Renaming or deletion of new fields would lead to a MINOR-Update. + Also changes to the internal format of a field are allowed. + Addition or renaming of certain files in the container format, or changing the container format would result in a MINOR-Update. +- **BUGFIX-Update:** + These updates promise forward compatibility with no effort of the parser implementation and backwards compatibility with small efforts of the parser implementaiton. + Addition of fields would lead to a BUGFIX-Update. + If validation with the schemas is done, the schema URL might need updating. + Support for the new fields should be added, but the parser shouldn't break if you don't do this. + Any parser supporting version `n.m.i` should also work for any version `n.m.j`. + +## Container Format + +The container format is [TAR](https://en.wikipedia.org/wiki/Tar_(computing)) with [ZSTD](https://github.com/facebook/zstd) compression and optional [OpenPGP](https://www.openpgp.org/) encryption. +The encryption may be symmetric or asymmetric. + +A change to the container or encryption format would require a bump to the MAJOR-Version. + +This container contains a number of JSON-Files. +The format of each of these files is defined as JSON-Schema. +Note that the schemas in general are rather strict and do not reflect the compatibility promises from above. +This is done on purpose to have a more meaningful format-definition. +For actual validation, the schemas defined in the `meta.json` should be used (see below). +The only exception from this is the `meta.schema.json`, which is more lax to allow for early validation of the meta-file. + +### meta.json + +This file contains meta information about the report. +The `meta.json` has stricter promises regarding compatibility than the other files, as it is essential for parser implementations. +The `meta.json` will validate correctly against the [meta.schema.json of version 1.0.0](https://yapscan.targodan.de/reportFormat/v1.0.0/meta.schema.json) for **any update except a MAJOR-Update**. +This means only the addition of fields to this file is allowed, not removal, renaming or changing of contents. + +Latest Schema: [meta.schema.json](https://yapscan.targodan.de/reportFormat/v1.0.0/meta.schema.json) / [meta.schema.html](https://yapscan.targodan.de/reportFormat/v1.0.0/meta.schema.html) + +### stats.json + +This file contains statistic information about the scan. + +Latest Schema: [stats.schema.json](https://yapscan.targodan.de/reportFormat/v1.0.0/stats.schema.json) / [stats.schema.html](https://yapscan.targodan.de/reportFormat/v1.0.0/stats.schema.html) + +### systeminfo.json + +This file contains information about the scanned system. + +Latest Schema: [systeminfo.schema.json](https://yapscan.targodan.de/reportFormat/v1.0.0/systeminfo.schema.json) / [systeminfo.schema.html](https://yapscan.targodan.de/reportFormat/v1.0.0/systeminfo.schema.html) + +### processes.json + +This file contains information about the scanned processes and their memory layouts. +There is one JSON-Object per line in this file (splitting on `'\n'` is safe). + +Latest Schema: [processes.schema.json](https://yapscan.targodan.de/reportFormat/v1.0.0/processes.schema.json) / [processes.schema.html](https://yapscan.targodan.de/reportFormat/v1.0.0/processes.schema.html) + +### memory-scans.json + +This file contains information about the scanned memory segments and any related yara rule matches. +There is one JSON-Object per line in this file (splitting on `'\n'` is safe). +It may be omitted if no memory was scanned. + +Latest Schema: [memory-scans.schema.json](https://yapscan.targodan.de/reportFormat/v1.0.0/memory-scans.schema.json) / [memory-scans.schema.html](https://yapscan.targodan.de/reportFormat/v1.0.0/memory-scans.schema.html) + +### file-scans.json + +This file contains information about the scanned files and any related yara rule matches. +There is one JSON-Object per line in this file (splitting on `'\n'` is safe). +It may be omitted if no files were scanned. + +Latest Schema: [file-scans.schema.json](https://yapscan.targodan.de/reportFormat/v1.0.0/file-scans.schema.json) / [file-scans.schema.html](https://yapscan.targodan.de/reportFormat/v1.0.0/file-scans.schema.html) diff --git a/report/v1.0.0/datetime.schema.html b/report/v1.0.0/datetime.schema.html new file mode 100644 index 0000000..2e878c1 --- /dev/null +++ b/report/v1.0.0/datetime.schema.html @@ -0,0 +1 @@ +
Datetime in RFC3339 with timezone and added micro seconds.
Must match regular expression:^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}Z[+-]\d{2}:\d{2}$
\ No newline at end of file
diff --git a/report/v1.0.0/datetime.schema.json b/report/v1.0.0/datetime.schema.json
new file mode 100644
index 0000000..084fcd1
--- /dev/null
+++ b/report/v1.0.0/datetime.schema.json
@@ -0,0 +1,8 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://yapscan.targodan.de/reportFormat/v1.0.0/datetime.schema.json",
+ "title": "datetime",
+ "description": "Datetime in RFC3339 with timezone and added micro seconds.",
+ "type": "string",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{6}Z[+-]\\d{2}:\\d{2}$"
+}
\ No newline at end of file
diff --git a/report/v1.0.0/file-scans.schema.html b/report/v1.0.0/file-scans.schema.html
new file mode 100644
index 0000000..6ef2c5d
--- /dev/null
+++ b/report/v1.0.0/file-scans.schema.html
@@ -0,0 +1 @@
+ Scan results of file-scans. For each scanned file, where either a match was found or an error was emitted, one JSON object per line is stored.
Information about the scanned file
No Additional PropertiesThe path of the scanned file
MD5 hexdigest of the executable file, if the file could be read
SHA256 hexdigest of the executable file, if the file could be read
Contains information about matched rules. Is empty-array if no rules matched.
Information about a yara rule match
No Additional PropertiesThe exact strings of the yara rule, that were found, including their offsets in the memory segment.
The offset, where the string was found, relative to the start of the scanned memory segment or file. Note, this value can get very large. make sure your parser uses an int64.
The name of the matched string as defined in the yara rule
The namespace of the matched yara rule. This depends on how the rules where compiled.
The name of the matched yara rule
The error message or null if no error happened. Note, there may still be matches if an error happened.
Information about a yara rule match
No Additional PropertiesThe exact strings of the yara rule, that were found, including their offsets in the memory segment.
The offset, where the string was found, relative to the start of the scanned memory segment or file. Note, this value can get very large. make sure your parser uses an int64.
The name of the matched string as defined in the yara rule
The namespace of the matched yara rule. This depends on how the rules where compiled.
The name of the matched yara rule
Memory scan results. For each scanned memory section, where either a match was found or an error was emitted, one JSON object per line is stored.
No Additional PropertiesContains information about matched rules. Is empty-array if no rules matched.
Information about a yara rule match
No Additional PropertiesThe exact strings of the yara rule, that were found, including their offsets in the memory segment.
The offset, where the string was found, relative to the start of the scanned memory segment or file. Note, this value can get very large. make sure your parser uses an int64.
The name of the matched string as defined in the yara rule
The namespace of the matched yara rule. This depends on how the rules where compiled.
The name of the matched yara rule
PID of the scanned process
The error message or null if no error happened. Note, there may still be matches if an error happened.
The base address of the scanned memory segment. This can be used to resolve the memory segment information in the processes.json. Note, this value can get very large. make sure your parser uses an int64.
Metainformation about the yapscan report.
Version of Yapscan, used to generate the report
SchemaURLs for the files of the report. There is one schema link for each JSON file contained in the report.
All property whose name matches the following regular expression must respect the following conditions
Property name regular expression:\.json$
Version of the report format
Permissions of a memory segment
No Additional PropertiesTrue, if readable
True, if writable. If COW is true, this will be as well.
True, if the Copy-On-Write flag is set.
True, if executable
Information about running processes of a scanned system
No Additional PropertiesPID of the process
Bitness of the process
Path to the executable file of the process, if it could be determined
MD5 hexdigest of the executable file, if the file could be read
^[a-f0-9]{32}$
SHA256 hexdigest of the executable file, if the file could be read
^[a-f0-9]{64}$
Name of the user, which the process is executed under
Address of the parent segment. This is equal to baseAddress if the segment is a root segment. Note, this value can get very large. make sure your parser uses an int64.
Address of this segment. Note, this value can get very large. make sure your parser uses an int64.
The permissions, this segment was initialized with
No Additional PropertiesTrue, if readable
True, if writable. If COW is true, this will be as well.
True, if the Copy-On-Write flag is set.
True, if executable
The permissions, this segment had during time of the scan
Same definition as allocatedPermissionsSize of the segment in bytes. Note, this value can get very large. make sure your parser uses an int64.
The resident set size (RSS) of the segment in bytes. Only applicable on linux. Note, this value can get very large. make sure your parser uses an int64.
The state of the segment. Note that the state "reserve" is an approximation on linux; this will be set if the RSS is exactly zero.
The type of the memory segment
The path of the mapped file
MD5 hexdigest of the mapped file, if the file could be read
SHA256 hexdigest of the mapped file, if the file could be read
Statistic information about the Yapscan run
No Additional PropertiesDatetime of the start of the scan. Format is RFC3339 with added micro seconds.
Must match regular expression:^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}Z[+-]\d{2}:\d{2}$
Datetime of the start of the scan. Format is RFC3339 with added micro seconds.
Same definition as startThe currently free RAM in bytes. Note, this value can get very large. make sure your parser uses an int64.
The currently free swap in bytes. Note, this value can get very large. make sure your parser uses an int64.
The load average over the last minute, normalized over the number of CPUs, i.e. a value of 1.0 means the system is fully loaded. On linux this value can exceed 1.0, meaning processes are waiting for CPU time. Note, that on windows load checking start with the scan, thus this value will be inaccurate for the first minute of the scan.
Value must be greater or equal to 0.0
The load average over the last five minutes, normalized over the number of CPUs, i.e. a value of 1.0 means the system is fully loaded. On linux this value can exceed 1.0, meaning processes are waiting for CPU time. Note, that on windows load checking start with the scan, thus this value will be inaccurate for the first 5 minutes of the scan.
Value must be greater or equal to 0.0
The load average over the last fifteen minutes, normalized over the number of CPUs, i.e. a value of 1.0 means the system is fully loaded. On linux this value can exceed 1.0, meaning processes are waiting for CPU time. Note, that on windows load checking start with the scan, thus this value will be inaccurate for the first 15 minutes of the scan.
Value must be greater or equal to 0.0
Datetime of the stats snapshot. Format is RFC3339 with added micro seconds.
Same definition as startThe number of scanned processes
The number of scanned memory segments
Number of total bytes scanned from memory. Note, this value can get very large. make sure your parser uses an int64.
The number of scanned files
Number of total bytes scanned from files. Note, this value can get very large. make sure your parser uses an int64.
System information gathered by Yapscan
No Additional PropertiesTotal swap capacity in bytes. Note, this value can get very large. make sure your parser uses an int64.
The hostname of the scanned system
Operating system version
Total number of CPUs/Cores
The operating system's architecture
Name of the operating system
Flavour of the operating system
List of local IPs
Total installed RAM in bytes. Note, this value can get very large. make sure your parser uses an int64.