Add file system information to each event (#36065)

This includes: * Unix-like: * device * inode * Windows: * idxlo * idxhi * vol * All: * Fingerprint (if the fingerprint mode is enabled)
elastic · Jul 17, 2023 · e4d287f · e4d287f
1 parent 061cb88
commit e4d287f
Show file tree

Hide file tree

Showing 13 changed files with 321 additions and 59 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -348,6 +348,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
 - Add `clean_session` configuration setting for MQTT input. {pull}35806[16204]
 - Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734]
+- Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065]
 
 *Auditbeat*
  - Migration of system/package module storage from gob encoding to flatbuffer encoding in bolt db. {pull}34817[34817]
@@ -442,6 +443,3 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 
 
 ==== Known Issues
-
-
-
diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go
@@ -65,7 +65,7 @@ scanner:
  Op: loginp.OpCreate,
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
- Info: testFileInfo{path: basename, size: 5}, // 5 bytes written
+ Info: testFileInfo{name: basename, size: 5}, // 5 bytes written
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -88,7 +88,7 @@ scanner:
  Op: loginp.OpWrite,
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
- Info: testFileInfo{path: basename, size: 10}, // +5 bytes appended
+ Info: testFileInfo{name: basename, size: 10}, // +5 bytes appended
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -110,7 +110,7 @@ scanner:
  Op: loginp.OpRename,
  Descriptor: loginp.FileDescriptor{
  Filename: newFilename,
- Info: testFileInfo{path: newBasename, size: 10},
+ Info: testFileInfo{name: newBasename, size: 10},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -130,7 +130,7 @@ scanner:
  Op: loginp.OpTruncate,
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
- Info: testFileInfo{path: basename, size: 2},
+ Info: testFileInfo{name: basename, size: 2},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -150,7 +150,7 @@ scanner:
  Op: loginp.OpTruncate,
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
- Info: testFileInfo{path: basename, size: 2},
+ Info: testFileInfo{name: basename, size: 2},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -169,7 +169,7 @@ scanner:
  Op: loginp.OpDelete,
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
- Info: testFileInfo{path: basename, size: 2},
+ Info: testFileInfo{name: basename, size: 2},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -207,7 +207,7 @@ scanner:
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
  Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
- Info: testFileInfo{path: basename, size: 1024},
+ Info: testFileInfo{name: basename, size: 1024},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -238,7 +238,7 @@ scanner:
  Op: loginp.OpCreate,
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
- Info: testFileInfo{path: basename, size: 1024},
+ Info: testFileInfo{name: basename, size: 1024},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -278,7 +278,7 @@ scanner:
  Descriptor: loginp.FileDescriptor{
  Filename: filename,
  Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
- Info: testFileInfo{path: basename, size: 1024},
+ Info: testFileInfo{name: basename, size: 1024},
  },
  }
  requireEqualEvents(t, expEvent, e)
@@ -372,35 +372,35 @@ scanner:
  Filename: normalFilename,
  Info: testFileInfo{
  size: sizes[normalFilename],
- path: normalBasename,
+ name: normalBasename,
  },
  },
  undersizedFilename: {
  Filename: undersizedFilename,
  Info: testFileInfo{
  size: sizes[undersizedFilename],
- path: undersizedBasename,
+ name: undersizedBasename,
  },
  },
  excludedFilename: {
  Filename: excludedFilename,
  Info: testFileInfo{
  size: sizes[excludedFilename],
- path: excludedBasename,
+ name: excludedBasename,
  },
  },
  excludedIncludedFilename: {
  Filename: excludedIncludedFilename,
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  travelerSymlinkFilename: {
  Filename: travelerSymlinkFilename,
  Info: testFileInfo{
  size: sizes[travelerFilename],
- path: travelerSymlinkBasename,
+ name: travelerSymlinkBasename,
  },
  },
  },
@@ -421,28 +421,28 @@ scanner:
  Filename: normalFilename,
  Info: testFileInfo{
  size: sizes[normalFilename],
- path: normalBasename,
+ name: normalBasename,
  },
  },
  undersizedFilename: {
  Filename: undersizedFilename,
  Info: testFileInfo{
  size: sizes[undersizedFilename],
- path: undersizedBasename,
+ name: undersizedBasename,
  },
  },
  excludedFilename: {
  Filename: excludedFilename,
  Info: testFileInfo{
  size: sizes[excludedFilename],
- path: excludedBasename,
+ name: excludedBasename,
  },
  },
  excludedIncludedFilename: {
  Filename: excludedIncludedFilename,
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  },
@@ -464,21 +464,21 @@ scanner:
  Filename: normalFilename,
  Info: testFileInfo{
  size: sizes[normalFilename],
- path: normalBasename,
+ name: normalBasename,
  },
  },
  undersizedFilename: {
  Filename: undersizedFilename,
  Info: testFileInfo{
  size: sizes[undersizedFilename],
- path: undersizedBasename,
+ name: undersizedBasename,
  },
  },
  travelerSymlinkFilename: {
  Filename: travelerSymlinkFilename,
  Info: testFileInfo{
  size: sizes[travelerFilename],
- path: travelerSymlinkBasename,
+ name: travelerSymlinkBasename,
  },
  },
  },
@@ -495,14 +495,14 @@ scanner:
  Filename: normalFilename,
  Info: testFileInfo{
  size: sizes[normalFilename],
- path: normalBasename,
+ name: normalBasename,
  },
  },
  undersizedFilename: {
  Filename: undersizedFilename,
  Info: testFileInfo{
  size: sizes[undersizedFilename],
- path: undersizedBasename,
+ name: undersizedBasename,
  },
  },
  },
@@ -524,7 +524,7 @@ scanner:
  Filename: excludedIncludedFilename,
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  },
@@ -541,7 +541,7 @@ scanner:
  Filename: excludedIncludedFilename,
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  },
@@ -558,14 +558,14 @@ scanner:
  Filename: excludedIncludedFilename,
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  travelerSymlinkFilename: {
  Filename: travelerSymlinkFilename,
  Info: testFileInfo{
  size: sizes[travelerFilename],
- path: travelerSymlinkBasename,
+ name: travelerSymlinkBasename,
  },
  },
  },
@@ -587,31 +587,31 @@ scanner:
  Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a",
  Info: testFileInfo{
  size: sizes[normalFilename],
- path: normalBasename,
+ name: normalBasename,
  },
  },
  excludedFilename: {
  Filename: excludedFilename,
  Fingerprint: "bd151321c3bbdb44185414a1b56b5649a00206dd4792e7230db8904e43987336",
  Info: testFileInfo{
  size: sizes[excludedFilename],
- path: excludedBasename,
+ name: excludedBasename,
  },
  },
  excludedIncludedFilename: {
  Filename: excludedIncludedFilename,
  Fingerprint: "bfdb99a65297062658c26dfcea816d76065df2a2da2594bfd9b96e9e405da1c2",
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  travelerSymlinkFilename: {
  Filename: travelerSymlinkFilename,
  Fingerprint: "c4058942bffcea08810a072d5966dfa5c06eb79b902bf0011890dd8d22e1a5f8",
  Info: testFileInfo{
  size: sizes[travelerFilename],
- path: travelerSymlinkBasename,
+ name: travelerSymlinkBasename,
  },
  },
  },
@@ -633,7 +633,7 @@ scanner:
  Fingerprint: "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb",
  Info: testFileInfo{
  size: sizes[normalFilename],
- path: normalBasename,
+ name: normalBasename,
  },
  },
  // undersizedFilename got excluded because of the matching fingerprint
@@ -642,23 +642,23 @@ scanner:
  Fingerprint: "9c225a1e6a7df9c869499e923565b93937e88382bb9188145f117195cd41dcd1",
  Info: testFileInfo{
  size: sizes[excludedFilename],
- path: excludedBasename,
+ name: excludedBasename,
  },
  },
  excludedIncludedFilename: {
  Filename: excludedIncludedFilename,
  Fingerprint: "7985b2b9750bdd3c76903db408aff3859204d6334279eaf516ecaeb618a218d5",
  Info: testFileInfo{
  size: sizes[excludedIncludedFilename],
- path: excludedIncludedBasename,
+ name: excludedIncludedBasename,
  },
  },
  travelerSymlinkFilename: {
  Filename: travelerSymlinkFilename,
  Fingerprint: "da437600754a8eed6c194b7241b078679551c06c7dc89685a9a71be7829ad7e5",
  Info: testFileInfo{
  size: sizes[travelerFilename],
- path: travelerSymlinkBasename,
+ name: travelerSymlinkBasename,
  },
  },
  },

diff --git a/filebeat/input/filestream/identifier.go b/filebeat/input/filestream/identifier.go
@@ -59,7 +59,7 @@ type fileIdentifier interface {
 // fileSource implements the Source interface
 // It is required to identify and manage file sources.
 type fileSource struct {
- info loginp.FileDescriptor
+ desc loginp.FileDescriptor
  newPath string
  oldPath string
  truncated bool
@@ -109,7 +109,7 @@ func newINodeDeviceIdentifier(_ *conf.C) (fileIdentifier, error) {
 
 func (i *inodeDeviceIdentifier) GetSource(e loginp.FSEvent) fileSource {
  return fileSource{
- info: e.Descriptor,
+ desc: e.Descriptor,
  newPath: e.NewPath,
  oldPath: e.OldPath,
  truncated: e.Op == loginp.OpTruncate,
@@ -148,7 +148,7 @@ func (p *pathIdentifier) GetSource(e loginp.FSEvent) fileSource {
  path = e.OldPath
  }
  return fileSource{
- info: e.Descriptor,
+ desc: e.Descriptor,
  newPath: e.NewPath,
  oldPath: e.OldPath,
  truncated: e.Op == loginp.OpTruncate,

diff --git a/filebeat/input/filestream/identifier_fingerprint.go b/filebeat/input/filestream/identifier_fingerprint.go
@@ -35,7 +35,7 @@ func newFingerprintIdentifier(cfg *conf.C) (fileIdentifier, error) {
 
 func (i *fingerprintIdentifier) GetSource(e loginp.FSEvent) fileSource {
  return fileSource{
- info: e.Descriptor,
+ desc: e.Descriptor,
  newPath: e.NewPath,
  oldPath: e.OldPath,
  truncated: e.Op == loginp.OpTruncate,

diff --git a/filebeat/input/filestream/identifier_inode_deviceid.go b/filebeat/input/filestream/identifier_inode_deviceid.go
@@ -95,7 +95,7 @@ func (i *inodeMarkerIdentifier) markerContents() string {
 func (i *inodeMarkerIdentifier) GetSource(e loginp.FSEvent) fileSource {
  osstate := file.GetOSState(e.Descriptor.Info)
  return fileSource{
- info: e.Descriptor,
+ desc: e.Descriptor,
  newPath: e.NewPath,
  oldPath: e.OldPath,
  truncated: e.Op == loginp.OpTruncate,

diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go
@@ -228,7 +228,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo
 
  r = readfile.NewStripNewline(r, inp.readerConfig.LineTerminator)
 
- r = readfile.NewFilemeta(r, fs.newPath, offset)
+ r = readfile.NewFilemeta(r, fs.newPath, fs.desc.Info, fs.desc.Fingerprint, offset)
 
  r = inp.parsers.Create(r)
 

diff --git a/filebeat/input/filestream/prospector_test.go b/filebeat/input/filestream/prospector_test.go
@@ -724,13 +724,13 @@ func TestOnRenameFileIdentity(t *testing.T) {
 }
 
 type testFileInfo struct {
- path string
+ name string
  size int64
  time time.Time
  sys interface{}
 }
 
-func (t testFileInfo) Name() string { return t.path }
+func (t testFileInfo) Name() string { return t.name }
 func (t testFileInfo) Size() int64 { return t.size }
 func (t testFileInfo) Mode() os.FileMode { return 0 }
 func (t testFileInfo) ModTime() time.Time { return t.time }