Merge pull request #21 from ugcs/support-multiple-sgy

Enable work with multiple sgy files; enable use of sparse csv; reorder templates

UgcsGeotagger/App/App.csproj

@@ -47,58 +47,43 @@
  <None Update="Assets\ugcs-logo-32.ico">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\FTUTemplates\1-SkyhubWithDateInColumnRTK.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </None>
- <None Update="Mapping\FTUTemplates\2-SkyhubWithDateInName.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </None>
- <None Update="Mapping\FTUTemplates\3-SkyhubWithDateInColumn.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </None>
- <None Update="Mapping\FTUTemplates\MagArrow.yaml">
+ <None Update="Mapping\FTUTemplates\10-SkyhubWithExternalGNSS.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\FTUTemplates\Pergam.yaml">
+ <None Update="Mapping\FTUTemplates\20-SkyhubWithDateInColumnRTK.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\FTUTemplates\RTKLib.yaml">
+ <None Update="Mapping\FTUTemplates\30-SkyhubWithDateInName.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\FTUTemplates\MagDrone.yaml">
+ <None Update="Mapping\FTUTemplates\40-SkyhubWithDateInColumn.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\FTUTemplates\SkyhubRTK.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </None>
- <None Update="Mapping\FTUTemplates\SkyhubWithDateInName.yaml">
+ <None Update="Mapping\FTUTemplates\50-MagArrow.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\FTUTemplates\SkyhubWithDateInColumn.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
- </None>
- <None Update="Mapping\PSFTemplates\1-SkyhubRTK.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ <None Update="Mapping\FTUTemplates\50-MagDrone.yaml">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\2-RTKLibKlau.yaml">
+ <None Update="Mapping\FTUTemplates\50-Pergam.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\1-RTKLib.yaml">
+ <None Update="Mapping\PSFTemplates\10-SkyhubWithExternalGNSS.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\2-SkyhubWithDateInName.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ <None Update="Mapping\PSFTemplates\20-RTKLib.yaml">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\3-SkyhubWithDateInColumn.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ <None Update="Mapping\PSFTemplates\20-SkyhubRTK.yaml">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\SkyhubRTK.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ <None Update="Mapping\PSFTemplates\30-RTKLibKlau.yaml">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\SkyhubWithDateInColumn.yaml">
- <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ <None Update="Mapping\PSFTemplates\30-SkyhubWithDateInName.yaml">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
- <None Update="Mapping\PSFTemplates\SkyhubWithDateInName.yaml">
+ <None Update="Mapping\PSFTemplates\40-SkyhubWithDateInColumn.yaml">
  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
  </None>
  </ItemGroup>

UgcsGeotagger/App/Mapping/FTUTemplates/10-SkyhubWithExternalGNSS.yaml

@@ -0,0 +1,31 @@
+ # Mapping for positioning solution file
+ # Displayable name of Template
+ name: "SkyHub with external GNSS"
+ # Code of template
+ code: "skyhub-ext-gnss" 
+ # Type of template for parsing data. Options are CSV and ColumnsFixedWidth
+ file-type: CSV 
+ # Applies to the first 10 not empty lines of the file to detect template
+ match-regex: >-
+ ^\s*Elapsed,Date,Time,Pitch,Roll,Yaw,Latitude,Longitude,Altitude,Velocity,RTK Status,Latitude RTK,Longitude RTK,Altitude RTK,ALT:Altitude,ALT:Filtered Altitude,GNSS:Date,GNSS:Time,GNSS:Latitude,GNSS:Longitude
+ file-format:
+ # Applies to CSV type. 'true' if file has headers, otherwise false
+ has-header: true
+ # The symbol which uses to make a comment
+ comment-prefix: '#'
+ # The symbol used to separate the integer part from the fractional part of a number written in decimal form. Options are ',' and '.'
+ decimal-separator: '.'
+ # Applies to CSV type. The symbol to separate values from line in CSV file;
+ separator: ','
+ # Mapping data from source file to data. Requierd fields (latitude, longitude, time and date or date-time)
+ data-mapping:
+ latitude:
+ header: GNSS:Latitude
+ longitude:
+ header: GNSS:Longitude
+ time:
+ header: Time
+ format: 'HH:mm:ss.fff'
+ date:
+ header: Date
+ format: 'yyyy/MM/dd' 

UgcsGeotagger/App/Mapping/PSFTemplates/10-SkyhubWithExternalGNSS.yaml

@@ -0,0 +1,31 @@
+ # Mapping for positioning solution file
+ # Displayable name of Template
+ name: "SkyHub with external GNSS"
+ # Code of template
+ code: "skyhub-ext-gnss" 
+ # Type of template for parsing data. Options are CSV and ColumnsFixedWidth
+ file-type: CSV 
+ # Applies to the first 10 not empty lines of the file to detect template
+ match-regex: >-
+ ^\s*Elapsed,Date,Time,Pitch,Roll,Yaw,Latitude,Longitude,Altitude,Velocity,RTK Status,Latitude RTK,Longitude RTK,Altitude RTK,ALT:Altitude,ALT:Filtered Altitude,GNSS:Date,GNSS:Time,GNSS:Latitude,GNSS:Longitude
+ file-format:
+ # Applies to CSV type. 'true' if file has headers, otherwise false
+ has-header: true
+ # The symbol which uses to make a comment
+ comment-prefix: '#'
+ # The symbol used to separate the integer part from the fractional part of a number written in decimal form. Options are ',' and '.'
+ decimal-separator: '.'
+ # Applies to CSV type. The symbol to separate values from line in CSV file;
+ separator: ','
+ # Mapping data from source file to data. Requierd fields (latitude, longitude, time and date or date-time)
+ data-mapping:
+ latitude:
+ header: GNSS:Latitude
+ longitude:
+ header: GNSS:Longitude
+ time:
+ header: Time
+ format: 'HH:mm:ss.fff'
+ date:
+ header: Date
+ format: 'yyyy/MM/dd' 

UgcsGeotagger/App/Models/FileToUpdate.cs

@@ -32,18 +32,30 @@ private set
 
  public event Action<string> OnProcessingStatus;
 
- public FileToUpdate(string filePath, Template template) : base(filePath, template)
+ public FileToUpdate(string filePath, Template template, string linkedFile) : base(filePath, template)
  {
- FindLinkedFile(filePath);
+ LinkedFile = linkedFile;
  SegyParser = new SegYLogParser(template);
  }
 
  public SegYLogParser SegyParser { get; private set; }
 
- private void FindLinkedFile(string filePath)
+ public static List<FileToUpdate> GetFilesToUpdate(string filePath, Template template)
  {
+ List<FileToUpdate> filesToUpdate = new List<FileToUpdate>();
+ var linkedFiles = FileToUpdate.FindLinkedFiles(filePath, template.FileType);
+ linkedFiles.ForEach(lf => filesToUpdate.Add(new FileToUpdate(filePath, template, lf)));
+ return filesToUpdate;
+ }
+
+ private static List<string> FindLinkedFiles(string filePath, FileType Type)
+ {
+ List<string> filesList = new List<string>();
  if (Type == FileType.Segy || Type == FileType.Unknown)
- return;
+ {
+ filesList.Add(filePath);
+ return filesList;
+ }
  try
  {
  var directory = Path.GetDirectoryName(filePath);
@@ -59,8 +71,7 @@ private void FindLinkedFile(string filePath)
  var linkedFileDate = DateTime.ParseExact(m.Value, "yyyy-MM-dd-HH-mm-ss", CultureInfo.InvariantCulture);
  if (linkedFileDate == csvFileDate)
  {
- LinkedFile = f;
- break;
+ filesList.Add(f);
  }
  }
  }
@@ -69,6 +80,7 @@ private void FindLinkedFile(string filePath)
  {
  log.Error(e.Message);
  }
+ return filesList;
  }
 
  public void CheckCoveringStatus(List<PositioningSolutionFile> psfFiles)
@@ -108,10 +120,13 @@ public Task<string> UpdateCoordinates(CancellationTokenSource source, int timeOf
  OnProcessingStatus?.Invoke($"Start Processing {FileName}");
  var watch = System.Diagnostics.Stopwatch.StartNew();
 
+ // Remove coordinates with no values
+ List<IGeoCoordinates> cleanCoverageCoordinates = coverageCoordinates.Where(cc => cc.Latitude != null && cc.Longitude != null).ToList();
+
  var elapsedMs = watch.ElapsedMilliseconds;
  try
  {
- correctedCoordinates = Interpolator.CreateCorrectedCoordinates(Coordinates, coverageCoordinates, timeOffset, source);
+ correctedCoordinates = Interpolator.CreateCorrectedCoordinates(Coordinates, cleanCoverageCoordinates, timeOffset, source);
  var correctedFile = CreateFileWithPreciseSuffix(FilePath);
  var result = Parser.CreateFileWithCorrectedCoordinates(FilePath, correctedFile, correctedCoordinates, source);
  message = $"{FileName}: {result.CountOfReplacedLines} of {result.CountOfLines} were replaced;";

UgcsGeotagger/App/ViewModels/GeotaggerToolViewModel.cs

@@ -161,8 +161,7 @@ public void AddFiles(IEnumerable<string> files, string fileType)
  switch (fileType)
  {
  case DataFile.FileToUpdateAbbr:
- dataFile = new FileToUpdate(file, template);
- filesToUpdate.Add(dataFile as FileToUpdate);
+ FileToUpdate.GetFilesToUpdate(file, template).ForEach(ftu => filesToUpdate.Add(ftu));
  break;
 
  case DataFile.PositionSolutionFileAbbr:
@@ -256,8 +255,7 @@ private async void BrowseFolder()
  }
  else if (template?.FileType == FileType.CSV)
  {
- var ftu = new FileToUpdate(file, template);
- filesToUpdate.Add(ftu);
+ FileToUpdate.GetFilesToUpdate(file, template).ForEach(ftu => filesToUpdate.Add(ftu));
  }
  else
  {

UgcsGeotagger/FileParsers/Interpolator.cs

@@ -20,19 +20,31 @@ public static List<IGeoCoordinates> CreateCorrectedCoordinates(List<IGeoCoordina
  c.TimeInMs += timeOffset;
  }
  psfCoordinates.Sort((x, y) => x.TimeInMs.Value.CompareTo(y.TimeInMs.Value));
- var min = psfCoordinates.Min(c => c.TimeInMs);
- var max = psfCoordinates.Max(c => c.TimeInMs);
  var countOfReplacedLines = 0;
+
+ // Don't run with no coodrinates at all
+ if (psfCoordinates.Count < 1)
+ return correctedTraces;
+
  foreach (var coordinates in ftuCoordinates)
  {
  if (token.IsCancellationRequested)
  break;
- if (coordinates.TimeInMs < min || coordinates.TimeInMs > max)
- continue;
- var leftValue = psfCoordinates.Last(c => c.TimeInMs <= coordinates.TimeInMs);
+
+ // Get left coordinate, if any
+ var leftValue = psfCoordinates.LastOrDefault(c => c.TimeInMs <= coordinates.TimeInMs);
+ // If not, use first in the list (this can happen if we have data before the first RTK record)
+ if (leftValue == null)
+ leftValue = psfCoordinates.First();
  var leftBorderIndex = psfCoordinates.IndexOf(leftValue);
- leftBorderIndex -= leftBorderIndex == psfCoordinates.Count - 1 ? 1 : 0;
- var rightBorderIndex = leftBorderIndex + 1;
+
+ // Get right coordinate, again, if any
+ var rightValue = psfCoordinates.FirstOrDefault(c => c.TimeInMs > coordinates.TimeInMs);
+ // If not, use what we have as a left coordinate (we're at the end of the list)
+ if (rightValue == null)
+ rightValue = leftValue;
+ var rightBorderIndex = psfCoordinates.IndexOf(rightValue);
+
  if (psfCoordinates[rightBorderIndex].TimeInMs - psfCoordinates[leftBorderIndex].TimeInMs > MaxTimeDifferenceMs)
  continue;
  if (coordinates.TimeInMs.HasValue && psfCoordinates[leftBorderIndex].Latitude.HasValue && psfCoordinates[rightBorderIndex].Latitude.HasValue