Skip to content

Commit

Permalink
Some logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@yogilad
yogilad committed Nov 5, 2023
1 parent f25db50 commit ba8c5ca
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 14 deletions.
2 changes: 1 addition & 1 deletion Src/App/Klipboard/WorkersUx/IngestUx/IngestWorkerUxBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public IngestWorkerUxBase(ISettings settings, INotificationHelper notificationHe

public override async Task HandleFilesAsync(List<string> filesAndFolders, string? chosenOption)
{
using var ux = new IngestForm(GetMenuText(ClipboardContent.Files), devMode: true);
using var ux = new IngestForm(GetMenuText(ClipboardContent.Files), devMode: AppConstants.DevMode);

ux.ShowDialog();

Expand Down
14 changes: 12 additions & 2 deletions Src/App/Utils/Common/Logger.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
using Kusto.Cloud.Platform.Utils;
using System.Text.RegularExpressions;
using Kusto.Cloud.Platform.Utils;
using Serilog;
using Serilog.Context;
using Serilog.Formatting.Json;


namespace Klipboard.Utils
{
public static class Logger
{
public static Serilog.ILogger Log => s_log;
private static readonly Serilog.Core.Logger s_log;
private static readonly SerilogTraceListener.SerilogTraceListener s_traceListener;
private static readonly Regex s_hstringMatcher = new Regex("(^| ){1}h['\"](\\\\|\\'|\\\"|[^\"'])*[\"']", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex s_sigMatcher = new Regex("&sig=[a-zA-Z0-9%]*", RegexOptions.IgnoreCase | RegexOptions.Compiled);

static Logger()
{
Expand Down Expand Up @@ -63,5 +65,13 @@ public static void CloseLog()
s_log.Dispose();
}
}

public static string ObfuscateHiddentStrings(string input)
{
input = s_hstringMatcher.Replace(input, "h'*****'");
input = s_sigMatcher.Replace(input, "&sig=*****");

return input;
}
}
}
56 changes: 45 additions & 11 deletions Src/App/Utils/Helpers/KustoDatabaseHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ public void Dispose()
#region Public APIs
public async Task<(bool Success, string? BlobUri, string? Error)> TryUploadFileToEngineStagingAreaAsync(Stream dataStream, string upstreamFileName, FileFormatDefiniton formatDefintion, string? filePath = null)
{
var cmd = ".create tempstorage";

if (m_localhost)
{
if (filePath == null)
Expand All @@ -75,7 +77,9 @@ public void Dispose()

try
{
using var res = await m_engineAdminClient.Value.ExecuteControlCommandAsync(m_databaseName, ".create tempstorage");
Logger.Log.Information("KustoDatabaseHelper.TryUploadFileToEngineStagingAreaAsync: Executing control command: {0}", cmd);

using var res = await m_engineAdminClient.Value.ExecuteControlCommandAsync(m_databaseName, cmd);
res.Read();

var tempStorage = res.GetString(0);
Expand All @@ -85,6 +89,7 @@ public void Dispose()
}
catch (Exception ex)
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryUploadFileToEngineStagingAreaAsync: Failed to get engine staging account");
return (false, null, "Failed to get engine staging account: " + ex.Message);
}
}
Expand All @@ -97,6 +102,8 @@ public void Dispose()

try
{
Logger.Log.Information("KustoDatabaseHelper.TryGetBlobSchemeAsync: Executing control command: {0}", Logger.ObfuscateHiddentStrings(cmd));

using var res = await m_engineQueryClient.Value.ExecuteQueryAsync(m_databaseName, cmd, new ClientRequestProperties());
var tableScheme = new TableColumns();
var nameCol = res.GetOrdinal("ColumnName");
Expand All @@ -120,10 +127,12 @@ public void Dispose()
return (true, tableScheme, format, null);
}

Logger.Log.Error("KustoDatabaseHelper.TryGetBlobSchemeAsync: Scheme detection returned no results");
return (false, null, format, "Scheme detection returned no results");
}
catch (Exception ex)
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryGetBlobSchemeAsync: Failed to get engine staging account");
return (false, null, format, "Failed to get engine staging account: " + ex.Message);
}
}
Expand Down Expand Up @@ -162,37 +171,49 @@ public void Dispose()

// Create the table
try
{
{
Logger.Log.Information("KustoDatabaseHelper.TryCreateTableAync: Executing control command: {0}", createTableCommand);

using var res = await m_engineAdminClient.Value.ExecuteControlCommandAsync(m_databaseName, createTableCommand);
}
catch (Exception ex)
{
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryCreateTableAync: Failed to create table {0}", tableName);

return (false, $"Failed to create table '{tableName}': {ex.Message}");
}

// Set ingestion time batching
try
{
if (!string.IsNullOrWhiteSpace(alterIngestionBatchingCommand))
{
{
Logger.Log.Information("KustoDatabaseHelper.TryCreateTableAync: Executing control command: {0}",alterIngestionBatchingCommand);

using var res = await m_engineAdminClient.Value.ExecuteControlCommandAsync(m_databaseName, alterIngestionBatchingCommand);
}
}
catch (Exception ex)
{
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryCreateTableAync: Failed to create table {0}", tableName);

return (false, $"Failed to create table '{tableName}': {ex.Message}");
}

// Set auto delete policy
try
{
if (!string.IsNullOrWhiteSpace(setTableLifetimeCommand))
{
{
Logger.Log.Information("KustoDatabaseHelper.TryCreateTableAync: Executing control command: {0}",setTableLifetimeCommand);

using var res = await m_engineAdminClient.Value.ExecuteControlCommandAsync(m_databaseName, setTableLifetimeCommand);
}
}
catch (Exception ex)
{
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryCreateTableAync: Failed to create table {0}", tableName);

return (false, $"Failed to create table '{tableName}': {ex.Message}");
}

Expand All @@ -208,6 +229,8 @@ public void Dispose()
///////////////////////////////////////
if (!m_localhost)
{
Logger.Log.Information("KustoDatabaseHelper.TryDirectIngestStorageToTable: Direct ingest from storage: {0}", Logger.ObfuscateHiddentStrings(blobUriOrFile));

var res = await m_directIngestClient.Value.IngestFromStorageAsync(blobUriOrFile, ingestionProperties, sourceOptions);
var ingestStatus = res.GetIngestionStatusBySourceId(sourceOptions.SourceId);

Expand All @@ -232,6 +255,8 @@ public void Dispose()
isAsync: false,
extensions: ingestionProperties.GetIngestionProperties());

Logger.Log.Information("KustoDatabaseHelper.TryDirectIngestStorageToTable: Executing control command: {0}", command);

using var resultReader = await m_engineAdminClient.Value.ExecuteControlCommandAsync(m_databaseName, command).ConfigureAwait(false);

resultReader.Read();
Expand All @@ -243,7 +268,9 @@ public void Dispose()
}
}
catch (Exception ex)
{
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryDirectIngestStorageToTable: Failed to ingest stream to Kusto");

return (false, $"Failed to ingest stream to Kusto: {ex.Message}");
}
}
Expand All @@ -264,7 +291,9 @@ public IKustoIngestClient GetDirectIngestClient()
#region Private APIs
private static async Task<(bool Success, string? BlobUri, string? Error)> TryUploadStreamAync(string blobContainerUriStr, Stream dataStream, string upstreamFileName, FileFormatDefiniton formatDefiniton)
{
var disposeOfStream = false;
var disposeOfStream = false;

Logger.Log.Information("KustoDatabaseHelper.TryUploadStreamAsync");

if (!formatDefiniton.DoNotCompress)
{
Expand All @@ -289,6 +318,7 @@ public IKustoIngestClient GetDirectIngestClient()

if (blobResp.IsError)
{
Logger.Log.Error("KustoDatabaseHelper.TryUploadStreamAsync: Failed to upload blob: {0}", blobResp.ReasonPhrase);
return (false, null, "Failed to upload blob: " + blobResp.ReasonPhrase);
}

Expand All @@ -305,7 +335,9 @@ public IKustoIngestClient GetDirectIngestClient()
}

private static async Task<(bool Success, MemoryStream? MemoryStream, string? Error)> TryCreateZipStreamAsync(Stream dataStream, string upsteramFileName)
{
{
Logger.Log.Information("KustoDatabaseHelper.TryUploadStreamAsync");

try
{
var memoryStream = new MemoryStream();
Expand All @@ -329,7 +361,9 @@ public IKustoIngestClient GetDirectIngestClient()
return (true, memoryStream, null);
}
catch (Exception ex)
{
{
Logger.Log.Error(ex, "KustoDatabaseHelper.TryUploadStreamAsync: Failed to compress memory stream");

return (false, null, "Failed to compress memory stream: " + ex.Message);
}
}
Expand Down
2 changes: 2 additions & 0 deletions Src/App/Utils/Helpers/TabularDataHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ public static bool TryConvertTableToInlineQuery(string tableData, string delimit
};

queryBuilder.AppendLine("// Query Created With Klipboard (https://github.com/yogilad/Klipboard/wiki)");
queryBuilder.AppendLine("//");
queryBuilder.Append("let Klipboard = datatable");
queryBuilder.AppendLine(tableScheme.ToSchemaString(detectionMode));
queryBuilder.AppendLine("[");
Expand Down Expand Up @@ -424,6 +425,7 @@ public static bool TryConvertFreeTextToInlineQuery(string freeText, string? opti
var queryBuilder = new StringBuilder();

queryBuilder.AppendLine("// Query Created With Klipboard (https://github.com/yogilad/Klipboard/wiki)");
queryBuilder.AppendLine("//");
queryBuilder.AppendLine("let Klipboard = datatable(['Line']:string)");
queryBuilder.AppendLine("[");

Expand Down
1 change: 1 addition & 0 deletions Src/App/Workers/Ingest/TempTableWorker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ private void InvokeTempTableQuery(string tempTableName, IProgressNotificationUpd
var target = GetQuickActionTarget();
var query = new StringBuilder()
.AppendLine("// Query Created With Klipboard (https://github.com/yogilad/Klipboard/wiki)")
.AppendLine("//")
.Append("['")
.Append(tempTableName)
.AppendLine("']")
Expand Down
1 change: 1 addition & 0 deletions Src/App/Workers/Query/ExternalDataQueryWorker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ private async Task HandleStreamAsync(Stream dataStream,
var queryBuilder = new StringBuilder();

queryBuilder.AppendLine("// Query Created With Klipboard (https://github.com/yogilad/Klipboard/wiki)");
queryBuilder.AppendLine("//");
queryBuilder.AppendLine("let Klipboard =");
queryBuilder.Append("externaldata");
queryBuilder.AppendLine(schemaStr);
Expand Down

0 comments on commit ba8c5ca

Please sign in to comment.