Skip to content

Commit

Permalink
Fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidJGapCR
DavidJGapCR committed Sep 27, 2024
1 parent 30c089d commit 28c5891
Show file tree
Hide file tree
Showing 15 changed files with 280 additions and 268 deletions.
2 changes: 1 addition & 1 deletion src/EdFi.Tools.ApiPublisher.Connections.Api/ApiClientManagement/EdFiApiClientProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public EdFiApiClient GetApiClient()
if (!_apiClient.IsValueCreated)
{
// Establish connection to API
_logger.Information($"Initializing API client '{_apiClient.Value.ConnectionDetails.Name}'...");
_logger.Information("Initializing API client '{ConnectionDetailsName}'...", _apiClient.Value.ConnectionDetails.Name);
}

return _apiClient.Value;
Expand Down
6 changes: 3 additions & 3 deletions ...Connections.Api/Configuration/Enhancers/EdFiApiConnectionsConfigurationBuilderEnhancer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ IEnumerable<KeyValuePair<string, string>> GetEnhancedConnectionConfigurationValu
ApiConnectionDetails connection,
ConnectionRole connectionType)
{
_logger.Debug($"{connectionType} connection details are not fully defined.");
_logger.Debug("{ConnectionType} connection details are not fully defined.", connectionType);

if (string.IsNullOrEmpty(connection.Name))
{
Expand All @@ -87,8 +87,8 @@ IEnumerable<KeyValuePair<string, string>> CreateNamedConnectionConfigurationValu
string apiConnectionName,
ConnectionRole connectionRole)
{
_logger.Debug(
$"Obtaining {connectionRole.ToString().ToLower()} API connection details for connection '{apiConnectionName}' using '{_namedApiConnectionDetailsReader.GetType().Name}'.");
_logger.Debug("Obtaining {ConnectionRole} API connection details for connection '{ApiConnectionName}' using '{NamedApiConnectionDetailsReaderName}'.",
connectionRole.ToString().ToLower(), apiConnectionName, _namedApiConnectionDetailsReader.GetType().Name);

var namedApiConnectionDetails =
_namedApiConnectionDetailsReader.GetNamedApiConnectionDetails(apiConnectionName, configurationStoreSection);
Expand Down
22 changes: 10 additions & 12 deletions ....Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC
getByIdDelay,
(result, ts, retryAttempt, ctx) =>
{
_logger.Warning(
$"Retrying GET for resource item '{resourceItemUrl}' from source failed with status '{result.Result.StatusCode}'. Retrying... (retry #{retryAttempt} of {_options.MaxRetryAttempts} with {ts.TotalSeconds:N1}s delay)");
_logger.Warning("Retrying GET for resource item '{ResourceItemUrl}' from source failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)",
resourceItemUrl, result.Result.StatusCode, retryAttempt, _options.MaxRetryAttempts, ts.TotalSeconds);
});
IAsyncPolicy<HttpResponseMessage> policy = isRateLimitingEnabled ? Policy.WrapAsync(_rateLimiter?.GetRateLimitingPolicy(), retryPolicy) : retryPolicy;
try
Expand All @@ -62,13 +62,10 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC
{
getByIdAttempts++;

if (getByIdAttempts > 1)
if (getByIdAttempts > 1 && _logger.IsEnabled(LogEventLevel.Debug))
{
if (_logger.IsEnabled(LogEventLevel.Debug))
{
_logger.Debug(
$"GET for missing dependency '{resourceItemUrl}' reference from source attempt #{getByIdAttempts}.");
}
_logger.Debug("GET for missing dependency '{ResourceItemUrl}' reference from source attempt #{GetByIdAttempts}.",
resourceItemUrl, getByIdAttempts);
}

return sourceEdFiApiClient.HttpClient.GetAsync(
Expand All @@ -94,15 +91,16 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC
}
else
{
_logger.Warning(
$"GET request from source API for '{resourceItemUrl}' reference failed with status '{getByIdResponse.StatusCode}': {responseContent}");
_logger.Warning("GET request from source API for '{ResourceItemUrl}' reference failed with status '{StatusCode}': {ResponseContent}",
resourceItemUrl, getByIdResponse.StatusCode, responseContent);

return (false, null);
}
}
catch (RateLimitRejectedException)
catch (RateLimitRejectedException ex)
{
_logger.Fatal($"{sourceEdFiApiClient.DataManagementApiSegment}{resourceItemUrl}: Rate limit exceeded. Please try again later.");
_logger.Fatal(ex, "{DataManagementApiSegment}{ResourceItemUrl}: Rate limit exceeded. Please try again later.",
sourceEdFiApiClient.DataManagementApiSegment, resourceItemUrl);
return (false, null);
}
//----------------------------------------------------------------------------------------------
Expand Down
9 changes: 6 additions & 3 deletions ...blisher.Connections.Api/Metadata/Dependencies/EdFiApiGraphMLDependencyMetadataProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiCl
string dependenciesRequestUri = $"metadata/{edFiApiClient.DataManagementApiSegment}/dependencies";

// Get the resource dependencies from the target
_logger.Information($"Getting dependencies from API at {edFiApiClient.HttpClient.BaseAddress}{dependenciesRequestUri}...");
_logger.Information("Getting dependencies from API at {BaseAddress}{DependenciesRequestUri}...",
edFiApiClient.HttpClient.BaseAddress, dependenciesRequestUri);

var dependencyRequest = new HttpRequestMessage(HttpMethod.Get, dependenciesRequestUri);
dependencyRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/graphml"));
Expand All @@ -42,7 +43,8 @@ public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiCl

if (!dependencyResponse.IsSuccessStatusCode)
{
_logger.Error($"Ed-Fi ODS API request for dependencies to '{dependencyRequest.RequestUri}' returned '{dependencyResponse.StatusCode}' with content:{Environment.NewLine}{dependencyResponseContent}");
_logger.Error("Ed-Fi ODS API request for dependencies to '{RequestUri}' returned '{StatusCode}' with content:{NewLine}{DependencyResponseContent}",
dependencyRequest.RequestUri, dependencyResponse.StatusCode, Environment.NewLine, dependencyResponseContent);
throw new Exception("Resource dependencies could not be obtained.");
}

Expand All @@ -55,7 +57,8 @@ public EdFiApiGraphMLDependencyMetadataProvider(IEdFiApiClientProvider edFiApiCl
}
catch (Exception ex)
{
_logger.Error($"Unable to parse dependency response as GraphML: {dependencyResponseContent}{Environment.NewLine}{ex}");
_logger.Error(ex, "Unable to parse dependency response as GraphML: {DependencyResponseContent}{NewLine}{Ex}",
dependencyResponseContent, Environment.NewLine, ex);
throw new Exception("Resource dependencies could not be obtained.");
}
}
Expand Down
3 changes: 2 additions & 1 deletion ...ls.ApiPublisher.Connections.Api/Metadata/Versioning/EdFiApiVersionMetadataProviderBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ JObject GetVersionObject(string versionJson)
try
{
versionObject = JObject.Parse(versionJson);
_logger.Information($"{_role} version information: {versionObject.ToString(Formatting.Indented)}");
_logger.Information("{Role} version information: {VersionObject}",
_role, versionObject.ToString(Formatting.Indented));
}
catch (Exception)
{
Expand Down
14 changes: 8 additions & 6 deletions ....ApiPublisher.Connections.Api/Processing/Source/Capabilities/EdFiApiSourceCapabilities.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,18 @@ public async Task<bool> SupportsKeyChangesAsync(string probeResourceKey)

string probeUrl = $"{edFiApiClient.DataManagementApiSegment}{probeResourceKey}{EdFiApiConstants.KeyChangesPathSuffix}";

_logger.Debug($"Probing source API for key changes support at '{probeUrl}'.");
_logger.Debug("Probing source API for key changes support at '{ProbeUrl}'.", probeUrl);

var probeResponse = await edFiApiClient.HttpClient.GetAsync($"{probeUrl}?limit=1").ConfigureAwait(false);

if (probeResponse.IsSuccessStatusCode)
{
_logger.Debug($"Probe response status was '{probeResponse.StatusCode}'.");
_logger.Debug("Probe response status was '{StatusCode}'.", probeResponse.StatusCode);
return true;
}

_logger.Warning($"Request to Source API for the '{EdFiApiConstants.KeyChangesPathSuffix}' child resource was unsuccessful (response status was '{probeResponse.StatusCode}'). Key change processing cannot be performed.");
_logger.Warning("Request to Source API for the '{KeyChangesPathSuffix}' child resource was unsuccessful (response status was '{StatusCode}'). Key change processing cannot be performed.",
EdFiApiConstants.KeyChangesPathSuffix, probeResponse.StatusCode);

return false;
}
Expand All @@ -49,17 +50,18 @@ public async Task<bool> SupportsDeletesAsync(string probeResourceKey)
// Probe for deletes support
string probeUrl = $"{edFiApiClient.DataManagementApiSegment}{probeResourceKey}{EdFiApiConstants.DeletesPathSuffix}";

_logger.Debug($"Probing source API for deletes support at '{probeUrl}'.");
_logger.Debug("Probing source API for deletes support at '{ProbeUrl}'.", probeUrl);

var probeResponse = await edFiApiClient.HttpClient.GetAsync($"{probeUrl}?limit=1").ConfigureAwait(false);

if (probeResponse.IsSuccessStatusCode)
{
_logger.Debug($"Probe response status was '{probeResponse.StatusCode}'.");
_logger.Debug("Probe response status was '{StatusCode}'.", probeResponse.StatusCode);
return true;
}

_logger.Warning($"Request to Source API for the '{EdFiApiConstants.DeletesPathSuffix}' child resource was unsuccessful (response status was '{probeResponse.StatusCode}'). Delete processing cannot be performed.");
_logger.Warning("Request to Source API for the '{DeletesPathSuffix}' child resource was unsuccessful (response status was '{StatusCode}'). Delete processing cannot be performed.",
EdFiApiConstants.DeletesPathSuffix, probeResponse.StatusCode);

return false;
}
Expand Down
22 changes: 12 additions & 10 deletions ...Publisher.Connections.Api/Processing/Source/Isolation/EdFiApiSourceIsolationApplicator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using EdFi.Tools.ApiPublisher.Core.Isolation;
using Newtonsoft.Json.Linq;
using Serilog;
using System.Globalization;
using System.Net;
using Version = EdFi.Tools.ApiPublisher.Core.Helpers.Version;

Expand Down Expand Up @@ -69,16 +70,16 @@ private async Task<string> GetSourceSnapshotIdentifierAsync(EdFiApiClient source

if (snapshotsResponse.StatusCode == HttpStatusCode.NotFound)
{
_logger.Warning(
$"Source API at '{sourceApiClient.HttpClient.BaseAddress}' does not support the necessary isolation for reliable API publishing. Errors may occur, or some data may not be published without causing failures.");
_logger.Warning("Source API at '{BaseAddress}' does not support the necessary isolation for reliable API publishing. Errors may occur, or some data may not be published without causing failures.",
sourceApiClient.HttpClient.BaseAddress);

return null;
}

if (snapshotsResponse.StatusCode == HttpStatusCode.Forbidden)
{
_logger.Warning(
$"The API publisher does not have permissions to access the source API's 'snapshots' resource at '{sourceApiClient.HttpClient.BaseAddress}{snapshotsRelativePath}'. Make sure that the source API is using a correctly configured claim set for your API Publisher's API client.");
_logger.Warning("The API publisher does not have permissions to access the source API's 'snapshots' resource at '{BaseAddress}{SnapshotsRelativePath}'. Make sure that the source API is using a correctly configured claim set for your API Publisher's API client.",
sourceApiClient.HttpClient.BaseAddress, snapshotsRelativePath);

return null;
}
Expand All @@ -99,8 +100,8 @@ private async Task<string> GetSourceSnapshotIdentifierAsync(EdFiApiClient source
if (!snapshotResponseArray.Any())
{
// No snapshots available.
_logger.Warning(
$"Snapshots are supported, but no snapshots are available from source API at '{sourceApiClient.HttpClient.BaseAddress}{snapshotsRelativePath}'.");
_logger.Warning("Snapshots are supported, but no snapshots are available from source API at '{BaseAddress}{SnapshotsRelativePath}'.",
sourceApiClient.HttpClient.BaseAddress, snapshotsRelativePath);

return null;
}
Expand All @@ -111,7 +112,7 @@ private async Task<string> GetSourceSnapshotIdentifierAsync(EdFiApiClient source
string snapshotIdentifier = jt["snapshotIdentifier"].Value<string>();
string snapshotDateTimeText = jt["snapshotDateTime"].Value<string>();

if (!DateTime.TryParse(snapshotDateTimeText, out var snapshotDateTimeValue))
if (!DateTime.TryParse(snapshotDateTimeText, new CultureInfo("en-US"), out var snapshotDateTimeValue))
{
snapshotDateTimeValue = DateTime.MinValue;
}
Expand All @@ -126,15 +127,16 @@ private async Task<string> GetSourceSnapshotIdentifierAsync(EdFiApiClient source
.OrderByDescending(x => x.SnapshotDateTime)
.First();

_logger.Information($"Using snapshot identifier '{snapshot.SnapshotIdentifier}' created at '{snapshot.SnapshotDateTime}'.");
_logger.Information("Using snapshot identifier '{SnapshotIdentifier}' created at '{SnapshotDateTime}'.",
snapshot.SnapshotIdentifier, snapshot.SnapshotDateTime);

return snapshot.SnapshotIdentifier;
}

string errorResponseText = await snapshotsResponse.Content.ReadAsStringAsync().ConfigureAwait(false);

_logger.Error(
$"Unable to get snapshot identifier from API at '{sourceApiClient.HttpClient.BaseAddress}{snapshotsRelativePath}'. Request for available snapshots returned status '{snapshotsResponse.StatusCode}' with message body: {errorResponseText}");
_logger.Error("Unable to get snapshot identifier from API at '{BaseAddress}{SnapshotsRelativePath}'. Request for available snapshots returned status '{StatusCode}' with message body: {ErrorResponseText}",
sourceApiClient.HttpClient.BaseAddress, snapshotsRelativePath, snapshotsResponse.StatusCode, errorResponseText);

return null;
}
Expand Down
Loading

0 comments on commit 28c5891

Please sign in to comment.