diff --git a/docs/API-Publisher-Configuration.md b/docs/API-Publisher-Configuration.md index 87c4c24..c481b37 100644 --- a/docs/API-Publisher-Configuration.md +++ b/docs/API-Publisher-Configuration.md @@ -27,8 +27,9 @@ Defines general behavior of the Ed-Fi API Publisher. | Options:UseChangeVersionPaging
`--useChangeVersionPaging` | Indicates whether or not to use change version paging.
(_Default value: false_) | | Options:ChangeVersionPagingWindowSize
`--changeVersionPagingWindowSize` | Indicates the change version paging window size.
(_Default value: 25000_) | | Options:EnableRateLimit
`--enableRateLimit` | Indicates whether or not to use rate limiting.
(_Default value: false_) | -| Options:RateLimitNumberExecutions
`--rateLimitNumberExecutions` | Indicates the maximum number of executions allowed within the defined time window.
(_Default value: 100_) | +| Options:RateLimitNumberExecutions
`--rateLimitNumberExecutions` | Indicates the maximum number of executions allowed within the defined time window.
(_Default value: 30_) | | Options:RateLimitTimeSeconds
`--rateLimitTimeSeconds` | Indicates the the time span for the rate limit in seconds.
(_Default value: 1_) | +| Options:RateLimitMaxRetries
`--rateLimitMaxRetries` | Indicates the number of times the Ed-Fi API publisher will attempt to _resend_ a request, rejected by rate limiting, to the source or destination APIs before determining that the failure is permanent.
(_Default value: 10_) | ## API Connections diff --git a/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json b/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json index 4c86649..223ef28 100644 --- a/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json +++ b/src/EdFi.Tools.ApiPublisher.Cli/apiPublisherSettings.json @@ -13,8 +13,9 @@ "useChangeVersionPaging": false, "changeVersionPagingWindowSize": 25000, "enableRateLimit": false, - "rateLimitNumberExecutions": 100, + "rateLimitNumberExecutions": 30, "rateLimitTimeSeconds": 1, + "rateLimitMaxRetries": 10, "useReversePaging": false }, "authorizationFailureHandling": [ diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs index e95d4c3..52f0847 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/DependencyResolution/ApiSourceResourceItemProvider.cs @@ -167,7 +167,7 @@ public ApiSourceResourceItemProvider(ISourceEdFiApiClientProvider sourceEdFiApiC } catch (RateLimitRejectedException) { - _logger.Warning($"{sourceEdFiApiClient.DataManagementApiSegment}{resourceItemUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{sourceEdFiApiClient.DataManagementApiSegment}{resourceItemUrl}: Rate limit exceeded. Please try again later."); return (false, null); } //---------------------------------------------------------------------------------------------- diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs index 8b1af62..f85871c 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/Counting/EdFiApiSourceTotalCountProvider.cs @@ -74,8 +74,7 @@ public EdFiApiSourceTotalCountProvider(ISourceEdFiApiClientProvider sourceEdFiAp string requestUri = $"{edFiApiClient.DataManagementApiSegment}{resourceUrl}?offset=0&limit=1&totalCount=true{changeWindowQueryStringParameters}"; - - return RequestHelpers.SendGetRequestAsync(edFiApiClient, resourceUrl, requestUri, ct).Result; + return await RequestHelpers.SendGetRequestAsync(edFiApiClient, resourceUrl, requestUri, ct); }, new Context(), cancellationToken); string responseContent = null; @@ -137,7 +136,7 @@ await HandleResourceCountRequestErrorAsync(resourceUrl, errorHandlingBlock, apiR } catch (RateLimitRejectedException) { - _logger.Warning($"{edFiApiClient.DataManagementApiSegment}{resourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{edFiApiClient.DataManagementApiSegment}{resourceUrl}: Rate limit exceeded. Please try again later."); return (false, 0); } } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs index 0751214..54e5301 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Source/MessageHandlers/EdFiApiStreamResourcePageMessageHandler.cs @@ -199,7 +199,7 @@ public async Task> HandleStreamResourcePageAsyn } catch (RateLimitRejectedException) { - _logger.Warning($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); } break; } diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs index 11e4988..c2220e1 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/ChangeResourceKeyProcessingBlocksFactory.cs @@ -235,7 +235,7 @@ private TransformManyBlock CreateG } catch (RateLimitRejectedException) { - _logger.Warning($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{message.ResourceUrl}: Rate limit exceeded. Please try again later."); throw; } catch (Exception ex) @@ -367,7 +367,7 @@ private TransformManyBlock CreateChangeKeyBl } catch (RateLimitRejectedException) { - _logger.Warning($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); throw; } catch (Exception ex) diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs index 482862b..3f54170 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/DeleteResourceProcessingBlocksFactory.cs @@ -187,7 +187,7 @@ private TransformManyBlock CreateG } catch (RateLimitRejectedException) { - _logger.Warning($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); throw; } catch (Exception ex) @@ -315,7 +315,7 @@ private TransformManyBlock CreateDeleteReso } catch (RateLimitRejectedException) { - _logger.Warning($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); + _logger.Fatal($"{msg.ResourceUrl}: Rate limit exceeded. Please try again later."); throw; } catch (Exception ex) diff --git a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs index 85dd4bd..d00a9a8 100644 --- a/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Connections.Api/Processing/Target/Blocks/PostResourceProcessingBlocksFactory.cs @@ -413,8 +413,8 @@ await HandlePostItemMessage( } catch (RateLimitRejectedException) { - _logger.Warning($"{postItemMessage.ResourceUrl}: Rate limit exceeded. Please try again later."); - throw; + _logger.Fatal($"{postItemMessage.ResourceUrl}: Rate limit exceeded. Please try again later."); + return Enumerable.Empty(); } catch (Exception ex) { diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs index a388a74..594b5a9 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ApiPublisherSettings.cs @@ -91,10 +91,12 @@ public int MaxDegreeOfParallelismForPostResourceItem public bool EnableRateLimit { get; set; } = false; - public int RateLimitNumberExecutions { get; set; } = 100; + public int RateLimitNumberExecutions { get; set; } = 30; public double RateLimitTimeSeconds { get; set; } = 1; + public int RateLimitMaxRetries { get; set; } = 5; + public bool UseReversePaging { get; set; } = false; } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs index 6016a62..b32ca5a 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/ConfigurationBuilderFactory.cs @@ -75,6 +75,7 @@ public IConfigurationBuilder Create(string[] commandLineArgs) ["--enableRateLimit"] = "Options:EnableRateLimit", ["--rateLimitNumberExecutions"] = "Options:RateLimitNumberExecutions", ["--rateLimitTimeSeconds"] = "Options:RateLimitTimeSeconds", + ["--rateLimitMaxRetries"] = "Options:RateLimitMaxRetries", ["--useReversePaging"] = "Options:UseReversePaging", diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/IRateLimiting.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/IRateLimiting.cs index 50e54ae..809d856 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/IRateLimiting.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/IRateLimiting.cs @@ -2,6 +2,7 @@ // Licensed to the Ed-Fi Alliance under one or more agreements. // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. +using Polly; using Polly.RateLimit; using System; using System.Threading.Tasks; @@ -10,6 +11,6 @@ namespace EdFi.Tools.ApiPublisher.Core.Configuration; public interface IRateLimiting { - AsyncRateLimitPolicy GetRateLimitingPolicy(); + IAsyncPolicy GetRateLimitingPolicy(); Task ExecuteAsync(Func> action); } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs b/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs index 7e4fee2..ce202d2 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Configuration/PollyRateLimiter.cs @@ -8,12 +8,15 @@ using System.Threading.Tasks; using System.Net.Http; using System.Threading.RateLimiting; +using Serilog; namespace EdFi.Tools.ApiPublisher.Core.Configuration; public class PollyRateLimiter : IRateLimiting { - private readonly AsyncRateLimitPolicy _rateLimiter; + private readonly IAsyncPolicy _rateLimiter; + private readonly IAsyncPolicy _retryPolicyForRateLimit; + private readonly ILogger _logger = Log.ForContext(typeof(PollyRateLimiter)); public PollyRateLimiter(Options options) { @@ -21,15 +24,33 @@ public PollyRateLimiter(Options options) options.RateLimitNumberExecutions, TimeSpan.FromSeconds(options.RateLimitTimeSeconds), options.RateLimitNumberExecutions); + _retryPolicyForRateLimit = Policy + .Handle() + .WaitAndRetryAsync(options.RateLimitMaxRetries, // Number of retries + retryAttempt => TimeSpan.FromSeconds(options.RateLimitTimeSeconds), + (exception, timeSpan, retryCount, context) => + { + var delay = TimeSpan.FromSeconds(options.RateLimitTimeSeconds); + _logger.Warning($"Retry {retryCount} due to rate limit exceeded. Waiting {delay.TotalSeconds} seconds before next retry."); + } + ); } public async Task ExecuteAsync(Func> action) { - return await _rateLimiter.ExecuteAsync(action); + try + { + return await _rateLimiter.ExecuteAsync(action); + } + catch (RateLimitRejectedException) { + _logger.Fatal("Rate limit exceeded. Please try again later."); + throw; + } + } - public AsyncRateLimitPolicy GetRateLimitingPolicy() + public IAsyncPolicy GetRateLimitingPolicy() { - return _rateLimiter; + return Policy.WrapAsync(_retryPolicyForRateLimit, _rateLimiter); } } diff --git a/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs b/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs index c169a47..b3efc7d 100644 --- a/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs +++ b/src/EdFi.Tools.ApiPublisher.Core/Processing/ChangeProcessor.cs @@ -15,6 +15,7 @@ using EdFi.Tools.ApiPublisher.Core.Processing.Messages; using EdFi.Tools.ApiPublisher.Core.Versioning; using Newtonsoft.Json; +using Polly.RateLimit; using Serilog; using Serilog.Events; using System; @@ -192,6 +193,11 @@ await _sourceIsolationApplicator.ApplySourceSnapshotIdentifierAsync(configuratio await UpdateChangeVersionAsync(configuration, changeWindow) .ConfigureAwait(false); } + catch (RateLimitRejectedException ex) + { + _logger.Fatal(ex.Message); + throw; + } catch (Exception ex) { _logger.Fatal($"An unhandled exception occurred during processing: {ex}"); diff --git a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs index 33428b5..dc04718 100644 --- a/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs +++ b/src/EdFi.Tools.ApiPublisher.Tests/Processing/RateLimitingTests.cs @@ -54,6 +54,7 @@ public void RateLimitedMethod_Should_Throw_RateLimiterRejectedException_On_Overl options.EnableRateLimit = true; options.RateLimitNumberExecutions = 5; options.RateLimitTimeSeconds = 1; + options.RateLimitMaxRetries = 1; var rateLimiter = new PollyRateLimiter(options); var methodToTest = new MockRateLimitingMethod(rateLimiter);