Add failed HTTP request retry policy (#259)

clients/imodels-client-authoring/src/IModelsClient.ts

@@ -4,7 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 import "reflect-metadata";
 
-import { AxiosRestClient } from "@itwin/imodels-client-management/lib/base/internal";
+import { AxiosRestClient, AxiosRetryPolicy, ExponentialBackoffAlgorithm } from "@itwin/imodels-client-management/lib/base/internal";
+import { Constants } from "@itwin/imodels-client-management/lib/Constants";
 import { AzureClientStorage, BlockBlobClientWrapperFactory } from "@itwin/object-storage-azure";
 import { ClientStorage } from "@itwin/object-storage-core";
 
@@ -103,12 +104,21 @@ export class IModelsClient extends ManagementIModelsClient {
   private static fillAuthoringClientConfiguration(
     options: IModelsClientOptions | undefined
   ): RecursiveRequired<IModelsClientOptions> {
+    const retryPolicy = options?.retryPolicy ?? new AxiosRetryPolicy({
+      maxRetries: Constants.retryPolicy.maxRetries,
+      backoffAlgorithm: new ExponentialBackoffAlgorithm({
+        baseDelayInMs: Constants.retryPolicy.baseDelayInMs,
+        factor: Constants.retryPolicy.delayFactor
+      })
+    });
+
     return {
       api: this.fillApiConfiguration(options?.api),
-      restClient: options?.restClient ?? new AxiosRestClient(IModelsErrorParser.parse),
+      restClient: options?.restClient ?? new AxiosRestClient(IModelsErrorParser.parse, retryPolicy),
       localFileSystem: options?.localFileSystem ?? new NodeLocalFileSystem(),
       cloudStorage: options?.cloudStorage ?? new AzureClientStorage(new BlockBlobClientWrapperFactory()),
-      headers: options?.headers ?? {}
+      headers: options?.headers ?? {},
+      retryPolicy
     };
   }
 }

clients/imodels-client-management/src/Constants.ts

@@ -6,18 +6,28 @@ export class Constants {
   public static api = {
     baseUrl: "https://api.bentley.com/imodels",
     version: "itwin-platform.v2"
-  };
+  } as const;
 
   public static headers = {
     accept: "Accept",
     authorization: "Authorization",
     contentType: "Content-Type",
     prefer: "Prefer",
     location: "Location"
-  };
+  } as const;
 
   public static time = {
     sleepPeriodInMs: 1000,
-    iModelInitiazationTimeOutInMs: 5 * 60 * 1000
-  };
+    iModelInitializationTimeOutInMs: 5 * 60 * 1000
+  } as const;
+
+  public static httpStatusCodes = {
+    internalServerError: 500
+  } as const;
+
+  public static retryPolicy = {
+    maxRetries: 3,
+    baseDelayInMs: 300,
+    delayFactor: 3
+  } as const;
 }

clients/imodels-client-management/src/IModelsClient.ts

@@ -2,8 +2,8 @@
  * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
  * See LICENSE.md in the project root for license terms and full copyright notice.
  *--------------------------------------------------------------------------------------------*/
-import { AxiosRestClient, IModelsErrorParser } from "./base/internal";
-import { ApiOptions, HeaderFactories, RecursiveRequired, RestClient } from "./base/types";
+import { AxiosRestClient, AxiosRetryPolicy, ExponentialBackoffAlgorithm, IModelsErrorParser } from "./base/internal";
+import { ApiOptions, HeaderFactories, HttpRequestRetryPolicy, RecursiveRequired, RestClient } from "./base/types";
 import { Constants } from "./Constants";
 import { BriefcaseOperations, ChangesetOperations, IModelOperations, NamedVersionOperations, OperationOperations, ThumbnailOperations, UserOperations, UserPermissionOperations } from "./operations";
 import { ChangesetExtendedDataOperations } from "./operations/changeset-extended-data/ChangesetExtendedDataOperations";
@@ -23,6 +23,13 @@ export interface IModelsClientOptions {
   api?: ApiOptions;
   /** Additional headers to add to each request. See {@link HeaderFactories}. */
   headers?: HeaderFactories;
+  /**
+   * Retry policy that is used with {@link AxiosRestClient} when HTTP request sending fails.
+   * If `undefined` the default implementation is used with exponential backoff algorithm
+   * (3 retries with incremental sleep durations of 300ms, 900ms and 2700ms).
+   * See {@link AxiosRetryPolicy} and {@link ExponentialBackoffAlgorithm}.
+   */
+  retryPolicy?: HttpRequestRetryPolicy;
 }
 
 /**
@@ -103,10 +110,19 @@ export class IModelsClient {
   private static fillManagementClientConfiguration(
     options: IModelsClientOptions | undefined
   ): RecursiveRequired<IModelsClientOptions> {
+    const retryPolicy = options?.retryPolicy ?? new AxiosRetryPolicy({
+      maxRetries: Constants.retryPolicy.maxRetries,
+      backoffAlgorithm: new ExponentialBackoffAlgorithm({
+        baseDelayInMs: Constants.retryPolicy.baseDelayInMs,
+        factor: Constants.retryPolicy.delayFactor
+      })
+    });
+
     return {
       api: this.fillApiConfiguration(options?.api),
-      restClient: options?.restClient ?? new AxiosRestClient(IModelsErrorParser.parse),
-      headers: options?.headers ?? {}
+      restClient: options?.restClient ?? new AxiosRestClient(IModelsErrorParser.parse, retryPolicy),
+      headers: options?.headers ?? {},
+      retryPolicy
     };
   }
 

clients/imodels-client-management/src/base/internal/AxiosRestClient.ts

@@ -4,10 +4,12 @@
  *--------------------------------------------------------------------------------------------*/
 import axios, { AxiosRequestConfig, AxiosResponse } from "axios";
 
+import { HttpRequestRetryPolicy } from "../types";
 import { ContentType, HttpGetRequestParams, HttpRequestParams, HttpRequestWithBinaryBodyParams, HttpRequestWithJsonBodyParams, HttpResponse, RestClient } from "../types/RestClient";
 
 import { AxiosResponseHeadersAdapter } from "./AxiosResponseHeadersAdapter";
 import { ResponseInfo } from "./IModelsErrorParser";
+import { sleep } from "./UtilityFunctions";
 
 /**
  * Function that is called if the HTTP request fails and which returns an error that will be thrown by one of the
@@ -17,10 +19,14 @@ export type ParseErrorFunc = (response: ResponseInfo, originalError: Error & { c
 
 /** Default implementation for {@link RestClient} interface that uses `axios` library for sending the requests. */
 export class AxiosRestClient implements RestClient {
+  private static readonly retryCountUpperBound = 10;
+
   private _parseErrorFunc: ParseErrorFunc;
+  private _retryPolicy: HttpRequestRetryPolicy | null;
 
-  constructor(parseErrorFunc: ParseErrorFunc) {
+  constructor(parseErrorFunc: ParseErrorFunc, retryPolicy: HttpRequestRetryPolicy | null) {
     this._parseErrorFunc = parseErrorFunc;
+    this._retryPolicy = retryPolicy;
   }
 
   public sendGetRequest<TBody>(params: HttpGetRequestParams & { responseType: ContentType.Json }): Promise<HttpResponse<TBody>>;
@@ -78,7 +84,7 @@ export class AxiosRestClient implements RestClient {
 
   private async executeRequest<TBody>(requestFunc: () => Promise<AxiosResponse<TBody>>): Promise<HttpResponse<TBody>> {
     try {
-      const response = await requestFunc();
+      const response = await this.executeWithRetry(requestFunc);
 
       return {
         body: response.data,
@@ -92,5 +98,28 @@ export class AxiosRestClient implements RestClient {
       throw error;
     }
   }
+
+  private async executeWithRetry<TBody>(requestFunc: () => Promise<AxiosResponse<TBody>>): Promise<AxiosResponse<TBody>> {
+    let retriesInvoked = 0;
+    for (;;) {
+      try {
+        return await requestFunc();
+      } catch (error: unknown) {
+        if (
+          this._retryPolicy === null ||
+          retriesInvoked >= this._retryPolicy.maxRetries ||
+          retriesInvoked >= AxiosRestClient.retryCountUpperBound ||
+          !(await this._retryPolicy.shouldRetry({ retriesInvoked, error }))
+        ) {
+          throw error;
+        }
+
+        const sleepDurationInMs = this._retryPolicy.getSleepDurationInMs({ retriesInvoked: retriesInvoked++ });
+        if (sleepDurationInMs > 0) {
+          await sleep(sleepDurationInMs);
+        }
+      }
+    }
+  }
 }
 

clients/imodels-client-management/src/base/internal/AxiosRetryPolicy.ts

@@ -0,0 +1,37 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+import { isAxiosError } from "axios";
+
+import { Constants } from "../../Constants";
+import { GetSleepDurationInMsParams, HttpRequestRetryPolicy, ShouldRetryParams } from "../types";
+
+import { BackoffAlgorithm } from "./ExponentialBackoffAlgorithm";
+
+/** Default implementation for {@link HttpRequestRetryPolicy}. */
+export class AxiosRetryPolicy implements HttpRequestRetryPolicy {
+  private readonly _backoffAlgorithm: BackoffAlgorithm;
+
+  public constructor(params: {
+    maxRetries: number;
+    backoffAlgorithm: BackoffAlgorithm;
+  }) {
+    this.maxRetries = params.maxRetries;
+    this._backoffAlgorithm = params.backoffAlgorithm;
+  }
+
+  public readonly maxRetries: number;
+
+  public shouldRetry(params: ShouldRetryParams): boolean {
+    if (isAxiosError(params.error) && params.error.response?.status != null) {
+      return params.error.response.status >= Constants.httpStatusCodes.internalServerError;
+    }
+
+    return true;
+  }
+
+  public getSleepDurationInMs(params: GetSleepDurationInMsParams): number {
+    return this._backoffAlgorithm.getSleepDurationInMs(params.retriesInvoked);
+  }
+}

clients/imodels-client-management/src/base/internal/ExponentialBackoffAlgorithm.ts

@@ -0,0 +1,31 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+/** Backoff algorithm for calculating sleep time after a failed HTTP request. */
+export interface BackoffAlgorithm {
+  /** Calculates the sleep duration in milliseconds. */
+  getSleepDurationInMs: (attempt: number) => number;
+}
+
+/**
+ * Exponential backoff algorithm for calculating sleep time after a failed HTTP request.
+ * Default implementation for {@link BackoffAlgorithm}.
+ */
+export class ExponentialBackoffAlgorithm implements BackoffAlgorithm {
+  private readonly _baseDelayInMs: number;
+  private readonly _factor: number;
+
+  public constructor(params: {
+    baseDelayInMs: number;
+    factor: number;
+  }) {
+    this._baseDelayInMs = params.baseDelayInMs;
+    this._factor = params.factor;
+  }
+
+  public getSleepDurationInMs(attempt: number): number {
+    return Math.pow(this._factor, attempt) * this._baseDelayInMs;
+  }
+}

clients/imodels-client-management/src/base/internal/UtilityFunctions.ts

@@ -15,7 +15,7 @@ export async function waitForCondition(params: {
   timeOutInMs?: number;
 }): Promise<void> {
   const sleepPeriodInMs = Constants.time.sleepPeriodInMs;
-  const timeOutInMs = params.timeOutInMs ?? Constants.time.iModelInitiazationTimeOutInMs;
+  const timeOutInMs = params.timeOutInMs ?? Constants.time.iModelInitializationTimeOutInMs;
 
   for (let retries = Math.ceil(timeOutInMs / sleepPeriodInMs); retries > 0; --retries) {
     const isTargetStateReached = await params.conditionToSatisfy();

clients/imodels-client-management/src/base/internal/index.ts

@@ -10,3 +10,5 @@ export * from "./IModelsErrorParser";
 export * from "./OperationsBase";
 export * from "./UtilityTypes";
 export * from "./UtilityFunctions";
+export * from "./AxiosRetryPolicy";
+export * from "./ExponentialBackoffAlgorithm";

clients/imodels-client-management/src/base/types/HttpRequestRetryPolicy.ts

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ * Copyright (c) Bentley Systems, Incorporated. All rights reserved.
+ * See LICENSE.md in the project root for license terms and full copyright notice.
+ *--------------------------------------------------------------------------------------------*/
+
+export interface ShouldRetryParams {
+  /** The number of already invoked retries, starting from 0. */
+  retriesInvoked: number;
+  /** The error that was thrown when sending the HTTP request. */
+  error: unknown;
+}
+
+export interface GetSleepDurationInMsParams {
+  /** The number of already invoked retries, starting from 0. */
+  retriesInvoked: number;
+}
+
+/** A policy for handling failed HTTP requests. */
+export interface HttpRequestRetryPolicy {
+  /** The maximum number of HTTP request retries. */
+  get maxRetries(): number;
+
+  /** Returns `true` if HTTP request should be retried, `false` otherwise. */
+  shouldRetry(params: ShouldRetryParams): boolean | Promise<boolean>;
+
+  /** Gets the duration to sleep in milliseconds before resending the HTTP request. */
+  getSleepDurationInMs: (params: GetSleepDurationInMsParams) => number;
+}

clients/imodels-client-management/src/base/types/index.ts

@@ -21,3 +21,4 @@ export * from "./RestClient";
 export * from "./CommonInterfaces";
 export * from "./UtilityTypes";
 export * from "./IModelsErrorInterfaces";
+export * from "./HttpRequestRetryPolicy";

common/changes/@itwin/imodels-client-authoring/ab-add-retry-policy_2024-06-18-06-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@itwin/imodels-client-authoring",
+      "comment": "Add failed HTTP request retry policy.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@itwin/imodels-client-authoring"
+}

common/changes/@itwin/imodels-client-management/ab-add-retry-policy_2024-06-18-06-49.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@itwin/imodels-client-management",
+      "comment": "Add failed HTTP request retry policy.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@itwin/imodels-client-management"
+}