Skip to content

Commit

Permalink
Merge pull request data-integrations#35 from cloudsufi/feat/retry-dev…
Browse files Browse the repository at this point in the history 
…-failsafe

[PLUGIN-1783] Adding Retry with dev.failsafe
  • Loading branch information
@psainics
psainics authored May 13, 2024
2 parents 3bc7a6e + 40c9834 commit 02d8aa8
Show file tree
Hide file tree
Showing 14 changed files with 363 additions and 66 deletions.
13 changes: 6 additions & 7 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
<apache.olingo.v2>2.0.0</apache.olingo.v2>
<wiremock.version>2.27.2</wiremock.version>
<hydrator.version>2.7.0</hydrator.version>
<failsafe.version>3.3.2</failsafe.version>
<testSourceLocation>${project.basedir}/src/test/java/</testSourceLocation>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
Expand All @@ -61,7 +62,11 @@
</repositories>

<dependencies>

<dependency>
<groupId>dev.failsafe</groupId>
<artifactId>failsafe</artifactId>
<version>${failsafe.version}</version>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-api</artifactId>
Expand Down Expand Up @@ -365,12 +370,6 @@
<version>2.0.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>2.0.0</version>
</dependency>

</dependencies>

<build>
Expand Down
10 changes: 7 additions & 3 deletions src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ List<String> listEntities() throws TransportException, IOException {
URL dataURL = HttpUrl.parse(config.getBaseURL()).newBuilder().build().url();
SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config);
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsEntity
(dataURL, MediaType.APPLICATION_JSON, METADATA);
(dataURL, MediaType.APPLICATION_JSON);
try (InputStream inputStream = responseContainer.getResponseStream()) {
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
String result = reader.lines().collect(Collectors.joining(""));
Expand Down Expand Up @@ -225,7 +225,11 @@ private InputStream callEntityData(long top, String entityName)
addQueryParameter(TOP_OPTION, String.valueOf(top)).addQueryParameter(SELECT_OPTION, selectFields.toString())
.build().url();
SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config);
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL);
SuccessFactorsResponseContainer responseContainer =
successFactorsHttpClient.callSuccessFactorsWithRetry(
dataURL, MediaType.APPLICATION_JSON, SuccessFactorsPluginConfig.DEFAULT_INITIAL_RETRY_DURATION_SECONDS,
SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_DURATION_SECONDS,
SuccessFactorsPluginConfig.DEFAULT_RETRY_MULTIPLIER, SuccessFactorsPluginConfig.DEFAULT_MAX_RETRY_COUNT);

ExceptionParser.checkAndThrowException("", responseContainer);
return responseContainer.getResponseStream();
Expand Down Expand Up @@ -255,7 +259,7 @@ private InputStream getMetaDataStream(String entity) throws TransportException,
.addPathSegment(METADATACALL).build().url();
SuccessFactorsTransporter successFactorsHttpClient = new SuccessFactorsTransporter(config);
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient
.callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML, METADATA);
.callSuccessFactorsEntity(metadataURL, MediaType.APPLICATION_XML);
return responseContainer.getResponseStream();
}
}
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/io/cdap/plugin/successfactors/connector/SuccessFactorsConnectorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import io.cdap.cdap.api.annotation.Name;
import io.cdap.cdap.api.plugin.PluginConfig;
import io.cdap.cdap.etl.api.FailureCollector;
import io.cdap.plugin.successfactors.common.exception.SuccessFactorsServiceException;
import io.cdap.plugin.successfactors.common.exception.TransportException;
import io.cdap.plugin.successfactors.common.util.ResourceConstants;
import io.cdap.plugin.successfactors.common.util.SuccessFactorsUtil;
Expand Down Expand Up @@ -153,7 +152,7 @@ public void validateConnection(FailureCollector collector) {
SuccessFactorsResponseContainer responseContainer = null;
try {
responseContainer =
successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON, TEST);
successFactorsHttpClient.callSuccessFactorsEntity(testerURL, MediaType.APPLICATION_JSON);
} catch (TransportException e) {
LOG.error("Unable to fetch the response", e);
collector.addFailure("Unable to call SuccessFactorsEntity",
Expand Down
120 changes: 118 additions & 2 deletions src/main/java/io/cdap/plugin/successfactors/source/config/SuccessFactorsPluginConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ public class SuccessFactorsPluginConfig extends PluginConfig {
private static final String COMMON_ACTION = ResourceConstants.ERR_MISSING_PARAM_OR_MACRO_ACTION.getMsgForKey();
private static final Pattern PATTERN = Pattern.compile("\\(.*\\)");
private static final String SAP_SUCCESSFACTORS_ENTITY_NAME = "Entity Name";
private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration";
private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration";
private static final String NAME_RETRY_MULTIPLIER = "retryMultiplier";
private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount";
public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 2;
public static final int DEFAULT_RETRY_MULTIPLIER = 2;
public static final int DEFAULT_MAX_RETRY_COUNT = 3;
public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 10;

@Macro
@Name(ENTITY_NAME)
Expand Down Expand Up @@ -128,6 +136,30 @@ public class SuccessFactorsPluginConfig extends PluginConfig {
@Description("The existing connection to use.")
private SuccessFactorsConnectorConfig connection;

@Name(NAME_INITIAL_RETRY_DURATION)
@Description("Time taken for the first retry. Default is 2 seconds.")
@Nullable
@Macro
private Integer initialRetryDuration;

@Name(NAME_MAX_RETRY_DURATION)
@Description("Maximum time in seconds retries can take. Default is 300 seconds.")
@Nullable
@Macro
private Integer maxRetryDuration;

@Name(NAME_MAX_RETRY_COUNT)
@Description("Maximum number of retries allowed. Default is 3.")
@Nullable
@Macro
private Integer maxRetryCount;

@Name(NAME_RETRY_MULTIPLIER)
@Description("Multiplier for exponential backoff. Default is 2.")
@Nullable
@Macro
private Integer retryMultiplier;

@VisibleForTesting
public SuccessFactorsPluginConfig(String referenceName,
String baseURL,
Expand All @@ -142,7 +174,11 @@ public SuccessFactorsPluginConfig(String referenceName,
@Nullable String selectOption,
@Nullable String expandOption,
@Nullable String additionalQueryParameters,
String paginationType) {
String paginationType,
@Nullable Integer initialRetryDuration,
@Nullable Integer maxRetryDuration,
@Nullable Integer retryMultiplier,
@Nullable Integer maxRetryCount) {
this.connection = new SuccessFactorsConnectorConfig(username, password, baseURL, proxyUrl, proxyPassword,
proxyUsername);
this.referenceName = referenceName;
Expand All @@ -153,6 +189,10 @@ public SuccessFactorsPluginConfig(String referenceName,
this.expandOption = expandOption;
this.paginationType = paginationType;
this.additionalQueryParameters = additionalQueryParameters;
this.initialRetryDuration = initialRetryDuration;
this.maxRetryDuration = maxRetryDuration;
this.retryMultiplier = retryMultiplier;
this.maxRetryCount = maxRetryCount;
}
@Nullable
public SuccessFactorsConnectorConfig getConnection() {
Expand Down Expand Up @@ -210,6 +250,22 @@ public String getAdditionalQueryParameters() {
return this.additionalQueryParameters;
}

public int getInitialRetryDuration() {
return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration;
}

public int getMaxRetryDuration() {
return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration;
}

public int getRetryMultiplier() {
return retryMultiplier == null ? DEFAULT_RETRY_MULTIPLIER : retryMultiplier;
}

public int getMaxRetryCount() {
return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount;
}

/**
* Checks if the call to SuccessFactors service is required for metadata creation.
* condition parameters: ['host' | 'serviceName' | 'entityName' | 'username' | 'password']
Expand Down Expand Up @@ -243,6 +299,7 @@ public void validatePluginParameters(FailureCollector failureCollector) {
validateMandatoryParameters(failureCollector);
validateBasicCredentials(failureCollector);
validateEntityParameter(failureCollector);
validateRetryConfiguration(failureCollector);
failureCollector.getOrThrowException();
}

Expand Down Expand Up @@ -292,6 +349,43 @@ private void validateEntityParameter(FailureCollector failureCollector) {
}
}

/**
* Validates the retry configuration.
*
* @param failureCollector {@code FailureCollector}
*/
public void validateRetryConfiguration(FailureCollector failureCollector) {
if (containsMacro(NAME_INITIAL_RETRY_DURATION) || containsMacro(NAME_MAX_RETRY_DURATION) ||
containsMacro(NAME_MAX_RETRY_COUNT) || containsMacro(NAME_RETRY_MULTIPLIER)) {
return;
}
if (initialRetryDuration != null && initialRetryDuration <= 0) {
failureCollector.addFailure("Initial retry duration must be greater than 0.",
"Please specify a valid initial retry duration.")
.withConfigProperty(NAME_INITIAL_RETRY_DURATION);
}
if (maxRetryDuration != null && maxRetryDuration <= 0) {
failureCollector.addFailure("Max retry duration must be greater than 0.",
"Please specify a valid max retry duration.")
.withConfigProperty(NAME_MAX_RETRY_DURATION);
}
if (maxRetryCount != null && maxRetryCount <= 0) {
failureCollector.addFailure("Max retry count must be greater than 0.",
"Please specify a valid max retry count.")
.withConfigProperty(NAME_MAX_RETRY_COUNT);
}
if (retryMultiplier != null && retryMultiplier <= 1) {
failureCollector.addFailure("Retry multiplier must be strictly greater than 1.",
"Please specify a valid retry multiplier.")
.withConfigProperty(NAME_RETRY_MULTIPLIER);
}
if (maxRetryDuration != null && initialRetryDuration != null && maxRetryDuration <= initialRetryDuration) {
failureCollector.addFailure("Max retry duration must be greater than initial retry duration.",
"Please specify a valid max retry duration.")
.withConfigProperty(NAME_MAX_RETRY_DURATION);
}
}

/**
* Helper class to simplify {@link SuccessFactorsPluginConfig} class creation.
*/
Expand All @@ -310,6 +404,10 @@ public static class Builder {
private String proxyUrl;
private String proxyUsername;
private String proxyPassword;
private Integer initialRetryDuration;
private Integer maxRetryDuration;
private Integer retryMultiplier;
private Integer maxRetryCount;

public Builder referenceName(String referenceName) {
this.referenceName = referenceName;
Expand Down Expand Up @@ -379,11 +477,29 @@ public Builder additionalQueryParameters(@Nullable String additionalQueryParamet
return this;
}

public Builder setInitialRetryDuration(Integer initialRetryDuration) {
this.initialRetryDuration = initialRetryDuration;
return this;
}
public Builder setMaxRetryDuration(Integer maxRetryDuration) {
this.maxRetryDuration = maxRetryDuration;
return this;
}
public Builder setRetryMultiplier(Integer retryMultiplier) {
this.retryMultiplier = retryMultiplier;
return this;
}
public Builder setMaxRetryCount(Integer maxRetryCount) {
this.maxRetryCount = maxRetryCount;
return this;
}

public SuccessFactorsPluginConfig build() {
return new SuccessFactorsPluginConfig(referenceName, baseURL, entityName, associateEntityName, username,
password, proxyUrl, proxyUsername, proxyPassword,
filterOption, selectOption, expandOption, additionalQueryParameters,
paginationType);
paginationType, initialRetryDuration, maxRetryDuration,
retryMultiplier, maxRetryCount);
}
}
}
13 changes: 9 additions & 4 deletions src/main/java/io/cdap/plugin/successfactors/source/service/SuccessFactorsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public SuccessFactorsService(SuccessFactorsPluginConfig pluginConfig,
public void checkSuccessFactorsURL() throws TransportException, SuccessFactorsServiceException {

SuccessFactorsResponseContainer responseContainer =
successFactorsHttpClient.callSuccessFactorsEntity(urlContainer.getTesterURL(), MediaType.APPLICATION_JSON, TEST);
successFactorsHttpClient.callSuccessFactorsEntity(urlContainer.getTesterURL(), MediaType.APPLICATION_JSON);

ExceptionParser.checkAndThrowException(ResourceConstants.ERR_FAILED_ENTITY_VALIDATION.getMsgForKey(),
responseContainer);
Expand Down Expand Up @@ -163,7 +163,9 @@ private SuccessFactorsEntityProvider fetchServiceMetadata(InputStream metadataSt
*/
private InputStream callEntityMetadata() throws TransportException {
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient
.callSuccessFactorsEntity(urlContainer.getMetadataURL(), MediaType.APPLICATION_XML, METADATA);
.callSuccessFactorsWithRetry(urlContainer.getMetadataURL(), MediaType.APPLICATION_XML, pluginConfig
.getInitialRetryDuration(), pluginConfig.getMaxRetryDuration(), pluginConfig.getRetryMultiplier(),
pluginConfig.getMaxRetryCount());
return responseContainer.getResponseStream();
}

Expand Down Expand Up @@ -320,7 +322,10 @@ private InputStream callEntityData(@Nullable Long skip, @Nullable Long top)
} else {
dataURL = urlContainer.getDataFetchURL(skip, top);
}
SuccessFactorsResponseContainer responseContainer = successFactorsHttpClient.callSuccessFactorsWithRetry(dataURL);
SuccessFactorsResponseContainer responseContainer =
successFactorsHttpClient.callSuccessFactorsWithRetry(
dataURL, MediaType.APPLICATION_JSON, pluginConfig.getInitialRetryDuration(), pluginConfig.getMaxRetryDuration(),
pluginConfig.getRetryMultiplier(), pluginConfig.getMaxRetryCount());

ExceptionParser.checkAndThrowException("", responseContainer);
return responseContainer.getResponseStream();
Expand All @@ -337,7 +342,7 @@ public List<String> getNonNavigationalProperties() throws TransportException, Su

/**
* Filter the data stream after removing the expanded entity data.
*
*
* Data stream after conversion to JSON has the following format:
* "d": {
* "results": [
Expand Down
Loading

0 comments on commit 02d8aa8

Please sign in to comment.