From a8f0bb32987c8cee9e1405b09b69ce310957a6a2 Mon Sep 17 00:00:00 2001 From: Lironrad <64735199+Lironrad@users.noreply.github.com> Date: Thu, 14 Jul 2022 11:40:17 +0300 Subject: [PATCH 01/16] [src] new feature: single task nodes --- JenkinsWiki.adoc | 5 + .../spotinst/cloud/AwsSpotinstCloud.java | 4 +- .../spotinst/cloud/BaseSpotinstCloud.java | 114 ++++++++++++------ .../cloud/SpotGlobalExecutorOverride.java | 4 + .../spotinst/slave/SpotinstComputer.java | 46 ++++++- .../slave/SpotinstNonLocalizable.java | 12 ++ .../slave/SpotinstSingleTaskOfflineCause.java | 9 ++ .../cloud/BaseSpotinstCloud/config.jelly | 9 +- 8 files changed, 165 insertions(+), 38 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/slave/SpotinstNonLocalizable.java create mode 100644 src/main/java/hudson/plugins/spotinst/slave/SpotinstSingleTaskOfflineCause.java diff --git a/JenkinsWiki.adoc b/JenkinsWiki.adoc index 1bbf5cfa..8e994761 100644 --- a/JenkinsWiki.adoc +++ b/JenkinsWiki.adoc @@ -40,6 +40,11 @@ termination" [SpotinstPlugin-Versionhistory] == Version history +[SpotinstPlugin-Version2.2.8(Jul14,2022)] +=== Version 2.2.8 (Jul 14, 2022) + +* Added 'Single Task Nodes' feature + [SpotinstPlugin-Version2.2.7(Jan17,2022)] === Version 2.2.7 (Jan 17, 2022) diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index b82bf72e..8cbd91f8 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -195,7 +195,7 @@ protected Integer getDefaultExecutorsNumber(String instanceType) { //region Private Methods @Override - protected Integer getNumOfExecutors(String instanceType) { + protected int getOverridedNumberOfExecutors(String instanceType) { Integer retVal; if (executorsByInstanceType == null) { @@ -207,7 +207,7 @@ protected Integer getNumOfExecutors(String instanceType) { LOGGER.info(String.format("We have a weight definition for this type of %s", retVal)); } else { - retVal = super.getNumOfExecutors(instanceType); + retVal = NO_OVERRIDED_NUM_OF_EXECUTORS; } return retVal; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index e02e70c2..4071b270 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -1,5 +1,6 @@ package hudson.plugins.spotinst.cloud; +import edu.umd.cs.findbugs.annotations.CheckForNull; import hudson.DescriptorExtensionList; import hudson.model.*; import hudson.model.labels.LabelAtom; @@ -14,6 +15,7 @@ import hudson.tools.ToolLocationNodeProperty; import jenkins.model.Jenkins; import org.apache.commons.lang.BooleanUtils; +import org.kohsuke.stapler.DataBoundSetter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,25 +32,27 @@ public abstract class BaseSpotinstCloud extends Cloud { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); - protected String accountId; - protected String groupId; - protected Map pendingInstances; - protected Map slaveInstancesDetailsByInstanceId; - private String labelString; - private String idleTerminationMinutes; - private String workspaceDir; - private Set labelSet; - private SlaveUsageEnum usage; - private String tunnel; - private String vmargs; - private EnvironmentVariablesNodeProperty environmentVariables; - private ToolLocationNodeProperty toolLocations; - private Boolean shouldUseWebsocket; - private Boolean shouldRetriggerBuilds; - private ComputerConnector computerConnector; - private ConnectionMethodEnum connectionMethod; - private Boolean shouldUsePrivateIp; - private SpotGlobalExecutorOverride globalExecutorOverride; + protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; + protected String accountId; + protected String groupId; + protected Map pendingInstances; + protected Map slaveInstancesDetailsByInstanceId; + private String labelString; + private String idleTerminationMinutes; + private String workspaceDir; + private Set labelSet; + private SlaveUsageEnum usage; + private String tunnel; + private String vmargs; + private EnvironmentVariablesNodeProperty environmentVariables; + private ToolLocationNodeProperty toolLocations; + private Boolean shouldUseWebsocket; + private Boolean shouldRetriggerBuilds; + private Boolean isSingleTaskNodesEnabled; + private ComputerConnector computerConnector; + private ConnectionMethodEnum connectionMethod; + private Boolean shouldUsePrivateIp; + private SpotGlobalExecutorOverride globalExecutorOverride; //endregion //region Constructor @@ -106,7 +110,6 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat else { this.globalExecutorOverride = new SpotGlobalExecutorOverride(false, 1); } - } //endregion @@ -585,28 +588,43 @@ protected PendingExecutorsCounts getPendingExecutors(ProvisionRequest request) { protected Integer getNumOfExecutors(String instanceType) { Integer retVal; + boolean isSingleTaskNodesEnabled = getIsSingleTaskNodesEnabled(); - Integer globalOverrideExecutorsNumber = getExecutorsFromGlobalOverride(); - - if (globalOverrideExecutorsNumber != null) { - LOGGER.debug(String.format("Overriding executors for instance type %s to be %s", instanceType, - globalOverrideExecutorsNumber)); - retVal = globalOverrideExecutorsNumber; + if (isSingleTaskNodesEnabled) { + retVal = 1; } else { - Integer defaultNumberOfExecutors = getDefaultExecutorsNumber(instanceType); + int overridedNumOfExecutors = getOverridedNumberOfExecutors(instanceType); + boolean isNumOfExecutorsOverrided = overridedNumOfExecutors != NO_OVERRIDED_NUM_OF_EXECUTORS; - if (defaultNumberOfExecutors != null) { - retVal = defaultNumberOfExecutors; + if(isNumOfExecutorsOverrided){ + retVal = overridedNumOfExecutors; } else { - retVal = 1; - String warningMsg = String.format( - "Failed to determine # of executors for instance type %s, defaulting to %s executor(s). Group ID: %s", - instanceType, retVal, this.getGroupId()); - LOGGER.warn(warningMsg); + Integer globalOverrideExecutorsNumber = getExecutorsFromGlobalOverride(); + + if (globalOverrideExecutorsNumber != null) { + LOGGER.debug(String.format("Overriding executors for instance type %s to be %s", instanceType, + globalOverrideExecutorsNumber)); + retVal = globalOverrideExecutorsNumber; + } + else { + Integer defaultNumberOfExecutors = getDefaultExecutorsNumber(instanceType); + + if (defaultNumberOfExecutors != null) { + retVal = defaultNumberOfExecutors; + } + else { + retVal = 1; + String warningMsg = String.format( + "Failed to determine # of executors for instance type %s, defaulting to %s executor(s). Group ID: %s", + instanceType, retVal, this.getGroupId()); + LOGGER.warn(warningMsg); + } + } } + } LOGGER.debug(String.format("instance type executors number was set to %s", retVal)); @@ -614,6 +632,10 @@ protected Integer getNumOfExecutors(String instanceType) { return retVal; } + protected int getOverridedNumberOfExecutors(String instanceType) { + return NO_OVERRIDED_NUM_OF_EXECUTORS; + } + protected Integer getPendingThreshold() { return Constants.PENDING_INSTANCE_TIMEOUT_IN_MINUTES; } @@ -734,6 +756,30 @@ public SpotGlobalExecutorOverride getGlobalExecutorOverride() { public void setGlobalExecutorOverride(SpotGlobalExecutorOverride globalExecutorOverride) { this.globalExecutorOverride = globalExecutorOverride; } + + + public Boolean getIsSingleTaskNodesEnabled() { + if (this.isSingleTaskNodesEnabled == null) { + return false; + } + else { + return isSingleTaskNodesEnabled; + } + } + + @DataBoundSetter + public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { + this.isSingleTaskNodesEnabled = isSingleTaskNodesEnabled; + + // if enabled, enable and override GlobalExecutorOverride to 1 + // better clarity to user, avoid race conditions + boolean shouldDisableGlobalExecutors = isSingleTaskNodesEnabled != null && isSingleTaskNodesEnabled && this.globalExecutorOverride != null; + if (shouldDisableGlobalExecutors) { + this.globalExecutorOverride.setIsEnabled(false); + } + } + + //endregion //region Abstract Methods diff --git a/src/main/java/hudson/plugins/spotinst/cloud/SpotGlobalExecutorOverride.java b/src/main/java/hudson/plugins/spotinst/cloud/SpotGlobalExecutorOverride.java index bfc0e0aa..82411ad1 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/SpotGlobalExecutorOverride.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/SpotGlobalExecutorOverride.java @@ -56,5 +56,9 @@ public Integer getExecutors() { public boolean getIsEnabled() { return isEnabled; } + + public void setIsEnabled(boolean isEnabled) { + this.isEnabled = isEnabled; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index f40fa93d..37cb2ee2 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -1,11 +1,16 @@ package hudson.plugins.spotinst.slave; +import hudson.model.Executor; import hudson.model.Node; +import hudson.model.Queue; +import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; import hudson.slaves.SlaveComputer; import org.kohsuke.stapler.HttpRedirect; import org.kohsuke.stapler.HttpResponse; import org.kohsuke.stapler.HttpResponses; import org.kohsuke.stapler.interceptor.RequirePOST; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; @@ -15,7 +20,46 @@ public class SpotinstComputer extends SlaveComputer { //region Members - private long launchTime; + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstComputer.class); + private long launchTime; + //endregion + + //region overrides + // better than JobListener because in complicated pipelines Jenkins is the initial Node and + // we know the Node is a SpotinstNode (I'm not using that word). + @Override + public void taskAccepted(Executor executor, Queue.Task task) { + super.taskAccepted(executor, task); + SpotinstSlave spotinstNode = this.getNode(); + + if (spotinstNode != null) { + BaseSpotinstCloud spotinstCloud = spotinstNode.getSpotinstCloud(); + + if (spotinstCloud != null) { + if (spotinstCloud.getIsSingleTaskNodesEnabled()) { + String msg = String.format( + "Node %s has accepted a job and 'Single Task Nodes' setting on Cloud %s is on. Node will not accept any more jobs.", + spotinstNode.getNodeName(), spotinstCloud.getDisplayName()); + LOGGER.info(msg); + this.setAcceptingTasks(false); + // I see much better responsiveness from Jenkins in case of setting offline, both have same effect + // should also be decided with UI because each combo looks different in the node list. + SpotinstNonLocalizable spotinstNonLocalizable = new SpotinstNonLocalizable(msg); + SpotinstSingleTaskOfflineCause spotinstSingleTaskOfflineCause = new SpotinstSingleTaskOfflineCause(spotinstNonLocalizable); + this.setTemporarilyOffline(true,spotinstSingleTaskOfflineCause); + } + } + else { + LOGGER.error(String.format( + "Node %s has accepted a job but can't determine 'Single Task Nodes' setting because SpotinstNode's SpotinstCloud appears to be null.", + spotinstNode.getNodeName())); + } + } else { + LOGGER.error(String.format( + "Executor of Node %s has accepted a job but can't determine 'Single Task Nodes' setting because SpotinstNode is null.", executor.getOwner().getName())); + } + } + //endregion //region Constructor diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstNonLocalizable.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstNonLocalizable.java new file mode 100644 index 00000000..18908409 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstNonLocalizable.java @@ -0,0 +1,12 @@ +package hudson.plugins.spotinst.slave; + +import jenkins.util.NonLocalizable; + +import java.util.Locale; + +class SpotinstNonLocalizable extends NonLocalizable { + + public SpotinstNonLocalizable(String nonLocalizable) { + super(nonLocalizable); + } +} diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSingleTaskOfflineCause.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSingleTaskOfflineCause.java new file mode 100644 index 00000000..96243fe3 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSingleTaskOfflineCause.java @@ -0,0 +1,9 @@ +package hudson.plugins.spotinst.slave; + +import hudson.slaves.OfflineCause; + +public class SpotinstSingleTaskOfflineCause extends OfflineCause.SimpleOfflineCause { + public SpotinstSingleTaskOfflineCause(SpotinstNonLocalizable nonLocalizable) { + super(nonLocalizable); + } +} diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly index ed5536ae..e14eb586 100644 --- a/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly +++ b/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly @@ -116,9 +116,16 @@ - + + + + Each node will perform at most one job and get terminated after "Idle minutes before termination" (see setting above). + Enabling this setting effectively means: override the Default executor count setting and ignore any instance-weight overrides if those are set. + + + From cffe57505b21aa9feb6fa1726dff678bad597f6c Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Tue, 18 Oct 2022 12:32:08 +0300 Subject: [PATCH 02/16] temp --- pom.xml | 5 + .../plugins/spotinst/api/SpotinstApi.java | 72 ++++++++- .../spotinst/api/infra/RestClient.java | 34 ++++ .../spotinst/cloud/AwsSpotinstCloud.java | 81 ++++++---- .../spotinst/cloud/BaseSpotinstCloud.java | 153 ++++++++++++++++-- .../spotinst/common/SpotinstContext.java | 40 ++++- .../jobs/SpotinstGroupsOwnerMonitor.java | 88 ++++++++++ .../model/redis/RedisDeleteKeyResponse.java | 6 + .../model/redis/RedisGetValueResponse.java | 6 + .../model/redis/RedisSetKeyRequest.java | 39 +++++ .../model/redis/RedisSetKeyResponse.java | 6 + .../plugins/spotinst/repos/IRedisRepo.java | 12 ++ .../plugins/spotinst/repos/RedisRepo.java | 59 +++++++ .../plugins/spotinst/repos/RepoManager.java | 10 ++ .../plugins/spotinst/slave/SpotinstSlave.java | 77 +++++---- 15 files changed, 608 insertions(+), 80 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java create mode 100644 src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java create mode 100644 src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java diff --git a/pom.xml b/pom.xml index 2e552473..11257804 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,11 @@ plain-credentials 1.7 + + org.apache.commons + commons-collections4 + 4.4 + diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index f1a6fa41..14e3a4c7 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -7,6 +7,10 @@ import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.model.azure.*; import hudson.plugins.spotinst.model.gcp.*; +import hudson.plugins.spotinst.model.redis.RedisDeleteKeyResponse; +import hudson.plugins.spotinst.model.redis.RedisGetValueResponse; +import hudson.plugins.spotinst.model.redis.RedisSetKeyRequest; +import hudson.plugins.spotinst.model.redis.RedisSetKeyResponse; import jenkins.model.Jenkins; import org.apache.commons.httpclient.HttpStatus; import org.slf4j.Logger; @@ -19,7 +23,7 @@ public class SpotinstApi { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstApi.class); - private final static String SPOTINST_API_HOST = "https://api.spotinst.io"; + private final static String SPOTINST_API_HOST = "http://localhost:3100"; private final static String HEADER_AUTH = "Authorization"; private final static String AUTH_PREFIX = "Bearer "; private final static String HEADER_CONTENT_TYPE = "Content-Type"; @@ -313,6 +317,72 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI } //endregion + //Redis + public static T getRedisValue(String groupId, String accountId) throws ApiException { + T retVal = null; + + Map headers = buildHeaders(); + + Map queryParams = buildQueryParams(accountId); + + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin", headers, queryParams); + + RedisGetValueResponse redisValue = getCastedResponse(response, RedisGetValueResponse.class); + + if (redisValue.getResponse().getItems().size() > 0) { + retVal = redisValue.getResponse().getItems().get(0); + } + + return retVal; + } + + public static String setRedisKey(String groupId, String accountId, String orchestratorIdentifier, Integer ttl) throws ApiException { + String retVal = null; + + Map headers = buildHeaders(); + + Map queryParams = buildQueryParams(accountId); + + RedisSetKeyRequest request = new RedisSetKeyRequest(); + request.setGroupId(groupId); + request.setOrchestratorIdentifier(orchestratorIdentifier); + request.setTtl(ttl); + + String body = JsonMapper.toJson(request); + + RestResponse response = + RestClient.sendPost(SPOTINST_API_HOST + "/aws/ec2/group/jenkinsPlugin", body, headers, queryParams); + + RedisSetKeyResponse redisValue = getCastedResponse(response, RedisSetKeyResponse.class); + + if (redisValue.getResponse().getItems().size() > 0) { + retVal = redisValue.getResponse().getItems().get(0); + } + + return retVal; + } + + public static Integer deleteRedisKey(String groupId, String accountId) throws ApiException { + Integer retVal = null; + + Map headers = buildHeaders(); + + Map queryParams = buildQueryParams(accountId); + + RestResponse response = + RestClient.sendDelete(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin", headers, queryParams); + + RedisDeleteKeyResponse redisValue = getCastedResponse(response, RedisDeleteKeyResponse.class); + + if (redisValue.getResponse().getItems().size() > 0) { + retVal = redisValue.getResponse().getItems().get(0); + } + + return retVal; + } + //endregion + //region Private Methods private static String buildUserAgent() { String retVal = null; diff --git a/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java b/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java index f5dbded5..5c6c9d7d 100644 --- a/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java +++ b/src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java @@ -61,6 +61,40 @@ public static RestResponse sendPut(String url, String body, Map return retVal; } + + public static RestResponse sendDelete(String url, Map headers, + Map queryParams) throws ApiException { + + HttpDelete getRequest = new HttpDelete(url); + addQueryParams(getRequest, queryParams); + addHeaders(getRequest, headers); + RestResponse retVal = sendRequest(getRequest); + + return retVal; + } + + public static RestResponse sendPost(String url, String body, Map headers, + Map queryParams) throws ApiException { + + HttpPost postRequest = new HttpPost(url); + + if (body != null) { + StringEntity entity = null; + try { + entity = new StringEntity(body); + } + catch (UnsupportedEncodingException e) { + LOGGER.error("Exception when building put body", e); + } + postRequest.setEntity(entity); + } + + addQueryParams(postRequest, queryParams); + addHeaders(postRequest, headers); + RestResponse retVal = sendRequest(postRequest); + + return retVal; + } //endregion //region Private Methods diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 8cbd91f8..dd2649de 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -6,6 +6,7 @@ import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.ConnectionMethodEnum; import hudson.plugins.spotinst.common.SpotAwsInstanceTypesHelper; +import hudson.plugins.spotinst.common.SpotinstContext; import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.repos.IAwsGroupRepo; import hudson.plugins.spotinst.repos.RepoManager; @@ -117,55 +118,77 @@ public Boolean detachInstance(String instanceId) { @Override public void syncGroupInstances() { - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - if (instancesResponse.isRequestSucceed()) { - List instances = instancesResponse.getValue(); - LOGGER.info(String.format("There are %s instances in group %s", instances.size(), groupId)); + if (isGroupBelongToCloud) { + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); - Map slaveInstancesDetailsByInstanceId = new HashMap<>(); + if (instancesResponse.isRequestSucceed()) { + List instances = instancesResponse.getValue(); + LOGGER.info(String.format("There are %s instances in group %s", instances.size(), groupId)); - for (AwsGroupInstance instance : instances) { - SlaveInstanceDetails instanceDetails = SlaveInstanceDetails.build(instance); - slaveInstancesDetailsByInstanceId.put(instanceDetails.getInstanceId(), instanceDetails); - } + Map slaveInstancesDetailsByInstanceId = new HashMap<>(); + + for (AwsGroupInstance instance : instances) { + SlaveInstanceDetails instanceDetails = SlaveInstanceDetails.build(instance); + slaveInstancesDetailsByInstanceId.put(instanceDetails.getInstanceId(), instanceDetails); + } - this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId); + this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId); - addNewSlaveInstances(instances); - removeOldSlaveInstances(instances); + addNewSlaveInstances(instances); + removeOldSlaveInstances(instances); + } else { + LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, + instancesResponse.getErrors())); + } } - else { - LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, - instancesResponse.getErrors())); + else{ + try { + handleGroupDosNotBelongToCloud(groupId); + } + catch (Exception e) { + LOGGER.error(e.getMessage()); + } } } + @Override public Map getInstanceIpsById() { Map retVal = new HashMap<>(); - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - if (instancesResponse.isRequestSucceed()) { - List instances = instancesResponse.getValue(); + if (isGroupBelongToCloud) { + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); - for (AwsGroupInstance instance : instances) { - if (this.getShouldUsePrivateIp()) { - retVal.put(instance.getInstanceId(), instance.getPrivateIp()); - } - else { - retVal.put(instance.getInstanceId(), instance.getPublicIp()); + if (instancesResponse.isRequestSucceed()) { + List instances = instancesResponse.getValue(); + + for (AwsGroupInstance instance : instances) { + if (this.getShouldUsePrivateIp()) { + retVal.put(instance.getInstanceId(), instance.getPrivateIp()); + } else { + retVal.put(instance.getInstanceId(), instance.getPublicIp()); + } } + } else { + LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, + instancesResponse.getErrors())); } } - else { - LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, - instancesResponse.getErrors())); + else{ + try { + handleGroupDosNotBelongToCloud(groupId); + } + catch (Exception e) { + LOGGER.warn(e.getMessage()); + } } return retVal; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 4071b270..b3b09e49 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -1,11 +1,13 @@ package hudson.plugins.spotinst.cloud; -import edu.umd.cs.findbugs.annotations.CheckForNull; import hudson.DescriptorExtensionList; import hudson.model.*; import hudson.model.labels.LabelAtom; +import hudson.plugins.spotinst.api.infra.ApiResponse; import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.*; +import hudson.plugins.spotinst.repos.IRedisRepo; +import hudson.plugins.spotinst.repos.RepoManager; import hudson.plugins.spotinst.slave.*; import hudson.plugins.sshslaves.SSHConnector; import hudson.slaves.*; @@ -19,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.servlet.ServletException; import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; @@ -33,6 +36,8 @@ public abstract class BaseSpotinstCloud extends Cloud { private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; + private static final Integer REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS = 60 * 3; + public static final String REDIS_OK_STATUS = "OK"; protected String accountId; protected String groupId; protected Map pendingInstances; @@ -110,6 +115,21 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat else { this.globalExecutorOverride = new SpotGlobalExecutorOverride(false, 1); } + + boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + + if (isGroupBelongToCloud == false) { + try { + handleGroupDosNotBelongToCloud(this.groupId); + } + catch (Exception e){ + LOGGER.error(e.getMessage()); + } + } + else { + SpotinstContext.getInstance().getGroupsInUse().put(groupId,accountId); + } + } //endregion @@ -119,29 +139,39 @@ public Collection provision(Label label, int excessWorkload) { ProvisionRequest request = new ProvisionRequest(label, excessWorkload); LOGGER.info(String.format("Got provision slave request: %s", JsonMapper.toJson(request))); + boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - setNumOfNeededExecutors(request); + if (isGroupBelongToCloud) { - if (request.getExecutors() > 0) { - LOGGER.info(String.format("Need to scale up %s units", request.getExecutors())); + setNumOfNeededExecutors(request); - List slaves = provisionSlaves(request); + if (request.getExecutors() > 0) { + LOGGER.info(String.format("Need to scale up %s units", request.getExecutors())); - if (slaves.size() > 0) { - for (final SpotinstSlave slave : slaves) { - try { - Jenkins.getInstance().addNode(slave); - } - catch (IOException e) { - LOGGER.error(String.format("Failed to create node for slave: %s", slave.getInstanceId()), e); + List slaves = provisionSlaves(request); + + if (slaves.size() > 0) { + for (final SpotinstSlave slave : slaves) { + + try { + Jenkins.getInstance().addNode(slave); + } catch (IOException e) { + LOGGER.error(String.format("Failed to create node for slave: %s", slave.getInstanceId()), e); + } } } + } + else { + LOGGER.info("No need to scale up new slaves, there are some that are initiating"); } - } + } else { - LOGGER.info("No need to scale up new slaves, there are some that are initiating"); - } + try { + handleGroupDosNotBelongToCloud(groupId); + }catch (Exception e){ + LOGGER.warn(e.getMessage()); + } } return Collections.emptyList(); } @@ -434,6 +464,33 @@ private boolean isGlobalExecutorOverrideValid() { return retVal; } + + private void removeFromSuspendedGroupFetching(String groupId) { + boolean isGroupExistInSuspendedGroupFetching = SpotinstContext.getInstance().getSuspendedGroupFetching().containsKey(groupId); + + if(isGroupExistInSuspendedGroupFetching){ + SpotinstContext.getInstance().getSuspendedGroupFetching().remove(groupId); + } + } + + private void addGroupToRedis(String groupId, String accountId, String orchestratorIdentifier) { + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisSetKeyResponse = redisRepo.setKey(groupId, accountId, orchestratorIdentifier,REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); + + if (redisSetKeyResponse.isRequestSucceed()) { + String redisResponseSetKeyValue = redisSetKeyResponse.getValue(); + + if(redisResponseSetKeyValue.equals(REDIS_OK_STATUS)){ + LOGGER.info(String.format("Successfully added group %s to redis memory", groupId)); + } + else{ + LOGGER.error(String.format("Failed adding group %s to redis memory", groupId)); + } + } + else { + LOGGER.error("redis request failed"); + } + } //endregion //region Protected Methods @@ -643,6 +700,71 @@ protected Integer getPendingThreshold() { protected Integer getSlaveOfflineThreshold() { return Constants.SLAVE_OFFLINE_THRESHOLD_IN_MINUTES; } + + public void handleGroupDosNotBelongToCloud(String groupId) throws ServletException, IOException { + boolean isGroupExistInSuspendedGroupFetching = SpotinstContext.getInstance().getSuspendedGroupFetching().containsKey(groupId); + String message; + if(isGroupExistInSuspendedGroupFetching){ + message = String.format("Group %s is might be in use by other Jenkins orchestrator, please make sure it belong to this Jenkins orchestrator and try again after 3 minutes", groupId); + LOGGER.warn(message); + } + else{ + //pop up message in jenkins UI (ONLY after fetching groupId retries pass the ttl for key in redis is expired) + message = String.format("Group %s is in use by other Jenkins orchestrator", groupId); + LOGGER.error(message); + sendError(message); + //throw new ServletException(message); + } + } + + public void syncGroupsOwner(String groupId, String accountId) { + LOGGER.info(String.format("try fetching orchestrator identifier for group %s from redis", groupId)); + + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); + String orchestratorIdentifier = SpotinstContext.getInstance().getOrchestratorIdentifier(); + + if (redisGetValueResponse.isRequestSucceed()) { + //redis response might return in different types + if (redisGetValueResponse.getValue() instanceof String) { + String redisResponseValue = (String) redisGetValueResponse.getValue(); + + if (redisResponseValue != null) { + boolean isGroupBelongToOrchestrator = redisResponseValue.equals(orchestratorIdentifier); + boolean isGroupExistInLocalCache = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); + + if (isGroupBelongToOrchestrator) { + LOGGER.info(String.format("group %s belong to orchestrator with identifier %s", groupId, orchestratorIdentifier)); + + if (isGroupExistInLocalCache == false) { + SpotinstContext.getInstance().getGroupsInUse().put(groupId,accountId); + } + + removeFromSuspendedGroupFetching(groupId); + + //expand TTL of the current orchestrator in redis + addGroupToRedis(groupId, accountId, orchestratorIdentifier); + } + else { + LOGGER.info(String.format("group %s does not belong to orchestrator with identifier %s", groupId, orchestratorIdentifier)); + + if (isGroupExistInLocalCache) { + SpotinstContext.getInstance().getGroupsInUse().remove(groupId); + } + } + } + else { + LOGGER.warn("redis response value return null"); + } + + } + //there is no orchestrator for the given group in redis, should take ownership + else { + addGroupToRedis(groupId, accountId, orchestratorIdentifier); + removeFromSuspendedGroupFetching(groupId); + } + } + } //endregion //region Getters / Setters @@ -779,7 +901,6 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { } } - //endregion //region Abstract Methods diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index 9a059e11..8ef82d59 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -2,9 +2,10 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.plugins.spotinst.model.aws.AwsInstanceType; +import org.apache.commons.collections4.map.PassiveExpiringMap; +import org.apache.commons.lang.RandomStringUtils; -import java.util.Date; -import java.util.List; +import java.util.*; /** * Created by ohadmuchnik on 24/05/2016. @@ -12,11 +13,15 @@ public class SpotinstContext { //region Members - private static SpotinstContext instance; - private String spotinstToken; - private String accountId; - private List awsInstanceTypes; - private Date awsInstanceTypesLastUpdate; + private static SpotinstContext instance; + private String spotinstToken; + private String accountId; + private List awsInstanceTypes; + private Date awsInstanceTypesLastUpdate; + private String orchestratorIdentifier; + private Map groupsInUse; + private PassiveExpiringMap suspendedGroupFetching; + private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = 1000 * 60 * 2; //endregion public static SpotinstContext getInstance() { @@ -63,6 +68,27 @@ public Date getAwsInstanceTypesLastUpdate() { public void setAwsInstanceTypesLastUpdate(Date awsInstanceTypesLastUpdate) { this.awsInstanceTypesLastUpdate = awsInstanceTypesLastUpdate; } + + public String getOrchestratorIdentifier() { + if(orchestratorIdentifier == null){ + orchestratorIdentifier = RandomStringUtils.randomAlphanumeric(10); + } + return orchestratorIdentifier; + } + + public PassiveExpiringMap getSuspendedGroupFetching() { + if (suspendedGroupFetching == null) { + suspendedGroupFetching = new PassiveExpiringMap<>(SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS); + } + return suspendedGroupFetching; + } + + public Map getGroupsInUse() { + if (groupsInUse == null) { + groupsInUse = new HashMap<>(); + } + return groupsInUse; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java new file mode 100644 index 00000000..42d3a71a --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java @@ -0,0 +1,88 @@ +package hudson.plugins.spotinst.jobs; + +import hudson.Extension; +import hudson.model.AsyncPeriodicWork; +import hudson.model.TaskListener; +import hudson.plugins.spotinst.api.infra.ApiResponse; +import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.common.SpotinstContext; +import hudson.plugins.spotinst.repos.IRedisRepo; +import hudson.plugins.spotinst.repos.RepoManager; +import hudson.slaves.Cloud; +import jenkins.model.Jenkins; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Created by ohadmuchnik on 25/05/2016. + */ +@Extension +public class SpotinstGroupsOwnerMonitor extends AsyncPeriodicWork { + + //region Members + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstGroupsOwnerMonitor.class); + public static final Integer JOB_INTERVAL_IN_SECONDS = 60; + final long recurrencePeriod; + //endregion + + //region Constructor + public SpotinstGroupsOwnerMonitor() { + super("Groups Monitor"); + recurrencePeriod = TimeUnit.SECONDS.toMillis(JOB_INTERVAL_IN_SECONDS); + } + //endregion + + //region Public Methods + @Override + protected void execute(TaskListener taskListener) { + List cloudList = Jenkins.getInstance().clouds; + + if (cloudList != null && cloudList.size() > 0) { + for (Cloud cloud : cloudList) { + Map groupsNoLongerInUse = SpotinstContext.getInstance().getGroupsInUse(); + + if (cloud instanceof BaseSpotinstCloud) { + BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + String groupId = spotinstCloud.getGroupId(); + String accountId = spotinstCloud.getAccountId(); + groupsNoLongerInUse.remove(groupId); + + if(groupId != null && accountId != null){ + spotinstCloud.syncGroupsOwner(groupId, accountId); + } + } + + deallocateGroupsNoLongerInUse(groupsNoLongerInUse); + } + } + } + + private void deallocateGroupsNoLongerInUse(Map groupsNoLongerInUse) { + for (Map.Entry entry : groupsNoLongerInUse.entrySet()){ + String groupId = entry.getKey(); + String accountId = entry.getValue(); + + SpotinstContext.getInstance().getGroupsInUse().remove(groupId, accountId); + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); + + if (redisGetValueResponse.isRequestSucceed()) { + LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); + } + else { + LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); + } + } + + } + + @Override + public long getRecurrencePeriod() { + return recurrencePeriod; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java new file mode 100644 index 00000000..28f0369a --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.redis; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class RedisDeleteKeyResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java new file mode 100644 index 00000000..de187296 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.redis; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class RedisGetValueResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java b/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java new file mode 100644 index 00000000..7b123164 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java @@ -0,0 +1,39 @@ +package hudson.plugins.spotinst.model.redis; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class RedisSetKeyRequest { + + //region Memebers + String groupId; + String orchestratorIdentifier; + Integer ttl; + //endregion + + //region Getters & Setters + public String getGroupId() { + return groupId; + } + + public String getOrchestratorIdentifier() { + return orchestratorIdentifier; + } + + public Integer getTtl() { + return ttl; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public void setOrchestratorIdentifier(String orchestratorIdentifier) { + this.orchestratorIdentifier = orchestratorIdentifier; + } + + public void setTtl(Integer ttl) { + this.ttl = ttl; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java new file mode 100644 index 00000000..83664ead --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.redis; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class RedisSetKeyResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java b/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java new file mode 100644 index 00000000..e4f9ba07 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java @@ -0,0 +1,12 @@ +package hudson.plugins.spotinst.repos; + +import hudson.plugins.spotinst.api.infra.ApiResponse; + +public interface IRedisRepo { + ApiResponse setKey(String groupId,String accountId, String orchestratorIdentifier,Integer ttl); + + ApiResponse getValue(String groupId, String accountId); + + ApiResponse deleteKey(String groupId, String accountId); + +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java b/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java new file mode 100644 index 00000000..ee477ae2 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java @@ -0,0 +1,59 @@ +package hudson.plugins.spotinst.repos; + +import hudson.plugins.spotinst.api.SpotinstApi; +import hudson.plugins.spotinst.api.infra.ApiException; +import hudson.plugins.spotinst.api.infra.ApiResponse; +import hudson.plugins.spotinst.api.infra.ExceptionHelper; + +public class RedisRepo implements IRedisRepo { + @Override + public ApiResponse setKey(String groupId, String accountId, String orchestratorIdentifier, Integer ttl) { + ApiResponse retVal; + + try { + String isKeySet = SpotinstApi.setRedisKey(groupId, accountId, orchestratorIdentifier, ttl); + + retVal = new ApiResponse<>(isKeySet); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } + + @Override + public ApiResponse getValue(String groupId, String accountId) { + ApiResponse retVal; + + try { + Object orchestratorIdentifier = SpotinstApi.getRedisValue(groupId, accountId); + + retVal = new ApiResponse<>(orchestratorIdentifier); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } + + @Override + public ApiResponse deleteKey(String groupId, String accountId) { + ApiResponse retVal; + + try { + Integer isKeyDeleted = SpotinstApi.deleteRedisKey(groupId, accountId); + + retVal = new ApiResponse<>(isKeyDeleted); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java b/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java index 2392347e..0a0ba788 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java +++ b/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java @@ -10,6 +10,7 @@ public class RepoManager { private IAzureGroupRepo azureGroupRepo; private IAzureVmGroupRepo azureVmGroupRepo; private IAwsInstanceTypesRepo awsInstanceTypesRepo; + private IRedisRepo redisRepo; //endregion //region Constructor @@ -19,6 +20,7 @@ private RepoManager() { this.azureGroupRepo = new AzureGroupRepo(); this.azureVmGroupRepo = new AzureVmGroupRepo(); this.awsInstanceTypesRepo = new AwsInstanceTypesRepo(); + this.redisRepo = new RedisRepo(); } private static RepoManager instance = new RepoManager(); @@ -68,5 +70,13 @@ public IAwsInstanceTypesRepo getAwsInstanceTypesRepo() { public void setAwsInstanceTypesRepo(IAwsInstanceTypesRepo awsInstanceTypesRepo) { this.awsInstanceTypesRepo = awsInstanceTypesRepo; } + + public IRedisRepo getRedisRepo() { + return redisRepo; + } + + public void setRedisRepo(IRedisRepo redisRepo) { + this.redisRepo = redisRepo; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 51c556c4..ef37f8d3 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -3,6 +3,7 @@ import hudson.Extension; import hudson.model.*; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.common.SpotinstContext; import hudson.slaves.*; import jenkins.model.Jenkins; import net.sf.json.JSONObject; @@ -173,42 +174,64 @@ public Node asNode() { //region Public Methods public void terminate() { - Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); - - if (isTerminated) { - LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); - removeIfInPending(); - try { - Jenkins.getInstance().removeNode(this); + String groupId = getSpotinstCloud().getGroupId(); + boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().contains(groupId); + + if (isGroupBelongToCloud) { + Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + + if (isTerminated) { + LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); + removeIfInPending(); + try { + Jenkins.getInstance().removeNode(this); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); } - catch (IOException e) { - e.printStackTrace(); - } - } - else { - LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); } + else{ + try { + getSpotinstCloud().handleGroupDosNotBelongToCloud(groupId); + }catch (Exception e){ + LOGGER.warn(e.getMessage()); + } } } public Boolean forceTerminate() { - Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + Boolean retVal = false; - if (isTerminated) { - LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); - removeIfInPending(); - } - else { - LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); - } + String groupId = getSpotinstCloud().getGroupId(); + boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().contains(groupId); - try { - Jenkins.get().removeNode(this); - } - catch (IOException e) { - e.printStackTrace(); + if (isGroupBelongToCloud) { + Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + + if (isTerminated) { + LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); + removeIfInPending(); + } else { + LOGGER.error(String.format("Failed to terminate instance: %s", getInstanceId())); + } + + try { + Jenkins.get().removeNode(this); + } catch (IOException e) { + e.printStackTrace(); + } + + retVal = isTerminated; } + else{ + try { + getSpotinstCloud().handleGroupDosNotBelongToCloud(groupId); + }catch (Exception e){ + LOGGER.warn(e.getMessage()); + } } - return isTerminated; + return retVal; } private void removeIfInPending() { From b252a2d7afcc712702ed8dc7f56f76fd975a1d7e Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Tue, 18 Oct 2022 12:33:30 +0300 Subject: [PATCH 03/16] temp --- .../java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java | 1 - .../java/hudson/plugins/spotinst/slave/SpotinstComputer.java | 4 ---- 2 files changed, 5 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index b3b09e49..7f11f546 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -713,7 +713,6 @@ public void handleGroupDosNotBelongToCloud(String groupId) throws ServletExcepti message = String.format("Group %s is in use by other Jenkins orchestrator", groupId); LOGGER.error(message); sendError(message); - //throw new ServletException(message); } } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index 37cb2ee2..8c34a1b8 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -25,8 +25,6 @@ public class SpotinstComputer extends SlaveComputer { //endregion //region overrides - // better than JobListener because in complicated pipelines Jenkins is the initial Node and - // we know the Node is a SpotinstNode (I'm not using that word). @Override public void taskAccepted(Executor executor, Queue.Task task) { super.taskAccepted(executor, task); @@ -42,8 +40,6 @@ public void taskAccepted(Executor executor, Queue.Task task) { spotinstNode.getNodeName(), spotinstCloud.getDisplayName()); LOGGER.info(msg); this.setAcceptingTasks(false); - // I see much better responsiveness from Jenkins in case of setting offline, both have same effect - // should also be decided with UI because each combo looks different in the node list. SpotinstNonLocalizable spotinstNonLocalizable = new SpotinstNonLocalizable(msg); SpotinstSingleTaskOfflineCause spotinstSingleTaskOfflineCause = new SpotinstSingleTaskOfflineCause(spotinstNonLocalizable); this.setTemporarilyOffline(true,spotinstSingleTaskOfflineCause); From 38bdde35719a24528e5cc8030f87ea6030555c90 Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Wed, 19 Oct 2022 11:50:28 +0300 Subject: [PATCH 04/16] temp --- .../plugins/spotinst/api/SpotinstApi.java | 4 +- .../spotinst/cloud/AwsSpotinstCloud.java | 12 ++-- .../spotinst/cloud/BaseSpotinstCloud.java | 72 ++++++++++--------- .../spotinst/common/SpotinstContext.java | 10 +-- .../jobs/SpotinstGroupsOwnerMonitor.java | 29 ++++---- .../model/redis/RedisSetKeyRequest.java | 10 +-- .../plugins/spotinst/repos/IRedisRepo.java | 2 +- .../plugins/spotinst/repos/RedisRepo.java | 8 +-- .../plugins/spotinst/slave/SpotinstSlave.java | 12 ++-- 9 files changed, 84 insertions(+), 75 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index 14e3a4c7..1ddf030f 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -337,7 +337,7 @@ public static T getRedisValue(String groupId, String accountId) throws ApiEx return retVal; } - public static String setRedisKey(String groupId, String accountId, String orchestratorIdentifier, Integer ttl) throws ApiException { + public static String setRedisKey(String groupId, String accountId, String controllerIdentifier, Integer ttl) throws ApiException { String retVal = null; Map headers = buildHeaders(); @@ -346,7 +346,7 @@ public static String setRedisKey(String groupId, String accountId, String orches RedisSetKeyRequest request = new RedisSetKeyRequest(); request.setGroupId(groupId); - request.setOrchestratorIdentifier(orchestratorIdentifier); + request.setControllerIdentifier(controllerIdentifier); request.setTtl(ttl); String body = JsonMapper.toJson(request); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index dd2649de..89f64fb8 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -118,9 +118,9 @@ public Boolean detachInstance(String instanceId) { @Override public void syncGroupInstances() { - boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - if (isGroupBelongToCloud) { + if (isGroupManagedByThisController) { IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); @@ -148,7 +148,7 @@ public void syncGroupInstances() { } else{ try { - handleGroupDosNotBelongToCloud(groupId); + handleGroupDosNotManageByThisController(groupId); } catch (Exception e) { LOGGER.error(e.getMessage()); @@ -161,9 +161,9 @@ public void syncGroupInstances() { public Map getInstanceIpsById() { Map retVal = new HashMap<>(); - boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - if (isGroupBelongToCloud) { + if (isGroupManagedByThisController) { IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); @@ -184,7 +184,7 @@ public Map getInstanceIpsById() { } else{ try { - handleGroupDosNotBelongToCloud(groupId); + handleGroupDosNotManageByThisController(groupId); } catch (Exception e) { LOGGER.warn(e.getMessage()); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 7f11f546..0fe7e482 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -116,11 +116,11 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat this.globalExecutorOverride = new SpotGlobalExecutorOverride(false, 1); } - boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByOtherController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - if (isGroupBelongToCloud == false) { + if (isGroupManagedByOtherController) { try { - handleGroupDosNotBelongToCloud(this.groupId); + handleGroupDosNotManageByThisController(this.groupId); } catch (Exception e){ LOGGER.error(e.getMessage()); @@ -139,9 +139,9 @@ public Collection provision(Label label, int excessWorkload) { ProvisionRequest request = new ProvisionRequest(label, excessWorkload); LOGGER.info(String.format("Got provision slave request: %s", JsonMapper.toJson(request))); - boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - if (isGroupBelongToCloud) { + if (isGroupManagedByThisController) { setNumOfNeededExecutors(request); @@ -168,7 +168,7 @@ public Collection provision(Label label, int excessWorkload) { } else { try { - handleGroupDosNotBelongToCloud(groupId); + handleGroupDosNotManageByThisController(groupId); }catch (Exception e){ LOGGER.warn(e.getMessage()); } } @@ -473,9 +473,9 @@ private void removeFromSuspendedGroupFetching(String groupId) { } } - private void addGroupToRedis(String groupId, String accountId, String orchestratorIdentifier) { + private void addGroupToRedis(String groupId, String accountId, String controllerIdentifier) { IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisSetKeyResponse = redisRepo.setKey(groupId, accountId, orchestratorIdentifier,REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); + ApiResponse redisSetKeyResponse = redisRepo.setKey(groupId, accountId, controllerIdentifier,REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); if (redisSetKeyResponse.isRequestSucceed()) { String redisResponseSetKeyValue = redisSetKeyResponse.getValue(); @@ -701,27 +701,31 @@ protected Integer getSlaveOfflineThreshold() { return Constants.SLAVE_OFFLINE_THRESHOLD_IN_MINUTES; } - public void handleGroupDosNotBelongToCloud(String groupId) throws ServletException, IOException { + public void handleGroupDosNotManageByThisController(String groupId) { boolean isGroupExistInSuspendedGroupFetching = SpotinstContext.getInstance().getSuspendedGroupFetching().containsKey(groupId); String message; if(isGroupExistInSuspendedGroupFetching){ - message = String.format("Group %s is might be in use by other Jenkins orchestrator, please make sure it belong to this Jenkins orchestrator and try again after 3 minutes", groupId); + message = String.format("Group %s is might be in use by other Jenkins controller, please make sure it belong to this Jenkins controller and try again after 3 minutes", groupId); LOGGER.warn(message); } else{ - //pop up message in jenkins UI (ONLY after fetching groupId retries pass the ttl for key in redis is expired) - message = String.format("Group %s is in use by other Jenkins orchestrator", groupId); + //TODO Liron - pop up message in jenkins UI (ONLY after fetching groupId retries pass the ttl for key in redis is expired) + message = String.format("Group %s is in use by other Jenkins controller", groupId); LOGGER.error(message); - sendError(message); + +// StaplerRequest req = new RequestImpl(); +// StaplerResponse rsp = new RequestImpl(); +// sendError(message); } } public void syncGroupsOwner(String groupId, String accountId) { - LOGGER.info(String.format("try fetching orchestrator identifier for group %s from redis", groupId)); + LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); + boolean isGroupExistInLocalCache = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); - String orchestratorIdentifier = SpotinstContext.getInstance().getOrchestratorIdentifier(); + String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); if (redisGetValueResponse.isRequestSucceed()) { //redis response might return in different types @@ -729,23 +733,15 @@ public void syncGroupsOwner(String groupId, String accountId) { String redisResponseValue = (String) redisGetValueResponse.getValue(); if (redisResponseValue != null) { - boolean isGroupBelongToOrchestrator = redisResponseValue.equals(orchestratorIdentifier); - boolean isGroupExistInLocalCache = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); - - if (isGroupBelongToOrchestrator) { - LOGGER.info(String.format("group %s belong to orchestrator with identifier %s", groupId, orchestratorIdentifier)); - - if (isGroupExistInLocalCache == false) { - SpotinstContext.getInstance().getGroupsInUse().put(groupId,accountId); - } - - removeFromSuspendedGroupFetching(groupId); + boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); - //expand TTL of the current orchestrator in redis - addGroupToRedis(groupId, accountId, orchestratorIdentifier); + if (isGroupBelongToController) { + handleGroupManagedByThisController(groupId, accountId, isGroupExistInLocalCache, controllerIdentifier); } else { - LOGGER.info(String.format("group %s does not belong to orchestrator with identifier %s", groupId, orchestratorIdentifier)); + LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, controllerIdentifier)); + + SpotinstContext.getInstance().getSuspendedGroupFetching().put(groupId, accountId); if (isGroupExistInLocalCache) { SpotinstContext.getInstance().getGroupsInUse().remove(groupId); @@ -755,15 +751,25 @@ public void syncGroupsOwner(String groupId, String accountId) { else { LOGGER.warn("redis response value return null"); } - } - //there is no orchestrator for the given group in redis, should take ownership + //there is no controller for the given group in redis, should take ownership else { - addGroupToRedis(groupId, accountId, orchestratorIdentifier); - removeFromSuspendedGroupFetching(groupId); + handleGroupManagedByThisController(groupId, accountId, isGroupExistInLocalCache, controllerIdentifier); } } } + + private void handleGroupManagedByThisController(String groupId, String accountId, boolean isGroupExistInLocalCache, String controllerIdentifier) { + LOGGER.info(String.format("group %s belong to controller with identifier %s", groupId, controllerIdentifier)); + + if (isGroupExistInLocalCache == false) { + SpotinstContext.getInstance().getGroupsInUse().put(groupId, accountId); + } + + removeFromSuspendedGroupFetching(groupId); + //expand TTL of the current controller in redis + addGroupToRedis(groupId, accountId, controllerIdentifier); + } //endregion //region Getters / Setters diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index 8ef82d59..c78c241f 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -18,7 +18,7 @@ public class SpotinstContext { private String accountId; private List awsInstanceTypes; private Date awsInstanceTypesLastUpdate; - private String orchestratorIdentifier; + private String controllerIdentifier; private Map groupsInUse; private PassiveExpiringMap suspendedGroupFetching; private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = 1000 * 60 * 2; @@ -69,11 +69,11 @@ public void setAwsInstanceTypesLastUpdate(Date awsInstanceTypesLastUpdate) { this.awsInstanceTypesLastUpdate = awsInstanceTypesLastUpdate; } - public String getOrchestratorIdentifier() { - if(orchestratorIdentifier == null){ - orchestratorIdentifier = RandomStringUtils.randomAlphanumeric(10); + public String getControllerIdentifier() { + if(controllerIdentifier == null){ + controllerIdentifier = RandomStringUtils.randomAlphanumeric(10); } - return orchestratorIdentifier; + return controllerIdentifier; } public PassiveExpiringMap getSuspendedGroupFetching() { diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java index 42d3a71a..aef22b80 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java @@ -13,6 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -39,24 +40,26 @@ public SpotinstGroupsOwnerMonitor() { //region Public Methods @Override protected void execute(TaskListener taskListener) { - List cloudList = Jenkins.getInstance().clouds; + synchronized (this) { + List cloudList = Jenkins.getInstance().clouds; - if (cloudList != null && cloudList.size() > 0) { - for (Cloud cloud : cloudList) { - Map groupsNoLongerInUse = SpotinstContext.getInstance().getGroupsInUse(); + if (cloudList != null && cloudList.size() > 0) { + for (Cloud cloud : cloudList) { + Map groupsNoLongerInUse = new HashMap<>(SpotinstContext.getInstance().getGroupsInUse()); - if (cloud instanceof BaseSpotinstCloud) { - BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; - String groupId = spotinstCloud.getGroupId(); - String accountId = spotinstCloud.getAccountId(); - groupsNoLongerInUse.remove(groupId); + if (cloud instanceof BaseSpotinstCloud) { + BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + String groupId = spotinstCloud.getGroupId(); + String accountId = spotinstCloud.getAccountId(); + groupsNoLongerInUse.remove(groupId); - if(groupId != null && accountId != null){ - spotinstCloud.syncGroupsOwner(groupId, accountId); + if (groupId != null && accountId != null) { + spotinstCloud.syncGroupsOwner(groupId, accountId); + } } - } - deallocateGroupsNoLongerInUse(groupsNoLongerInUse); + deallocateGroupsNoLongerInUse(groupsNoLongerInUse); + } } } } diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java b/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java index 7b123164..c54c1db6 100644 --- a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java +++ b/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java @@ -7,7 +7,7 @@ public class RedisSetKeyRequest { //region Memebers String groupId; - String orchestratorIdentifier; + String controllerIdentifier; Integer ttl; //endregion @@ -16,8 +16,8 @@ public String getGroupId() { return groupId; } - public String getOrchestratorIdentifier() { - return orchestratorIdentifier; + public String getControllerIdentifier() { + return controllerIdentifier; } public Integer getTtl() { @@ -28,8 +28,8 @@ public void setGroupId(String groupId) { this.groupId = groupId; } - public void setOrchestratorIdentifier(String orchestratorIdentifier) { - this.orchestratorIdentifier = orchestratorIdentifier; + public void setControllerIdentifier(String controllerIdentifier) { + this.controllerIdentifier = controllerIdentifier; } public void setTtl(Integer ttl) { diff --git a/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java b/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java index e4f9ba07..5305df54 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java +++ b/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java @@ -3,7 +3,7 @@ import hudson.plugins.spotinst.api.infra.ApiResponse; public interface IRedisRepo { - ApiResponse setKey(String groupId,String accountId, String orchestratorIdentifier,Integer ttl); + ApiResponse setKey(String groupId,String accountId, String controllerIdentifier,Integer ttl); ApiResponse getValue(String groupId, String accountId); diff --git a/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java b/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java index ee477ae2..96a6a254 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java +++ b/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java @@ -7,11 +7,11 @@ public class RedisRepo implements IRedisRepo { @Override - public ApiResponse setKey(String groupId, String accountId, String orchestratorIdentifier, Integer ttl) { + public ApiResponse setKey(String groupId, String accountId, String controllerIdentifier, Integer ttl) { ApiResponse retVal; try { - String isKeySet = SpotinstApi.setRedisKey(groupId, accountId, orchestratorIdentifier, ttl); + String isKeySet = SpotinstApi.setRedisKey(groupId, accountId, controllerIdentifier, ttl); retVal = new ApiResponse<>(isKeySet); @@ -28,9 +28,9 @@ public ApiResponse getValue(String groupId, String accountId) { ApiResponse retVal; try { - Object orchestratorIdentifier = SpotinstApi.getRedisValue(groupId, accountId); + Object controllerIdentifier = SpotinstApi.getRedisValue(groupId, accountId); - retVal = new ApiResponse<>(orchestratorIdentifier); + retVal = new ApiResponse<>(controllerIdentifier); } catch (ApiException e) { diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index ef37f8d3..9dc3acf9 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -175,9 +175,9 @@ public Node asNode() { //region Public Methods public void terminate() { String groupId = getSpotinstCloud().getGroupId(); - boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().contains(groupId); + boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); - if (isGroupBelongToCloud) { + if (isGroupManagedByThisController) { Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); if (isTerminated) { @@ -194,7 +194,7 @@ public void terminate() { } else{ try { - getSpotinstCloud().handleGroupDosNotBelongToCloud(groupId); + getSpotinstCloud().handleGroupDosNotManageByThisController(groupId); }catch (Exception e){ LOGGER.warn(e.getMessage()); } } @@ -204,9 +204,9 @@ public Boolean forceTerminate() { Boolean retVal = false; String groupId = getSpotinstCloud().getGroupId(); - boolean isGroupBelongToCloud = SpotinstContext.getInstance().getGroupsInUse().contains(groupId); + boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); - if (isGroupBelongToCloud) { + if (isGroupManagedByThisController) { Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); if (isTerminated) { @@ -226,7 +226,7 @@ public Boolean forceTerminate() { } else{ try { - getSpotinstCloud().handleGroupDosNotBelongToCloud(groupId); + getSpotinstCloud().handleGroupDosNotManageByThisController(groupId); }catch (Exception e){ LOGGER.warn(e.getMessage()); } } From df4a97422236de5ac97b8c09875a40e6b89ea7dc Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Sun, 23 Oct 2022 10:13:31 +0300 Subject: [PATCH 05/16] high level review --- .../spotinst/cloud/AwsSpotinstCloud.java | 12 +- .../spotinst/cloud/BaseSpotinstCloud.java | 147 +++++++++--------- .../GroupsManagedByControllerMonitor.java | 96 ++++++++++++ .../plugins/spotinst/cloud/PluginImpl.java | 59 +++++++ .../SpotinstCloudCommunicationState.java | 41 +++++ .../spotinst/common/SpotinstContext.java | 30 ++-- .../jobs/SpotinstGroupsOwnerMonitor.java | 39 +++-- .../spotinst/jobs/SpotinstSyncInstances.java | 14 ++ .../plugins/spotinst/slave/SpotinstSlave.java | 12 +- .../message.jelly | 27 ++++ .../message.properties | 4 + 11 files changed, 357 insertions(+), 124 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java create mode 100644 src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java create mode 100644 src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 89f64fb8..e1060a0a 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -6,7 +6,6 @@ import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.ConnectionMethodEnum; import hudson.plugins.spotinst.common.SpotAwsInstanceTypesHelper; -import hudson.plugins.spotinst.common.SpotinstContext; import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.repos.IAwsGroupRepo; import hudson.plugins.spotinst.repos.RepoManager; @@ -118,7 +117,7 @@ public Boolean detachInstance(String instanceId) { @Override public void syncGroupInstances() { - boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); @@ -161,7 +160,7 @@ public void syncGroupInstances() { public Map getInstanceIpsById() { Map retVal = new HashMap<>(); - boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); @@ -183,12 +182,7 @@ public Map getInstanceIpsById() { } } else{ - try { - handleGroupDosNotManageByThisController(groupId); - } - catch (Exception e) { - LOGGER.warn(e.getMessage()); - } + handleGroupDosNotManageByThisController(groupId); } return retVal; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 0fe7e482..55dcb687 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -21,7 +21,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletException; import java.io.IOException; import java.util.*; import java.util.concurrent.TimeUnit; @@ -36,8 +35,8 @@ public abstract class BaseSpotinstCloud extends Cloud { private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; - private static final Integer REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS = 60 * 3; - public static final String REDIS_OK_STATUS = "OK"; + private static final Integer REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS = 60 * 3; + public static final String REDIS_OK_STATUS = "OK"; protected String accountId; protected String groupId; protected Map pendingInstances; @@ -116,20 +115,9 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat this.globalExecutorOverride = new SpotGlobalExecutorOverride(false, 1); } - boolean isGroupManagedByOtherController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); - - if (isGroupManagedByOtherController) { - try { - handleGroupDosNotManageByThisController(this.groupId); - } - catch (Exception e){ - LOGGER.error(e.getMessage()); - } - } - else { - SpotinstContext.getInstance().getGroupsInUse().put(groupId,accountId); + if (groupId != null && accountId != null) { + syncGroupsOwner(this); } - } //endregion @@ -139,7 +127,7 @@ public Collection provision(Label label, int excessWorkload) { ProvisionRequest request = new ProvisionRequest(label, excessWorkload); LOGGER.info(String.format("Got provision slave request: %s", JsonMapper.toJson(request))); - boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(this.groupId); + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { @@ -161,17 +149,14 @@ public Collection provision(Label label, int excessWorkload) { } } } - } + } else { LOGGER.info("No need to scale up new slaves, there are some that are initiating"); } - } + } else { - try { - handleGroupDosNotManageByThisController(groupId); - }catch (Exception e){ - LOGGER.warn(e.getMessage()); - } } + handleGroupDosNotManageByThisController(groupId); + } return Collections.emptyList(); } @@ -377,6 +362,19 @@ public SlaveInstanceDetails getSlaveDetails(String instanceId) { return retVal; } + + public Boolean isCloudReadyForGroupCommunication(String groupId){ + Boolean retVal = null; + + SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); + + if(state != null){ + if(state.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)); + retVal = true; + } + + return retVal; + } //endregion //region Private Methods @@ -465,14 +463,6 @@ private boolean isGlobalExecutorOverrideValid() { return retVal; } - private void removeFromSuspendedGroupFetching(String groupId) { - boolean isGroupExistInSuspendedGroupFetching = SpotinstContext.getInstance().getSuspendedGroupFetching().containsKey(groupId); - - if(isGroupExistInSuspendedGroupFetching){ - SpotinstContext.getInstance().getSuspendedGroupFetching().remove(groupId); - } - } - private void addGroupToRedis(String groupId, String accountId, String controllerIdentifier) { IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); ApiResponse redisSetKeyResponse = redisRepo.setKey(groupId, accountId, controllerIdentifier,REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); @@ -491,6 +481,18 @@ private void addGroupToRedis(String groupId, String accountId, String controller LOGGER.error("redis request failed"); } } + + private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(String groupId) { + SpotinstCloudCommunicationState state = null; + + for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance().getCloudsInitializationState().entrySet()) { + if(cloudsInitializationStateEntry.getKey().groupId.equals(groupId)){ + state = cloudsInitializationStateEntry.getValue(); + } + } + + return state; + } //endregion //region Protected Methods @@ -702,73 +704,66 @@ protected Integer getSlaveOfflineThreshold() { } public void handleGroupDosNotManageByThisController(String groupId) { - boolean isGroupExistInSuspendedGroupFetching = SpotinstContext.getInstance().getSuspendedGroupFetching().containsKey(groupId); - String message; - if(isGroupExistInSuspendedGroupFetching){ - message = String.format("Group %s is might be in use by other Jenkins controller, please make sure it belong to this Jenkins controller and try again after 3 minutes", groupId); - LOGGER.warn(message); + boolean isGroupExistInCandidateGroupsForControllerOwnership = SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().containsKey(groupId); + + if(isGroupExistInCandidateGroupsForControllerOwnership){ + SpotinstContext.getInstance().getCloudsInitializationState().put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); } else{ - //TODO Liron - pop up message in jenkins UI (ONLY after fetching groupId retries pass the ttl for key in redis is expired) - message = String.format("Group %s is in use by other Jenkins controller", groupId); - LOGGER.error(message); - -// StaplerRequest req = new RequestImpl(); -// StaplerResponse rsp = new RequestImpl(); -// sendError(message); + SpotinstContext.getInstance().getCloudsInitializationState().put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); } } - public void syncGroupsOwner(String groupId, String accountId) { + public void syncGroupsOwner(BaseSpotinstCloud cloud) { + String groupId = cloud.getGroupId(); + String accountId = cloud.getAccountId(); LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); - boolean isGroupExistInLocalCache = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); - String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); + String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); - if (redisGetValueResponse.isRequestSucceed()) { - //redis response might return in different types - if (redisGetValueResponse.getValue() instanceof String) { - String redisResponseValue = (String) redisGetValueResponse.getValue(); + if (redisGetValueResponse.isRequestSucceed()) { + //redis response might return in different types + if (redisGetValueResponse.getValue() instanceof String) { + String redisResponseValue = (String) redisGetValueResponse.getValue(); - if (redisResponseValue != null) { - boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); + if (redisResponseValue != null) { + boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); - if (isGroupBelongToController) { - handleGroupManagedByThisController(groupId, accountId, isGroupExistInLocalCache, controllerIdentifier); + if (isGroupBelongToController) { +; handleGroupManagedByThisController(cloud, controllerIdentifier); + } + else { + LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, controllerIdentifier)); + SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().put(groupId, accountId); + SpotinstContext.getInstance().getCloudsInitializationState().replace(cloud, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + } } else { - LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, controllerIdentifier)); - - SpotinstContext.getInstance().getSuspendedGroupFetching().put(groupId, accountId); - - if (isGroupExistInLocalCache) { - SpotinstContext.getInstance().getGroupsInUse().remove(groupId); - } + LOGGER.warn("redis response value return null"); } } + //there is no controller for the given group in redis, should take ownership else { - LOGGER.warn("redis response value return null"); + handleGroupManagedByThisController(cloud, controllerIdentifier); } } - //there is no controller for the given group in redis, should take ownership - else { - handleGroupManagedByThisController(groupId, accountId, isGroupExistInLocalCache, controllerIdentifier); - } - } } - private void handleGroupManagedByThisController(String groupId, String accountId, boolean isGroupExistInLocalCache, String controllerIdentifier) { - LOGGER.info(String.format("group %s belong to controller with identifier %s", groupId, controllerIdentifier)); + private void handleGroupManagedByThisController(BaseSpotinstCloud cloud, String controllerIdentifier) { + LOGGER.info(String.format("group %s belong to controller with identifier %s", cloud.getGroupId(), controllerIdentifier)); + SpotinstCloudCommunicationState previousCloudState = SpotinstContext.getInstance().getCloudsInitializationState().get(cloud); - if (isGroupExistInLocalCache == false) { - SpotinstContext.getInstance().getGroupsInUse().put(groupId, accountId); + if(previousCloudState != null){ + if(previousCloudState.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY) == false){ + SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); + } } - removeFromSuspendedGroupFetching(groupId); + SpotinstContext.getInstance().getCloudsInitializationState().put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); //expand TTL of the current controller in redis - addGroupToRedis(groupId, accountId, controllerIdentifier); + addGroupToRedis(cloud.getGroupId(), cloud.getAccountId(), controllerIdentifier); } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java b/src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java new file mode 100644 index 00000000..db6c1f71 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java @@ -0,0 +1,96 @@ +package hudson.plugins.spotinst.cloud; + +import hudson.Extension; +import hudson.model.AdministrativeMonitor; +import hudson.plugins.spotinst.common.SpotinstCloudCommunicationState; +import hudson.plugins.spotinst.common.SpotinstContext; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Extension +public class GroupsManagedByControllerMonitor extends AdministrativeMonitor { + + //region Members + private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); + List spotinstCloudsCommunicationFailures; + List spotinstCloudsCommunicationInitializing; + List spotinstCloudsCommunicationReady; + //endregion + + //region Overridden Public Methods + @Override + public boolean isActivated() { + return isSpotinstCloudsCommunicationFailuresExist() || isSpotinstCloudsCommunicationInitializingExist() + || isSpotinstCloudsCommunicationReadyExist(); + } + + @Override + public String getDisplayName() { + return "Spotinst Clouds Communication Monitor"; + } + //endregion + + //region getters & setters + public boolean isSpotinstCloudsCommunicationFailuresExist() { + return SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + } + + public boolean isSpotinstCloudsCommunicationInitializingExist() { + return SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + } + + public boolean isSpotinstCloudsCommunicationReadyExist() { + return SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); + } + + public String getSpotinstCloudsCommunicationFailures() { + String retVal; + + spotinstCloudsCommunicationFailures = getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + retVal = spotinstCloudsCommunicationFailures.stream().collect(Collectors.joining(", ")); + + return retVal; + } + + public String getSpotinstCloudsCommunicationInitializing() { + String retVal; + + spotinstCloudsCommunicationInitializing = getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + retVal = spotinstCloudsCommunicationInitializing.stream().collect(Collectors.joining(", ")); + + return retVal; + } + + public String getSpotinstCloudsCommunicationReady() { + String retVal; + + spotinstCloudsCommunicationReady = getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); + retVal = spotinstCloudsCommunicationReady.stream().collect(Collectors.joining(", ")); + + return retVal; + } + + private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState state) { + List retVal = new ArrayList<>(); + + for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance().getCloudsInitializationState().entrySet()) { + if (cloudsInitializationStateEntry.getValue().equals(state)) { + BaseSpotinstCloud cloud = cloudsInitializationStateEntry.getKey(); + + if (cloud.getGroupId() != null) { + retVal.add(cloud.getGroupId()); + } + } + } + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java b/src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java new file mode 100644 index 00000000..fc774b8c --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/PluginImpl.java @@ -0,0 +1,59 @@ +package hudson.plugins.spotinst.cloud; + +import hudson.Extension; +import hudson.Plugin; +import hudson.model.Describable; +import hudson.model.Descriptor; +import jenkins.model.Jenkins; + +import java.io.IOException; +import java.util.logging.Logger; + +@Extension +public class PluginImpl extends Plugin implements Describable { + private static final Logger LOGGER = Logger.getLogger(PluginImpl.class.getName()); + + // Whether the SshHostKeyVerificationAdministrativeMonitor should show messages when we have templates using + // accept-new or check-new-soft strategies + private long dismissInsecureMessages; + + public void saveDismissInsecureMessages(long dismissInsecureMessages) { + this.dismissInsecureMessages = dismissInsecureMessages; + try { + save(); + } catch (IOException io) { + LOGGER.warning("There was a problem saving that you want to dismiss all messages related to insecure EC2 templates"); + } + } + + public long getDismissInsecureMessages() { + return dismissInsecureMessages; + } + + public DescriptorImpl getDescriptor() { + return (DescriptorImpl) Jenkins.get().getDescriptorOrDie(getClass()); + } + + public static PluginImpl get() { + return Jenkins.get().getPlugin(PluginImpl.class); + } + + @Extension + public static final class DescriptorImpl extends Descriptor { + @Override + public String getDisplayName() { + return "Spotinst PluginImpl"; + } + } + + @Override + public void postInitialize() throws IOException { + // backward compatibility with the legacy class name +// Jenkins.XSTREAM.alias("hudson.plugins.ec2.EC2Cloud", AmazonEC2Cloud.class); +// Jenkins.XSTREAM.alias("hudson.plugins.ec2.EC2Slave", EC2OndemandSlave.class); +// // backward compatibility with the legacy instance type +// Jenkins.XSTREAM.registerConverter(new InstanceTypeConverter()); + + load(); + } +} diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java new file mode 100644 index 00000000..bbb387cd --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstCloudCommunicationState.java @@ -0,0 +1,41 @@ +package hudson.plugins.spotinst.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public enum SpotinstCloudCommunicationState { + SPOTINST_CLOUD_COMMUNICATION_INITIALIZING("SPOTINST CLOUD COMMUNICATION INITIALIZING"), + SPOTINST_CLOUD_COMMUNICATION_FAILED("SPOTINST CLOUD COMMUNICATION FAILED"), + SPOTINST_CLOUD_COMMUNICATION_READY("SPOTINST CLOUD COMMUNICATION READY"); + + // region members + private static final Logger LOGGER = LoggerFactory.getLogger(CredentialsMethodEnum.class); + + private String name; + // endregion + + SpotinstCloudCommunicationState(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public SpotinstCloudCommunicationState fromName(String name){ + SpotinstCloudCommunicationState retVal = null; + + for (SpotinstCloudCommunicationState enumValues : SpotinstCloudCommunicationState.values()) { + if (enumValues.name.equals(name)) { + retVal = enumValues; + break; + } + } + + if (retVal == null) { + LOGGER.warn(String.format( + "Tried to create CredentialsMethodEnum for name: %s, but we don't support such type ", name)); + } + return retVal; + } +} diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index c78c241f..f9d7abf3 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -1,6 +1,7 @@ package hudson.plugins.spotinst.common; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; import hudson.plugins.spotinst.model.aws.AwsInstanceType; import org.apache.commons.collections4.map.PassiveExpiringMap; import org.apache.commons.lang.RandomStringUtils; @@ -19,9 +20,12 @@ public class SpotinstContext { private List awsInstanceTypes; private Date awsInstanceTypesLastUpdate; private String controllerIdentifier; - private Map groupsInUse; - private PassiveExpiringMap suspendedGroupFetching; - private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = 1000 * 60 * 2; + private Map groupsManageByController; + private List groupsDoesNotManageByController; + private PassiveExpiringMap candidateGroupsForControllerOwnership; + private Map cloudsInitializationState; + + private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = 1000 * 60 * 4; //endregion public static SpotinstContext getInstance() { @@ -73,22 +77,26 @@ public String getControllerIdentifier() { if(controllerIdentifier == null){ controllerIdentifier = RandomStringUtils.randomAlphanumeric(10); } + return controllerIdentifier; } - public PassiveExpiringMap getSuspendedGroupFetching() { - if (suspendedGroupFetching == null) { - suspendedGroupFetching = new PassiveExpiringMap<>(SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS); + public PassiveExpiringMap getCandidateGroupsForControllerOwnership() { + if (candidateGroupsForControllerOwnership == null) { + candidateGroupsForControllerOwnership = new PassiveExpiringMap<>(SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS); } - return suspendedGroupFetching; + + return candidateGroupsForControllerOwnership; } - public Map getGroupsInUse() { - if (groupsInUse == null) { - groupsInUse = new HashMap<>(); + public Map getCloudsInitializationState() { + if (cloudsInitializationState == null) { + cloudsInitializationState = new HashMap<>(); } - return groupsInUse; + + return cloudsInitializationState; } + //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java index aef22b80..269c744b 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java @@ -13,9 +13,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; /** @@ -42,45 +40,46 @@ public SpotinstGroupsOwnerMonitor() { protected void execute(TaskListener taskListener) { synchronized (this) { List cloudList = Jenkins.getInstance().clouds; + Set cloudsFromContext = SpotinstContext.getInstance().getCloudsInitializationState().keySet(); + Set cloudsNoLongerExist = new HashSet<>(cloudsFromContext); if (cloudList != null && cloudList.size() > 0) { for (Cloud cloud : cloudList) { - Map groupsNoLongerInUse = new HashMap<>(SpotinstContext.getInstance().getGroupsInUse()); if (cloud instanceof BaseSpotinstCloud) { BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; String groupId = spotinstCloud.getGroupId(); String accountId = spotinstCloud.getAccountId(); - groupsNoLongerInUse.remove(groupId); + cloudsNoLongerExist.remove(spotinstCloud); if (groupId != null && accountId != null) { - spotinstCloud.syncGroupsOwner(groupId, accountId); + spotinstCloud.syncGroupsOwner(spotinstCloud); } } - deallocateGroupsNoLongerInUse(groupsNoLongerInUse); + deallocateGroupsNoLongerInUse(cloudsNoLongerExist); } } } } - private void deallocateGroupsNoLongerInUse(Map groupsNoLongerInUse) { - for (Map.Entry entry : groupsNoLongerInUse.entrySet()){ - String groupId = entry.getKey(); - String accountId = entry.getValue(); + private void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { + for (BaseSpotinstCloud cloud : cloudsNoLongerExist){ + String groupId = cloud.getGroupId(); + String accountId = cloud.getAccountId(); + SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); - SpotinstContext.getInstance().getGroupsInUse().remove(groupId, accountId); - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); + if (groupId != null && accountId != null) { + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); - if (redisGetValueResponse.isRequestSucceed()) { - LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); - } - else { - LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); + if (redisGetValueResponse.isRequestSucceed()) { + LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); + } else { + LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); + } } } - } @Override diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java index 75f1804d..690310d1 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java @@ -1,11 +1,14 @@ package hudson.plugins.spotinst.jobs; import hudson.Extension; +import hudson.ExtensionList; import hudson.model.AsyncPeriodicWork; +import hudson.model.PeriodicWork; import hudson.model.TaskListener; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; import hudson.slaves.Cloud; import jenkins.model.Jenkins; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +25,7 @@ public class SpotinstSyncInstances extends AsyncPeriodicWork { private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncInstances.class); public static final Integer JOB_INTERVAL_IN_MINUTES = 1; final long recurrencePeriod; + boolean isJobExecuted; //endregion //region Constructor @@ -40,6 +44,16 @@ protected void execute(TaskListener taskListener) { for (Cloud cloud : cloudList) { if (cloud instanceof BaseSpotinstCloud) { BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + + if(this.isJobExecuted == false) { + ExtensionList spotinstGroupsOwnerMonitorPeriodicWork = SpotinstGroupsOwnerMonitor.all(); + + if (CollectionUtils.isNotEmpty(spotinstGroupsOwnerMonitorPeriodicWork)) { + spotinstGroupsOwnerMonitorPeriodicWork.get(0).run(); + this.isJobExecuted = true; + } + } + try { spotinstCloud.syncGroupInstances(); } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 9dc3acf9..3fcd8be8 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -3,7 +3,6 @@ import hudson.Extension; import hudson.model.*; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; -import hudson.plugins.spotinst.common.SpotinstContext; import hudson.slaves.*; import jenkins.model.Jenkins; import net.sf.json.JSONObject; @@ -175,7 +174,7 @@ public Node asNode() { //region Public Methods public void terminate() { String groupId = getSpotinstCloud().getGroupId(); - boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); + boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); @@ -204,7 +203,7 @@ public Boolean forceTerminate() { Boolean retVal = false; String groupId = getSpotinstCloud().getGroupId(); - boolean isGroupManagedByThisController = SpotinstContext.getInstance().getGroupsInUse().containsKey(groupId); + boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); @@ -225,11 +224,8 @@ public Boolean forceTerminate() { retVal = isTerminated; } else{ - try { - getSpotinstCloud().handleGroupDosNotManageByThisController(groupId); - }catch (Exception e){ - LOGGER.warn(e.getMessage()); - } } + getSpotinstCloud().handleGroupDosNotManageByThisController(groupId); + } return retVal; } diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly new file mode 100644 index 00000000..b4eaf6fe --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly @@ -0,0 +1,27 @@ + + +
+ ${%Critical.Title} +

+ + ${%CGroupsDoesNotManageByController(it.spotinstCloudsCommunicationFailures)} +

+ + ${%Explanation(rootURL)} +

+ + + + ${%GroupsOfInitializingClouds(it.spotinstCloudsCommunicationInitializing)} +

+ + + + ${%GroupsOfSuccessfulInitializingClouds(it.spotinstCloudsCommunicationReady)} +

+ + +

+

+

+ diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties b/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties new file mode 100644 index 00000000..7e1d6df7 --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties @@ -0,0 +1,4 @@ +Critical.Title=Spotinst plugin - groups does not manage by this Jenkins controller +Critical.GroupsDoesNotManageByController=The following Elastigroups %s is already connected to a different Jenkins controller: {0} +Critical.Explanation= Please use another Elastigroup or delete the Elastigroup's configuration from the other controller. Visit \ + Jenkins configure page and change the text box elastigroup id. \ No newline at end of file From d42c9d5ed79dd7e58c33427ff0b935639786ae8c Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Tue, 25 Oct 2022 11:54:21 +0300 Subject: [PATCH 06/16] temp --- .../spotinst/cloud/AwsSpotinstCloud.java | 5 --- .../spotinst/cloud/BaseSpotinstCloud.java | 20 ++++++----- ...> SpotinstCloudsCommunicationMonitor.java} | 10 +++--- .../common/SpotinstRestartListener.java | 36 +++++++++++++++++++ ...itor.java => SpotinstSyncGroupsOwner.java} | 26 +++++++++----- .../spotinst/jobs/SpotinstSyncInstances.java | 2 +- .../message.jelly | 27 -------------- .../message.properties | 4 --- .../message.jelly | 35 ++++++++++++++++++ .../message.properties | 6 ++++ 10 files changed, 113 insertions(+), 58 deletions(-) rename src/main/java/hudson/plugins/spotinst/cloud/{GroupsManagedByControllerMonitor.java => SpotinstCloudsCommunicationMonitor.java} (84%) create mode 100644 src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java rename src/main/java/hudson/plugins/spotinst/jobs/{SpotinstGroupsOwnerMonitor.java => SpotinstSyncGroupsOwner.java} (78%) delete mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly delete mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index e1060a0a..b91538ee 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -146,12 +146,7 @@ public void syncGroupInstances() { } } else{ - try { handleGroupDosNotManageByThisController(groupId); - } - catch (Exception e) { - LOGGER.error(e.getMessage()); - } } } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 55dcb687..2836f762 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -17,6 +17,7 @@ import hudson.tools.ToolLocationNodeProperty; import jenkins.model.Jenkins; import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.DataBoundSetter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,7 +116,7 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat this.globalExecutorOverride = new SpotGlobalExecutorOverride(false, 1); } - if (groupId != null && accountId != null) { + if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { syncGroupsOwner(this); } } @@ -363,14 +364,17 @@ public SlaveInstanceDetails getSlaveDetails(String instanceId) { return retVal; } - public Boolean isCloudReadyForGroupCommunication(String groupId){ - Boolean retVal = null; + public boolean isCloudReadyForGroupCommunication(String groupId){ + boolean retVal = false; - SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); + if(StringUtils.isNotEmpty(groupId)) { + SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); - if(state != null){ - if(state.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)); - retVal = true; + if (state != null) { + if (state.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)) { + retVal = true; + } + } } return retVal; @@ -486,7 +490,7 @@ private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(St SpotinstCloudCommunicationState state = null; for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance().getCloudsInitializationState().entrySet()) { - if(cloudsInitializationStateEntry.getKey().groupId.equals(groupId)){ + if(cloudsInitializationStateEntry.getKey().getGroupId().equals(groupId)){ state = cloudsInitializationStateEntry.getValue(); } } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java b/src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java similarity index 84% rename from src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java rename to src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java index db6c1f71..f7262aa4 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java @@ -5,17 +5,17 @@ import hudson.plugins.spotinst.common.SpotinstCloudCommunicationState; import hudson.plugins.spotinst.common.SpotinstContext; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @Extension -public class GroupsManagedByControllerMonitor extends AdministrativeMonitor { +public class SpotinstCloudsCommunicationMonitor extends AdministrativeMonitor { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); @@ -39,7 +39,9 @@ public String getDisplayName() { //region getters & setters public boolean isSpotinstCloudsCommunicationFailuresExist() { - return SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + boolean isCloudsWithFailureStateExist = SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + boolean isCloudsWithGroupIdExist = CollectionUtils.isNotEmpty(getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED)); + return isCloudsWithFailureStateExist && isCloudsWithGroupIdExist; } public boolean isSpotinstCloudsCommunicationInitializingExist() { @@ -84,7 +86,7 @@ private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunic if (cloudsInitializationStateEntry.getValue().equals(state)) { BaseSpotinstCloud cloud = cloudsInitializationStateEntry.getKey(); - if (cloud.getGroupId() != null) { + if (StringUtils.isNotEmpty(cloud.getGroupId())) { retVal.add(cloud.getGroupId()); } } diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java new file mode 100644 index 00000000..31544b3d --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java @@ -0,0 +1,36 @@ +package hudson.plugins.spotinst.common; + +import hudson.model.RestartListener; +import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.jobs.SpotinstSyncGroupsOwner; +import hudson.slaves.Cloud; +import jenkins.model.Jenkins; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class SpotinstRestartListener extends RestartListener { + @Override + public boolean isReadyToRestart() throws IOException, InterruptedException { + return true; + } + + @Override + public void onRestart() { + SpotinstSyncGroupsOwner groupsOwnerJob = new SpotinstSyncGroupsOwner(); + List cloudList = Jenkins.getInstance().clouds; + Set cloudSet = new HashSet<>(); + + if (cloudList != null && cloudList.size() > 0) { + for (Cloud cloud : cloudList) { + if (cloud instanceof BaseSpotinstCloud) { + cloudSet.add((BaseSpotinstCloud) cloud); + } + } + } + + groupsOwnerJob.deallocateGroupsNoLongerInUse(cloudSet); + } +} diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java similarity index 78% rename from src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java rename to src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java index 269c744b..38b91e18 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstGroupsOwnerMonitor.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java @@ -10,6 +10,7 @@ import hudson.plugins.spotinst.repos.RepoManager; import hudson.slaves.Cloud; import jenkins.model.Jenkins; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,17 +21,17 @@ * Created by ohadmuchnik on 25/05/2016. */ @Extension -public class SpotinstGroupsOwnerMonitor extends AsyncPeriodicWork { +public class SpotinstSyncGroupsOwner extends AsyncPeriodicWork { //region Members - private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstGroupsOwnerMonitor.class); + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncGroupsOwner.class); public static final Integer JOB_INTERVAL_IN_SECONDS = 60; final long recurrencePeriod; //endregion //region Constructor - public SpotinstGroupsOwnerMonitor() { - super("Groups Monitor"); + public SpotinstSyncGroupsOwner() { + super("Sync Groups Owner"); recurrencePeriod = TimeUnit.SECONDS.toMillis(JOB_INTERVAL_IN_SECONDS); } //endregion @@ -52,24 +53,26 @@ protected void execute(TaskListener taskListener) { String accountId = spotinstCloud.getAccountId(); cloudsNoLongerExist.remove(spotinstCloud); - if (groupId != null && accountId != null) { + if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { spotinstCloud.syncGroupsOwner(spotinstCloud); } } - - deallocateGroupsNoLongerInUse(cloudsNoLongerExist); } } + + deallocateGroupsNoLongerInUse(cloudsNoLongerExist); } } - private void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { + public void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { + Map groupsAccountMapping = extractGroupsToDeallocate(); + for (BaseSpotinstCloud cloud : cloudsNoLongerExist){ String groupId = cloud.getGroupId(); String accountId = cloud.getAccountId(); SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); - if (groupId != null && accountId != null) { + if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); @@ -82,6 +85,11 @@ private void deallocateGroupsNoLongerInUse(Set cloudsNoLonger } } + private Map extractGroupsToDeallocate() { + HashMap retVal = new HashMap<>(); + return retVal; + } + @Override public long getRecurrencePeriod() { return recurrencePeriod; diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java index 690310d1..6b6c858f 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncInstances.java @@ -46,7 +46,7 @@ protected void execute(TaskListener taskListener) { BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; if(this.isJobExecuted == false) { - ExtensionList spotinstGroupsOwnerMonitorPeriodicWork = SpotinstGroupsOwnerMonitor.all(); + ExtensionList spotinstGroupsOwnerMonitorPeriodicWork = SpotinstSyncGroupsOwner.all(); if (CollectionUtils.isNotEmpty(spotinstGroupsOwnerMonitorPeriodicWork)) { spotinstGroupsOwnerMonitorPeriodicWork.get(0).run(); diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly deleted file mode 100644 index b4eaf6fe..00000000 --- a/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.jelly +++ /dev/null @@ -1,27 +0,0 @@ - - -
- ${%Critical.Title} -

- - ${%CGroupsDoesNotManageByController(it.spotinstCloudsCommunicationFailures)} -

- - ${%Explanation(rootURL)} -

- - - - ${%GroupsOfInitializingClouds(it.spotinstCloudsCommunicationInitializing)} -

- - - - ${%GroupsOfSuccessfulInitializingClouds(it.spotinstCloudsCommunicationReady)} -

- - -

-

-

- diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties b/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties deleted file mode 100644 index 7e1d6df7..00000000 --- a/src/main/resources/hudson/plugins/spotinst/cloud/GroupsManagedByControllerMonitor/message.properties +++ /dev/null @@ -1,4 +0,0 @@ -Critical.Title=Spotinst plugin - groups does not manage by this Jenkins controller -Critical.GroupsDoesNotManageByController=The following Elastigroups %s is already connected to a different Jenkins controller: {0} -Critical.Explanation= Please use another Elastigroup or delete the Elastigroup's configuration from the other controller. Visit \ - Jenkins configure page and change the text box elastigroup id. \ No newline at end of file diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly new file mode 100644 index 00000000..bdd0b60a --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly @@ -0,0 +1,35 @@ + + +
+ + ${%Title} +

+ + ${%SpotinstCloudsCommunicationFailures(it.spotinstCloudsCommunicationFailures)} +

+ + ${%Explanation(rootURL)} +

+ +

+ +
+ + ${%Title} +

+ + ${%SpotinstCloudsCommunicationInitializing(it.spotinstCloudsCommunicationInitializing)} +

+ +

+ +
+ + ${%Title} +

+ + ${%SpotinstCloudsCommunicationReady(it.spotinstCloudsCommunicationReady)} +

+ +

+ diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties new file mode 100644 index 00000000..e92f5309 --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties @@ -0,0 +1,6 @@ +Title=Spotinst Clouds Communication Monitor +SpotinstCloudsCommunicationInitializing=Communication of Spotinst clouds with the following Elastigroups is initializing: {0} +SpotinstCloudsCommunicationReady=Spotinst clouds with the following Elastigroups are ready for communication: {0} +SpotinstCloudsCommunicationFailures=The following Elastigroups are already connected to a different Jenkins controller: {0} +Explanation= Please use another Elastigroup or delete the Elastigroup's configuration from the other controller. Visit \ + Jenkins configure page and change the text box elastigroup id. \ No newline at end of file From 7dbadd41e17be3e5ee63d200e2ca9cb2bd139e45 Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Tue, 25 Oct 2022 15:32:21 +0300 Subject: [PATCH 07/16] first working version (without restart handling --- .../plugins/spotinst/common/SpotinstRestartListener.java | 1 + .../plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java index 31544b3d..ac36ca12 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Set; +//TODO Liron - this part not working yet public class SpotinstRestartListener extends RestartListener { @Override public boolean isReadyToRestart() throws IOException, InterruptedException { diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java index 38b91e18..ae540ed8 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java @@ -65,8 +65,6 @@ protected void execute(TaskListener taskListener) { } public void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { - Map groupsAccountMapping = extractGroupsToDeallocate(); - for (BaseSpotinstCloud cloud : cloudsNoLongerExist){ String groupId = cloud.getGroupId(); String accountId = cloud.getAccountId(); @@ -85,11 +83,6 @@ public void deallocateGroupsNoLongerInUse(Set cloudsNoLongerE } } - private Map extractGroupsToDeallocate() { - HashMap retVal = new HashMap<>(); - return retVal; - } - @Override public long getRecurrencePeriod() { return recurrencePeriod; From 401a0706bba6fa31d3d1f199c8a2db1388b9bbe9 Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Wed, 26 Oct 2022 13:32:46 +0300 Subject: [PATCH 08/16] test for restart listener --- .../spotinst/common/SpotinstRestartListener.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java index ac36ca12..a6c3a5ff 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java @@ -5,6 +5,8 @@ import hudson.plugins.spotinst.jobs.SpotinstSyncGroupsOwner; import hudson.slaves.Cloud; import jenkins.model.Jenkins; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.HashSet; @@ -13,6 +15,18 @@ //TODO Liron - this part not working yet public class SpotinstRestartListener extends RestartListener { + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstRestartListener.class); + + //TODO Liron - check if necessary + public static SpotinstRestartListener getInstance() { + return Jenkins.get() + .getExtensionList(RestartListener.class) + .get(SpotinstRestartListener.class); + } + + //TODO Liron - check if necessary + public SpotinstRestartListener(){} + @Override public boolean isReadyToRestart() throws IOException, InterruptedException { return true; @@ -32,6 +46,7 @@ public void onRestart() { } } + LOGGER.info(String.format("deallocating %s Spotinst clouds", cloudSet.size())); groupsOwnerJob.deallocateGroupsNoLongerInUse(cloudSet); } } From bb5c5ce8f2e413686d4b1bdbbdf82a690bf6f998 Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Mon, 31 Oct 2022 13:40:34 +0200 Subject: [PATCH 09/16] change messages --- .../cloud/SpotinstCloudsCommunicationMonitor/message.jelly | 2 ++ .../SpotinstCloudsCommunicationMonitor/message.properties | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly index bdd0b60a..47caa25b 100644 --- a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.jelly @@ -28,6 +28,8 @@ ${%Title}

+ rootURL + ${%SpotinstCloudsCommunicationReady(it.spotinstCloudsCommunicationReady)}

diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties index e92f5309..06e30210 100644 --- a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor/message.properties @@ -2,5 +2,4 @@ Title=Spotinst Clouds Communication Monitor SpotinstCloudsCommunicationInitializing=Communication of Spotinst clouds with the following Elastigroups is initializing: {0} SpotinstCloudsCommunicationReady=Spotinst clouds with the following Elastigroups are ready for communication: {0} SpotinstCloudsCommunicationFailures=The following Elastigroups are already connected to a different Jenkins controller: {0} -Explanation= Please use another Elastigroup or delete the Elastigroup's configuration from the other controller. Visit \ - Jenkins configure page and change the text box elastigroup id. \ No newline at end of file +Explanation= Please use another Elastigroup or delete the Elastigroup's configuration from the other controller. \ No newline at end of file From de0b5be3ed3bd2b4d7a8492b1dc7b9554e41d249 Mon Sep 17 00:00:00 2001 From: Lironrad <64735199+Lironrad@users.noreply.github.com> Date: Thu, 29 Dec 2022 14:08:16 +0200 Subject: [PATCH 10/16] Jenkins plugin instance type search (#27) Added capability to search for instances by instance types --- JenkinsWiki.adoc | 5 + .../spotinst/cloud/AwsSpotinstCloud.java | 13 +- .../cloud/SpotinstInstanceWeight.java | 174 +++++++++++++++--- .../AwsSpotinstCloudInstanceTypeMonitor.java | 83 +++++++++ .../AwsInstanceTypeSelectMethodEnum.java | 37 ++++ .../cloud/SpotinstInstanceWeight/config.jelly | 16 +- .../message.jelly | 15 ++ .../message.properties | 3 + 8 files changed, 316 insertions(+), 30 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor.java create mode 100644 src/main/java/hudson/plugins/spotinst/common/AwsInstanceTypeSelectMethodEnum.java create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.jelly create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.properties diff --git a/JenkinsWiki.adoc b/JenkinsWiki.adoc index 8e994761..9d4c93ad 100644 --- a/JenkinsWiki.adoc +++ b/JenkinsWiki.adoc @@ -40,6 +40,11 @@ termination" [SpotinstPlugin-Versionhistory] == Version history +[SpotinstPlugin-Version2.2.9(Dec29,2022)] +=== Version 2.2.9 (Dec 29, 2022) + +* Added Search Instance Type by Input feature + [SpotinstPlugin-Version2.2.8(Jul14,2022)] === Version 2.2.8 (Jul 14, 2022) diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 8cbd91f8..387080ac 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -32,6 +32,7 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud { private static final String CLOUD_URL = "aws/ec2"; protected Map executorsByInstanceType; private List executorsForTypes; + private List invalidInstanceTypes; //endregion //region Constructor @@ -370,13 +371,19 @@ private void addSpotinstSlave(AwsGroupInstance instance) { private void initExecutorsByInstanceType() { this.executorsByInstanceType = new HashMap<>(); + this.invalidInstanceTypes = new LinkedList<>(); if (this.executorsForTypes != null) { for (SpotinstInstanceWeight instance : this.executorsForTypes) { if (instance.getExecutors() != null) { Integer executors = instance.getExecutors(); - String type = instance.getAwsInstanceTypeFromAPI(); + String type = instance.getAwsInstanceTypeFromAPIInput(); this.executorsByInstanceType.put(type, executors); + + if(instance.getIsValid() == false){ + LOGGER.error(String.format("Invalid type \'%s\' in group \'%s\'", type, this.getGroupId())); + invalidInstanceTypes.add(type); + } } } } @@ -387,6 +394,10 @@ private void initExecutorsByInstanceType() { public List getExecutorsForTypes() { return executorsForTypes; } + + public List getInvalidInstanceTypes() { + return this.invalidInstanceTypes; + } //endregion //region Classes diff --git a/src/main/java/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight.java b/src/main/java/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight.java index 2d184d93..24f15c82 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight.java @@ -1,9 +1,11 @@ package hudson.plugins.spotinst.cloud; import hudson.Extension; +import hudson.model.AutoCompletionCandidates; import hudson.model.Describable; import hudson.model.Descriptor; import hudson.plugins.spotinst.common.AwsInstanceTypeEnum; +import hudson.plugins.spotinst.common.AwsInstanceTypeSelectMethodEnum; import hudson.plugins.spotinst.common.SpotAwsInstanceTypesHelper; import hudson.plugins.spotinst.common.SpotinstContext; import hudson.plugins.spotinst.model.aws.AwsInstanceType; @@ -12,8 +14,10 @@ import jenkins.model.Jenkins; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.QueryParameter; import java.util.List; +import java.util.stream.Stream; import static hudson.plugins.spotinst.api.SpotinstApi.validateToken; @@ -22,10 +26,13 @@ */ public class SpotinstInstanceWeight implements Describable { //region Members - private Integer executors; - private String awsInstanceTypeFromAPI; + private Integer executors; + private String awsInstanceTypeFromAPI; + private String awsInstanceTypeFromAPISearch; + private AwsInstanceTypeSelectMethodEnum selectMethod; + private boolean isValid; //Deprecated - private AwsInstanceTypeEnum awsInstanceType; + private AwsInstanceTypeEnum awsInstanceType; //endregion //region Constructors @@ -47,6 +54,67 @@ public Descriptor getDescriptor() { return retVal; } + + @Override + public String toString() { + return "SpotinstInstanceWeight:{ " + "Pick: " + this.awsInstanceTypeFromAPI + ", " + "Search: " + + this.awsInstanceTypeFromAPISearch + ", " + "type: " + this.awsInstanceType + ", " + "executors: " + + this.executors + " }"; + + } + //endregion + + //region Methods + public String getAwsInstanceTypeFromAPIInput() { + String type; + AwsInstanceTypeSelectMethodEnum selectMethod = getSelectMethod(); + + if (selectMethod == AwsInstanceTypeSelectMethodEnum.SEARCH) { + type = getAwsInstanceTypeFromAPISearch(); + } + else { + type = getAwsInstanceTypeFromAPI(); + } + + return type; + } + //endregion + + //region Private Methods + private String getAwsInstanceTypeByName(String awsInstanceTypeFromAPIName) { + String retVal = null; + + if (awsInstanceTypeFromAPIName != null) { + + /* + If the user Previously chosen was a type that not exist in the hard coded list + and did not configure the token right, we will present the chosen type and set the default vCPU to 1 + The descriptor of this class will show a warning message will note the user that something is wrong, + and point to authentication fix before saving this configuration. + */ + List types = SpotAwsInstanceTypesHelper.getAllInstanceTypes(); + isValid = types.stream().anyMatch(i -> i.getInstanceType().equals(awsInstanceTypeFromAPIName)); + + if (isValid == false) { + if (getSelectMethod() != AwsInstanceTypeSelectMethodEnum.SEARCH) { + AwsInstanceType instanceType = new AwsInstanceType(); + instanceType.setInstanceType(awsInstanceTypeFromAPIName); + instanceType.setvCPU(1); + SpotinstContext.getInstance().getAwsInstanceTypes().add(instanceType); + } + } + + retVal = awsInstanceTypeFromAPIName; + + } + else { + if (awsInstanceType != null) { + retVal = awsInstanceType.getValue(); + } + } + + return retVal; + } //endregion //region Classes @@ -71,13 +139,36 @@ public ListBoxModel doFillAwsInstanceTypeFromAPIItems() { return retVal; } + public AutoCompletionCandidates doAutoCompleteAwsInstanceTypeFromAPISearch(@QueryParameter String value) { + AutoCompletionCandidates retVal = new AutoCompletionCandidates(); + List allAwsInstanceTypes = SpotAwsInstanceTypesHelper.getAllInstanceTypes(); + Stream allTypes = + allAwsInstanceTypes.stream().map(AwsInstanceType::getInstanceType); + Stream matchingTypes = allTypes.filter(type -> type.startsWith(value)); + matchingTypes.forEach(retVal::add); + + return retVal; + } + public FormValidation doCheckAwsInstanceTypeFromAPI() { + FormValidation retVal = CheckAccountIdAndToken(); + + return retVal; + } + + public FormValidation doCheckAwsInstanceTypeFromAPISearch() { + FormValidation retVal = CheckAccountIdAndToken(); + + return retVal; + } + + private FormValidation CheckAccountIdAndToken() { FormValidation retVal = null; String accountId = SpotinstContext.getInstance().getAccountId(); String token = SpotinstContext.getInstance().getSpotinstToken(); int isValid = validateToken(token, accountId); - Boolean isInstanceTypesListUpdate = SpotAwsInstanceTypesHelper.isInstanceTypesListUpdate(); + boolean isInstanceTypesListUpdate = SpotAwsInstanceTypesHelper.isInstanceTypesListUpdate(); if (isValid != 0 || isInstanceTypesListUpdate == false) { retVal = FormValidation.error( @@ -99,42 +190,73 @@ public AwsInstanceTypeEnum getAwsInstanceType() { return awsInstanceType; } + public String getAwsInstanceTypeFromAPI() { + String retVal; + + if (selectMethod != AwsInstanceTypeSelectMethodEnum.SEARCH) { + retVal = getAwsInstanceTypeByName(this.awsInstanceTypeFromAPI); + } + else { + retVal = this.awsInstanceTypeFromAPI; + } + + return retVal; + } + @DataBoundSetter public void setAwsInstanceTypeFromAPI(String awsInstanceTypeFromAPI) { this.awsInstanceTypeFromAPI = awsInstanceTypeFromAPI; + + if (selectMethod != AwsInstanceTypeSelectMethodEnum.SEARCH) { + this.awsInstanceTypeFromAPISearch = awsInstanceTypeFromAPI; + } } - public String getAwsInstanceTypeFromAPI() { - String retVal = null; + public String getAwsInstanceTypeFromAPISearch() { + String retVal; - if (this.awsInstanceTypeFromAPI != null) { + if (selectMethod == AwsInstanceTypeSelectMethodEnum.SEARCH) { + retVal = getAwsInstanceTypeByName(this.awsInstanceTypeFromAPISearch); + } + else { + retVal = this.awsInstanceTypeFromAPISearch; + } - /* - If the user Previously chosen was a type that not exist in the hard coded list - and did not configure the token right, we will present the chosen type and set the default vCPU to 1 - The descriptor of this class will show a warning message will note the user that something is wrong, - and point to authentication fix before saving this configuration. - */ - List types = SpotAwsInstanceTypesHelper.getAllInstanceTypes(); - boolean isTypeInList = types.stream().anyMatch(i -> i.getInstanceType().equals(this.awsInstanceTypeFromAPI)); + return retVal; + } - if (isTypeInList == false) { - AwsInstanceType instanceType = new AwsInstanceType(); - instanceType.setInstanceType(awsInstanceTypeFromAPI); - instanceType.setvCPU(1); - SpotinstContext.getInstance().getAwsInstanceTypes().add(instanceType); - } + @DataBoundSetter + public void setAwsInstanceTypeFromAPISearch(String awsInstanceTypeFromAPISearch) { + this.awsInstanceTypeFromAPISearch = awsInstanceTypeFromAPISearch; + + if (selectMethod == AwsInstanceTypeSelectMethodEnum.SEARCH) { + this.awsInstanceTypeFromAPI = awsInstanceTypeFromAPISearch; + } + } - retVal = awsInstanceTypeFromAPI; + public AwsInstanceTypeSelectMethodEnum getSelectMethod() { + AwsInstanceTypeSelectMethodEnum retVal = AwsInstanceTypeSelectMethodEnum.PICK; + if (selectMethod != null) { + retVal = selectMethod; + } + + return retVal; + } + + @DataBoundSetter + public void setSelectMethod(AwsInstanceTypeSelectMethodEnum selectMethod) { + + if (selectMethod == null) { + this.selectMethod = AwsInstanceTypeSelectMethodEnum.PICK; } else { - if(awsInstanceType != null){ - retVal = awsInstanceType.getValue(); - } + this.selectMethod = selectMethod; } + } - return retVal; + public boolean getIsValid() { + return this.isValid; } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor.java b/src/main/java/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor.java new file mode 100644 index 00000000..180c70e7 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor.java @@ -0,0 +1,83 @@ +package hudson.plugins.spotinst.cloud.monitor; + +import hudson.Extension; +import hudson.model.AdministrativeMonitor; +import hudson.plugins.spotinst.cloud.AwsSpotinstCloud; +import hudson.slaves.Cloud; +import jenkins.model.Jenkins; +import org.apache.commons.collections.CollectionUtils; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Extension +public class AwsSpotinstCloudInstanceTypeMonitor extends AdministrativeMonitor { + //region members + Map> invalidInstancesByGroupId; + //endregion + + //region Overrides + @Override + public boolean isActivated() { + boolean retVal; + initInvalidInstances(); + retVal = hasInvalidInstanceType(); + return retVal; + } + + @Override + public String getDisplayName() { + return "Aws Spotinst Cloud Instance Type Monitor"; + } + //endregion + + //region Methods + public boolean hasInvalidInstanceType() { + return invalidInstancesByGroupId.isEmpty() == false; + } + //endregion + + //region getters & setters + public String getInvalidInstancesByGroupId() { + Stream invalidInstancesForOutput = + invalidInstancesByGroupId.keySet().stream().map(this::generateAlertMessage); + String retVal = invalidInstancesForOutput.collect(Collectors.joining(", ")); + + return retVal; + } + //endregion + + //region private Methods + private void initInvalidInstances() { + invalidInstancesByGroupId = new HashMap<>(); + Jenkins jenkinsInstance = Jenkins.getInstance(); + List clouds = jenkinsInstance != null ? jenkinsInstance.clouds : new LinkedList<>(); + List awsClouds = clouds.stream().filter(cloud -> cloud instanceof AwsSpotinstCloud) + .map(awsCloud -> (AwsSpotinstCloud) awsCloud) + .collect(Collectors.toList()); + + awsClouds.forEach(awsCloud -> { + String elastigroupId = awsCloud.getGroupId(); + List invalidTypes = awsCloud.getInvalidInstanceTypes(); + + if (CollectionUtils.isEmpty(invalidTypes) == false) { + invalidInstancesByGroupId.put(elastigroupId, invalidTypes); + } + }); + } + + private String generateAlertMessage(String group) { + StringBuilder retVal = new StringBuilder(); + retVal.append('\'').append(group).append('\'').append(": ["); + + List InvalidInstancesByGroup = invalidInstancesByGroupId.get(group); + Stream InvalidInstancesForAlert = + InvalidInstancesByGroup.stream().map(invalidInstance -> '\'' + invalidInstance + '\''); + + String instances = InvalidInstancesForAlert.collect(Collectors.joining(", ")); + retVal.append(instances).append(']'); + return retVal.toString(); + } + //region +} diff --git a/src/main/java/hudson/plugins/spotinst/common/AwsInstanceTypeSelectMethodEnum.java b/src/main/java/hudson/plugins/spotinst/common/AwsInstanceTypeSelectMethodEnum.java new file mode 100644 index 00000000..c647d816 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/AwsInstanceTypeSelectMethodEnum.java @@ -0,0 +1,37 @@ +package hudson.plugins.spotinst.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public enum AwsInstanceTypeSelectMethodEnum { + PICK("PICK"), + SEARCH("SEARCH"); + + private final String name; + + private static final Logger LOGGER = LoggerFactory.getLogger(AwsInstanceTypeSelectMethodEnum.class); + + AwsInstanceTypeSelectMethodEnum(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static AwsInstanceTypeSelectMethodEnum fromName(String name) { + AwsInstanceTypeSelectMethodEnum retVal = null; + for (AwsInstanceTypeSelectMethodEnum selectMethodEnum : AwsInstanceTypeSelectMethodEnum.values()) { + if (selectMethodEnum.name.equals(name)) { + retVal = selectMethodEnum; + break; + } + } + + if (retVal == null) { + LOGGER.error("Tried to create select method type enum for: " + name + ", but we don't support such type "); + } + + return retVal; + } +} diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight/config.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight/config.jelly index f342aade..12bd0dd5 100644 --- a/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight/config.jelly +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotinstInstanceWeight/config.jelly @@ -1,8 +1,18 @@ - - - + + + + + + + + + + + diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.jelly new file mode 100644 index 00000000..031334d3 --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.jelly @@ -0,0 +1,15 @@ + + +

+ + ${%Title} +

+ + ${%InvalidInstanceType(it.invalidInstancesByGroupId)} +

+ + ${%Explanation(rootURL)} +

+ +

+ \ No newline at end of file diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.properties b/src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.properties new file mode 100644 index 00000000..d6f361f8 --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/monitor/AwsSpotinstCloudInstanceTypeMonitor/message.properties @@ -0,0 +1,3 @@ +Title=Invalid AWS Instance types +InvalidInstanceType=Cloud has settings for invalid instance types for elastigroup Groups & Types: {0} +Explanation= Please use another Instance type or delete the types' configuration from the cloud configuration. \ No newline at end of file From a5537bcae272ff30246cc75863975ec03e1c1ac3 Mon Sep 17 00:00:00 2001 From: sitay93 <116492420+sitay93@users.noreply.github.com> Date: Thu, 29 Dec 2022 14:14:15 +0200 Subject: [PATCH 11/16] InstanceType_by_user_input JenkinsWiki.adoc (#28) --- JenkinsWiki.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/JenkinsWiki.adoc b/JenkinsWiki.adoc index 9d4c93ad..d6c02b28 100644 --- a/JenkinsWiki.adoc +++ b/JenkinsWiki.adoc @@ -43,7 +43,7 @@ termination" [SpotinstPlugin-Version2.2.9(Dec29,2022)] === Version 2.2.9 (Dec 29, 2022) -* Added Search Instance Type by Input feature +* Added capability to search for instances by instance types [SpotinstPlugin-Version2.2.8(Jul14,2022)] === Version 2.2.8 (Jul 14, 2022) From 799e5e90236ff658b14d098673d9245f5bbd30f6 Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Thu, 5 Jan 2023 11:33:05 +0200 Subject: [PATCH 12/16] rollback api endpoint --- src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index 1ddf030f..f487a163 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -23,7 +23,7 @@ public class SpotinstApi { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstApi.class); - private final static String SPOTINST_API_HOST = "http://localhost:3100"; + private final static String SPOTINST_API_HOST = "https://api.spotinst.io"; private final static String HEADER_AUTH = "Authorization"; private final static String AUTH_PREFIX = "Bearer "; private final static String HEADER_CONTENT_TYPE = "Content-Type"; From 4b63f7be8b6d772823331ebd7f1110594e6dbc5b Mon Sep 17 00:00:00 2001 From: Liron Arad Date: Wed, 11 Jan 2023 13:54:28 +0200 Subject: [PATCH 13/16] temp commit --- .../hudson/plugins/spotinst/api/SpotinstApi.java | 11 +++++++---- .../plugins/spotinst/cloud/AzureSpotCloud.java | 2 ++ .../plugins/spotinst/cloud/GcpSpotinstCloud.java | 2 ++ .../plugins/spotinst/common/SpotinstContext.java | 14 +++++++++++++- .../spotinst/jobs/SpotinstSyncGroupsOwner.java | 2 +- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index f487a163..b8fb7d3e 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -23,7 +23,7 @@ public class SpotinstApi { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstApi.class); - private final static String SPOTINST_API_HOST = "https://api.spotinst.io"; + private final static String SPOTINST_API_HOST = "http://localhost:3100"; private final static String HEADER_AUTH = "Authorization"; private final static String AUTH_PREFIX = "Bearer "; private final static String HEADER_CONTENT_TYPE = "Content-Type"; @@ -318,6 +318,7 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI //endregion //Redis + //TODO Liron - change name to getLock public static T getRedisValue(String groupId, String accountId) throws ApiException { T retVal = null; @@ -337,7 +338,8 @@ public static T getRedisValue(String groupId, String accountId) throws ApiEx return retVal; } - public static String setRedisKey(String groupId, String accountId, String controllerIdentifier, Integer ttl) throws ApiException { + //TODO Liron - change method name lock + public static String setRedisKey(String lockKey, String accountId, String lockValue, Integer ttl) throws ApiException { String retVal = null; Map headers = buildHeaders(); @@ -345,8 +347,8 @@ public static String setRedisKey(String groupId, String accountId, String contro Map queryParams = buildQueryParams(accountId); RedisSetKeyRequest request = new RedisSetKeyRequest(); - request.setGroupId(groupId); - request.setControllerIdentifier(controllerIdentifier); + request.setGroupId(lockKey); + request.setControllerIdentifier(lockValue); request.setTtl(ttl); String body = JsonMapper.toJson(request); @@ -363,6 +365,7 @@ public static String setRedisKey(String groupId, String accountId, String contro return retVal; } + //TODO Liron - change name to unlock public static Integer deleteRedisKey(String groupId, String accountId) throws ApiException { Integer retVal = null; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index f7bf5a5d..4180a61a 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -108,6 +108,7 @@ protected Integer getPendingThreshold() { @Override public void syncGroupInstances() { + //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); ApiResponse> instancesResponse = azureVmGroupRepo.getGroupVms(groupId, this.accountId); @@ -136,6 +137,7 @@ public void syncGroupInstances() { @Override public Map getInstanceIpsById() { + //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); Map retVal = new HashMap<>(); IAzureVmGroupRepo awsGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index 773a22b8..713da903 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -116,6 +116,7 @@ public Boolean detachInstance(String instanceId) { @Override public void syncGroupInstances() { + //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); IGcpGroupRepo gcpGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); ApiResponse> instancesResponse = gcpGroupRepo.getGroupInstances(groupId, this.accountId); @@ -147,6 +148,7 @@ public void syncGroupInstances() { public Map getInstanceIpsById() { Map retVal = new HashMap<>(); + //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); IGcpGroupRepo awsGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index f9d7abf3..d515479b 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -24,8 +24,8 @@ public class SpotinstContext { private List groupsDoesNotManageByController; private PassiveExpiringMap candidateGroupsForControllerOwnership; private Map cloudsInitializationState; + private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = generateSuspendedGroupFetchingTime(); - private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = 1000 * 60 * 4; //endregion public static SpotinstContext getInstance() { @@ -35,6 +35,15 @@ public static SpotinstContext getInstance() { return instance; } + private static Integer generateSuspendedGroupFetchingTime() { + Integer retVal; + + Integer redisTimeToLeaveInSeconds = getRedisTimeToLeave(); + retVal = 1000 * redisTimeToLeaveInSeconds + 1; + + return retVal; + } + //region Public Methods public String getSpotinstToken() { return spotinstToken; @@ -97,6 +106,9 @@ public Map getCloudsInitiali return cloudsInitializationState; } + public static Integer getRedisTimeToLeave() { + return 60 * 3; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java index ae540ed8..f1d74ee4 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java @@ -25,7 +25,7 @@ public class SpotinstSyncGroupsOwner extends AsyncPeriodicWork { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncGroupsOwner.class); - public static final Integer JOB_INTERVAL_IN_SECONDS = 60; + public static final Integer JOB_INTERVAL_IN_SECONDS = SpotinstContext.getInstance().getRedisTimeToLeave() / 3; final long recurrencePeriod; //endregion From 387fac3fae6c8de9c197dd49548d9bea11937e85 Mon Sep 17 00:00:00 2001 From: sitay Date: Thu, 12 Jan 2023 14:04:46 +0200 Subject: [PATCH 14/16] one orchestrator check if candidates has been initialized --- .../spotinst/cloud/BaseSpotinstCloud.java | 117 +++++++++++------- 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 2836f762..f2534eca 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -35,9 +35,9 @@ public abstract class BaseSpotinstCloud extends Cloud { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); - protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; + protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; private static final Integer REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS = 60 * 3; - public static final String REDIS_OK_STATUS = "OK"; + public static final String REDIS_OK_STATUS = "OK"; protected String accountId; protected String groupId; protected Map pendingInstances; @@ -145,8 +145,10 @@ public Collection provision(Label label, int excessWorkload) { try { Jenkins.getInstance().addNode(slave); - } catch (IOException e) { - LOGGER.error(String.format("Failed to create node for slave: %s", slave.getInstanceId()), e); + } + catch (IOException e) { + LOGGER.error(String.format("Failed to create node for slave: %s", slave.getInstanceId()), + e); } } } @@ -364,10 +366,10 @@ public SlaveInstanceDetails getSlaveDetails(String instanceId) { return retVal; } - public boolean isCloudReadyForGroupCommunication(String groupId){ + public boolean isCloudReadyForGroupCommunication(String groupId) { boolean retVal = false; - if(StringUtils.isNotEmpty(groupId)) { + if (StringUtils.isNotEmpty(groupId)) { SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); if (state != null) { @@ -468,16 +470,17 @@ private boolean isGlobalExecutorOverrideValid() { } private void addGroupToRedis(String groupId, String accountId, String controllerIdentifier) { - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisSetKeyResponse = redisRepo.setKey(groupId, accountId, controllerIdentifier,REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisSetKeyResponse = + redisRepo.setKey(groupId, accountId, controllerIdentifier, REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); if (redisSetKeyResponse.isRequestSucceed()) { String redisResponseSetKeyValue = redisSetKeyResponse.getValue(); - if(redisResponseSetKeyValue.equals(REDIS_OK_STATUS)){ + if (redisResponseSetKeyValue.equals(REDIS_OK_STATUS)) { LOGGER.info(String.format("Successfully added group %s to redis memory", groupId)); } - else{ + else { LOGGER.error(String.format("Failed adding group %s to redis memory", groupId)); } } @@ -489,8 +492,10 @@ private void addGroupToRedis(String groupId, String accountId, String controller private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(String groupId) { SpotinstCloudCommunicationState state = null; - for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance().getCloudsInitializationState().entrySet()) { - if(cloudsInitializationStateEntry.getKey().getGroupId().equals(groupId)){ + for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance() + .getCloudsInitializationState() + .entrySet()) { + if (cloudsInitializationStateEntry.getKey().getGroupId().equals(groupId)) { state = cloudsInitializationStateEntry.getValue(); } } @@ -657,10 +662,10 @@ protected Integer getNumOfExecutors(String instanceType) { retVal = 1; } else { - int overridedNumOfExecutors = getOverridedNumberOfExecutors(instanceType); + int overridedNumOfExecutors = getOverridedNumberOfExecutors(instanceType); boolean isNumOfExecutorsOverrided = overridedNumOfExecutors != NO_OVERRIDED_NUM_OF_EXECUTORS; - if(isNumOfExecutorsOverrided){ + if (isNumOfExecutorsOverrided) { retVal = overridedNumOfExecutors; } else { @@ -708,64 +713,81 @@ protected Integer getSlaveOfflineThreshold() { } public void handleGroupDosNotManageByThisController(String groupId) { - boolean isGroupExistInCandidateGroupsForControllerOwnership = SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().containsKey(groupId); + boolean isGroupExistInCandidateGroupsForControllerOwnership = + SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().containsKey(groupId); - if(isGroupExistInCandidateGroupsForControllerOwnership){ - SpotinstContext.getInstance().getCloudsInitializationState().put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + if (isGroupExistInCandidateGroupsForControllerOwnership) { + SpotinstContext.getInstance().getCloudsInitializationState() + .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); } - else{ - SpotinstContext.getInstance().getCloudsInitializationState().put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + else { + SpotinstContext.getInstance().getCloudsInitializationState() + .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); } } public void syncGroupsOwner(BaseSpotinstCloud cloud) { - String groupId = cloud.getGroupId(); + String groupId = cloud.getGroupId(); String accountId = cloud.getAccountId(); LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); - String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); + String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); - if (redisGetValueResponse.isRequestSucceed()) { - //redis response might return in different types - if (redisGetValueResponse.getValue() instanceof String) { - String redisResponseValue = (String) redisGetValueResponse.getValue(); + if (redisGetValueResponse.isRequestSucceed()) { + //redis response might return in different types + if (redisGetValueResponse.getValue() instanceof String) { + String redisResponseValue = (String) redisGetValueResponse.getValue(); - if (redisResponseValue != null) { - boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); + if (redisResponseValue != null) { + boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); - if (isGroupBelongToController) { -; handleGroupManagedByThisController(cloud, controllerIdentifier); - } - else { - LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, controllerIdentifier)); - SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().put(groupId, accountId); - SpotinstContext.getInstance().getCloudsInitializationState().replace(cloud, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); - } + if (isGroupBelongToController) { + ; + handleGroupManagedByThisController(cloud, controllerIdentifier); } else { - LOGGER.warn("redis response value return null"); + LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, + controllerIdentifier)); + boolean isContainsCandidates = + SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() + .containsKey(groupId); + + if (isContainsCandidates == false) { + SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() + .put(groupId, accountId); + SpotinstContext.getInstance().getCloudsInitializationState().replace(cloud, + SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + } } } - //there is no controller for the given group in redis, should take ownership else { - handleGroupManagedByThisController(cloud, controllerIdentifier); + LOGGER.warn("redis response value return null"); } } + //there is no controller for the given group in redis, should take ownership + else { + handleGroupManagedByThisController(cloud, controllerIdentifier); + } + } } private void handleGroupManagedByThisController(BaseSpotinstCloud cloud, String controllerIdentifier) { - LOGGER.info(String.format("group %s belong to controller with identifier %s", cloud.getGroupId(), controllerIdentifier)); - SpotinstCloudCommunicationState previousCloudState = SpotinstContext.getInstance().getCloudsInitializationState().get(cloud); - - if(previousCloudState != null){ - if(previousCloudState.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY) == false){ + LOGGER.info(String.format("group %s belong to controller with identifier %s", cloud.getGroupId(), + controllerIdentifier)); + SpotinstCloudCommunicationState previousCloudState = + SpotinstContext.getInstance().getCloudsInitializationState().get(cloud); + + if (previousCloudState != null) { + if (previousCloudState.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY) == + false) { SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); } } - SpotinstContext.getInstance().getCloudsInitializationState().put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); + SpotinstContext.getInstance().getCloudsInitializationState() + .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); //expand TTL of the current controller in redis addGroupToRedis(cloud.getGroupId(), cloud.getAccountId(), controllerIdentifier); } @@ -899,7 +921,8 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { // if enabled, enable and override GlobalExecutorOverride to 1 // better clarity to user, avoid race conditions - boolean shouldDisableGlobalExecutors = isSingleTaskNodesEnabled != null && isSingleTaskNodesEnabled && this.globalExecutorOverride != null; + boolean shouldDisableGlobalExecutors = + isSingleTaskNodesEnabled != null && isSingleTaskNodesEnabled && this.globalExecutorOverride != null; if (shouldDisableGlobalExecutors) { this.globalExecutorOverride.setIsEnabled(false); } From 56d616d4149e95f60b8b4e22950dda00868b934e Mon Sep 17 00:00:00 2001 From: sitay Date: Wed, 18 Jan 2023 13:15:29 +0200 Subject: [PATCH 15/16] one orchestrator Ziv's rejects --- .../spotinst/cloud/AwsSpotinstCloud.java | 63 +++-- .../spotinst/cloud/AzureSpotCloud.java | 11 +- .../spotinst/cloud/AzureSpotinstCloud.java | 4 +- .../spotinst/cloud/BaseSpotinstCloud.java | 235 ++++++++++++++---- .../spotinst/cloud/GcpSpotinstCloud.java | 5 +- .../SpotinstCloudsCommunicationMonitor.java | 98 -------- .../spotinst/cloud/helpers/TimeHelper.java | 42 ++++ .../SpotinstCloudsCommunicationMonitor.java | 127 ++++++++++ .../spotinst/common/GroupStateTracker.java | 46 ++++ .../spotinst/common/SpotinstContext.java | 56 ++--- .../common/SpotinstRestartListener.java | 2 +- .../plugins/spotinst/common/TimeUtils.java | 2 +- .../jobs/SpotinstSyncGroupsOwner.java | 95 +++++-- .../plugins/spotinst/slave/SpotinstSlave.java | 8 +- 14 files changed, 547 insertions(+), 247 deletions(-) delete mode 100644 src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java create mode 100644 src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java create mode 100644 src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java create mode 100644 src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 393e9a7f..778b3c0a 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -18,6 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.*; @@ -32,7 +33,7 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud { private static final String CLOUD_URL = "aws/ec2"; protected Map executorsByInstanceType; private List executorsForTypes; - private List invalidInstanceTypes; + private List invalidInstanceTypes; //endregion //region Constructor @@ -117,37 +118,31 @@ public Boolean detachInstance(String instanceId) { } @Override - public void syncGroupInstances() { - boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); - - if (isGroupManagedByThisController) { - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + protected void handleSyncGroupInstances() { + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); - if (instancesResponse.isRequestSucceed()) { - List instances = instancesResponse.getValue(); - LOGGER.info(String.format("There are %s instances in group %s", instances.size(), groupId)); + if (instancesResponse.isRequestSucceed()) { + List instances = instancesResponse.getValue(); + LOGGER.info(String.format("There are %s instances in group %s", instances.size(), groupId)); - Map slaveInstancesDetailsByInstanceId = new HashMap<>(); + Map slaveInstancesDetailsByInstanceId = new HashMap<>(); - for (AwsGroupInstance instance : instances) { - SlaveInstanceDetails instanceDetails = SlaveInstanceDetails.build(instance); - slaveInstancesDetailsByInstanceId.put(instanceDetails.getInstanceId(), instanceDetails); - } + for (AwsGroupInstance instance : instances) { + SlaveInstanceDetails instanceDetails = SlaveInstanceDetails.build(instance); + slaveInstancesDetailsByInstanceId.put(instanceDetails.getInstanceId(), instanceDetails); + } - this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId); + this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId); - addNewSlaveInstances(instances); - removeOldSlaveInstances(instances); + addNewSlaveInstances(instances); + removeOldSlaveInstances(instances); - } else { - LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, - instancesResponse.getErrors())); - } } - else{ - handleGroupDosNotManageByThisController(groupId); + else { + LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, + instancesResponse.getErrors())); } } @@ -159,8 +154,9 @@ public Map getInstanceIpsById() { boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = + awsGroupRepo.getGroupInstances(groupId, this.accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -168,17 +164,19 @@ public Map getInstanceIpsById() { for (AwsGroupInstance instance : instances) { if (this.getShouldUsePrivateIp()) { retVal.put(instance.getInstanceId(), instance.getPrivateIp()); - } else { + } + else { retVal.put(instance.getInstanceId(), instance.getPublicIp()); } } - } else { + } + else { LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, - instancesResponse.getErrors())); + instancesResponse.getErrors())); } } - else{ - handleGroupDosNotManageByThisController(groupId); + else { + handleGroupDoesNotManageByThisController(accountId, groupId); } return retVal; @@ -392,7 +390,7 @@ private void initExecutorsByInstanceType() { String type = instance.getAwsInstanceTypeFromAPIInput(); this.executorsByInstanceType.put(type, executors); - if(instance.getIsValid() == false){ + if (instance.getIsValid() == false) { LOGGER.error(String.format("Invalid type \'%s\' in group \'%s\'", type, this.getGroupId())); invalidInstanceTypes.add(type); } @@ -416,6 +414,7 @@ public List getInvalidInstanceTypes() { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot AWS Elastigroup"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index 4180a61a..b842560e 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -107,8 +108,7 @@ protected Integer getPendingThreshold() { } @Override - public void syncGroupInstances() { - //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + protected void handleSyncGroupInstances() { IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); ApiResponse> instancesResponse = azureVmGroupRepo.getGroupVms(groupId, this.accountId); @@ -195,7 +195,7 @@ private List handleNewVms(List newVms, S } private SpotinstSlave handleNewVm(String vmName, String vmSize, String label) { - Integer executors = getNumOfExecutors(vmSize); + Integer executors = getNumOfExecutors(vmSize); addToPending(vmName, executors, PendingInstance.StatusEnum.PENDING, label); SpotinstSlave retVal = buildSpotinstSlave(vmName, vmSize, String.valueOf(executors)); return retVal; @@ -262,8 +262,8 @@ private void addSpotinstSlave(AzureGroupVm vm) { SpotinstSlave slave = null; if (vm.getVmName() != null) { - String vmSize = vm.getVmSize(); - Integer executors = getNumOfExecutors(vmSize); + String vmSize = vm.getVmSize(); + Integer executors = getNumOfExecutors(vmSize); slave = buildSpotinstSlave(vm.getVmName(), vmSize, String.valueOf(executors)); } @@ -294,6 +294,7 @@ private Boolean isSlaveExistForInstance(AzureGroupVm vm) { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot Azure Elastigroup"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java index 99e25a99..d7611af9 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java @@ -21,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.*; @@ -88,7 +89,7 @@ public Boolean detachInstance(String instanceId) { } @Override - public void syncGroupInstances() { + protected void handleSyncGroupInstances() { } @@ -322,6 +323,7 @@ private void addToGroupPending(ProvisionRequest request) { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot Azure LPVM (old)"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index f2534eca..486f7946 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -5,6 +5,7 @@ import hudson.model.labels.LabelAtom; import hudson.plugins.spotinst.api.infra.ApiResponse; import hudson.plugins.spotinst.api.infra.JsonMapper; +import hudson.plugins.spotinst.cloud.helpers.TimeHelper; import hudson.plugins.spotinst.common.*; import hudson.plugins.spotinst.repos.IRedisRepo; import hudson.plugins.spotinst.repos.RepoManager; @@ -27,6 +28,9 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING; +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY; + /** * Created by ohadmuchnik on 25/05/2016. */ @@ -158,7 +162,7 @@ public Collection provision(Label label, int excessWorkload) { } } else { - handleGroupDosNotManageByThisController(groupId); + handleGroupDoesNotManageByThisController(accountId, groupId); } return Collections.emptyList(); @@ -216,7 +220,7 @@ public void monitorInstances() { Integer pendingThreshold = getPendingThreshold(); Boolean isPendingOverThreshold = - TimeUtils.isTimePassed(pendingInstance.getCreatedAt(), pendingThreshold); + TimeUtils.isTimePassedInMinutes(pendingInstance.getCreatedAt(), pendingThreshold); if (isPendingOverThreshold) { LOGGER.info(String.format( @@ -347,7 +351,7 @@ protected void terminateOfflineSlaves(SpotinstSlave slave, String slaveInstanceI Date slaveCreatedAt = slave.getCreatedAt(); - Boolean isOverOfflineThreshold = TimeUtils.isTimePassed(slaveCreatedAt, offlineThreshold); + Boolean isOverOfflineThreshold = TimeUtils.isTimePassedInMinutes(slaveCreatedAt, offlineThreshold); if (isSlaveOffline && isSlaveConnecting == false && isOverOfflineThreshold && temporarilyOffline == false && isOverIdleThreshold) { @@ -366,14 +370,31 @@ public SlaveInstanceDetails getSlaveDetails(String instanceId) { return retVal; } + // public boolean isCloudReadyForGroupCommunication(String groupId) { + // boolean retVal = false; + // + // if (StringUtils.isNotEmpty(groupId)) { + // SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); + // + // if (state != null) { + // if (state.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)) { + // retVal = true; + // } + // } + // } + // + // return retVal; + // } + public boolean isCloudReadyForGroupCommunication(String groupId) { boolean retVal = false; if (StringUtils.isNotEmpty(groupId)) { - SpotinstCloudCommunicationState state = getCloudInitializationResultByGroupId(groupId); + GroupStateTracker stateDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); - if (state != null) { - if (state.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)) { + if (stateDetails != null && stateDetails.getState() != null) { + if (stateDetails.getState() + .equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)) { retVal = true; } } @@ -387,8 +408,8 @@ public boolean isCloudReadyForGroupCommunication(String groupId) { private synchronized List provisionSlaves(ProvisionRequest request) { LOGGER.info(String.format("Scale up group: %s with %s workload units", groupId, request.getExecutors())); - List slaves = scaleUp(request); - return slaves; + List retVal = scaleUp(request); + return retVal; } private void setNumOfNeededExecutors(ProvisionRequest request) { @@ -470,7 +491,7 @@ private boolean isGlobalExecutorOverrideValid() { } private void addGroupToRedis(String groupId, String accountId, String controllerIdentifier) { - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); ApiResponse redisSetKeyResponse = redisRepo.setKey(groupId, accountId, controllerIdentifier, REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); @@ -489,19 +510,19 @@ private void addGroupToRedis(String groupId, String accountId, String controller } } - private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(String groupId) { - SpotinstCloudCommunicationState state = null; - - for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance() - .getCloudsInitializationState() - .entrySet()) { - if (cloudsInitializationStateEntry.getKey().getGroupId().equals(groupId)) { - state = cloudsInitializationStateEntry.getValue(); - } - } - - return state; - } + // private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(String groupId) { + // SpotinstCloudCommunicationState state = null; + // + // for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance() + // .getCloudsInitializationState() + // .entrySet()) { + // if (cloudsInitializationStateEntry.getKey().getGroupId().equals(groupId)) { + // state = cloudsInitializationStateEntry.getValue(); + // } + // } + // + // return state; + // } //endregion //region Protected Methods @@ -712,20 +733,102 @@ protected Integer getSlaveOfflineThreshold() { return Constants.SLAVE_OFFLINE_THRESHOLD_IN_MINUTES; } - public void handleGroupDosNotManageByThisController(String groupId) { - boolean isGroupExistInCandidateGroupsForControllerOwnership = - SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().containsKey(groupId); + // public void handleGroupDosNotManageByThisController(String groupId) { + // boolean isGroupExistInCandidateGroupsForControllerOwnership = + // SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership().containsKey(groupId); + // + // if (isGroupExistInCandidateGroupsForControllerOwnership) { + // Boolean hasValidState = SpotinstContext.getInstance().getCloudsInitializationState().containsKey(this); + // + // if (hasValidState == false) { + // SpotinstContext.getInstance().getCloudsInitializationState() + // .put(this, SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + // } + // } + // else { + // SpotinstContext.getInstance().getCloudsInitializationState() + // .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + // } + // } + + public void handleGroupDoesNotManageByThisController(String accountId, String groupId) { + GroupStateTracker groupStateDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (groupStateDetails == null || groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY)) { + if (groupStateDetails == null) { + groupStateDetails = new GroupStateTracker(); + } + + groupStateDetails.setGroupId(groupId); + groupStateDetails.setAccountId(accountId); + groupStateDetails.setState(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + Date now = new Date(); + groupStateDetails.setTimeStamp(now); - if (isGroupExistInCandidateGroupsForControllerOwnership) { - SpotinstContext.getInstance().getCloudsInitializationState() - .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); } - else { - SpotinstContext.getInstance().getCloudsInitializationState() - .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + else if (groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING)) { + Date timeStamp = groupStateDetails.getTimeStamp(); + boolean shouldFail = TimeHelper.isTimePassedInSeconds(timeStamp); + + if (shouldFail) { + groupStateDetails.setState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); + } } } + // public void syncGroupsOwner(BaseSpotinstCloud cloud) { + // String groupId = cloud.getGroupId(); + // String accountId = cloud.getAccountId(); + // LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); + // + // IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + // ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); + // String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); + // + // if (redisGetValueResponse.isRequestSucceed()) { + // //redis response might return in different types + // if (redisGetValueResponse.getValue() instanceof String) { + // String redisResponseValue = (String) redisGetValueResponse.getValue(); + // + // if (redisResponseValue != null) { + // boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); + // + // if (isGroupBelongToController) { + // handleGroupManagedByThisController(cloud, controllerIdentifier); + // } + // else { + // LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, + // controllerIdentifier)); + // boolean isContainsCandidates = + // SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() + // .containsKey(groupId); + // + // if (isContainsCandidates == false) { + // SpotinstCloudCommunicationState cloudState = + // SpotinstContext.getInstance().getCloudsInitializationState().get(cloud); + // + // if (cloudState == null) { + // SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() + // .put(groupId, accountId); + // SpotinstContext.getInstance().getCloudsInitializationState() + // .put(cloud, SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + // } + // } + // } + // } + // else { + // LOGGER.warn("redis response value return null"); + // } + // } + // //there is no controller for the given group in redis, should take ownership + // else { + // handleGroupManagedByThisController(cloud, controllerIdentifier); + // } + // } + // } + public void syncGroupsOwner(BaseSpotinstCloud cloud) { String groupId = cloud.getGroupId(); String accountId = cloud.getAccountId(); @@ -744,50 +847,61 @@ public void syncGroupsOwner(BaseSpotinstCloud cloud) { boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); if (isGroupBelongToController) { - ; handleGroupManagedByThisController(cloud, controllerIdentifier); } else { LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, controllerIdentifier)); - boolean isContainsCandidates = - SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() - .containsKey(groupId); - - if (isContainsCandidates == false) { - SpotinstContext.getInstance().getCandidateGroupsForControllerOwnership() - .put(groupId, accountId); - SpotinstContext.getInstance().getCloudsInitializationState().replace(cloud, - SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + GroupStateTracker groupStateDetails = + SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (groupStateDetails == null) { + groupStateDetails = new GroupStateTracker(); + groupStateDetails.setGroupId(groupId); + groupStateDetails.setAccountId(accountId); + groupStateDetails.setState(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + Date now = new Date(); + groupStateDetails.setTimeStamp(now); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); } } } - else { - LOGGER.warn("redis response value return null"); - } } - //there is no controller for the given group in redis, should take ownership else { - handleGroupManagedByThisController(cloud, controllerIdentifier); + LOGGER.warn("redis response value return null"); } } + //there is no controller for the given group in redis, should take ownership + else { + handleGroupManagedByThisController(cloud, controllerIdentifier); + } } + private void handleGroupManagedByThisController(BaseSpotinstCloud cloud, String controllerIdentifier) { LOGGER.info(String.format("group %s belong to controller with identifier %s", cloud.getGroupId(), controllerIdentifier)); - SpotinstCloudCommunicationState previousCloudState = - SpotinstContext.getInstance().getCloudsInitializationState().get(cloud); + GroupStateTracker cloudDetails = + SpotinstContext.getInstance().getConnectionStateByGroupId().get(cloud.getGroupId()); - if (previousCloudState != null) { - if (previousCloudState.equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY) == - false) { - SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); + if (cloudDetails != null) { + + SpotinstCloudCommunicationState cloudState = cloudDetails.getState(); + + if (cloudState != null && cloudState.equals(SPOTINST_CLOUD_COMMUNICATION_READY) == false) { + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(cloud.getGroupId()); } } + else { + cloudDetails = new GroupStateTracker(); + } - SpotinstContext.getInstance().getCloudsInitializationState() - .put(this, SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); + cloudDetails.setGroupId(cloud.getGroupId()); + cloudDetails.setAccountId(cloud.getAccountId()); + cloudDetails.setState(SPOTINST_CLOUD_COMMUNICATION_READY); + Date now = new Date(); + cloudDetails.setTimeStamp(now); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(cloud.getGroupId(), cloudDetails); //expand TTL of the current controller in redis addGroupToRedis(cloud.getGroupId(), cloud.getAccountId(), controllerIdentifier); } @@ -937,7 +1051,18 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { public abstract String getCloudUrl(); - public abstract void syncGroupInstances(); + public void syncGroupInstances(){ + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + + if (isGroupManagedByThisController) { + handleSyncGroupInstances(); + } + else{ + handleGroupDoesNotManageByThisController(accountId, groupId); + } + } + + protected abstract void handleSyncGroupInstances(); public abstract Map getInstanceIpsById(); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index 713da903..c95e57ae 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -22,6 +22,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.HashMap; import java.util.LinkedList; @@ -115,8 +116,7 @@ public Boolean detachInstance(String instanceId) { } @Override - public void syncGroupInstances() { - //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + protected void handleSyncGroupInstances() { IGcpGroupRepo gcpGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); ApiResponse> instancesResponse = gcpGroupRepo.getGroupInstances(groupId, this.accountId); @@ -292,6 +292,7 @@ private void addSpotinstGcpSlave(GcpGroupInstance instance) { @Extension public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl { + @Nonnull @Override public String getDisplayName() { return "Spot GCP Elastigroup"; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java b/src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java deleted file mode 100644 index f7262aa4..00000000 --- a/src/main/java/hudson/plugins/spotinst/cloud/SpotinstCloudsCommunicationMonitor.java +++ /dev/null @@ -1,98 +0,0 @@ -package hudson.plugins.spotinst.cloud; - -import hudson.Extension; -import hudson.model.AdministrativeMonitor; -import hudson.plugins.spotinst.common.SpotinstCloudCommunicationState; -import hudson.plugins.spotinst.common.SpotinstContext; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Extension -public class SpotinstCloudsCommunicationMonitor extends AdministrativeMonitor { - - //region Members - private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); - List spotinstCloudsCommunicationFailures; - List spotinstCloudsCommunicationInitializing; - List spotinstCloudsCommunicationReady; - //endregion - - //region Overridden Public Methods - @Override - public boolean isActivated() { - return isSpotinstCloudsCommunicationFailuresExist() || isSpotinstCloudsCommunicationInitializingExist() - || isSpotinstCloudsCommunicationReadyExist(); - } - - @Override - public String getDisplayName() { - return "Spotinst Clouds Communication Monitor"; - } - //endregion - - //region getters & setters - public boolean isSpotinstCloudsCommunicationFailuresExist() { - boolean isCloudsWithFailureStateExist = SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); - boolean isCloudsWithGroupIdExist = CollectionUtils.isNotEmpty(getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED)); - return isCloudsWithFailureStateExist && isCloudsWithGroupIdExist; - } - - public boolean isSpotinstCloudsCommunicationInitializingExist() { - return SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); - } - - public boolean isSpotinstCloudsCommunicationReadyExist() { - return SpotinstContext.getInstance().getCloudsInitializationState().containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); - } - - public String getSpotinstCloudsCommunicationFailures() { - String retVal; - - spotinstCloudsCommunicationFailures = getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); - retVal = spotinstCloudsCommunicationFailures.stream().collect(Collectors.joining(", ")); - - return retVal; - } - - public String getSpotinstCloudsCommunicationInitializing() { - String retVal; - - spotinstCloudsCommunicationInitializing = getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); - retVal = spotinstCloudsCommunicationInitializing.stream().collect(Collectors.joining(", ")); - - return retVal; - } - - public String getSpotinstCloudsCommunicationReady() { - String retVal; - - spotinstCloudsCommunicationReady = getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); - retVal = spotinstCloudsCommunicationReady.stream().collect(Collectors.joining(", ")); - - return retVal; - } - - private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState state) { - List retVal = new ArrayList<>(); - - for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance().getCloudsInitializationState().entrySet()) { - if (cloudsInitializationStateEntry.getValue().equals(state)) { - BaseSpotinstCloud cloud = cloudsInitializationStateEntry.getKey(); - - if (StringUtils.isNotEmpty(cloud.getGroupId())) { - retVal.add(cloud.getGroupId()); - } - } - } - - return retVal; - } - //endregion -} diff --git a/src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java b/src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java new file mode 100644 index 00000000..7c19ff88 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/helpers/TimeHelper.java @@ -0,0 +1,42 @@ +package hudson.plugins.spotinst.cloud.helpers; + +import java.util.Calendar; +import java.util.Date; + +public class TimeHelper { + //region members + private static final Integer redisTimeToLeaveInSeconds = 60 * 3; + private static final Integer miliToSeconds = 1000; + private static final Integer MILI_TO_SECONDS = 1000; + public static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = generateSuspendedGroupFetchingTime(); + //endregion + + //region methods + public static Boolean isTimePassedInSeconds(Date from) { + Boolean retVal = false; + Date now = new Date(); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(from); + calendar.add(Calendar.SECOND, SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS / MILI_TO_SECONDS); + Date timeToPass = calendar.getTime(); + + if (now.after(timeToPass)) { + retVal = true; + } + + return retVal; + } + + public static Integer getRedisTimeToLeaveInSeconds() { + return redisTimeToLeaveInSeconds; + } + //endregion + + //region private methods + private static Integer generateSuspendedGroupFetchingTime() { + Integer retVal = miliToSeconds * redisTimeToLeaveInSeconds + 10; + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java b/src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java new file mode 100644 index 00000000..347a234b --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/monitor/SpotinstCloudsCommunicationMonitor.java @@ -0,0 +1,127 @@ +package hudson.plugins.spotinst.cloud.monitor; + +import hudson.Extension; +import hudson.model.AdministrativeMonitor; +import hudson.plugins.spotinst.common.GroupStateTracker; +import hudson.plugins.spotinst.common.SpotinstCloudCommunicationState; +import hudson.plugins.spotinst.common.SpotinstContext; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.*; + +@Extension +public class SpotinstCloudsCommunicationMonitor extends AdministrativeMonitor { + + //region Members + List spotinstCloudsCommunicationFailures; + List spotinstCloudsCommunicationInitializing; + //endregion + + //region Overridden Public Methods + @Override + public boolean isActivated() { + return isSpotinstCloudsCommunicationFailuresExist() || isSpotinstCloudsCommunicationInitializingExist(); + } + + @Override + public String getDisplayName() { + return "Spotinst Clouds Communication Monitor"; + } + //endregion + + //region getters & setters +// public boolean isSpotinstCloudsCommunicationFailuresExist() { +// boolean isCloudsWithFailureStateExist = SpotinstContext.getInstance().getCloudsInitializationState() +// .containsValue( +// SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); +// boolean isCloudsWithGroupIdExist = CollectionUtils.isNotEmpty(getGroupsIdByCloudInitializationState( +// SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED)); +// return isCloudsWithFailureStateExist && isCloudsWithGroupIdExist; +// } + + public boolean isSpotinstCloudsCommunicationFailuresExist() { + return isSpotinstCloudsCommunicationStateExist(SPOTINST_CLOUD_COMMUNICATION_FAILED); + } + +// public boolean isSpotinstCloudsCommunicationInitializingExist() { +// return SpotinstContext.getInstance().getCloudsInitializationState() +// .containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); +// } + + public boolean isSpotinstCloudsCommunicationInitializingExist() { + return isSpotinstCloudsCommunicationStateExist(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + } + +// public boolean isSpotinstCloudsCommunicationReadyExist() { +// return SpotinstContext.getInstance().getCloudsInitializationState() +// .containsValue(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY); +// } + + private boolean isSpotinstCloudsCommunicationStateExist(SpotinstCloudCommunicationState state){ + Collection allCurrentGroups = + SpotinstContext.getInstance().getConnectionStateByGroupId().values(); + + boolean isCloudsWithReadyStateExist = allCurrentGroups.stream().anyMatch( + groupDetails -> state.equals( + groupDetails.getState())); + + return isCloudsWithReadyStateExist; + } + + public String getSpotinstCloudsCommunicationFailures() { + String retVal; + + spotinstCloudsCommunicationFailures = getGroupsIdByCloudInitializationState( + SPOTINST_CLOUD_COMMUNICATION_FAILED); + retVal = String.join(", ", spotinstCloudsCommunicationFailures); + + return retVal; + } + + public String getSpotinstCloudsCommunicationInitializing() { + String retVal; + + spotinstCloudsCommunicationInitializing = getGroupsIdByCloudInitializationState( + SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); + retVal = String.join(", ", spotinstCloudsCommunicationInitializing); + + return retVal; + } + +// private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState state) { +// List retVal = new ArrayList<>(); +// +// for (Map.Entry cloudsInitializationStateEntry : SpotinstContext.getInstance() +// .getCloudsInitializationState() +// .entrySet()) { +// if (cloudsInitializationStateEntry.getValue().equals(state)) { +// BaseSpotinstCloud cloud = cloudsInitializationStateEntry.getKey(); +// +// if (StringUtils.isNotEmpty(cloud.getGroupId())) { +// retVal.add(cloud.getGroupId()); +// } +// } +// } +// +// return retVal; +// } + + private List getGroupsIdByCloudInitializationState(SpotinstCloudCommunicationState state) { + List retVal = new ArrayList<>(); + + Collection allCurrentGroups = + SpotinstContext.getInstance().getConnectionStateByGroupId().values(); + allCurrentGroups.forEach(group -> { + if(state.equals(group.getState()) && StringUtils.isNotEmpty(group.getGroupId())){ + retVal.add(group.getGroupId()); + } + }); + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java b/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java new file mode 100644 index 00000000..0263d003 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java @@ -0,0 +1,46 @@ +package hudson.plugins.spotinst.common; + +import java.util.Date; + +public class GroupStateTracker { + //region members + private String groupId; + private String accountId; + private SpotinstCloudCommunicationState state; + private Date timeStamp; + //endregion + + //region getters & setters + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + public String getAccountId() { + return accountId; + } + + public void setAccountId(String accountId) { + this.accountId = accountId; + } + + public SpotinstCloudCommunicationState getState() { + return state; + } + + public void setState(SpotinstCloudCommunicationState state) { + this.state = state; + } + + public Date getTimeStamp() { + return timeStamp; + } + + public void setTimeStamp(Date timeStamp) { + this.timeStamp = timeStamp; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index d515479b..06b6cc6a 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -1,9 +1,7 @@ package hudson.plugins.spotinst.common; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; import hudson.plugins.spotinst.model.aws.AwsInstanceType; -import org.apache.commons.collections4.map.PassiveExpiringMap; import org.apache.commons.lang.RandomStringUtils; import java.util.*; @@ -20,12 +18,9 @@ public class SpotinstContext { private List awsInstanceTypes; private Date awsInstanceTypesLastUpdate; private String controllerIdentifier; - private Map groupsManageByController; - private List groupsDoesNotManageByController; - private PassiveExpiringMap candidateGroupsForControllerOwnership; - private Map cloudsInitializationState; - private static final Integer SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS = generateSuspendedGroupFetchingTime(); - + private Map connectionStateByGroupId; +// private PassiveExpiringMap candidateGroupsForControllerOwnership; +// private Map cloudsInitializationState; //endregion public static SpotinstContext getInstance() { @@ -35,15 +30,6 @@ public static SpotinstContext getInstance() { return instance; } - private static Integer generateSuspendedGroupFetchingTime() { - Integer retVal; - - Integer redisTimeToLeaveInSeconds = getRedisTimeToLeave(); - retVal = 1000 * redisTimeToLeaveInSeconds + 1; - - return retVal; - } - //region Public Methods public String getSpotinstToken() { return spotinstToken; @@ -90,24 +76,28 @@ public String getControllerIdentifier() { return controllerIdentifier; } - public PassiveExpiringMap getCandidateGroupsForControllerOwnership() { - if (candidateGroupsForControllerOwnership == null) { - candidateGroupsForControllerOwnership = new PassiveExpiringMap<>(SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS); +// public PassiveExpiringMap getCandidateGroupsForControllerOwnership() { +// if (candidateGroupsForControllerOwnership == null) { +// candidateGroupsForControllerOwnership = new PassiveExpiringMap<>(SUSPENDED_GROUP_FETCHING_TIME_TO_LIVE_IN_MILLIS); +// } +// +// return candidateGroupsForControllerOwnership; +// } +// +// public Map getCloudsInitializationState() { +// if (cloudsInitializationState == null) { +// cloudsInitializationState = new HashMap<>(); +// } +// +// return cloudsInitializationState; +// } + + public Map getConnectionStateByGroupId() { + if(connectionStateByGroupId == null){ + connectionStateByGroupId = new HashMap<>(); } - return candidateGroupsForControllerOwnership; - } - - public Map getCloudsInitializationState() { - if (cloudsInitializationState == null) { - cloudsInitializationState = new HashMap<>(); - } - - return cloudsInitializationState; - } - - public static Integer getRedisTimeToLeave() { - return 60 * 3; + return connectionStateByGroupId; } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java index a6c3a5ff..a0d9012c 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstRestartListener.java @@ -47,6 +47,6 @@ public void onRestart() { } LOGGER.info(String.format("deallocating %s Spotinst clouds", cloudSet.size())); - groupsOwnerJob.deallocateGroupsNoLongerInUse(cloudSet); + groupsOwnerJob.deallocateCloudsNoLongerInUse(cloudSet);//TODO: execute instead? } } diff --git a/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java b/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java index 5d7a8c92..5b1c5379 100644 --- a/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java +++ b/src/main/java/hudson/plugins/spotinst/common/TimeUtils.java @@ -8,7 +8,7 @@ * Created by ohadmuchnik on 21/03/2017. */ public class TimeUtils { - public static Boolean isTimePassed(Date from, Integer minutes) { + public static Boolean isTimePassedInMinutes(Date from, Integer minutes) { Boolean retVal = false; Date now = new Date(); Calendar calendar = Calendar.getInstance(); diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java index f1d74ee4..f66884aa 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java @@ -5,6 +5,8 @@ import hudson.model.TaskListener; import hudson.plugins.spotinst.api.infra.ApiResponse; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.cloud.helpers.TimeHelper; +import hudson.plugins.spotinst.common.GroupStateTracker; import hudson.plugins.spotinst.common.SpotinstContext; import hudson.plugins.spotinst.repos.IRedisRepo; import hudson.plugins.spotinst.repos.RepoManager; @@ -16,6 +18,7 @@ import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; /** * Created by ohadmuchnik on 25/05/2016. @@ -24,34 +27,65 @@ public class SpotinstSyncGroupsOwner extends AsyncPeriodicWork { //region Members - private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncGroupsOwner.class); - public static final Integer JOB_INTERVAL_IN_SECONDS = SpotinstContext.getInstance().getRedisTimeToLeave() / 3; + private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstSyncGroupsOwner.class); + private static final Integer redisToJobRatio = 3; + public final Integer JOB_INTERVAL_IN_SECONDS; + final long recurrencePeriod; //endregion //region Constructor public SpotinstSyncGroupsOwner() { super("Sync Groups Owner"); + JOB_INTERVAL_IN_SECONDS = TimeHelper.getRedisTimeToLeaveInSeconds() / redisToJobRatio; recurrencePeriod = TimeUnit.SECONDS.toMillis(JOB_INTERVAL_IN_SECONDS); } //endregion //region Public Methods + // @Override + // protected void execute(TaskListener taskListener) { + // synchronized (this) { + // List cloudList = Jenkins.getInstance().clouds; + // Set cloudsFromContext = + // SpotinstContext.getInstance().getCloudsInitializationState().keySet(); + // Set cloudsNoLongerExist = new HashSet<>(cloudsFromContext); + // + // if (cloudList != null && cloudList.size() > 0) { + // for (Cloud cloud : cloudList) { + // + // if (cloud instanceof BaseSpotinstCloud) { + // BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; + // String groupId = spotinstCloud.getGroupId(); + // String accountId = spotinstCloud.getAccountId(); + // cloudsNoLongerExist.remove(spotinstCloud); + // + // if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + // spotinstCloud.syncGroupsOwner(spotinstCloud); + // } + // } + // } + // } + // + // deallocateGroupsNoLongerInUse(cloudsNoLongerExist); + // } + // } + @Override protected void execute(TaskListener taskListener) { synchronized (this) { - List cloudList = Jenkins.getInstance().clouds; - Set cloudsFromContext = SpotinstContext.getInstance().getCloudsInitializationState().keySet(); - Set cloudsNoLongerExist = new HashSet<>(cloudsFromContext); + List cloudList = Jenkins.getInstance().clouds; + Set groupsFromContext = SpotinstContext.getInstance().getConnectionStateByGroupId().keySet(); + Set groupsNoLongerExist = new HashSet<>(groupsFromContext); if (cloudList != null && cloudList.size() > 0) { for (Cloud cloud : cloudList) { if (cloud instanceof BaseSpotinstCloud) { BaseSpotinstCloud spotinstCloud = (BaseSpotinstCloud) cloud; - String groupId = spotinstCloud.getGroupId(); - String accountId = spotinstCloud.getAccountId(); - cloudsNoLongerExist.remove(spotinstCloud); + String groupId = spotinstCloud.getGroupId(); + String accountId = spotinstCloud.getAccountId(); + groupsNoLongerExist.remove(groupId); if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { spotinstCloud.syncGroupsOwner(spotinstCloud); @@ -59,24 +93,51 @@ protected void execute(TaskListener taskListener) { } } } - - deallocateGroupsNoLongerInUse(cloudsNoLongerExist); + + deallocateGroupsNoLongerInUse(groupsNoLongerExist); } } - public void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { - for (BaseSpotinstCloud cloud : cloudsNoLongerExist){ - String groupId = cloud.getGroupId(); - String accountId = cloud.getAccountId(); - SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); + // public void deallocateGroupsNoLongerInUse(Set cloudsNoLongerExist) { + // for (BaseSpotinstCloud cloud : cloudsNoLongerExist) { + // String groupId = cloud.getGroupId(); + // String accountId = cloud.getAccountId(); + // SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); + // + // if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { + // IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + // ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); + // + // if (redisGetValueResponse.isRequestSucceed()) { + // LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); + // } + // else { + // LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); + // } + // } + // } + // } + + public void deallocateCloudsNoLongerInUse(Set cloudsNoLongerExist) { + Set groupsNoLongerExist = + cloudsNoLongerExist.stream().map(BaseSpotinstCloud::getGroupId).collect(Collectors.toSet()); + deallocateGroupsNoLongerInUse(groupsNoLongerExist); + } + + private void deallocateGroupsNoLongerInUse(Set groupsNoLongerExist) { + for (String groupId : groupsNoLongerExist) { + GroupStateTracker groupDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + String accountId = groupDetails.getAccountId(); + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); if (redisGetValueResponse.isRequestSucceed()) { LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); - } else { + } + else { LOGGER.error(String.format("Failed to remove group %s from redis", groupId)); } } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 3fcd8be8..c52457d6 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -10,6 +10,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; import java.util.Date; import java.util.List; @@ -174,6 +175,7 @@ public Node asNode() { //region Public Methods public void terminate() { String groupId = getSpotinstCloud().getGroupId(); + String accountId = getSpotinstCloud().getAccountId(); boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { @@ -193,7 +195,7 @@ public void terminate() { } else{ try { - getSpotinstCloud().handleGroupDosNotManageByThisController(groupId); + getSpotinstCloud().handleGroupDoesNotManageByThisController(accountId, groupId); }catch (Exception e){ LOGGER.warn(e.getMessage()); } } @@ -203,6 +205,7 @@ public Boolean forceTerminate() { Boolean retVal = false; String groupId = getSpotinstCloud().getGroupId(); + String accountId = getSpotinstCloud().getAccountId(); boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { @@ -224,7 +227,7 @@ public Boolean forceTerminate() { retVal = isTerminated; } else{ - getSpotinstCloud().handleGroupDosNotManageByThisController(groupId); + getSpotinstCloud().handleGroupDoesNotManageByThisController(accountId, groupId); } return retVal; @@ -254,6 +257,7 @@ public void onSlaveConnected() { @Extension public static class DescriptorImpl extends SlaveDescriptor { + @Nonnull @Override public String getDisplayName() { return "Spot Node"; From 375ba212ab21f5e521e3c461f7a87f39487088d1 Mon Sep 17 00:00:00 2001 From: sitay Date: Thu, 19 Jan 2023 10:32:03 +0200 Subject: [PATCH 16/16] one orchestrator Ziv's rejects --- .../plugins/spotinst/api/SpotinstApi.java | 136 ++++++------ .../spotinst/cloud/AwsSpotinstCloud.java | 41 ++-- .../spotinst/cloud/AzureSpotCloud.java | 5 +- .../spotinst/cloud/AzureSpotinstCloud.java | 16 +- .../spotinst/cloud/BaseSpotinstCloud.java | 204 +++++++++--------- .../spotinst/cloud/GcpSpotinstCloud.java | 5 +- .../plugins/spotinst/common/Constants.java | 2 + .../spotinst/common/GroupStateTracker.java | 31 ++- .../spotinst/common/SpotinstContext.java | 2 +- .../jobs/SpotinstSyncGroupsOwner.java | 10 +- ...va => GetGroupControllerLockResponse.java} | 2 +- ...t.java => LockGroupControllerRequest.java} | 28 ++- ....java => LockGroupControllerResponse.java} | 2 +- ...ava => UnlockGroupControllerResponse.java} | 2 +- .../plugins/spotinst/repos/ILockRepo.java | 12 ++ .../plugins/spotinst/repos/IRedisRepo.java | 12 -- .../repos/{RedisRepo.java => LockRepo.java} | 20 +- .../plugins/spotinst/repos/RepoManager.java | 12 +- 18 files changed, 268 insertions(+), 274 deletions(-) rename src/main/java/hudson/plugins/spotinst/model/redis/{RedisGetValueResponse.java => GetGroupControllerLockResponse.java} (57%) rename src/main/java/hudson/plugins/spotinst/model/redis/{RedisSetKeyRequest.java => LockGroupControllerRequest.java} (66%) rename src/main/java/hudson/plugins/spotinst/model/redis/{RedisDeleteKeyResponse.java => LockGroupControllerResponse.java} (58%) rename src/main/java/hudson/plugins/spotinst/model/redis/{RedisSetKeyResponse.java => UnlockGroupControllerResponse.java} (57%) create mode 100644 src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java delete mode 100644 src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java rename src/main/java/hudson/plugins/spotinst/repos/{RedisRepo.java => LockRepo.java} (57%) diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index b8fb7d3e..f64ab14c 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -7,10 +7,10 @@ import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.model.azure.*; import hudson.plugins.spotinst.model.gcp.*; -import hudson.plugins.spotinst.model.redis.RedisDeleteKeyResponse; -import hudson.plugins.spotinst.model.redis.RedisGetValueResponse; -import hudson.plugins.spotinst.model.redis.RedisSetKeyRequest; -import hudson.plugins.spotinst.model.redis.RedisSetKeyResponse; +import hudson.plugins.spotinst.model.redis.UnlockGroupControllerResponse; +import hudson.plugins.spotinst.model.redis.GetGroupControllerLockResponse; +import hudson.plugins.spotinst.model.redis.LockGroupControllerRequest; +import hudson.plugins.spotinst.model.redis.LockGroupControllerResponse; import jenkins.model.Jenkins; import org.apache.commons.httpclient.HttpStatus; import org.slf4j.Logger; @@ -93,8 +93,9 @@ public static AwsScaleUpResult awsScaleUp(String groupId, int adjustment, String Map queryParams = buildQueryParams(accountId); queryParams.put(QUERY_PARAM_ADJUSTMENT, String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/scale/up", null, headers, queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/scale/up", null, headers, + queryParams); AwsScaleUpResponse scaleUpResponse = getCastedResponse(response, AwsScaleUpResponse.class); @@ -129,13 +130,12 @@ public static List getAllAwsInstanceTypes(String accountId) thr List retVal; Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - queryParams.put("distinctTypesList","true"); + queryParams.put("distinctTypesList", "true"); - RestResponse response = - RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/instanceType", headers, queryParams); + RestResponse response = RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/instanceType", headers, queryParams); - AwsInstanceTypesResponse - allAwsInstanceTypesResponse = getCastedResponse(response, AwsInstanceTypesResponse.class); + AwsInstanceTypesResponse allAwsInstanceTypesResponse = + getCastedResponse(response, AwsInstanceTypesResponse.class); retVal = allAwsInstanceTypesResponse.getResponse().getItems(); @@ -153,8 +153,9 @@ public static GcpScaleUpResult gcpScaleUp(String groupId, int adjustment, String Map queryParams = buildQueryParams(accountId); queryParams.put(QUERY_PARAM_ADJUSTMENT, String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/scale/up", null, headers, queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/scale/up", null, headers, + queryParams); GcpScaleUpResponse scaleUpResponse = getCastedResponse(response, GcpScaleUpResponse.class); @@ -175,9 +176,9 @@ public static Boolean gcpDetachInstance(String groupId, String instanceName, Str request.setShouldTerminateInstances(true); String body = JsonMapper.toJson(request); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/detachInstances", body, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/detachInstances", body, headers, + queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -210,8 +211,9 @@ public static List getAzureGroupInstances(String groupId, Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - RestResponse response = RestClient - .sendGet(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/status", headers, queryParams); + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/status", headers, + queryParams); AzureGroupInstancesResponse instancesResponse = getCastedResponse(response, AzureGroupInstancesResponse.class); @@ -228,9 +230,9 @@ public static Boolean azureScaleUp(String groupId, int adjustment, String accoun Map queryParams = buildQueryParams(accountId); queryParams.put("adjustment", String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/scale/up", null, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/scale/up", null, headers, + queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -247,9 +249,9 @@ public static Boolean azureDetachInstance(String groupId, String instanceId, Str request.setShouldDecrementTargetCapacity(true); String body = JsonMapper.toJson(request); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/detachInstances", body, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/detachInstances", body, + headers, queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -264,9 +266,9 @@ public static AzureGroupStatus getAzureVmGroupStatus(String groupId, String acco Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - RestResponse response = RestClient - .sendGet(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/status", headers, - queryParams); + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/status", + headers, queryParams); AzureGroupStatusResponse vmsResponse = getCastedResponse(response, AzureGroupStatusResponse.class); @@ -285,9 +287,9 @@ public static List azureVmScaleUp(String groupId, int a Map queryParams = buildQueryParams(accountId); queryParams.put("adjustment", String.valueOf(adjustment)); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/scale/up", null, headers, - queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/scale/up", + null, headers, queryParams); AzureScaleUpResponse scaleUpResponse = getCastedResponse(response, AzureScaleUpResponse.class); @@ -307,9 +309,9 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI request.setShouldDecrementTargetCapacity(true); request.setShouldTerminateVms(true); String body = JsonMapper.toJson(request); - RestResponse response = RestClient - .sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/detachVms", body, - headers, queryParams); + RestResponse response = + RestClient.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/detachVms", + body, headers, queryParams); getCastedResponse(response, ApiEmptyResponse.class); Boolean retVal = true; @@ -318,65 +320,57 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI //endregion //Redis - //TODO Liron - change name to getLock - public static T getRedisValue(String groupId, String accountId) throws ApiException { - T retVal = null; + public static String getGroupLockValueById(String groupId, String accountId) throws ApiException { + String retVal = null; - Map headers = buildHeaders(); + Map headers = buildHeaders(); - Map queryParams = buildQueryParams(accountId); + Map queryParams = buildQueryParams(accountId); RestResponse response = - RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin", headers, queryParams); + RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin/lock", headers, + queryParams); - RedisGetValueResponse redisValue = getCastedResponse(response, RedisGetValueResponse.class); + GetGroupControllerLockResponse lockResponse = getCastedResponse(response, GetGroupControllerLockResponse.class); - if (redisValue.getResponse().getItems().size() > 0) { - retVal = redisValue.getResponse().getItems().get(0); + if (CollectionUtils.isEmpty(lockResponse.getResponse().getItems()) == false) { + retVal = lockResponse.getResponse().getItems().get(0); } return retVal; } - //TODO Liron - change method name lock - public static String setRedisKey(String lockKey, String accountId, String lockValue, Integer ttl) throws ApiException { - String retVal = null; - - Map headers = buildHeaders(); - - Map queryParams = buildQueryParams(accountId); - - RedisSetKeyRequest request = new RedisSetKeyRequest(); - request.setGroupId(lockKey); - request.setControllerIdentifier(lockValue); - request.setTtl(ttl); - + public static String LockGroupController(String lockKey, String accountId, String lockValue, + Integer ttl) throws ApiException { + String retVal = null; + Map headers = buildHeaders(); + Map queryParams = buildQueryParams(accountId); + LockGroupControllerRequest request = new LockGroupControllerRequest(lockKey, lockValue, ttl); String body = JsonMapper.toJson(request); RestResponse response = - RestClient.sendPost(SPOTINST_API_HOST + "/aws/ec2/group/jenkinsPlugin", body, headers, queryParams); - - RedisSetKeyResponse redisValue = getCastedResponse(response, RedisSetKeyResponse.class); - - if (redisValue.getResponse().getItems().size() > 0) { - retVal = redisValue.getResponse().getItems().get(0); + RestClient.sendPost(SPOTINST_API_HOST + "/aws/ec2/group/jenkinsPlugin/lock", body, headers, + queryParams); + + LockGroupControllerResponse lockControllerValue = + getCastedResponse(response, LockGroupControllerResponse.class); + //TODO: check optimizer and service response + if (lockControllerValue.getResponse().getItems().size() > 0) { + retVal = lockControllerValue.getResponse().getItems().get(0); } return retVal; } - //TODO Liron - change name to unlock - public static Integer deleteRedisKey(String groupId, String accountId) throws ApiException { - Integer retVal = null; - - Map headers = buildHeaders(); - - Map queryParams = buildQueryParams(accountId); + public static Integer UnlockGroupController(String groupId, String accountId) throws ApiException { + Integer retVal = null; + Map headers = buildHeaders(); + Map queryParams = buildQueryParams(accountId); RestResponse response = - RestClient.sendDelete(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin", headers, queryParams); - - RedisDeleteKeyResponse redisValue = getCastedResponse(response, RedisDeleteKeyResponse.class); + RestClient.sendDelete(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin/lock", headers, + queryParams); + UnlockGroupControllerResponse redisValue = getCastedResponse(response, UnlockGroupControllerResponse.class); if (redisValue.getResponse().getItems().size() > 0) { retVal = redisValue.getResponse().getItems().get(0); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 778b3c0a..66b2b05f 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -137,8 +137,6 @@ protected void handleSyncGroupInstances() { addNewSlaveInstances(instances); removeOldSlaveInstances(instances); - - } else { LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, @@ -148,35 +146,26 @@ protected void handleSyncGroupInstances() { @Override - public Map getInstanceIpsById() { - Map retVal = new HashMap<>(); - - boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); - - if (isGroupManagedByThisController) { - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = - awsGroupRepo.getGroupInstances(groupId, this.accountId); + public Map handleGetInstanceIpsById() { + Map retVal = new HashMap<>(); + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, accountId); - if (instancesResponse.isRequestSucceed()) { - List instances = instancesResponse.getValue(); + if (instancesResponse.isRequestSucceed()) { + List instances = instancesResponse.getValue(); - for (AwsGroupInstance instance : instances) { - if (this.getShouldUsePrivateIp()) { - retVal.put(instance.getInstanceId(), instance.getPrivateIp()); - } - else { - retVal.put(instance.getInstanceId(), instance.getPublicIp()); - } + for (AwsGroupInstance instance : instances) { + if (this.getShouldUsePrivateIp()) { + retVal.put(instance.getInstanceId(), instance.getPrivateIp()); + } + else { + retVal.put(instance.getInstanceId(), instance.getPublicIp()); } - } - else { - LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, - instancesResponse.getErrors())); } } else { - handleGroupDoesNotManageByThisController(accountId, groupId); + LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId, + instancesResponse.getErrors())); } return retVal; @@ -218,7 +207,7 @@ protected int getOverridedNumberOfExecutors(String instanceType) { LOGGER.info(String.format("We have a weight definition for this type of %s", retVal)); } else { - retVal = NO_OVERRIDED_NUM_OF_EXECUTORS; + retVal = NO_OVERRIDDEN_NUM_OF_EXECUTORS; } return retVal; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index b842560e..4b496ee1 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -136,12 +136,11 @@ protected void handleSyncGroupInstances() { } @Override - public Map getInstanceIpsById() { - //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + public Map handleGetInstanceIpsById() { Map retVal = new HashMap<>(); IAzureVmGroupRepo awsGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupVms(groupId, this.accountId); + ApiResponse> instancesResponse = awsGroupRepo.getGroupVms(groupId, accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java index d7611af9..c8160efd 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java @@ -74,7 +74,7 @@ public Boolean detachInstance(String instanceId) { Boolean retVal = false; IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); ApiResponse detachInstanceResponse = - azureGroupRepo.detachInstance(groupId, instanceId, this.accountId); + azureGroupRepo.detachInstance(groupId, instanceId, accountId); if (detachInstanceResponse.isRequestSucceed()) { LOGGER.info(String.format("Instance %s detached", instanceId)); @@ -88,13 +88,18 @@ public Boolean detachInstance(String instanceId) { return retVal; } + @Override + public void syncGroupInstances() { + + } + @Override protected void handleSyncGroupInstances() { } @Override - public Map getInstanceIpsById() { + public Map getInstanceIpsById() {//TODO: check if is different Map retVal = new HashMap<>(); IAzureGroupRepo awsGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); @@ -105,7 +110,7 @@ public Map getInstanceIpsById() { List instances = instancesResponse.getValue(); for (AzureGroupInstance instance : instances) { - if (this.getShouldUsePrivateIp()) { + if (getShouldUsePrivateIp()) { retVal.put(instance.getInstanceId(), instance.getPrivateIp()); } else { @@ -122,6 +127,11 @@ public Map getInstanceIpsById() { return retVal; } + @Override + protected Map handleGetInstanceIpsById() { + return null; + } + @Override public void monitorInstances() { IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 486f7946..6e55e94f 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -7,7 +7,7 @@ import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.cloud.helpers.TimeHelper; import hudson.plugins.spotinst.common.*; -import hudson.plugins.spotinst.repos.IRedisRepo; +import hudson.plugins.spotinst.repos.ILockRepo; import hudson.plugins.spotinst.repos.RepoManager; import hudson.plugins.spotinst.slave.*; import hudson.plugins.sshslaves.SSHConnector; @@ -28,8 +28,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING; -import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY; +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.*; /** * Created by ohadmuchnik on 25/05/2016. @@ -39,9 +38,7 @@ public abstract class BaseSpotinstCloud extends Cloud { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(BaseSpotinstCloud.class); - protected static final int NO_OVERRIDED_NUM_OF_EXECUTORS = -1; - private static final Integer REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS = 60 * 3; - public static final String REDIS_OK_STATUS = "OK"; + protected static final int NO_OVERRIDDEN_NUM_OF_EXECUTORS = -1; protected String accountId; protected String groupId; protected Map pendingInstances; @@ -121,7 +118,7 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat } if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { - syncGroupsOwner(this); + syncGroupsOwner(); } } //endregion @@ -135,18 +132,14 @@ public Collection provision(Label label, int excessWorkload) { boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { - setNumOfNeededExecutors(request); if (request.getExecutors() > 0) { LOGGER.info(String.format("Need to scale up %s units", request.getExecutors())); - - List slaves = provisionSlaves(request); if (slaves.size() > 0) { for (final SpotinstSlave slave : slaves) { - try { Jenkins.getInstance().addNode(slave); } @@ -392,11 +385,8 @@ public boolean isCloudReadyForGroupCommunication(String groupId) { if (StringUtils.isNotEmpty(groupId)) { GroupStateTracker stateDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); - if (stateDetails != null && stateDetails.getState() != null) { - if (stateDetails.getState() - .equals(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_READY)) { - retVal = true; - } + if (stateDetails != null) { + retVal = stateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY); } } @@ -490,24 +480,30 @@ private boolean isGlobalExecutorOverrideValid() { return retVal; } - private void addGroupToRedis(String groupId, String accountId, String controllerIdentifier) { - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisSetKeyResponse = - redisRepo.setKey(groupId, accountId, controllerIdentifier, REDIS_ENTRY_TIME_TO_LIVE_IN_SECONDS); + private Boolean LockGroupController(String controllerIdentifier) { + Boolean retVal = null; + ILockRepo lockRepo = RepoManager.getInstance().getLockRepo(); + ApiResponse lockGroupControllerResponse = + lockRepo.Lock(groupId, accountId, controllerIdentifier, Constants.LOCK_TIME_TO_LIVE_IN_SECONDS); + + if (lockGroupControllerResponse.isRequestSucceed()) { + String responseValue = lockGroupControllerResponse.getValue(); - if (redisSetKeyResponse.isRequestSucceed()) { - String redisResponseSetKeyValue = redisSetKeyResponse.getValue(); + if (Constants.LOCK_OK_STATUS.equals(responseValue)) { + LOGGER.info(String.format("Successfully locked group %s controller", groupId)); - if (redisResponseSetKeyValue.equals(REDIS_OK_STATUS)) { - LOGGER.info(String.format("Successfully added group %s to redis memory", groupId)); + retVal = true; } else { - LOGGER.error(String.format("Failed adding group %s to redis memory", groupId)); + LOGGER.error(String.format("Failed locking group %s controller", groupId)); + retVal = false; } } else { - LOGGER.error("redis request failed"); + LOGGER.error("lock request failed"); } + + return retVal; } // private SpotinstCloudCommunicationState getCloudInitializationResultByGroupId(String groupId) { @@ -684,7 +680,7 @@ protected Integer getNumOfExecutors(String instanceType) { } else { int overridedNumOfExecutors = getOverridedNumberOfExecutors(instanceType); - boolean isNumOfExecutorsOverrided = overridedNumOfExecutors != NO_OVERRIDED_NUM_OF_EXECUTORS; + boolean isNumOfExecutorsOverrided = overridedNumOfExecutors != NO_OVERRIDDEN_NUM_OF_EXECUTORS; if (isNumOfExecutorsOverrided) { retVal = overridedNumOfExecutors; @@ -722,7 +718,7 @@ protected Integer getNumOfExecutors(String instanceType) { } protected int getOverridedNumberOfExecutors(String instanceType) { - return NO_OVERRIDED_NUM_OF_EXECUTORS; + return NO_OVERRIDDEN_NUM_OF_EXECUTORS; } protected Integer getPendingThreshold() { @@ -755,24 +751,14 @@ public void handleGroupDoesNotManageByThisController(String accountId, String gr GroupStateTracker groupStateDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); if (groupStateDetails == null || groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY)) { - if (groupStateDetails == null) { - groupStateDetails = new GroupStateTracker(); - } - - groupStateDetails.setGroupId(groupId); - groupStateDetails.setAccountId(accountId); - groupStateDetails.setState(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); - Date now = new Date(); - groupStateDetails.setTimeStamp(now); - + groupStateDetails = new GroupStateTracker(groupId, accountId); SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); } else if (groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING)) { - Date timeStamp = groupStateDetails.getTimeStamp(); - boolean shouldFail = TimeHelper.isTimePassedInSeconds(timeStamp); + boolean shouldFail = TimeHelper.isTimePassedInSeconds(groupStateDetails.getTimeStamp()); if (shouldFail) { - groupStateDetails.setState(SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_FAILED); + groupStateDetails.setState(SPOTINST_CLOUD_COMMUNICATION_FAILED); SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); } } @@ -783,7 +769,7 @@ else if (groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_INITIA // String accountId = cloud.getAccountId(); // LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); // - // IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + // ILockRepo redisRepo = RepoManager.getInstance().getRedisRepo(); // ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); // String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); // @@ -829,81 +815,86 @@ else if (groupStateDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_INITIA // } // } - public void syncGroupsOwner(BaseSpotinstCloud cloud) { - String groupId = cloud.getGroupId(); - String accountId = cloud.getAccountId(); - LOGGER.info(String.format("try fetching controller identifier for group %s from redis", groupId)); - - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisGetValueResponse = redisRepo.getValue(groupId, accountId); - String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); + public void syncGroupsOwner() { + ILockRepo lockRepo = RepoManager.getInstance().getLockRepo(); + ApiResponse lockGroupControllerResponse = lockRepo.getLockValueById(groupId, accountId); + String controllerIdentifier = SpotinstContext.getInstance().getControllerIdentifier(); - if (redisGetValueResponse.isRequestSucceed()) { - //redis response might return in different types - if (redisGetValueResponse.getValue() instanceof String) { - String redisResponseValue = (String) redisGetValueResponse.getValue(); + if (lockGroupControllerResponse.isRequestSucceed()) { + String lockGroupControllerValue = lockGroupControllerResponse.getValue(); + boolean isGroupBelongToNone = lockGroupControllerValue == null; - if (redisResponseValue != null) { - boolean isGroupBelongToController = redisResponseValue.equals(controllerIdentifier); - - if (isGroupBelongToController) { - handleGroupManagedByThisController(cloud, controllerIdentifier); - } - else { - LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, - controllerIdentifier)); - GroupStateTracker groupStateDetails = - SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); - - if (groupStateDetails == null) { - groupStateDetails = new GroupStateTracker(); - groupStateDetails.setGroupId(groupId); - groupStateDetails.setAccountId(accountId); - groupStateDetails.setState(SPOTINST_CLOUD_COMMUNICATION_INITIALIZING); - Date now = new Date(); - groupStateDetails.setTimeStamp(now); - SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, groupStateDetails); - } - } - } + if (isGroupBelongToNone) { + handleGroupManagedByNone(controllerIdentifier); } else { - LOGGER.warn("redis response value return null"); + boolean isGroupBelongToController = controllerIdentifier.equals(lockGroupControllerValue); + + if (isGroupBelongToController) { + handleGroupManagedByThisController(controllerIdentifier); + } + else { + LOGGER.info(String.format("group %s does not belong to controller with identifier %s", groupId, + controllerIdentifier)); + handleGroupDoesNotManageByThisController(accountId, groupId); + } } } - //there is no controller for the given group in redis, should take ownership else { - handleGroupManagedByThisController(cloud, controllerIdentifier); + LOGGER.error("group locking service failed to get lock for groupId {}, accountId {}.", groupId, accountId); } } - - private void handleGroupManagedByThisController(BaseSpotinstCloud cloud, String controllerIdentifier) { - LOGGER.info(String.format("group %s belong to controller with identifier %s", cloud.getGroupId(), - controllerIdentifier)); - GroupStateTracker cloudDetails = - SpotinstContext.getInstance().getConnectionStateByGroupId().get(cloud.getGroupId()); + private void handleGroupManagedByNone(String controllerIdentifier) { + LOGGER.info(String.format("group %s belong to controller with identifier %s", groupId, controllerIdentifier)); + GroupStateTracker cloudDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); if (cloudDetails != null) { - SpotinstCloudCommunicationState cloudState = cloudDetails.getState(); - if (cloudState != null && cloudState.equals(SPOTINST_CLOUD_COMMUNICATION_READY) == false) { - SpotinstContext.getInstance().getConnectionStateByGroupId().remove(cloud.getGroupId()); + if (cloudState.equals(SPOTINST_CLOUD_COMMUNICATION_READY) == false) { + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); } } else { - cloudDetails = new GroupStateTracker(); + cloudDetails = new GroupStateTracker(groupId, accountId); + } + + Boolean hasLock = LockGroupController(controllerIdentifier); + + if (hasLock != null) { + if (hasLock) { + cloudDetails.setState(SPOTINST_CLOUD_COMMUNICATION_READY); + } + else { + cloudDetails = new GroupStateTracker(groupId, accountId); + } + } + + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, cloudDetails); + } + + private void handleGroupManagedByThisController(String controllerIdentifier) { + LOGGER.info(String.format("group %s belong to controller with identifier %s", groupId, controllerIdentifier)); + GroupStateTracker cloudDetails = SpotinstContext.getInstance().getConnectionStateByGroupId().get(groupId); + + if (cloudDetails != null) { + boolean isCloudNotReady = cloudDetails.getState().equals(SPOTINST_CLOUD_COMMUNICATION_READY) == false; + + if (isCloudNotReady) { + SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); + } } - cloudDetails.setGroupId(cloud.getGroupId()); - cloudDetails.setAccountId(cloud.getAccountId()); + cloudDetails = new GroupStateTracker(groupId, accountId); cloudDetails.setState(SPOTINST_CLOUD_COMMUNICATION_READY); - Date now = new Date(); - cloudDetails.setTimeStamp(now); - SpotinstContext.getInstance().getConnectionStateByGroupId().put(cloud.getGroupId(), cloudDetails); + SpotinstContext.getInstance().getConnectionStateByGroupId().put(groupId, cloudDetails); //expand TTL of the current controller in redis - addGroupToRedis(cloud.getGroupId(), cloud.getAccountId(), controllerIdentifier); + boolean isLockRevive = LockGroupController(controllerIdentifier); + + if (isLockRevive == false) { + LOGGER.warn("could not expand lock ttl for group {}", groupId); + } } //endregion @@ -1051,20 +1042,35 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { public abstract String getCloudUrl(); - public void syncGroupInstances(){ + public void syncGroupInstances() { boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); if (isGroupManagedByThisController) { handleSyncGroupInstances(); } - else{ + else { handleGroupDoesNotManageByThisController(accountId, groupId); } } protected abstract void handleSyncGroupInstances(); - public abstract Map getInstanceIpsById(); + public Map getInstanceIpsById() { + Map retVal; + boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); + + if (isGroupManagedByThisController) { + retVal = handleGetInstanceIpsById(); + } + else { + handleGroupDoesNotManageByThisController(accountId, groupId); + retVal = new HashMap<>(); + } + + return retVal; + } + + protected abstract Map handleGetInstanceIpsById(); protected abstract Integer getDefaultExecutorsNumber(String instanceType); //endregion diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index c95e57ae..af53400c 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -145,12 +145,11 @@ protected void handleSyncGroupInstances() { } @Override - public Map getInstanceIpsById() { + public Map handleGetInstanceIpsById() { Map retVal = new HashMap<>(); - //TODO Liron - add boolean isGroupManagedByThisController = isCloudReadyForGroupCommunication(groupId); IGcpGroupRepo awsGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); diff --git a/src/main/java/hudson/plugins/spotinst/common/Constants.java b/src/main/java/hudson/plugins/spotinst/common/Constants.java index 9693d635..bcf70270 100644 --- a/src/main/java/hudson/plugins/spotinst/common/Constants.java +++ b/src/main/java/hudson/plugins/spotinst/common/Constants.java @@ -10,4 +10,6 @@ public class Constants { public static final Integer REST_CLIENT_CONNECT_TIMEOUT_IN_SECONDS = 120; public static final Integer REST_CLIENT_CONNECTION_REQUEST_TIMEOUT_IN_SECONDS = 120; public static final Integer REST_CLIENT_SOCKET_TIMEOUT_IN_SECONDS = 120; + public static final String LOCK_OK_STATUS = "OK"; + public static final Integer LOCK_TIME_TO_LIVE_IN_SECONDS = 60 * 3; } diff --git a/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java b/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java index 0263d003..0c0f80e6 100644 --- a/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java +++ b/src/main/java/hudson/plugins/spotinst/common/GroupStateTracker.java @@ -2,12 +2,23 @@ import java.util.Date; +import static hudson.plugins.spotinst.common.SpotinstCloudCommunicationState.SPOTINST_CLOUD_COMMUNICATION_INITIALIZING; + public class GroupStateTracker { //region members - private String groupId; - private String accountId; - private SpotinstCloudCommunicationState state; - private Date timeStamp; + private final String groupId; + private final String accountId; + private SpotinstCloudCommunicationState state; + private final Date timeStamp; + //endregion + + //region Constructor + public GroupStateTracker(String groupId, String accountId) { + this.groupId = groupId; + this.accountId = accountId; + state = SPOTINST_CLOUD_COMMUNICATION_INITIALIZING; + timeStamp = new Date(); + } //endregion //region getters & setters @@ -15,18 +26,10 @@ public String getGroupId() { return groupId; } - public void setGroupId(String groupId) { - this.groupId = groupId; - } - public String getAccountId() { return accountId; } - public void setAccountId(String accountId) { - this.accountId = accountId; - } - public SpotinstCloudCommunicationState getState() { return state; } @@ -38,9 +41,5 @@ public void setState(SpotinstCloudCommunicationState state) { public Date getTimeStamp() { return timeStamp; } - - public void setTimeStamp(Date timeStamp) { - this.timeStamp = timeStamp; - } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java index 06b6cc6a..2f9e6f96 100644 --- a/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java +++ b/src/main/java/hudson/plugins/spotinst/common/SpotinstContext.java @@ -68,7 +68,7 @@ public void setAwsInstanceTypesLastUpdate(Date awsInstanceTypesLastUpdate) { this.awsInstanceTypesLastUpdate = awsInstanceTypesLastUpdate; } - public String getControllerIdentifier() { + public String getControllerIdentifier() {//TODO: verify with Ziv if(controllerIdentifier == null){ controllerIdentifier = RandomStringUtils.randomAlphanumeric(10); } diff --git a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java index f66884aa..fc397c65 100644 --- a/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java +++ b/src/main/java/hudson/plugins/spotinst/jobs/SpotinstSyncGroupsOwner.java @@ -8,7 +8,7 @@ import hudson.plugins.spotinst.cloud.helpers.TimeHelper; import hudson.plugins.spotinst.common.GroupStateTracker; import hudson.plugins.spotinst.common.SpotinstContext; -import hudson.plugins.spotinst.repos.IRedisRepo; +import hudson.plugins.spotinst.repos.ILockRepo; import hudson.plugins.spotinst.repos.RepoManager; import hudson.slaves.Cloud; import jenkins.model.Jenkins; @@ -88,7 +88,7 @@ protected void execute(TaskListener taskListener) { groupsNoLongerExist.remove(groupId); if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { - spotinstCloud.syncGroupsOwner(spotinstCloud); + spotinstCloud.syncGroupsOwner(); } } } @@ -105,7 +105,7 @@ protected void execute(TaskListener taskListener) { // SpotinstContext.getInstance().getCloudsInitializationState().remove(cloud); // // if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { - // IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); + // ILockRepo redisRepo = RepoManager.getInstance().getRedisRepo(); // ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); // // if (redisGetValueResponse.isRequestSucceed()) { @@ -131,8 +131,8 @@ private void deallocateGroupsNoLongerInUse(Set groupsNoLongerExist) { SpotinstContext.getInstance().getConnectionStateByGroupId().remove(groupId); if (StringUtils.isNotEmpty(groupId) && StringUtils.isNotEmpty(accountId)) { - IRedisRepo redisRepo = RepoManager.getInstance().getRedisRepo(); - ApiResponse redisGetValueResponse = redisRepo.deleteKey(groupId, accountId); + ILockRepo redisRepo = RepoManager.getInstance().getLockRepo(); + ApiResponse redisGetValueResponse = redisRepo.Unlock(groupId, accountId); if (redisGetValueResponse.isRequestSucceed()) { LOGGER.info(String.format("Successfully removed group %s from redis", groupId)); diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/GetGroupControllerLockResponse.java similarity index 57% rename from src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java rename to src/main/java/hudson/plugins/spotinst/model/redis/GetGroupControllerLockResponse.java index de187296..f8482288 100644 --- a/src/main/java/hudson/plugins/spotinst/model/redis/RedisGetValueResponse.java +++ b/src/main/java/hudson/plugins/spotinst/model/redis/GetGroupControllerLockResponse.java @@ -2,5 +2,5 @@ import hudson.plugins.spotinst.api.infra.BaseItemsResponse; -public class RedisGetValueResponse extends BaseItemsResponse { +public class GetGroupControllerLockResponse extends BaseItemsResponse { } diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerRequest.java similarity index 66% rename from src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java rename to src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerRequest.java index c54c1db6..ce11981f 100644 --- a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyRequest.java +++ b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerRequest.java @@ -3,12 +3,20 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) -public class RedisSetKeyRequest { +public class LockGroupControllerRequest { //region Memebers - String groupId; - String controllerIdentifier; - Integer ttl; + private final String groupId; + private final String controllerIdentifier; + private final Integer ttl; + //endregion + + //region Constructor + public LockGroupControllerRequest(String groupId, String controllerIdentifier, Integer ttl) { + this.groupId = groupId; + this.controllerIdentifier = controllerIdentifier; + this.ttl = ttl; + } //endregion //region Getters & Setters @@ -23,17 +31,5 @@ public String getControllerIdentifier() { public Integer getTtl() { return ttl; } - - public void setGroupId(String groupId) { - this.groupId = groupId; - } - - public void setControllerIdentifier(String controllerIdentifier) { - this.controllerIdentifier = controllerIdentifier; - } - - public void setTtl(Integer ttl) { - this.ttl = ttl; - } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerResponse.java similarity index 58% rename from src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java rename to src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerResponse.java index 28f0369a..53c4efd2 100644 --- a/src/main/java/hudson/plugins/spotinst/model/redis/RedisDeleteKeyResponse.java +++ b/src/main/java/hudson/plugins/spotinst/model/redis/LockGroupControllerResponse.java @@ -2,5 +2,5 @@ import hudson.plugins.spotinst.api.infra.BaseItemsResponse; -public class RedisDeleteKeyResponse extends BaseItemsResponse { +public class LockGroupControllerResponse extends BaseItemsResponse { } diff --git a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java b/src/main/java/hudson/plugins/spotinst/model/redis/UnlockGroupControllerResponse.java similarity index 57% rename from src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java rename to src/main/java/hudson/plugins/spotinst/model/redis/UnlockGroupControllerResponse.java index 83664ead..ec148506 100644 --- a/src/main/java/hudson/plugins/spotinst/model/redis/RedisSetKeyResponse.java +++ b/src/main/java/hudson/plugins/spotinst/model/redis/UnlockGroupControllerResponse.java @@ -2,5 +2,5 @@ import hudson.plugins.spotinst.api.infra.BaseItemsResponse; -public class RedisSetKeyResponse extends BaseItemsResponse { +public class UnlockGroupControllerResponse extends BaseItemsResponse { } diff --git a/src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java b/src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java new file mode 100644 index 00000000..73975659 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/repos/ILockRepo.java @@ -0,0 +1,12 @@ +package hudson.plugins.spotinst.repos; + +import hudson.plugins.spotinst.api.infra.ApiResponse; + +public interface ILockRepo { + ApiResponse Lock(String groupId, String accountId, String controllerIdentifier, Integer ttl); + + ApiResponse getLockValueById(String groupId, String accountId); + + ApiResponse Unlock(String groupId, String accountId); + +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java b/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java deleted file mode 100644 index 5305df54..00000000 --- a/src/main/java/hudson/plugins/spotinst/repos/IRedisRepo.java +++ /dev/null @@ -1,12 +0,0 @@ -package hudson.plugins.spotinst.repos; - -import hudson.plugins.spotinst.api.infra.ApiResponse; - -public interface IRedisRepo { - ApiResponse setKey(String groupId,String accountId, String controllerIdentifier,Integer ttl); - - ApiResponse getValue(String groupId, String accountId); - - ApiResponse deleteKey(String groupId, String accountId); - -} diff --git a/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java b/src/main/java/hudson/plugins/spotinst/repos/LockRepo.java similarity index 57% rename from src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java rename to src/main/java/hudson/plugins/spotinst/repos/LockRepo.java index 96a6a254..87a4aa64 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/RedisRepo.java +++ b/src/main/java/hudson/plugins/spotinst/repos/LockRepo.java @@ -5,15 +5,16 @@ import hudson.plugins.spotinst.api.infra.ApiResponse; import hudson.plugins.spotinst.api.infra.ExceptionHelper; -public class RedisRepo implements IRedisRepo { +public class LockRepo implements ILockRepo { + @Override - public ApiResponse setKey(String groupId, String accountId, String controllerIdentifier, Integer ttl) { + public ApiResponse Lock(String groupId, String accountId, String controllerIdentifier, Integer ttl) { ApiResponse retVal; try { - String isKeySet = SpotinstApi.setRedisKey(groupId, accountId, controllerIdentifier, ttl); + String lockResult = SpotinstApi.LockGroupController(groupId, accountId, controllerIdentifier, ttl); - retVal = new ApiResponse<>(isKeySet); + retVal = new ApiResponse<>(lockResult); } catch (ApiException e) { @@ -24,12 +25,11 @@ public ApiResponse setKey(String groupId, String accountId, String contr } @Override - public ApiResponse getValue(String groupId, String accountId) { - ApiResponse retVal; + public ApiResponse getLockValueById(String groupId, String accountId) { + ApiResponse retVal; try { - Object controllerIdentifier = SpotinstApi.getRedisValue(groupId, accountId); - + String controllerIdentifier = SpotinstApi.getGroupLockValueById(groupId, accountId); retVal = new ApiResponse<>(controllerIdentifier); } @@ -41,11 +41,11 @@ public ApiResponse getValue(String groupId, String accountId) { } @Override - public ApiResponse deleteKey(String groupId, String accountId) { + public ApiResponse Unlock(String groupId, String accountId) { ApiResponse retVal; try { - Integer isKeyDeleted = SpotinstApi.deleteRedisKey(groupId, accountId); + Integer isKeyDeleted = SpotinstApi.UnlockGroupController(groupId, accountId); retVal = new ApiResponse<>(isKeyDeleted); diff --git a/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java b/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java index 0a0ba788..6d2a82fe 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java +++ b/src/main/java/hudson/plugins/spotinst/repos/RepoManager.java @@ -10,7 +10,7 @@ public class RepoManager { private IAzureGroupRepo azureGroupRepo; private IAzureVmGroupRepo azureVmGroupRepo; private IAwsInstanceTypesRepo awsInstanceTypesRepo; - private IRedisRepo redisRepo; + private ILockRepo lockRepo; //endregion //region Constructor @@ -20,7 +20,7 @@ private RepoManager() { this.azureGroupRepo = new AzureGroupRepo(); this.azureVmGroupRepo = new AzureVmGroupRepo(); this.awsInstanceTypesRepo = new AwsInstanceTypesRepo(); - this.redisRepo = new RedisRepo(); + this.lockRepo = new LockRepo(); } private static RepoManager instance = new RepoManager(); @@ -71,12 +71,12 @@ public void setAwsInstanceTypesRepo(IAwsInstanceTypesRepo awsInstanceTypesRepo) this.awsInstanceTypesRepo = awsInstanceTypesRepo; } - public IRedisRepo getRedisRepo() { - return redisRepo; + public ILockRepo getLockRepo() { + return lockRepo; } - public void setRedisRepo(IRedisRepo redisRepo) { - this.redisRepo = redisRepo; + public void setLockRepo(ILockRepo lockRepo) { + this.lockRepo = lockRepo; } //endregion }