Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[src] deny option to connect more then one jenkins controller for each elastigroup #84

Open
wants to merge 20 commits into
base: master
Choose a base branch
from
Open

[src] deny option to connect more then one jenkins controller for each elastigroup #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
a8f0bb3
[src] new feature: single task nodes
Lironrad Jul 14, 2022
409a78e
Merge branch 'jenkinsci:master' into master
adivos Jul 14, 2022
cffe575
temp
Lironrad Oct 18, 2022
b252a2d
temp
Lironrad Oct 18, 2022
38bdde3
temp
Lironrad Oct 19, 2022
df4a974
high level review
Lironrad Oct 23, 2022
d42c9d5
temp
Lironrad Oct 25, 2022
7dbadd4
first working version (without restart handling
Lironrad Oct 25, 2022
401a070
test for restart listener
Lironrad Oct 26, 2022
bb5c5ce
change messages
Lironrad Oct 31, 2022
de0b5be
Jenkins plugin instance type search (#27)
Lironrad Dec 29, 2022
a5537bc
InstanceType_by_user_input JenkinsWiki.adoc (#28)
sitay93 Dec 29, 2022
abf5771
Merge branch 'jenkinsci:master' into master
adivos Dec 29, 2022
799e5e9
rollback api endpoint
Lironrad Jan 5, 2023
43e00d2
Merge branch 'master' into one_orchestrator_only
Lironrad Jan 5, 2023
4b63f7b
temp commit
Lironrad Jan 11, 2023
387fac3
one orchestrator check if candidates has been initialized
sitay93 Jan 12, 2023
ee98642
Merge remote-tracking branch 'origin/one_orchestrator_only' into one_…
Lironrad Jan 12, 2023
56d616d
one orchestrator Ziv's rejects
sitay93 Jan 18, 2023
375ba21
one orchestrator Ziv's rejects
sitay93 Jan 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@
<artifactId>plain-credentials</artifactId>
<version>1.7</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
</dependencies>

<repositories>
Expand Down
127 changes: 97 additions & 30 deletions src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
import hudson.plugins.spotinst.model.aws.*;
import hudson.plugins.spotinst.model.azure.*;
import hudson.plugins.spotinst.model.gcp.*;
import hudson.plugins.spotinst.model.redis.UnlockGroupControllerResponse;
import hudson.plugins.spotinst.model.redis.GetGroupControllerLockResponse;
import hudson.plugins.spotinst.model.redis.LockGroupControllerRequest;
import hudson.plugins.spotinst.model.redis.LockGroupControllerResponse;
import jenkins.model.Jenkins;
import org.apache.commons.httpclient.HttpStatus;
import org.slf4j.Logger;
Expand All @@ -19,7 +23,7 @@ public class SpotinstApi {

//region Members
private static final Logger LOGGER = LoggerFactory.getLogger(SpotinstApi.class);
private final static String SPOTINST_API_HOST = "https://api.spotinst.io";
private final static String SPOTINST_API_HOST = "http://localhost:3100";
private final static String HEADER_AUTH = "Authorization";
private final static String AUTH_PREFIX = "Bearer ";
private final static String HEADER_CONTENT_TYPE = "Content-Type";
Expand Down Expand Up @@ -89,8 +93,9 @@ public static AwsScaleUpResult awsScaleUp(String groupId, int adjustment, String
Map<String, String> queryParams = buildQueryParams(accountId);
queryParams.put(QUERY_PARAM_ADJUSTMENT, String.valueOf(adjustment));

RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/scale/up", null, headers, queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/scale/up", null, headers,
queryParams);

AwsScaleUpResponse scaleUpResponse = getCastedResponse(response, AwsScaleUpResponse.class);

Expand Down Expand Up @@ -125,13 +130,12 @@ public static List<AwsInstanceType> getAllAwsInstanceTypes(String accountId) thr
List<AwsInstanceType> retVal;
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);
queryParams.put("distinctTypesList","true");
queryParams.put("distinctTypesList", "true");

RestResponse response =
RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/instanceType", headers, queryParams);
RestResponse response = RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/instanceType", headers, queryParams);

AwsInstanceTypesResponse
allAwsInstanceTypesResponse = getCastedResponse(response, AwsInstanceTypesResponse.class);
AwsInstanceTypesResponse allAwsInstanceTypesResponse =
getCastedResponse(response, AwsInstanceTypesResponse.class);

retVal = allAwsInstanceTypesResponse.getResponse().getItems();

Expand All @@ -149,8 +153,9 @@ public static GcpScaleUpResult gcpScaleUp(String groupId, int adjustment, String
Map<String, String> queryParams = buildQueryParams(accountId);
queryParams.put(QUERY_PARAM_ADJUSTMENT, String.valueOf(adjustment));

RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/scale/up", null, headers, queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/scale/up", null, headers,
queryParams);

GcpScaleUpResponse scaleUpResponse = getCastedResponse(response, GcpScaleUpResponse.class);

Expand All @@ -171,9 +176,9 @@ public static Boolean gcpDetachInstance(String groupId, String instanceName, Str
request.setShouldTerminateInstances(true);
String body = JsonMapper.toJson(request);

RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/detachInstances", body, headers,
queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + "/gcp/gce/group/" + groupId + "/detachInstances", body, headers,
queryParams);

getCastedResponse(response, ApiEmptyResponse.class);
Boolean retVal = true;
Expand Down Expand Up @@ -206,8 +211,9 @@ public static List<AzureGroupInstance> getAzureGroupInstances(String groupId,
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);

RestResponse response = RestClient
.sendGet(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/status", headers, queryParams);
RestResponse response =
RestClient.sendGet(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/status", headers,
queryParams);

AzureGroupInstancesResponse instancesResponse = getCastedResponse(response, AzureGroupInstancesResponse.class);

Expand All @@ -224,9 +230,9 @@ public static Boolean azureScaleUp(String groupId, int adjustment, String accoun
Map<String, String> queryParams = buildQueryParams(accountId);
queryParams.put("adjustment", String.valueOf(adjustment));

RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/scale/up", null, headers,
queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/scale/up", null, headers,
queryParams);

getCastedResponse(response, ApiEmptyResponse.class);
Boolean retVal = true;
Expand All @@ -243,9 +249,9 @@ public static Boolean azureDetachInstance(String groupId, String instanceId, Str
request.setShouldDecrementTargetCapacity(true);
String body = JsonMapper.toJson(request);

RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/detachInstances", body, headers,
queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + "/compute/azure/group/" + groupId + "/detachInstances", body,
headers, queryParams);

getCastedResponse(response, ApiEmptyResponse.class);
Boolean retVal = true;
Expand All @@ -260,9 +266,9 @@ public static AzureGroupStatus getAzureVmGroupStatus(String groupId, String acco
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);

RestResponse response = RestClient
.sendGet(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/status", headers,
queryParams);
RestResponse response =
RestClient.sendGet(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/status",
headers, queryParams);

AzureGroupStatusResponse vmsResponse = getCastedResponse(response, AzureGroupStatusResponse.class);

Expand All @@ -281,9 +287,9 @@ public static List<AzureScaleUpResultNewVm> azureVmScaleUp(String groupId, int a
Map<String, String> queryParams = buildQueryParams(accountId);
queryParams.put("adjustment", String.valueOf(adjustment));

RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/scale/up", null, headers,
queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/scale/up",
null, headers, queryParams);

AzureScaleUpResponse scaleUpResponse = getCastedResponse(response, AzureScaleUpResponse.class);

Expand All @@ -303,16 +309,77 @@ public static Boolean azureVmDetach(String groupId, String vmId, String accountI
request.setShouldDecrementTargetCapacity(true);
request.setShouldTerminateVms(true);
String body = JsonMapper.toJson(request);
RestResponse response = RestClient
.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/detachVms", body,
headers, queryParams);
RestResponse response =
RestClient.sendPut(SPOTINST_API_HOST + AZURE_VM_SERVICE_PREFIX + "/group/" + groupId + "/detachVms",
body, headers, queryParams);
getCastedResponse(response, ApiEmptyResponse.class);
Boolean retVal = true;

return retVal;
}
//endregion

//Redis
public static String getGroupLockValueById(String groupId, String accountId) throws ApiException {
String retVal = null;

Map<String, String> headers = buildHeaders();

Map<String, String> queryParams = buildQueryParams(accountId);

RestResponse response =
RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin/lock", headers,
queryParams);

GetGroupControllerLockResponse lockResponse = getCastedResponse(response, GetGroupControllerLockResponse.class);

if (CollectionUtils.isEmpty(lockResponse.getResponse().getItems()) == false) {
retVal = lockResponse.getResponse().getItems().get(0);
}

return retVal;
}

public static String LockGroupController(String lockKey, String accountId, String lockValue,
Integer ttl) throws ApiException {
String retVal = null;
Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);
LockGroupControllerRequest request = new LockGroupControllerRequest(lockKey, lockValue, ttl);
String body = JsonMapper.toJson(request);

RestResponse response =
RestClient.sendPost(SPOTINST_API_HOST + "/aws/ec2/group/jenkinsPlugin/lock", body, headers,
queryParams);

LockGroupControllerResponse lockControllerValue =
getCastedResponse(response, LockGroupControllerResponse.class);
//TODO: check optimizer and service response
if (lockControllerValue.getResponse().getItems().size() > 0) {
retVal = lockControllerValue.getResponse().getItems().get(0);
}

return retVal;
}

public static Integer UnlockGroupController(String groupId, String accountId) throws ApiException {
Integer retVal = null;

Map<String, String> headers = buildHeaders();
Map<String, String> queryParams = buildQueryParams(accountId);
RestResponse response =
RestClient.sendDelete(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/jenkinsPlugin/lock", headers,
queryParams);
UnlockGroupControllerResponse redisValue = getCastedResponse(response, UnlockGroupControllerResponse.class);

if (redisValue.getResponse().getItems().size() > 0) {
retVal = redisValue.getResponse().getItems().get(0);
}

return retVal;
}
//endregion

//region Private Methods
private static String buildUserAgent() {
String retVal = null;
Expand Down
34 changes: 34 additions & 0 deletions src/main/java/hudson/plugins/spotinst/api/infra/RestClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,40 @@ public static RestResponse sendPut(String url, String body, Map<String, String>

return retVal;
}

public static RestResponse sendDelete(String url, Map<String, String> headers,
Map<String, String> queryParams) throws ApiException {

HttpDelete getRequest = new HttpDelete(url);
addQueryParams(getRequest, queryParams);
addHeaders(getRequest, headers);
RestResponse retVal = sendRequest(getRequest);

return retVal;
}

public static RestResponse sendPost(String url, String body, Map<String, String> headers,
Map<String, String> queryParams) throws ApiException {

HttpPost postRequest = new HttpPost(url);

if (body != null) {
StringEntity entity = null;
try {
entity = new StringEntity(body);
}
catch (UnsupportedEncodingException e) {
LOGGER.error("Exception when building put body", e);
}
postRequest.setEntity(entity);
}

addQueryParams(postRequest, queryParams);
addHeaders(postRequest, headers);
RestResponse retVal = sendRequest(postRequest);

return retVal;
}
//endregion

//region Private Methods
Expand Down
22 changes: 11 additions & 11 deletions src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.*;

Expand All @@ -32,7 +33,7 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud {
private static final String CLOUD_URL = "aws/ec2";
protected Map<String, Integer> executorsByInstanceType;
private List<? extends SpotinstInstanceWeight> executorsForTypes;
private List<String> invalidInstanceTypes;
private List<String> invalidInstanceTypes;
//endregion

//region Constructor
Expand Down Expand Up @@ -117,7 +118,7 @@ public Boolean detachInstance(String instanceId) {
}

@Override
public void syncGroupInstances() {
protected void handleSyncGroupInstances() {
IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<List<AwsGroupInstance>> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId);

Expand All @@ -136,21 +137,19 @@ public void syncGroupInstances() {

addNewSlaveInstances(instances);
removeOldSlaveInstances(instances);


}
else {
LOGGER.error(String.format("Failed to get group %s instances. Errors: %s", groupId,
instancesResponse.getErrors()));
}
}

@Override
public Map<String, String> getInstanceIpsById() {
Map<String, String> retVal = new HashMap<>();

IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<List<AwsGroupInstance>> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId);
@Override
public Map<String, String> handleGetInstanceIpsById() {
Map<String, String> retVal = new HashMap<>();
IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo();
ApiResponse<List<AwsGroupInstance>> instancesResponse = awsGroupRepo.getGroupInstances(groupId, accountId);

if (instancesResponse.isRequestSucceed()) {
List<AwsGroupInstance> instances = instancesResponse.getValue();
Expand Down Expand Up @@ -208,7 +207,7 @@ protected int getOverridedNumberOfExecutors(String instanceType) {
LOGGER.info(String.format("We have a weight definition for this type of %s", retVal));
}
else {
retVal = NO_OVERRIDED_NUM_OF_EXECUTORS;
retVal = NO_OVERRIDDEN_NUM_OF_EXECUTORS;
}

return retVal;
Expand Down Expand Up @@ -380,7 +379,7 @@ private void initExecutorsByInstanceType() {
String type = instance.getAwsInstanceTypeFromAPIInput();
this.executorsByInstanceType.put(type, executors);

if(instance.getIsValid() == false){
if (instance.getIsValid() == false) {
LOGGER.error(String.format("Invalid type \'%s\' in group \'%s\'", type, this.getGroupId()));
invalidInstanceTypes.add(type);
}
Expand All @@ -404,6 +403,7 @@ public List<String> getInvalidInstanceTypes() {
@Extension
public static class DescriptorImpl extends BaseSpotinstCloud.DescriptorImpl {

@Nonnull
@Override
public String getDisplayName() {
return "Spot AWS Elastigroup";
Expand Down
Loading