Merge pull request #425 from FlowCI/develop

Develop

.gitignore

@@ -7,5 +7,4 @@ target/
 .settings/
 .classpath
 .factorypath
-*Proto.java
-.run/
+*Proto.java

.run/Application - 1.run.xml

@@ -0,0 +1,23 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Application - 1" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
+    <module name="core" />
+    <option name="SPRING_BOOT_MAIN_CLASS" value="com.flowci.core.Application" />
+    <option name="VM_PARAMETERS" value="-Xms1g -Xmx1g -Xmn600m -XX:-UseAdaptiveSizePolicy -XX:SurvivorRatio=4 -verbose:gc" />
+    <option name="ALTERNATIVE_JRE_PATH" />
+    <envs>
+      <env name="FLOWCI_LOG_LEVEL" value="DEBUG" />
+      <env name="FLOWCI_PLUGIN_URL" value="https://raw.githubusercontent.com/FlowCI/plugins/develop/repository.json" />
+      <env name="FLOWCI_RABBITMQ_URI" value="amqp://guest:guest@127.0.0.1:5672" />
+      <env name="FLOWCI_ZK_HOST" value="127.0.0.1" />
+      <env name="FLOWCI_AUTO_AGENT" value="false" />
+      <env name="FLOWCI_AGENT_VOLUMES" value="name=pyenv,dest=/ci/python,script=init.sh,image=flowci/pyenv,init=init-pyenv-volume.sh" />
+      <env name="FLOWCI_SERVER_PORT" value="8080" />
+      <env name="FLOWCI_TEMPLATES" value="https://raw.githubusercontent.com/FlowCI/templates/develop/templates.json" />
+      <env name="FLOWCI_AGENT_IMAGE" value="flowci/agent:dev" />
+      <env name="FLOWCI_SERVER_URL" value="http://192.168.31.173:8080" />
+    </envs>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>

build.sh

@@ -6,4 +6,4 @@ if [[ -n ${tag} ]]; then
   versionTag="-t flowci/core:$tag"
 fi
 
-docker build -f ./core/Dockerfile -t flowci/core:latest $versionTag ./core
+docker buildx build -f ./core/Dockerfile --platform linux/arm64,linux/amd64 --push -t flowci/core:latest $versionTag ./core

core/src/main/java/com/flowci/core/auth/config/AuthConfig.java

@@ -22,9 +22,10 @@
 import com.flowci.core.common.domain.Settings;
 import com.flowci.core.config.domain.ConfigAction;
 import com.flowci.core.flow.domain.FlowAction;
+import com.flowci.core.git.controller.GitActions;
 import com.flowci.core.job.domain.JobAction;
-import com.flowci.core.trigger.domain.TriggerOperations;
-import com.flowci.core.secret.domain.SecretAction;
+import com.flowci.core.trigger.domain.TriggerActions;
+import com.flowci.core.secret.controller.SecretActions;
 import com.flowci.core.user.domain.User;
 import com.flowci.core.user.domain.UserAction;
 import org.springframework.context.annotation.Bean;
@@ -43,19 +44,20 @@ public PermissionMap actionMap() {
         // admin
         permissionMap.add(User.Role.Admin, FlowAction.ALL);
         permissionMap.add(User.Role.Admin, JobAction.ALL);
-        permissionMap.add(User.Role.Admin, SecretAction.ALL);
+        permissionMap.add(User.Role.Admin, SecretActions.ALL);
         permissionMap.add(User.Role.Admin, AgentAction.ALL);
         permissionMap.add(User.Role.Admin, AgentHostAction.ALL);
         permissionMap.add(User.Role.Admin, UserAction.ALL);
         permissionMap.add(User.Role.Admin, ConfigAction.ALL);
-        permissionMap.add(User.Role.Admin, TriggerOperations.ALL);
+        permissionMap.add(User.Role.Admin, TriggerActions.ALL);
+        permissionMap.add(User.Role.Admin, GitActions.ALL);
         permissionMap.add(User.Role.Admin, Settings.Action.ALL);
 
         // developer
         permissionMap.add(User.Role.Developer,
                 FlowAction.GET, FlowAction.LIST, FlowAction.LIST_BRANCH, FlowAction.GET, FlowAction.GET_YML);
         permissionMap.add(User.Role.Developer, JobAction.ALL);
-        permissionMap.add(User.Role.Developer, SecretAction.LIST_NAME);
+        permissionMap.add(User.Role.Developer, SecretActions.LIST_NAME);
         permissionMap.add(User.Role.Developer, AgentAction.GET, AgentAction.LIST);
         permissionMap.add(User.Role.Developer, AgentHostAction.GET, AgentHostAction.LIST);
         permissionMap.add(User.Role.Developer, UserAction.CHANGE_PASSWORD, UserAction.UPDATE_AVATAR);

core/src/main/java/com/flowci/core/common/adviser/ExceptionAdviser.java

@@ -45,6 +45,7 @@
         "com.flowci.core.config",
         "com.flowci.core.trigger",
         "com.flowci.core.api",
+        "com.flowci.core.git",
         "com.flowci.core.common.controller"
 })
 public class ExceptionAdviser {

core/src/main/java/com/flowci/core/common/adviser/ResponseMessageAdviser.java

@@ -41,6 +41,7 @@
         "com.flowci.core.config",
         "com.flowci.core.trigger",
         "com.flowci.core.api",
+        "com.flowci.core.git",
         "com.flowci.core.common.controller"
 })
 public class ResponseMessageAdviser implements ResponseBodyAdvice {

core/src/main/java/com/flowci/core/common/config/AppConfig.java

@@ -130,11 +130,12 @@ public void multicastEvent(ApplicationEvent event, ResolvableType eventType) {
     }
 
     @Bean("httpClient")
-    public HttpClient httpClient() {
+    public HttpClient httpClient(TaskExecutor appTaskExecutor) {
         return HttpClient.newBuilder()
                 .version(HttpClient.Version.HTTP_1_1)
                 .followRedirects(HttpClient.Redirect.NORMAL)
                 .connectTimeout(Duration.ofSeconds(10))
+                .executor(appTaskExecutor)
                 .build();
     }
 }

core/src/main/java/com/flowci/core/common/config/MongoConfig.java

@@ -24,6 +24,8 @@
 import com.flowci.core.common.mongo.VariableMapConverter;
 import com.flowci.core.config.domain.SmtpConfig;
 import com.flowci.core.config.domain.TextConfig;
+import com.flowci.core.git.domain.GitConfig;
+import com.flowci.core.git.domain.GitConfigWithHost;
 import com.flowci.core.job.domain.JobItem;
 import com.flowci.core.secret.domain.*;
 import com.flowci.core.trigger.domain.EmailTrigger;
@@ -111,6 +113,8 @@ public MongoMappingContext mongoMappingContext(MongoCustomConversions customConv
         context.addEntity(EmailTrigger.class);
         context.addEntity(WebhookTrigger.class);
 
+        context.addEntity(GitConfigWithHost.class);
+
         return context;
     }
 

core/src/main/java/com/flowci/core/common/config/WebConfig.java

@@ -80,6 +80,7 @@ public void addInterceptors(InterceptorRegistry registry) {
                         .addPathPatterns("/configs/**")
                         .addPathPatterns("/triggers/**")
                         .addPathPatterns("/system/**")
+                        .addPathPatterns("/gitconfig/**")
                         .addPathPatterns("/auth/logout");
 
                 registry.addInterceptor(apiAuth)

core/src/main/java/com/flowci/core/common/domain/GitSource.java

@@ -27,4 +27,6 @@ public enum GitSource {
     GOGS,
 
     GITEE,
+
+    GERRIT
 }

core/src/main/java/com/flowci/core/common/domain/Variables.java

@@ -16,7 +16,7 @@
 
 package com.flowci.core.common.domain;
 
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.util.Collection;
 
@@ -42,11 +42,6 @@ public abstract static class Flow {
 
         public static final String Name = "FLOWCI_FLOW_NAME";
 
-        public static final String GitUrl = "FLOWCI_GIT_URL"; // set
-
-        public static final String GitRepo = "FLOWCI_GIT_REPO"; // set
-
-        public static final String GitCredential = "FLOWCI_GIT_CREDENTIAL"; // set
     }
 
     public abstract static class Job {
@@ -103,6 +98,12 @@ public abstract static class Agent {
 
     public abstract static class Git {
 
+        public static final String URL = "FLOWCI_GIT_URL"; // set
+
+        public static final String REPO_NAME = "FLOWCI_GIT_REPO"; // set
+
+        public static final String SECRET = "FLOWCI_GIT_CREDENTIAL"; // set
+
         public static final String EVENT_ID = "FLOWCI_GIT_EVENT_ID";
 
         /**
@@ -116,7 +117,7 @@ public abstract static class Git {
         public static final String EVENT = "FLOWCI_GIT_EVENT";
 
         /**
-         * Used for git clone
+         * Used for git clone, will be put to job context from gitclone plugin
          */
         public static final String COMMIT_ID = "FLOWCI_GIT_COMMIT_ID";
 
@@ -125,25 +126,32 @@ public abstract static class Git {
          */
         public static final String BRANCH = "FLOWCI_GIT_BRANCH";
 
+        /**
+         * For some git sources, they need repo id for write back commit status
+         */
+        public static final String REPO_ID = "FLOWCI_GIT_REPO_ID";
+
+        public static final Collection<String> GENERIC_VARS = ImmutableSet.<String>builder()
+                .add(SOURCE)
+                .add(COMMIT_ID)
+                .add(EVENT)
+                .add(EVENT_ID)
+                .add(REPO_ID)
+                .add(BRANCH)
+                .add(SECRET)
+                .add(REPO_NAME)
+                .add(URL)
+                .build();
+
         /**
          * Push / Tag variables
          */
         public static final String PUSH_AUTHOR = "FLOWCI_GIT_AUTHOR";
-
-        public static final String PUSH_BRANCH = "FLOWCI_GIT_BRANCH";
-
         public static final String PUSH_MESSAGE = "FLOWCI_GIT_COMMIT_MESSAGE";
-
         public static final String PUSH_COMMIT_TOTAL = "FLOWCI_GIT_COMMIT_TOTAL";
-
-        /**
-         * b64 json content
-         */
-        public static final String PUSH_COMMIT_LIST = "FLOWCI_GIT_COMMIT_LIST";
-
-        public static final Collection<String> PUSH_TAG_VARS = ImmutableList.<String>builder()
+        public static final String PUSH_COMMIT_LIST = "FLOWCI_GIT_COMMIT_LIST"; // b64 json
+        public static final Collection<String> PUSH_TAG_VARS = ImmutableSet.<String>builder()
                 .add(PUSH_AUTHOR)
-                .add(PUSH_BRANCH)
                 .add(PUSH_MESSAGE)
                 .add(PUSH_COMMIT_TOTAL)
                 .add(PUSH_COMMIT_LIST)
@@ -153,32 +161,19 @@ public abstract static class Git {
          * Variables for git pull(mr) request
          */
         public static final String PR_TITLE = "FLOWCI_GIT_PR_TITLE";
-
         public static final String PR_MESSAGE = "FLOWCI_GIT_PR_MESSAGE";
-
         public static final String PR_AUTHOR = "FLOWCI_GIT_AUTHOR";
-
         public static final String PR_URL = "FLOWCI_GIT_PR_URL";
-
         public static final String PR_TIME = "FLOWCI_GIT_PR_TIME";
-
         public static final String PR_NUMBER = "FLOWCI_GIT_PR_NUMBER";
-
         public static final String PR_IS_MERGED = "FLOWCI_GIT_PR_IS_MERGED";
-
         public static final String PR_HEAD_REPO_NAME = "FLOWCI_GIT_PR_HEAD_REPO_NAME";
-
         public static final String PR_HEAD_REPO_BRANCH = "FLOWCI_GIT_PR_HEAD_REPO_BRANCH";
-
         public static final String PR_HEAD_REPO_COMMIT = "FLOWCI_GIT_PR_HEAD_REPO_COMMIT";
-
         public static final String PR_BASE_REPO_NAME = "FLOWCI_GIT_PR_BASE_REPO_NAME";
-
         public static final String PR_BASE_REPO_BRANCH = "FLOWCI_GIT_PR_BASE_REPO_BRANCH";
-
         public static final String PR_BASE_REPO_COMMIT = "FLOWCI_GIT_PR_BASE_REPO_COMMIT";
-
-        public static final Collection<String> PR_VARS = ImmutableList.<String>builder()
+        public static final Collection<String> PR_VARS = ImmutableSet.<String>builder()
                 .add(PR_TITLE)
                 .add(PR_MESSAGE)
                 .add(PR_AUTHOR)
@@ -194,5 +189,42 @@ public abstract static class Git {
                 .add(PR_BASE_REPO_COMMIT)
                 .build();
 
+        /**
+         * Variables for gerrit patchset
+         */
+        public static final String PATCHSET_SUBJECT = "FLOWCI_GIT_PATCHSET_SUBJECT";
+        public static final String PATCHSET_MESSAGE = "FLOWCI_GIT_PATCHSET_MESSAGE";
+        public static final String PATCHSET_PROJECT = "FLOWCI_GIT_PATCHSET_PROJECT";
+        public static final String PATCHSET_BRANCH = "FLOWCI_GIT_PATCHSET_BRANCH";
+        public static final String PATCHSET_CHANGE_ID = "FLOWCI_GIT_PATCHSET_CHANGE_ID";
+        public static final String PATCHSET_CHANGE_NUM = "FLOWCI_GIT_PATCHSET_CHANGE_NUM";
+        public static final String PATCHSET_CHANGE_URL = "FLOWCI_GIT_PATCHSET_CHANGE_URL";
+        public static final String PATCHSET_CHANGE_STATUS = "FLOWCI_GIT_PATCHSET_CHANGE_STATUS";
+        public static final String PATCHSET_PATCH_NUM = "FLOWCI_GIT_PATCHSET_PATCH_NUM";
+        public static final String PATCHSET_PATCH_URL = "FLOWCI_GIT_PATCHSET_PATCH_URL";
+        public static final String PATCHSET_PATCH_REVISION = "FLOWCI_GIT_PATCHSET_PATCH_REVISION";
+        public static final String PATCHSET_PATCH_REF = "FLOWCI_GIT_PATCHSET_PATCH_REF";
+        public static final String PATCHSET_CREATE_TIME = "FLOWCI_GIT_PATCHSET_CREATE_TIME";
+        public static final String PATCHSET_INSERT_SIZE = "FLOWCI_GIT_PATCHSET_INSERT_SIZE";
+        public static final String PATCHSET_DELETE_SIZE = "FLOWCI_GIT_PATCHSET_DELETE_SIZE";
+        public static final String PATCHSET_AUTHOR = "FLOWCI_GIT_PATCHSET_AUTHOR";
+        public static final Collection<String> PATCHSET_VARS = ImmutableSet.<String>builder()
+                .add(PATCHSET_SUBJECT)
+                .add(PATCHSET_MESSAGE)
+                .add(PATCHSET_PROJECT)
+                .add(PATCHSET_BRANCH)
+                .add(PATCHSET_CHANGE_ID)
+                .add(PATCHSET_CHANGE_NUM)
+                .add(PATCHSET_CHANGE_URL)
+                .add(PATCHSET_CHANGE_STATUS)
+                .add(PATCHSET_PATCH_NUM)
+                .add(PATCHSET_PATCH_URL)
+                .add(PATCHSET_PATCH_REVISION)
+                .add(PATCHSET_PATCH_REF)
+                .add(PATCHSET_CREATE_TIME)
+                .add(PATCHSET_INSERT_SIZE)
+                .add(PATCHSET_DELETE_SIZE)
+                .add(PATCHSET_AUTHOR)
+                .build();
     }
 }

core/src/main/java/com/flowci/core/flow/controller/GitController.java

@@ -21,7 +21,7 @@
 import com.flowci.core.flow.domain.FlowAction;
 import com.flowci.core.flow.domain.FlowGitTest;
 import com.flowci.core.flow.service.FlowService;
-import com.flowci.core.flow.service.GitService;
+import com.flowci.core.flow.service.GitConnService;
 import com.flowci.util.StringHelper;
 import java.util.List;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -44,7 +44,7 @@ public class GitController {
     private FlowService flowService;
 
     @Autowired
-    private GitService gitService;
+    private GitConnService gitConnService;
 
     @PostMapping(value = "/{name}/git/test")
     @Action(FlowAction.GIT_TEST)
@@ -53,26 +53,26 @@ public void gitTest(@PathVariable String name, @Validated @RequestBody FlowGitTe
         String gitUrl = body.getGitUrl();
 
         if (body.hasCredential()) {
-            gitService.testConn(flow, gitUrl, body.getSecret());
+            gitConnService.testConn(flow, gitUrl, body.getSecret());
             return;
         }
 
         if (body.hasPrivateKey()) {
-            gitService.testConn(flow, gitUrl, body.getRsa());
+            gitConnService.testConn(flow, gitUrl, body.getRsa());
             return;
         }
 
         if (body.hasUsernamePassword()) {
-            gitService.testConn(flow, gitUrl, body.getAuth());
+            gitConnService.testConn(flow, gitUrl, body.getAuth());
         }
 
-        gitService.testConn(flow, gitUrl, StringHelper.EMPTY);
+        gitConnService.testConn(flow, gitUrl, StringHelper.EMPTY);
     }
 
     @GetMapping(value = "/{name}/git/branches")
     @Action(FlowAction.LIST_BRANCH)
     public List<String> listGitBranches(@PathVariable String name) {
         Flow flow = flowService.get(name);
-        return gitService.listGitBranch(flow);
+        return gitConnService.listGitBranch(flow);
     }
 }

core/src/main/java/com/flowci/core/flow/controller/StatsController.java

@@ -23,11 +23,12 @@
 import com.flowci.core.flow.service.FlowService;
 import com.flowci.core.flow.service.StatsService;
 import com.flowci.exception.ArgumentException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.web.bind.annotation.*;
 
 /**
  * @author yang
@@ -36,7 +37,7 @@
 @RequestMapping("/flows")
 public class StatsController {
 
-    private static final int MaxDays = 30;
+    private static final int MaxDays = 31;
 
     @Autowired
     private FlowService flowService;

core/src/main/java/com/flowci/core/flow/domain/Flow.java

@@ -109,11 +109,11 @@ public String pathName() {
     }
 
     public String getCredentialName() {
-        return findVar(Variables.Flow.GitCredential);
+        return findVar(Variables.Git.SECRET);
     }
 
     public String getGitUrl() {
-        return findVar(Variables.Flow.GitUrl);
+        return findVar(Variables.Git.URL);
     }
 
     /**