Merge pull request #23 from kaviththiranga/master

Fixes to cache & compiler
ballerina-attic · Nov 20, 2019 · c3d6534 · c3d6534
2 parents 7916b26 + 215c6c1
commit c3d6534
Show file tree

Hide file tree

Showing 20 changed files with 103 additions and 130 deletions.
diff --git a/api/compiler/Ballerina.lock b/api/compiler/Ballerina.lock
@@ -1,4 +1,4 @@
-org_name = "ballerina.io"
+org_name = "ballerina.playground"
 version = "1.0.0"
 lockfile_version = "1.0.0"
-ballerina_version = "1.0.2"
+ballerina_version = "1.0.3"
diff --git a/api/compiler/src/playground_compiler/compile.bal b/api/compiler/src/playground_compiler/compile.bal
@@ -34,37 +34,18 @@ function createSourceFile(string cacheId, string sourceCode) returns string|erro
 
 function compile(CompileData data) returns CompilerResponse|error {
     log:printDebug("Compiling request: " + data.toString());
-    string? cacheId = getCacheId(data.sourceCode, data.balVersion);
-    if (cacheId is string) {
-        boolean hasCachedOutputResult = hasCachedOutput(cacheId);
-        log:printDebug("Searching for cached output. Cache ID: " + cacheId);
-        if (hasCachedOutputResult) {
-            string? cachedOutput = getCachedOutput(cacheId);
-            if (cachedOutput is string) {
-                log:printDebug("Found cached output. " + cachedOutput);
-                return createDataResponse(cachedOutput);
-            } else {
-                log:printError("Empty cached output returned. " );
-                return createErrorResponse("Invalid cached output returned from cache.");
-            }
-        } else {
-            log:printDebug("Cached output not found.");
-            string sourceFile = check createSourceFile(cacheId, data.sourceCode);
-            string buildDir = check filepath:parent(sourceFile);
-            log:printDebug("Using " + sourceFile + " for compilation.");
-            string|error execStatus = execBallerinaCmd(buildDir, "build", "app.bal");
-            if (execStatus is error) {
-                log:printError("Error while executing compile command. ", execStatus);
-                return createErrorResponse(execStatus.reason());
-            } else {
-                log:printDebug("Finished executing compile command. Output: " + execStatus);
-                string jarPath = check filepath:build(buildDir, "app.jar");
-                setCachedOutput(cacheId, execStatus);
-                return createDataResponse(execStatus);
-            }
-        }
+    string cacheId = getCacheId(data.sourceCode, data.balVersion);
+    string sourceFile = check createSourceFile(cacheId, data.sourceCode);
+    string buildDir = check filepath:parent(sourceFile);
+    log:printDebug("Using " + sourceFile + " for compilation.");
+    string|error execStatus = execBallerinaCmd(buildDir, "build", "app.bal");
+    if (execStatus is error) {
+        log:printError("Error while executing compile command. ", execStatus);
+        return createErrorResponse(execStatus.reason());
     } else {
-        return createErrorResponse("Cannot access cache");
+        log:printDebug("Finished executing compile command. Output: " + execStatus);
+        string jarPath = check filepath:build(buildDir, "app.jar");
+        return createDataResponse(execStatus);
     }
 }
 

diff --git a/api/compiler/src/playground_compiler/utils/cache_utils.bal b/api/compiler/src/playground_compiler/utils/cache_utils.bal
@@ -1,21 +1,6 @@
 import ballerina/crypto;
-import ballerina/cache;
 
-cache:Cache inMemCache = new();
-
-function getCacheId(string sourceCode, string balVersion) returns string? {
+function getCacheId(string sourceCode, string balVersion) returns string {
     string cacheSource = sourceCode + balVersion;
     return crypto:hashMd5(cacheSource.toBytes()).toBase16();
-}
-
-function hasCachedOutput(string cacheId) returns boolean {
-   return inMemCache.hasKey(cacheId);
-}
-
-function getCachedOutput(string cacheId) returns string? {
-    return <string?> inMemCache.get(cacheId);
-}
-
-function setCachedOutput(string cacheId, string output) {
-   inMemCache.put(cacheId, output);
 }
diff --git a/api/controller/Ballerina.toml b/api/controller/Ballerina.toml
@@ -8,7 +8,7 @@ target = "java8"
     [[platform.libraries]]
     artifactId = "redis-client"
     version = "@project.version@"
-    path = "./libs/redis-client-1.0.0-SNAPSHOT.jar"
+    path = "./libs/redis-client-1.0.0-m1.jar"
     groupId = "ballerina"
     modules = ["playground_commons"]
 

diff --git a/api/controller/src/playground_commons/redis/natives.bal b/api/controller/src/playground_commons/redis/natives.bal
@@ -34,6 +34,11 @@ public function redisSet(string key, string value) = @java:Method {
     class:"org/ballerinalang/playground/cache/RedisCache"
 } external;
 
+public function redisRemove(string key) = @java:Method {
+    name: "remove",
+    class:"org/ballerinalang/playground/cache/RedisCache"
+} external;
+
 public function redisPushToList(string key, string value) = @java:Method {
     name: "pushToList",
     class:"org/ballerinalang/playground/cache/RedisCache"

diff --git a/api/controller/src/playground_controller/run.bal b/api/controller/src/playground_controller/run.bal
@@ -146,13 +146,17 @@ function run(http:WebSocketCaller caller, RunData data) returns error? {
             }
         }
     };
-    if (commons:redisContains(cacheId)) {
-        log:printDebug("Found cached responses. ");
-        string[] cachedResponses = commons:redisGetList(cacheId);
-        foreach string response in cachedResponses {
+    [boolean, boolean, string[]] checkCacheResult = checkCache(cacheId);
+    if (checkCacheResult[0] && checkCacheResult[1]) {
+        log:printDebug("Found valid cached responses. ");
+        foreach string response in checkCacheResult[2] {
             respHandler(response, false);
         }
     } else {
+        // invalidate cache entry if it exists & invalid
+        if (checkCacheResult[0] && !checkCacheResult[1]) {
+            commons:redisRemove(cacheId);
+        }
         log:printDebug("Cached responses not found. Compiling. ");
         error? compilerResult = invokeCompiler(respHandler, data, compilerCallBack);
         if (compilerResult is error) {
@@ -161,6 +165,30 @@ function run(http:WebSocketCaller caller, RunData data) returns error? {
     }
 }
 
+
+# Check if a valid cache exists and return cache if it exists.
+#
+# + cacheId - cacheId Parameter 
+# + return - [isCacheAvailable, isCacheValid, cache]
+function checkCache(string cacheId) returns [boolean, boolean, string[]] {
+    if (commons:redisContains(cacheId)) {
+        string[] cachedResponses = commons:redisGetList(cacheId);
+        string lastResponse = cachedResponses[cachedResponses.length() - 1];
+        json|error jsonResponse = lastResponse.fromJsonString();
+        if (jsonResponse is json) {
+            PlaygroundResponse|error response = PlaygroundResponse.constructFrom(jsonResponse);
+            if (response is PlaygroundResponse) {
+                boolean isValid = response.'type == ControlResponse && 
+                    (response.data == "Finished Executing."
+                        || response.data == "Finished Compiling with errors.");
+                return [true, isValid, cachedResponses];
+            }
+        }
+        return [true, false, cachedResponses];
+    }
+    return [false, false, []];
+}
+
 function createJSONResponse(PlaygroundResponse reponse) returns json|error {
     json jsonResp = check json.constructFrom(reponse);
     return jsonResp.toJsonString();

diff --git a/api/executor/Ballerina.lock b/api/executor/Ballerina.lock
@@ -1,4 +1,4 @@
-org_name = "ballerina.io"
+org_name = "ballerina.playground"
 version = "1.0.0"
 lockfile_version = "1.0.0"
 ballerina_version = "1.0.3"
diff --git a/api/executor/src/playground_executor/execute.bal b/api/executor/src/playground_executor/execute.bal
@@ -19,35 +19,17 @@ function getAppJar(string cacheId) returns string|error {
 
 function execute(ExecuteData data) returns ExecutorResponse|error {
     log:printDebug("Executing request: " + data.toString());
-    string? cacheId = getCacheId(data.sourceCode, data.balVersion);
-    if (cacheId is string) {
-        log:printDebug("Searching for cached output. Cache ID: " + cacheId);
-        boolean hasCachedOutputResult = hasCachedOutput(cacheId);
-        if (hasCachedOutputResult) {
-            string? cachedOutput = getCachedOutput(cacheId);
-            if (cachedOutput is string) {
-                log:printDebug("Found cached output. " + cachedOutput);
-                return createDataResponse(cachedOutput);
-            } else {
-                return createErrorResponse("Invalid cached output returned from cache.");
-            }
-        } else {
-            log:printDebug("Cached output not found.");
-            string appJar = check getAppJar(cacheId);
-            log:printDebug("Executing jar: " + appJar);
-            string cwd = check filepath:parent(appJar);
-            string|error execStatus = execJar(cwd, appJar);
-            if (execStatus is error) {
-                log:printError("Error while executing jar: " + execStatus.reason());
-                return createErrorResponse(execStatus.reason());
-            } else {
-                log:printDebug("Executed jar: " + execStatus);
-                setCachedOutput(cacheId, execStatus);
-                return createDataResponse(execStatus);
-            }
-        }
+    string cacheId = getCacheId(data.sourceCode, data.balVersion);
+    string appJar = check getAppJar(cacheId);
+    log:printDebug("Executing jar: " + appJar);
+    string cwd = check filepath:parent(appJar);
+    string|error execStatus = execJar(cwd, appJar);
+    if (execStatus is error) {
+        log:printError("Error while executing jar: " + execStatus.reason());
+        return createErrorResponse(execStatus.reason());
     } else {
-        return createErrorResponse("Cannot access cache");
+        log:printDebug("Executed jar: " + execStatus);
+        return createDataResponse(execStatus);
     }
 }
 

diff --git a/api/executor/src/playground_executor/utils/cache_utils.bal b/api/executor/src/playground_executor/utils/cache_utils.bal
@@ -1,21 +1,6 @@
 import ballerina/crypto;
-import ballerina/cache;
-
-cache:Cache inMemCache = new();
 
 function getCacheId(string sourceCode, string balVersion) returns string {
     string cacheSource = sourceCode + balVersion;
     return crypto:hashMd5(cacheSource.toBytes()).toBase16();
-}
-
-function hasCachedOutput(string cacheId) returns boolean {
-   return inMemCache.hasKey(cacheId);
-}
-
-function getCachedOutput(string cacheId) returns string? {
-    return <string?> inMemCache.get(cacheId);
-}
-
-function setCachedOutput(string cacheId, string output) {
-   inMemCache.put(cacheId, output);
 }
diff --git a/build.gradle b/build.gradle
@@ -16,6 +16,6 @@ allprojects {
     }
 }
 subprojects {
-    version = '1.0.0-SNAPSHOT'
+    version = '1.0.0-m1'
     ext.setProperty("ballerinaVersion", "1.0.1")
 }
diff --git a/docker/compiler/Dockerfile b/docker/compiler/Dockerfile
@@ -4,11 +4,12 @@ FROM ballerina/ballerina:$BALLERINA_VERSION
 
 LABEL maintainer "ballerina.io"
 
-COPY plugins/*.jar $BALLERINA_HOME/bre/lib
-
 WORKDIR /api
 COPY api/*.jar /api
 
+COPY plugins/*.jar $BALLERINA_HOME/bre/lib/
+RUN cp $BALLERINA_HOME/bre/lib/*.jar $(ballerina home)/bre/lib/
+
 USER root
 
 RUN chown -R ballerina:troupe /api && \

diff --git a/docker/compiler/build.gradle b/docker/compiler/build.gradle
@@ -7,6 +7,10 @@ configurations {
     }
 }
 
+def env = System.getenv()
+def gcpProjectID = env["BPG_GCP_PROJECT_ID"]
+def imageTag = 'gcr.io/' + gcpProjectID +'/playground-compiler:' + project.version
+
 dependencies {
     compilerPlugins project(":utils:compiler-plugin")
     api project(path: ":api:compiler", configuration: "apiJar")
@@ -35,14 +39,10 @@ task cloudBuild(type: Exec) {
     dependsOn copyCompilerPlugins
     inputs.files(inputFiles)
     environment("BALLERINA_VERSION", project.ext.ballerinaVersion)
-    commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", 
-        '--substitutions=_BALLERINA_VERSION=' + project.ext.ballerinaVersion, "."
+    commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml",
+        '--substitutions=_IMAGE_TAG=' + imageTag + ',_BALLERINA_VERSION=' + project.ext.ballerinaVersion, "."
 }
 
-def env = System.getenv()
-def gcpProjectID = env["BPG_GCP_PROJECT_ID"]
-def imageTag = 'gcr.io/' + gcpProjectID +'/playground-compiler:' + project.version
-
 task build(type: Exec) {
     dependsOn copyApi
     dependsOn copyCompilerPlugins

diff --git a/docker/compiler/cloudbuild.yaml b/docker/compiler/cloudbuild.yaml
@@ -1,6 +1,6 @@
 steps:
 - name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/playground-compiler','--build-arg', 
+  args: [ 'build', '-t', '$_IMAGE_TAG','--build-arg', 
             'BALLERINA_VERSION=$_BALLERINA_VERSION', '.' ]
 images:
-- 'gcr.io/$PROJECT_ID/playground-compiler'
+-  '$_IMAGE_TAG'
diff --git a/docker/controller/build.gradle b/docker/controller/build.gradle
@@ -24,16 +24,18 @@ def inputFiles = [
     project.file("Dockerfile")
 ]
 
+def env = System.getenv()
+def gcpProjectID = env["BPG_GCP_PROJECT_ID"]
+def imageTag = 'gcr.io/' + gcpProjectID +'/playground-controller:' + project.version
+
+
 task cloudBuild(type: Exec) {
     dependsOn copyApi
     inputs.files(inputFiles)
-    commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", "."
+    commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", 
+        '--substitutions=_IMAGE_TAG=' + imageTag, "."
 }
 
-def env = System.getenv()
-def gcpProjectID = env["BPG_GCP_PROJECT_ID"]
-def imageTag = 'gcr.io/' + gcpProjectID +'/playground-controller:' + project.version
-
 task build(type: Exec) {
     dependsOn copyApi
     inputs.files(inputFiles)

diff --git a/docker/controller/cloudbuild.yaml b/docker/controller/cloudbuild.yaml
@@ -1,5 +1,5 @@
 steps:
 - name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/playground-controller', '.' ]
+  args: [ 'build', '-t', '$_IMAGE_TAG', '.' ]
 images:
-- 'gcr.io/$PROJECT_ID/playground-controller'
+-  '$_IMAGE_TAG'
diff --git a/docker/executor/build.gradle b/docker/executor/build.gradle
@@ -24,16 +24,17 @@ def inputFiles = [
     project.file("Dockerfile")
 ]
 
+def env = System.getenv()
+def gcpProjectID = env["BPG_GCP_PROJECT_ID"]
+def imageTag = 'gcr.io/' + gcpProjectID +'/playground-executor:' + project.version
+
 task cloudBuild(type: Exec) {
     dependsOn copyApi
     inputs.files(inputFiles)
-    commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", "."
+    commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", 
+        '--substitutions=_IMAGE_TAG=' + imageTag, "."
 }
 
-def env = System.getenv()
-def gcpProjectID = env["BPG_GCP_PROJECT_ID"]
-def imageTag = 'gcr.io/' + gcpProjectID +'/playground-executor:' + project.version
-
 task build(type: Exec) {
     dependsOn copyApi
     inputs.files(inputFiles)

diff --git a/docker/executor/cloudbuild.yaml b/docker/executor/cloudbuild.yaml
@@ -1,5 +1,5 @@
 steps:
 - name: 'gcr.io/cloud-builders/docker'
-  args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/playground-executor', '.' ]
+  args: [ 'build', '-t', '$_IMAGE_TAG', '.' ]
 images:
-- 'gcr.io/$PROJECT_ID/playground-executor'
+- '$_IMAGE_TAG'
diff --git a/k8s/build.gradle b/k8s/build.gradle
@@ -2,7 +2,7 @@ def namespace = 'ballerina-playground-v2';
 
 task deployWebServer(type: Exec) {
     dependsOn(":docker:nginx:cloudBuild")
-    environment("RELEASE_VERSION", "latest") // replace latest with project.version
+    environment("RELEASE_VERSION", project.version)
     commandLine './deploy-web-server.sh'
 }
 
@@ -20,13 +20,13 @@ task undeployNfs(type: Exec) {
 
 task deployCompiler(type: Exec) {
     dependsOn(":docker:compiler:cloudBuild")
-    environment("RELEASE_VERSION", "latest") // replace latest with project.version
+    environment("RELEASE_VERSION", project.version)
     commandLine './deploy-compiler.sh'
 }
 
 task deployExecutor(type: Exec) {
     dependsOn(":docker:executor:cloudBuild")
-    environment("RELEASE_VERSION", "latest") // replace latest with project.version
+    environment("RELEASE_VERSION", project.version)
     commandLine './deploy-executor.sh'
 }
 
@@ -48,7 +48,7 @@ task undeployExecutor(type: Exec) {
 
 task deployController(type: Exec) {
     dependsOn(":docker:controller:cloudBuild")
-    environment("RELEASE_VERSION", "latest") // replace latest with project.version
+    environment("RELEASE_VERSION", project.version)
     commandLine './deploy-controller.sh'
 }
 
@@ -73,7 +73,6 @@ task undeployController(type: Exec) {
 }
 
 task deployIngressResources(type: Exec) {
-    environment("RELEASE_VERSION", "latest") // replace latest with project.version
     commandLine './deploy-ingress-resources.sh'
 }