reworking integration testing

.github/workflows/ci.yml

@@ -44,3 +44,4 @@ jobs:
       - run: ./ci/test_distribute_scenario.sh
       - run: ./ci/test_run_scenario.sh
       - run: ./ci/test_add_new_node.sh
+      - run: ./ci/stop_demo_cluster.sh

ci/docker-compose.yml

@@ -2,14 +2,9 @@ x-amoc-defaults: &amoc-defaults
   image: "amoc-arsenal"
   networks:
     - amoc-test-network
-  volumes:
-    - type: bind
-      source: ./extra_code_paths
-      target: /extra_code_paths
   environment:
     AMOC_GRAPHITE_HOST: '"graphite"'
     AMOC_NODES: "['amoc_arsenal@amoc-master']"
-    AMOC_EXTRA_CODE_PATHS: '["/extra_code_paths/test1", "/extra_code_paths/test2"]'
   healthcheck:
     test: "amoc_arsenal status"
 

ci/helper.sh

@@ -10,42 +10,36 @@ function enable_strict_mode() {
     IFS=$'\n\t'
 }
 
-function create_code_path() {
-    dir="${git_root}/ci/extra_code_paths/${1}"
-    [ -d "$dir" ] || return 1
-    erl_file="${dir}/${1}.erl"
-    dummy_scenario="${git_root}/ci/dummy_scenario.erl"
-    sed "s/-module(.*)./-module(${1})./" "$dummy_scenario" > "$erl_file"
-    erlc -o "$dir" "$erl_file"
-}
-
-function contain() {
-    local pipeline='cat -'
+function contains() {
+    local output="$(cat -)"
+    local ret= acc=0
     for pattern in "$@"; do
-        pipeline+=" | tee >(grep -q -e \"$pattern\"; echo \"+\$?\")"
+        ret="$(echo "$output" | grep -q -e "$pattern"; echo "$?")"
+        if [ "$ret" -ne "0" ]; then
+            [ "$(($acc))" -eq "0" ] && {
+                echo "contains FAILED"
+                echo "output: '${output}'"; }
+            echo "pattern is missing: '${pattern}'"
+        fi >&2
+        acc+="+${ret}"
     done
-    pipeline+=' >/dev/null'
-    local output="$(eval "$pipeline")"
-    test "$(($output))" -eq 0
+    test "$(($acc))" "-eq" "0"
 }
 
-function wait_for_cmd() {
-    local timeout="${1:-0}"
-    local cmd="${2:-true}"
-    shift 2
-    local full_cmd=("$cmd" "$@")
-    echo "Waiting for '${full_cmd[@]}'"
-    for i in $(seq 0 "${timeout}"); do
-        if "${full_cmd[@]}"; then
-            [ "$i" -ne 0 ] && echo
-            echo "Waiting is done after $i seconds"
-            return 0
-        fi
-        echo -n "."
+function retry() {
+    local n="$1" m="0"
+    local output=
+    shift 1
+    echo "waiting for '$@'"
+    until echo -n "." && output="$("$@" 2>&1)"; do
+        [ "$n" -gt "$m" ] || {
+            echo -e "\nfailed after '$m' retries\nlast output"
+            echo "${output}"
+            return 1; }
         sleep 1
+        m="$(( m + 1 ))"
     done
-    echo -e "\nKilled by timeout"
-    return 1
+    echo -e "\nsuccess after '$m' retries";
 }
 
 ######################
@@ -63,35 +57,35 @@ function amoc_container_port() {
     esac
 }
 
-docker_compose() {
+function docker_compose() {
     local compose_file="${git_root}/ci/docker-compose.yml"
     docker compose -p "amoc-demo-cluster" -f "$compose_file" "$@"
 }
 
-function amoc_eval() {
-    local exec_path="amoc_arsenal"
-    local service="$1"
-    shift 1
-    docker_compose exec -T "$service" "$exec_path" eval "$@"
-}
-
-function container_is_healthy() {
-   docker_compose ps $1 | contain "healthy"
+#############################
+## amoc REST API functions ##
+#############################
+function get_nodes() {
+    local servise="$1"
+    port="$(amoc_container_port "$servise")"
+    curl -s -X GET "http://localhost:${port}/nodes" -H  "accept: application/json"
 }
 
-function wait_for_healthcheck() {
-    local container=$1
-    wait_for_cmd 60 container_is_healthy "$container"
+function get_status() {
+    local servise="$1"
+    local port="$(amoc_container_port "$servise")"
+    curl -s -X GET "http://localhost:${port}/status" -H  "accept: application/json"
 }
 
+#################################
+## graphite REST API functions ##
+#################################
 function metrics_reported() {
-    local graphite_query="target=summarize(*.amoc.users.size,'1hour','max')&from=-1h&format=json"
-    local result="$(curl -s "http://localhost:8080/render/?${graphite_query}")"
-    echo "$result" | contain "$@"
+    curl -s "http://localhost:8080/metrics/find?query=*" | contains "$@"
 }
 
 function wait_for_metrics() {
-     wait_for_cmd 60 metrics_reported "$@"
+     retry 60 metrics_reported "$@"
 }
 
 ######################

ci/start_demo_cluster.sh

@@ -3,15 +3,9 @@
 source "$(dirname "$0")/helper.sh"
 enable_strict_mode
 
-create_code_path test1
-create_code_path test2
-
-docker_compose up -d amoc-{master,worker-1,worker-2} graphite grafana
-
-wait_for_healthcheck amoc-master
-wait_for_healthcheck amoc-worker-1
-wait_for_healthcheck amoc-worker-2
+docker_compose up --wait --wait-timeout 100 amoc-{master,worker-1,worker-2} graphite grafana
 
+## configure default grafana datasource
 json=( '{'
        '"name": "graphite",'
        '"access": "proxy",'
@@ -20,7 +14,6 @@ json=( '{'
        '"isDefault": true'
        '}' )
 
-## configure default grafana datasource
 curl 'http://admin:admin@localhost:3000/api/datasources' -X POST \
      -H 'Content-Type: application/json;charset=UTF-8' \
      -d "${json[*]}" -w "\n%{response_code}\n"

ci/stop_demo_cluster.sh

@@ -2,4 +2,5 @@
 
 source "$(dirname "$0")/helper.sh"
 enable_strict_mode
+
 docker_compose down

ci/test_add_new_node.sh

@@ -3,9 +3,25 @@
 source "$(dirname "$0")/helper.sh"
 enable_strict_mode
 
-docker_compose up -d amoc-worker-3
-wait_for_healthcheck amoc-worker-3
+docker_compose up --wait --wait-timeout 100 amoc-worker-3
 
-amoc_eval amoc-worker-3 "amoc_controller:get_status()." | contain dummy_scenario running
-amoc_eval amoc-worker-3 "binary_to_list(amoc_config:get(test))." | contain "test_value"
-amoc_eval amoc-worker-3 "dummy_helper:test_amoc_dist()." | contain 'amoc_dist_works_as_expected'
+echo "checking that clustering is done properly"
+node_list=( '"amoc_arsenal@amoc-'{"master","worker-1","worker-2","worker-3"}'":"up"' )
+get_nodes amoc-master | contains "${node_list[@]}"
+get_nodes amoc-worker-1 | contains "${node_list[@]}"
+get_nodes amoc-worker-2 | contains "${node_list[@]}"
+get_nodes amoc-worker-3 | contains "${node_list[@]}"
+
+echo "checking status of the nodes"
+worker_status=( '"amoc_status":"up"'
+                '"status":"running"'
+                '"scenario":"dummy_scenario"'
+                '"test":"<<\\"test_value\\">>"'
+                '"interarrival":"50"' )
+get_status amoc-worker-1 | contains "${worker_status[@]}" '"number_of_users":5'
+get_status amoc-worker-2 | contains "${worker_status[@]}" '"number_of_users":5'
+sleep 1 ## wait one second to ensure that noumber of users is indeed 0
+get_status amoc-worker-3 | contains "${worker_status[@]}" '"number_of_users":0'
+
+echo "checking that metrics are reported"
+wait_for_metrics "amoc-"{"master","worker-1","worker-2","worker-3"}

ci/test_amoc_cluster.sh

@@ -4,12 +4,15 @@ source "$(dirname "$0")/helper.sh"
 enable_strict_mode
 
 echo "checking that clustering is done properly"
-amoc_eval amoc-master "nodes()." | contain amoc-worker-1 amoc-worker-2
-amoc_eval amoc-worker-1 "nodes()." | contain amoc-master amoc-worker-2
-amoc_eval amoc-worker-2 "nodes()." | contain amoc-master amoc-worker-1
+node_list=( '"amoc_arsenal@amoc-'{"master","worker-1","worker-2"}'":"up"' )
+get_nodes amoc-master | contains "${node_list[@]}"
+get_nodes amoc-worker-1 | contains "${node_list[@]}"
+get_nodes amoc-worker-2 | contains "${node_list[@]}"
 
-echo "checking that AMOC_EXTRA_CODE_PATHS setting works as expected"
-amoc_eval amoc-master "amoc_code_server:does_scenario_exist(test1)." | contain true
-amoc_eval amoc-master "amoc_code_server:does_scenario_exist(test2)." | contain true
+echo "checking status of the nodes"
+get_status amoc-master | contains '"amoc_status":"up"' '"status":"idle"'
+get_status amoc-worker-1 | contains '"amoc_status":"up"' '"status":"idle"'
+get_status amoc-worker-2 | contains '"amoc_status":"up"' '"status":"idle"'
 
-wait_for_metrics "amoc-"{"master","worker-1","worker-2"}".amoc.users.size"
+echo "checking that metrics are reported"
+wait_for_metrics "amoc-"{"master","worker-1","worker-2"}

ci/test_distribute_scenario.sh

@@ -15,7 +15,7 @@ function get_scenarios() {
          --request GET "http://localhost:${port}/scenarios"
 }
 
-function list_scenarios_by_port() {
+function list_scenarios() {
     local result="$(get_scenarios "$1")"
     echo "Scenarios on the ${1} node: ${result}"
 }
@@ -24,7 +24,7 @@ function ensure_scenarios_installed() {
     local result="$(get_scenarios "$1")"
     echo "Scenarios on the ${1} node: ${result}"
     shift 1
-    echo "$result" | contain "$@"
+    echo "$result" | contains "$@"
 }
 
 function upload_module() {
@@ -34,15 +34,15 @@ function upload_module() {
          "http://localhost:${port}/scenarios/upload"
 }
 
-list_scenarios_by_port amoc-master
-list_scenarios_by_port amoc-worker-1
-list_scenarios_by_port amoc-worker-2
+list_scenarios amoc-master
+list_scenarios amoc-worker-1
+list_scenarios amoc-worker-2
 
 echo "Installing scenario and helper module on the amoc-master node"
-scenario_put="$(upload_module amoc-master "${scenario_name}.erl")"
-echo "Response for '${scenario_name}.erl': ${scenario_put}"
-helper_put="$(upload_module amoc-master "dummy_helper.erl")"
-echo "Response for 'dummy_helper.erl': ${helper_put}"
+scenario_uploading="$(upload_module amoc-master "${scenario_name}.erl")"
+echo "Response for '${scenario_name}.erl': ${scenario_uploading}"
+helper_uploading="$(upload_module amoc-worker-1 "dummy_helper.erl")"
+echo "Response for 'dummy_helper.erl': ${helper_uploading}"
 
-ensure_scenarios_installed amoc-worker-1 ${scenario_name}
-ensure_scenarios_installed amoc-worker-2 ${scenario_name}
+ensure_scenarios_installed amoc-worker-1 "$scenario_name"
+ensure_scenarios_installed amoc-worker-2 "$scenario_name"

ci/test_run_scenario.sh

@@ -13,12 +13,18 @@ run_scenario() {
          -s  -w "%{http_code}" -o /dev/null -d "$json_body" "http://localhost:${port}/execution/start"
 }
 
-result="$(run_scenario amoc-master dummy_scenario 10)"
+echo "executing dummy_scenario"
+run_scenario amoc-master dummy_scenario 10 | contains 200
 
-if [ "$result" = "200" ]; then
-    echo "Scenario executed"
-    exit 0
-else
-    echo "Scenario failed"
-    exit -1
-fi
+echo "checking status of the nodes"
+get_status amoc-master | contains '"amoc_status":"up"' \
+                                  '"status":"disabled"'
+sleep 1 ## 1 second is enought to start 5 users with 50ms interarrival timeout
+worker_status=( '"amoc_status":"up"'
+                '"status":"running"'
+                '"scenario":"dummy_scenario"'
+                '"number_of_users":5'
+                '"test":"<<\\"test_value\\">>"'
+                '"interarrival":"50"' )
+get_status amoc-worker-1 | contains "${worker_status[@]}"
+get_status amoc-worker-2 | contains "${worker_status[@]}"