basic test for service discovery (#44)

.github/actions/set-environment-variables/action.yaml

@@ -25,7 +25,9 @@ runs:
         newestVersionWithTimestamp="${newestVersion}"."$(date +%Y%m%d-%H%M%S)"
         printf "ARTIFACT_VERSION=%s\n" "${newestVersionWithTimestamp}" >> $GITHUB_ENV
         
-        printf "RELEASE_PATH=%s\n" "${{ github.workspace }}/build/Release" >> $GITHUB_ENV
+        printf "BUILD_PATH=%s\n" "${{ github.workspace }}/build/${buildType}" >> $GITHUB_ENV
+        
+        printf "TEST_PATH=%s\n" "${{ github.workspace }}/testing" >> $GITHUB_ENV
         
         printf "ARTIFACT_PATH=%s\n" "${{ github.workspace }}/artifact" >> $GITHUB_ENV
         

.github/actions/update-packages/action.yaml

@@ -7,4 +7,3 @@ runs:
       shell: bash
       run: |
         sudo apt-get update
-        sudo apt-get upgrade -y

.github/workflows/build-and-test-debug.yaml

@@ -20,11 +20,6 @@ jobs:
         with:
           build-type: Debug
 
-      - name: Set environment variables
-        uses: ./.github/actions/set-environment-variables
-        with:
-          build-type: Debug
-
       - name: Install requirements
         uses: ./.github/actions/install-requirements
 

.github/workflows/build-test-and-publish-release.yaml

@@ -46,10 +46,10 @@ jobs:
             local requirement="${1}"
             local requirementPath="${2}"
             local licenseUrl="${3}"
-            
+          
             printf "[INFO] Downloading %s license file from %s to %s\n" "${requirement}" "${licenseUrl}" "${requirementPath}"
             (cd "${requirementPath}" && curl "${licenseUrl}" --output LICENSE)
-            
+          
           }
           
           copyLicense() {
@@ -76,37 +76,37 @@ jobs:
               downloadLicense "${requirement}" "${requirementPath}" "https://raw.githubusercontent.com/protocolbuffers/protobuf/main/LICENSE"
               continue
             fi
-            
+          
             cacheDirectoryPrefix="$(printf "%s" "${requirement}" | cut -d'/' -f1 | cut -c1-5)"
             licenseDirPath="$(printf "%s" "${licenseDirPaths}" | grep -o "/home/runner/.conan2/p/[^ ]*${cacheDirectoryPrefix}[^ ]*/licenses")"
-            
+          
             licensePath="${licenseDirPath}/LICENSE"
             if [ -e "${licensePath}" ]; then
               copyLicense "${requirement}" "${requirementPath}" "${licensePath}"
               continue
             fi
-            
+          
             licensePath="${licenseDirPath}/LICENSE_1_0.txt"
             if [ -e "${licensePath}" ]; then
               copyLicense "${requirement}" "${requirementPath}" "${licensePath}"
               continue
             fi
-            
+          
             licensePath="${licenseDirPath}/LICENSE.rst"
             if [ -e "${licensePath}" ]; then
               copyLicense "${requirement}" "${requirementPath}" "${licensePath}"
               continue
             fi
-            
+          
             printf "[ERROR] No license file for %s has been found in %s\n" "${requirement}" "${licenseDirPath}"
             return 1
           done
 
       - name: Move protobuf model to artifact directory
-        run: mv ${{ env.RELEASE_PATH }}/libebpfdiscoveryproto/ebpfdiscoveryproto ${{ env.ARTIFACT_PATH }}/ebpfdiscoveryproto
+        run: mv ${{ env.BUILD_PATH }}/libebpfdiscoveryproto/ebpfdiscoveryproto ${{ env.ARTIFACT_PATH }}/ebpfdiscoveryproto
 
       - name: Move binaries to artifact directory
-        run: mv ${{ env.RELEASE_PATH }}/bin ${{ env.ARTIFACT_PATH }}/bin
+        run: mv ${{ env.BUILD_PATH }}/bin ${{ env.ARTIFACT_PATH }}/bin
 
       - name: Remove test binaries from artifact directory
         run: find ${{ env.ARTIFACT_PATH }}/* -name 'test*' -exec rm {} \;

testing/conftest.py

@@ -0,0 +1,46 @@
+import os
+import pytest
+import subprocess
+import glob
+
+from utils import discovered_service_has_clients, is_responsive
+from time import sleep
+
+
+def pytest_addoption(parser):
+    parser.addoption("--discovery_path", action="store", help="Path to eBPF Discovery binary")
+
+
+@pytest.fixture(scope="session")
+def discovery_path(pytestconfig):
+    discovery_path = pytestconfig.getoption("discovery_path")
+    assert discovery_path, "Path to eBPF discovery needs to be provided via --discovery_path"
+    return discovery_path
+
+
+@pytest.fixture(scope="session")
+def run_ebpf_discovery(discovery_path):
+    args = (discovery_path, "--interval", "2")
+    discovery = subprocess.Popen(args, stdout=subprocess.PIPE)
+    yield discovery
+
+    discovery.terminate()
+    while discovery.poll() is None:
+        sleep(0.5)
+    exit_code = discovery.returncode
+    assert not exit_code, "Discovery returned exit code: {}".format(exit_code)
+
+    discovery_root_dir = os.path.dirname(os.path.realpath(discovery_path))
+    log_files = glob.glob(discovery_root_dir+'/*.log')
+    assert log_files == [], "Discovery produced log files on exit: {}".format(log_files)
+
+
+@pytest.fixture(scope="session")
+def http_service(docker_ip, docker_services, run_ebpf_discovery):
+    port = docker_services.port_for("httpbin", 80)
+    url = "http://{}:{}/".format(docker_ip, port)
+    docker_services.wait_until_responsive(
+        timeout=30.0, pause=0.1, check=lambda: is_responsive(url)
+    )
+    assert discovered_service_has_clients(run_ebpf_discovery, url, 1, 0)
+    return url

testing/requirements.txt

@@ -0,0 +1,3 @@
+pytest~=7.4.3
+pytest-docker~=2.0.0
+requests~=2.31.0

testing/test_discovery.py

@@ -0,0 +1,13 @@
+from utils import discovered_service_has_clients, send_http_requests
+
+
+def test_service_discovery(run_ebpf_discovery, http_service):
+    url = http_service + "some/url"
+    requests_num = 5
+    send_http_requests(url, requests_num)
+    assert discovered_service_has_clients(run_ebpf_discovery, url, requests_num, 0)
+
+    url = http_service + "other/url"
+    requests_num = 10
+    send_http_requests(url, requests_num)
+    assert discovered_service_has_clients(run_ebpf_discovery, url, requests_num, 0)

testing/tests/docker-compose.yml

@@ -0,0 +1,6 @@
+version: '2'
+services:
+  httpbin:
+    image: "kennethreitz/httpbin"
+    ports:
+      - "8000:80"

testing/utils.py

@@ -0,0 +1,54 @@
+import json
+import logging
+import requests
+import subprocess
+import typing
+
+
+def send_http_requests(url: str, requests_num: int):
+    for i in range(requests_num):
+        requests.get(url)
+
+
+def is_responsive(url: str) -> bool:
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            return True
+    except (requests.ConnectionError, requests.ConnectTimeout, requests.Timeout):
+        return False
+
+
+def get_discovered_service_json(discovery: subprocess.Popen, url: str) -> typing.Optional[dict]:
+    def url_matches_endpoint(url, endpoint):
+        return url[len("http://"):] == endpoint
+
+    output = discovery.stdout.readline()
+    if not output:
+        return None
+    try:
+        services_json = json.loads(output)
+        for service in services_json["service"]:
+            if url_matches_endpoint(url, service["endpoint"]):
+                return service
+    except (json.decoder.JSONDecodeError, KeyError):
+        return None
+    return None
+
+
+def discovered_service_has_clients(discovery: subprocess.Popen, url: str, local_clients_number: int, external_clients_number: int) -> bool:
+    service = get_discovered_service_json(discovery, url)
+    if not service:
+        logging.warning("No discovered service for endpoint {}".format(url))
+        return False
+    if local_clients_number:
+        if service.get("internalClientsNumber", 0) != local_clients_number:
+            logging.warning("Internal clients number mismatch ({}!={}) for endpoint {}"
+                            .format(service.get("internalClientsNumber", "0"), local_clients_number, url))
+            return False
+    if external_clients_number:
+        if service.get("externalClientsNumber", 0) != external_clients_number:
+            logging.warning("External clients number mismatch ({}!={}) for endpoint {}"
+                            .format(service.get("externalClientsNumber", "0"), external_clients_number, url))
+            return False
+    return True