Merge pull request #400 from neilcook/docker

Docker and Regression Improvements

.github/workflows/pubtests.yml

@@ -0,0 +1,39 @@
+---
+name: Publish Test Results
+
+on:
+  workflow_run:
+    workflows: ["Run regression tests"]
+    types:
+      - completed
+permissions: {}
+
+jobs:
+  test-results:
+    name: Publish Test Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    permissions:
+      checks: write
+
+      # needed unless run with comment_mode: off
+      pull-requests: write
+
+      # required by download step to access artifacts API
+      actions: read
+
+    steps:
+      - name: Download and Extract Artifacts
+        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
+        with:
+          run_id: ${{ github.event.workflow_run.id }}
+          path: artifacts
+
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/**/**/*.xml"

.github/workflows/regression.yml

@@ -34,3 +34,19 @@ jobs:
           MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
       - run: make dist
       - run: make distcheck
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: Test Results
+          path: |
+            docker/tmp/*.xml
+  event_file:
+    name: "Event File"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

common/iputils.hh

@@ -142,7 +142,7 @@ union ComboAddress {
     }
   };
 
-  struct addressOnlyLessThan: public std::binary_function<ComboAddress, ComboAddress, bool>
+  struct addressOnlyLessThan
   {
     bool operator()(const ComboAddress& a, const ComboAddress& b) const
     {
@@ -157,7 +157,7 @@ union ComboAddress {
     }
   };
 
-  struct addressOnlyEqual: public std::binary_function<ComboAddress, ComboAddress, bool>
+  struct addressOnlyEqual
   {
     bool operator()(const ComboAddress& a, const ComboAddress& b) const
     {

common/misc.cc

@@ -857,12 +857,12 @@ unsigned int pdns_stou(const std::string& str, size_t * idx, int base)
 std::string getDirectoryPath(const std::string& filename)
 {
   boost::filesystem::path my_path(filename);
-  boost::filesystem::path branch_path = my_path.branch_path();
+  boost::filesystem::path branch_path = my_path.parent_path();
   return branch_path.empty() ? "." : branch_path.string();
 }
 
 std::string getFileFromPath(const std::string& filename)
 {
   boost::filesystem::path my_path(filename);
-  return my_path.leaf().string();
+  return my_path.filename().string();
 }

common/misc.hh

@@ -390,8 +390,8 @@ private:
     #endif
 };
 
-
-struct CIStringCompare: public std::binary_function<string, string, bool>  
+// FIXME400 this should probably go?
+struct CIStringCompare
 {
   bool operator()(const string& a, const string& b) const
   {
@@ -401,28 +401,32 @@ struct CIStringCompare: public std::binary_function<string, string, bool>
 
 struct CIStringComparePOSIX
 {
-   bool operator() (const std::string& lhs, const std::string& rhs)
-   {
-      std::string::const_iterator a,b;
-      const std::locale &loc = std::locale("POSIX");
-      a=lhs.begin();b=rhs.begin();
-      while(a!=lhs.end()) {
-          if (b==rhs.end() || std::tolower(*b,loc)<std::tolower(*a,loc)) return false;
-          else if (std::tolower(*a,loc)<std::tolower(*b,loc)) return true;
-          ++a;++b;
+  bool operator() (const std::string& lhs, const std::string& rhs) const
+  {
+    const std::locale &loc = std::locale("POSIX");
+    auto lhsIter = lhs.begin();
+    auto rhsIter = rhs.begin();
+    while (lhsIter != lhs.end()) {
+      if (rhsIter == rhs.end() || std::tolower(*rhsIter,loc) < std::tolower(*lhsIter,loc)) {
+        return false;
+      }
+      if (std::tolower(*lhsIter,loc) < std::tolower(*rhsIter,loc)) {
+        return true;
       }
-      return (b!=rhs.end());
-   }
+      ++lhsIter;++rhsIter;
+    }
+    return rhsIter != rhs.end();
+  }
 };
 
-struct CIStringPairCompare: public std::binary_function<pair<string, uint16_t>, pair<string,uint16_t>, bool>  
+struct CIStringPairCompare
 {
   bool operator()(const pair<string, uint16_t>& a, const pair<string, uint16_t>& b) const
   {
     if(pdns_ilexicographical_compare(a.first, b.first))
-	return true;
+      return true;
     if(pdns_ilexicographical_compare(b.first, a.first))
-	return false;
+      return false;
     return a.second < b.second;
   }
 };

docker/Makefile.am

@@ -5,7 +5,7 @@ REGRESSION_SERVICE = regression
 
 SUBDIRS= wforce_image
 
-$(COMPOSE_TARGET): $(COMPOSE_SOURCE) $(shell find ../common -type f) $(shell find ../wforce -type f)  $(shell find ../trackalert -type f) $(shell find ../ext -type f) $(shell find ../regression-tests -type f) $(shell find regression -type f)  $(shell find logstash -type f) 
+$(COMPOSE_TARGET): $(COMPOSE_SOURCE) $(shell find ../common -type f) $(shell find ../wforce -type f) $(shell find ../report_api -type f)  $(shell find ../trackalert -type f) $(shell find ../ext -type f) $(shell find ../regression-tests -type f) $(shell find regression -type f) $(shell find logstash -type f) 
 	$(DCMP) down -v
 	$(DCMP) build
 	touch $(COMPOSE_TARGET)
@@ -32,10 +32,10 @@ clean_docker:
 clean: clean_docker
 
 regression-gcc: build_image start
-	$(DCMP) exec -T $(REGRESSION_SERVICE) docker/regression/regression.sh gcc g++ --enable-sodium
+	$(DCMP) exec -T $(REGRESSION_SERVICE) docker/regression/regression.sh gcc g++ --enable-sodium /var/tmp/testlog/gcctest.xml
 
 regression-clang: build_image start
-	$(DCMP) exec -T $(REGRESSION_SERVICE) docker/regression/regression.sh clang clang++ --disable-sodium
+	$(DCMP) exec -T $(REGRESSION_SERVICE) docker/regression/regression.sh clang clang++ --disable-sodium /var/tmp/testlog/clangtest.xml
 
 regression-none:
 	echo "Regression tests skipped due to compiler 'none'"

docker/docker-compose.yml

@@ -50,6 +50,7 @@ services:
         - MAXMIND_LICENSE_KEY=${MAXMIND_LICENSE_KEY}
     volumes:
       - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
+      - ./tmp:/var/tmp/testlog
     links:
       - elasticsearch
       - logstash

docker/regression/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:focal as wforce_regression
+FROM ubuntu:jammy as wforce_regression
 
 ENV DEBIAN_FRONTEND noninteractive
 

docker/regression/regression.sh

@@ -2,27 +2,30 @@
 
 set -e
 
-if [ $# -ne 3 ]
+if [ $# -ne 4 ]
 then
     export MYCC=clang
     export MYCXX=clang++
     export SODIUM=
+    export TESTFILE=pytest.xml
 else
     export MYCC=$1
     export MYCXX=$2
     export SODIUM=$3
+    export TESTFILE=$4
 fi
 
 echo "CC=$MYCC"
 echo "CXX=$MYCXX"
+echo "TESTFILE=$TESTFILE"
 
 autoreconf -v -i -f
 ./configure --enable-trackalert --enable-systemd --disable-docker --enable-unit-tests --enable-asan --enable-ubsan $SODIUM --disable-silent-rules CC=$MYCC CXX=$MYCXX
 make clean
 make
 make check || (cat common/test-suite.log && false)
 cd regression-tests
-./runtests
+./runtests $TESTFILE
 cd ..
 make dist
 export WF_VERSION=`grep PACKAGE_VERSION Makefile | awk  '{ print $3}'`

docker/wforce_image/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim as wforce_build
+FROM debian:bookworm-slim as wforce_build
 
 RUN apt-get update && \
     apt-get dist-upgrade -y && \
@@ -11,14 +11,14 @@ RUN apt-get update && \
                libhiredis-dev \
                libmaxminddb-dev \
                liblua5.1-0-dev \
-               libluajit-5.1-dev \
+               libluajit2-5.1-dev \
                libprotobuf-dev \
                libssl-dev \
-               libsodium-dev \
                libsystemd-dev \
                libyaml-cpp-dev \
                libjsoncpp-dev \
                libz-dev \
+               prometheus-cpp-dev \
                uuid-dev \
                libtool \
                pkg-config \
@@ -27,19 +27,13 @@ RUN apt-get update && \
                wget \
                docker \
                docker-compose \
-               python3-pip \
-               python3-venv \
                net-tools \
                clang \
                cmake
 
 WORKDIR /wforce/
 RUN mkdir /wforce/install
 
-RUN git clone https://github.com/jupp0r/prometheus-cpp.git
-RUN cd prometheus-cpp && git checkout tags/v1.0.1 -b v1.0.1
-RUN cd prometheus-cpp && mkdir _build && cd _build && cmake .. -DBUILD_SHARED_LIBS=off -DENABLE_PULL=off -DENABLE_PUSH=off -DENABLE_COMPRESSION=off -DENABLE_TESTING=off && make && make install
-
 RUN git clone https://github.com/drogonframework/drogon.git
 RUN cd drogon && git checkout tags/v1.7.4 -b v1.7.4
 RUN cd drogon && git submodule init && git submodule update && mkdir _build && cd _build && cmake .. -DBUILD_REDIS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_ORM=OFF && make && make install
@@ -57,11 +51,11 @@ COPY docker/wforce_image/Makefile.am /wforce/docker/wforce_image
 COPY elk/ /wforce/elk/
 
 RUN autoreconf -ivf
-RUN ./configure --prefix /usr --enable-trackalert --disable-systemd --disable-docker --with-luajit --sysconfdir=/etc/wforce CC=clang CXX=clang++
+RUN ./configure --prefix /usr --enable-trackalert --disable-systemd --disable-sodium --disable-docker --with-luajit --sysconfdir=/etc/wforce CC=clang CXX=clang++
 RUN make clean
 RUN make install DESTDIR=/wforce/install
 
-FROM debian:bullseye-slim as wforce_image
+FROM debian:bookworm-slim as wforce_image
 
 WORKDIR /wforce/
 
@@ -80,13 +74,13 @@ RUN apt-get update && \
                libcurl4 \
                libgetdns10 \
                libhiredis0.14 \
-               libluajit-5.1-2 \
+               libluajit2-5.1-2 \
                libmaxminddb0 \
                libreadline8 \
-               libprotobuf23 \
-               libssl1.1 \
-               libsodium23 \
-               libyaml-cpp0.6 \
+               libprotobuf32 \
+               libprometheus-cpp-core1.0 \
+               libssl3 \
+               libyaml-cpp0.7 \
                libjsoncpp-dev \
                libz-dev \
                uuid-dev \

docs/docker/wforce_docker.md

@@ -119,52 +119,4 @@ Warning: this generates a lot of logs - you have been warned!
 
 ## Running the report_api webservice
 
-The report_api webservice is used to access report events stored in an
-Elasticsearch server. The REST API is documented using the OpenAPI
-specification at
-https://raw.githubusercontent.com/PowerDNS/weakforced/master/docs/swagger/report_api.7.yml
-
-To start the report_api webservice, you'll need to specify a custom
-command to the image, as follows:
-
-```
-% docker run -p 5000:5000 powerdns/wforce:2.4.0 /usr/bin/wforce-report-api-webserver
-[INFO  tini (1)] Spawned child process '/usr/bin/docker-entrypoint.sh' with pid '7'
-[2020-06-29 16:50:55 +0000] [8] [INFO] Starting gunicorn 19.9.0
-[2020-06-29 16:50:55 +0000] [8] [INFO] Listening at: http://0.0.0.0:5000 (8)
-[2020-06-29 16:50:55 +0000] [8] [INFO] Using worker: threads
-[2020-06-29 16:50:55 +0000] [11] [INFO] Booting worker with pid: 11
-[2020-06-29 16:50:55 +0000] [12] [INFO] Booting worker with pid: 12
-[2020-06-29 16:50:55 +0000] [13] [INFO] Booting worker with pid: 13
-[2020-06-29 16:50:55 +0000] [14] [INFO] Booting worker with pid: 14
-[2020-06-29 16:50:55 +0000] [15] [INFO] Booting worker with pid: 15
-[2020-06-29 16:50:55 +0000] [16] [INFO] Booting worker with pid: 16
-[2020-06-29 16:50:55 +0000] [17] [INFO] Booting worker with pid: 17
-[2020-06-29 16:50:55 +0000] [18] [INFO] Booting worker with pid: 18
-```
-
-The webservice runs under gunicorn. There are two configuration files
-that allow the webservice to be customized:
-
-* /etc/wforce-report-api/wforce-report-api-instance.conf - The
-  report_api configuration file. The default config file shown below.
-* /etc/wforce-report-api/wforce-report-api-web.conf - The gunicorn
-  configuration file; this will not normally need to be modified.
-
-### Default webservice configuration file
-
-```
-import logging
-
-ELASTICSEARCH_URL = ["http://elasticsearch:9200"]
-ELASTICSEARCH_TIMEOUT = 1
-ELASTICSEARCH_INDEX = "logstash-wforce*"
-AUTH_PASSWORD = "secret"
-LOG_LEVEL = logging.INFO
-LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-DEVICE_UNIQUE_ATTRS = ['os.family', 'browser.family', 'imapc.family', 'device.family', 'app.name']
-TRACKALERT_REDIS = True
-TRACKALERT_SERVER = 'localhost'
-TRACKALERT_PORT = 8085
-TRACKALERT_PASSWORD = 'super'
-```
+This is no longer supported in the wforce image (>2.8.0)

regression-tests/requirements.txt

@@ -1,5 +1,5 @@
 requests==2.31.0
-nose==1.3.7
+pytest==7.4.2
 click==6.7
 coverage==4.4.1
 elasticsearch>=7.0.0

regression-tests/runtests.py

@@ -103,8 +103,10 @@ def sighandler(signum, frame):
 
 try:
     print("")
-    p = subprocess.check_call(["nosetests", "--with-xunit"] + sys.argv[1:],
-                              env=test_env)
+    if len(sys.argv) > 1:
+        p = subprocess.check_call(["pytest", "--junitxml", sys.argv[1]], env=test_env)
+    else:
+        p = subprocess.check_call(["pytest", "--junitxml", "pytest.xml"], env=test_env)
 except subprocess.CalledProcessError as ex:
     rc = ex.returncode
 finally: