diff --git a/go.mod b/go.mod index 3ea237695..5a7f428ab 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,13 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/go-redis/redis/v9 v9.0.0-beta.1 github.com/google/go-containerregistry v0.19.1 + github.com/google/go-tpm v0.9.1 github.com/google/uuid v1.6.0 github.com/lf-edge/eden/eserver v0.0.0-20220711180217-6e2bfa9c3f67 github.com/lf-edge/eden/sdn/vm v0.0.0-00010101000000-000000000000 github.com/lf-edge/edge-containers v0.0.0-20240207093504-5dfda0619b80 github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff + github.com/lf-edge/eve/pkg/pillar v0.0.0-20240923082146-6d403aaa5513 github.com/mcuadros/go-lookup v0.0.0-20200831155250-80f87a4fa5ee github.com/moby/term v0.5.0 github.com/nerd2/gexto v0.0.0-20190529073929-39468ec063f6 @@ -73,9 +75,14 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect + github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.15.5 // indirect github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -89,8 +96,9 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.6 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/lf-edge/eve/libs/depgraph v0.0.0-20220711144346-0659e3b03496 // indirect - github.com/lf-edge/eve/pkg/pillar v0.0.0-20240923082146-6d403aaa5513 // indirect + github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d // indirect github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -120,6 +128,8 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.0 // indirect github.com/vbatts/tar-split v0.11.3 // indirect + github.com/vishvananda/netlink v1.2.1-beta.2 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.7 // indirect github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect diff --git a/go.sum b/go.sum index ee0dd5aa8..3bb2f8b3f 100644 --- a/go.sum +++ b/go.sum @@ -973,6 +973,8 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7 h1:25R+h8+75+zyY0+/6TfUsz/arhEltIwNMgDH6IGJ/tc= +github.com/eriknordmark/ipinfo v0.0.0-20230728132417-2d8f4da903d7/go.mod h1:m5kR+NOoKCuA5r6T+9f7q7VfPjPhHskhmxRAebb7avM= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -1004,6 +1006,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= @@ -1059,6 +1063,14 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= +github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-redis/redis/v9 v9.0.0-beta.1 h1:oW3jlPic5HhGUbYMH0lidnP+72BgsT+lCwlVud6o2Mc= github.com/go-redis/redis/v9 v9.0.0-beta.1/go.mod h1:6gNX1bXdwkpEG0M/hEBNK/Fp8zdyCkjwwKc6vBbfCDI= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= @@ -1161,6 +1173,8 @@ github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYV github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk= github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= +github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= +github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -1363,6 +1377,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= @@ -1373,15 +1389,13 @@ github.com/lf-edge/eden/eserver v0.0.0-20220711180217-6e2bfa9c3f67 h1:lUJ/IASqGL github.com/lf-edge/eden/eserver v0.0.0-20220711180217-6e2bfa9c3f67/go.mod h1:D8XwIk3t45EzFr7Yxf4EE3G9H+4H2nEkvte/2ItzkNk= github.com/lf-edge/edge-containers v0.0.0-20240207093504-5dfda0619b80 h1:kiqB1Rk8fmWci0idN68azRDJfPxCivD3zNDddWZocFw= github.com/lf-edge/edge-containers v0.0.0-20240207093504-5dfda0619b80/go.mod h1:4yXdumKdTzF0URMtxOl8Xnzdxnoy1QR+2dzfOr4CIZY= -github.com/lf-edge/eve-api/go v0.0.0-20240816135418-f858514b03a3 h1:RyGrCByT/Ip1DlqU37LfyG5MvZ76TIr3eJN7Sv+qbMA= -github.com/lf-edge/eve-api/go v0.0.0-20240816135418-f858514b03a3/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA= github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff h1:3uGTOvWQFQkIrlkFalmzUmXINnzmVOAn5Zx0ryBSzxQ= github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA= github.com/lf-edge/eve/libs/depgraph v0.0.0-20220711144346-0659e3b03496 h1:txHCOKhVsKIZKvKWzyIMe3J+ATKk61o4bADhsdLk42Y= github.com/lf-edge/eve/libs/depgraph v0.0.0-20220711144346-0659e3b03496/go.mod h1:8gtCaEwMJftnaP8PjjgRStLhOoHquzzlmYzj441QwpU= github.com/lf-edge/eve/libs/reconciler v0.0.0-20220711144346-0659e3b03496/go.mod h1:BWuSpe83TvdUmwqhSPa8/gGfapb91OkJ1jNHEnSypr4= -github.com/lf-edge/eve/pkg/pillar v0.0.0-20240901211357-262f5f152b36 h1:WQ2Mwznrp/H4tA+UBaHj7NF6/O2yhoCyijSY3JYlbQ4= -github.com/lf-edge/eve/pkg/pillar v0.0.0-20240901211357-262f5f152b36/go.mod h1:B2+BzX9yz7JtcY0ZDRun94KwjO9UjsK/gnBdYDIdgXA= +github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo= +github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d/go.mod h1:Nn3juMJJ1G8dyHOebdZyS4jOB/fuxAd5fIajBaWjHr8= github.com/lf-edge/eve/pkg/pillar v0.0.0-20240923082146-6d403aaa5513 h1:6eoKsOh15KAIGzZbLNlMhsOoPiMgoD6+VeuKsYsbqHY= github.com/lf-edge/eve/pkg/pillar v0.0.0-20240923082146-6d403aaa5513/go.mod h1:5zB+wlsiTBE2N/Jb0Zr7DaUJMG2OCcAP6X+dyftIdzo= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= @@ -1833,10 +1847,12 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= diff --git a/tests/aziot_legacy/Makefile b/tests/aziot_legacy/Makefile new file mode 100644 index 000000000..8cf09b708 --- /dev/null +++ b/tests/aziot_legacy/Makefile @@ -0,0 +1,76 @@ +DEBUG ?= "debug" + +# HOSTARCH is the host architecture +# ARCH is the target architecture +# we need to keep track of them separately +HOSTARCH ?= $(shell uname -m) +HOSTOS ?= $(shell uname -s | tr A-Z a-z) + +# canonicalized names for host architecture +override HOSTARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(HOSTARCH))) + +# unless otherwise set, I am building for my own architecture, i.e. not cross-compiling +# and for my OS +ARCH ?= $(HOSTARCH) +OS ?= $(HOSTOS) + +# canonicalized names for target architecture +override ARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(ARCH))) + +WORKDIR ?= $(CURDIR)/../../dist +TESTDIR := tests/$(shell basename $(CURDIR)) +BINDIR := $(WORKDIR)/bin +DATADIR := $(WORKDIR)/$(TESTDIR)/ +BIN := eden +LOCALBIN := $(BINDIR)/$(BIN)-$(OS)-$(ARCH) +TESTNAME := eden.aziot +TESTBIN := $(TESTNAME).test +TESTSCN := $(TESTNAME).tests.txt +LOCALTESTBIN := $(TESTBIN)-$(OS)-$(ARCH) +LINKDIR := ../../tests/aziot + +.DEFAULT_GOAL := help + +clean: + rm -rf $(LOCALTESTBIN) $(BINDIR)/$(TESTBIN) $(WORKDIR)/$(TESTSCN) $(CURDIR)/$(TESTBIN) $(BINDIR)/$(TESTBIN) + +$(BINDIR): + mkdir -p $@ +$(DATADIR): + mkdir -p $@ + +test: + $(LOCALBIN) test $(CURDIR) -v $(DEBUG) + +build: setup + +testbin: $(TESTBIN) +$(LOCALTESTBIN): $(BINDIR) *.go + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go test -c -ldflags "-s -w" -o $@ *.go + +$(TESTBIN): $(LOCALTESTBIN) + ln -sf $(LOCALTESTBIN) $(CURDIR)/$(TESTBIN) + +setup: testbin $(BINDIR) $(DATADIR) + cp -a $(LOCALTESTBIN) $(CURDIR)/$(TESTBIN) $(BINDIR) + cp -a *.yml $(TESTSCN) $(DATADIR) + +debug: + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go test -c -gcflags "all=-N -l" -o $@ *.go + dlv dap --listen=:12345 --headless=true --api-version=2 exec ./debug -- -test.v + +.PHONY: test build setup clean all testbin debug + +help: + @echo "EDEN is the harness for testing EVE and ADAM" + @echo + @echo "This Makefile automates commons tasks of EDEN testing" + @echo + @echo "Commonly used maintenance and development targets:" + @echo " build build test-binary (OS and ARCH options supported, for ex. OS=linux ARCH=arm64)" + @echo " setup setup of test environment" + @echo " test run tests" + @echo " clean cleanup of test harness" + @echo + @echo "You need install requirements for EVE (look at https://github.com/lf-edge/eve#install-dependencies)." + @echo "You need access to docker socket and installed qemu packages." diff --git a/tests/aziot_legacy/aziot_test.go b/tests/aziot_legacy/aziot_test.go new file mode 100644 index 000000000..19999cea9 --- /dev/null +++ b/tests/aziot_legacy/aziot_test.go @@ -0,0 +1,237 @@ +package aziot + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + tk "github.com/lf-edge/eden/pkg/evetestkit" + log "github.com/sirupsen/logrus" +) + +const ( + sshPort = "8027" + testScriptBasePath = "/home/ubuntu/" + projectName = "aziot-test" + aziotwait = 30 // seconds + appWait = 60 * 10 // 10 minutes + sshWait = 60 * 5 // 5 minutes +) + +var ( + appLink = "https://cloud-images.ubuntu.com/releases/20.04/release/ubuntu-20.04-server-cloudimg-amd64.img" + testScript = "scripts/test_ubuntu20.04_aziot_1.4.0.sh" + eveNode *tk.EveNode + // We need a shared access policy with the following permissions: + // Registration Status Read, Registration Status Write, Enrollment Read, Enrollment Write + // We can create a new policy in the Azure portal by going to : + // IoT Hub -> Device Provisioning Service (DPS) -> Shared access policies -> Add + // and then copy the connection string. + connectionString = os.Getenv("AZIOT_CONNECTION_STRING") + // The ID Scope is required to configure azure-iot in the VM, + // we can get it from the Azure IoT Hub -> Device Provisioning Service -> Overview + // and copy the "ID Scope". + aziotIDScope = os.Getenv("AZIOT_ID_SCOPE") +) + +func TestMain(m *testing.M) { + log.Println("Azure IOT Hub Legacy Test started") + defer log.Println("Azure IOT Hub Legacy Test finished") + + // Check for secrets, if not available don't bother running the tests. + if connectionString == "" { + log.Fatalf("AZIOT_CONNECTION_STRING environment variable is not set") + } + if aziotIDScope == "" { + log.Fatalf("AZIOT_ID_SCOPE environment variable is not set") + } + + node, err := tk.InitilizeTest(projectName, tk.WithControllerVerbosity("debug")) + if err != nil { + log.Fatalf("Failed to initialize test: %v", err) + } + + eveNode = node + res := m.Run() + os.Exit(res) +} + +func TestAzureIotTPMEndrolmentWithEveTools(t *testing.T) { + t.Log("TestAzureIotTPMEndrolmentWithEveTools started") + t.Log("Setup :\n\tAziot version 1.4.0 on Ubuntu-20.04-amd64\n\twith EVE-Tools and PTPM") + defer t.Log("TestAzureIotTPMEndrolmentWithEveTools finished") + + if !eveNode.EveIsTpmEnabled() { + t.Skip("TPM is not enabled, skipping test") + } + + testAzureIotEdge(t) +} + +func setupApp() (string, error) { + appName := tk.GetRandomAppName(projectName + "-") + pubPorts := []string{sshPort + ":22"} + pc := tk.GetDefaultVMConfig(appName, tk.AppDefaultCloudConfig, pubPorts) + err := eveNode.EveDeployApp(appLink, pc, + tk.WithSSH(tk.AppDefaultSSHUser, tk.AppDefaultSSHPass, sshPort)) + if err != nil { + return "", fmt.Errorf("failed to deploy app: %v", err) + } + + return appName, nil +} + +func waitForApp(t *testing.T, appName string) error { + // Wait for the app to start and ssh to be ready + t.Logf("Waiting for app %s to start...", appName) + err := eveNode.AppWaitForRunningState(appName, appWait) + if err != nil { + return fmt.Errorf("failed to wait for app to start: %v", err) + } + t.Logf("Waiting for ssh to be ready...") + err = eveNode.AppWaitForSSH(appName, sshWait) + if err != nil { + return fmt.Errorf("failed to wait for ssh: %v", err) + } + + t.Log("SSH connection established") + return nil +} + +func checkAziotServices(t *testing.T, appName string) { + // Check the status of the iotedge services + status, err := eveNode.AppSSHExec(appName, "sudo iotedge system status") + if err != nil { + t.Fatalf("Failed to get iotedge status: %v", err) + } + services, err := getAzureIoTServicesStatus(status) + if err != nil { + t.Fatalf("Failed to get Azure IoT services status: %v", err) + } + + // If all services are running we are good, otherwise fail the test + for service, status := range services { + if strings.ToLower(status) != "running" { + // Errorf calls Fail(), so we don't need to call it explicitly + t.Errorf("Service %s is not running", service) + } + } + t.Log("====================== SERVICES STATUS ======================") + for service, status := range services { + t.Logf("%s: \t\t%s\n", service, status) + } + + if t.Failed() { + // Get the aziot-tpmd logs, in one test we patch this service with eve-tools + // so good to have the logs for debugging. + command := "sudo iotedge system logs | grep aziot-tpmd" + tpmLog, err := eveNode.AppSSHExec(appName, command) + if err != nil { + t.Errorf("Failed to get aziot-tpmd logs: %v", err) + } else { + t.Log("====================== TPMD LOG ======================") + t.Log(tpmLog) + } + + // Get all the errors from the aziot logs + command = "sudo iotedge system logs | grep ERR | sed 's/.*ERR!] - //' | sort | uniq" + errors, err := eveNode.AppSSHExec(appName, command) + if err != nil { + t.Errorf("Failed to error logs: %v", err) + } else { + t.Log("====================== ERRORS ======================") + t.Log(errors) + } + } +} + +func testAzureIotEdge(t *testing.T) { + appName, err := setupApp() + if err != nil { + t.Fatalf("Failed to setup app: %v", err) + } + defer func() { + err = eveNode.AppStopAndRemove(appName) + if err != nil { + log.Errorf("Failed to stop and remove app: %v", err) + } + }() + + // Wait for the deployed app to appear in the list + time.Sleep(30 * time.Second) + err = waitForApp(t, appName) + if err != nil { + t.Fatalf("Failed to wait for app: %v", err) + } + + // Copy the test script to the VM + testScriptPath := testScriptBasePath + filepath.Base(testScript) + err = eveNode.AppSCPCopy(appName, testScript, testScriptPath) + if err != nil { + t.Fatalf("Failed to copy file to vm: %v", err) + } + t.Log("Test script copied to VM") + + // for this to test to work, we need to create an enrollment in the Azure IoT Hub, + // the enrolment should be created with the endorsement key of the TPM and + // since we are running EVE in QEMU with SWTPM, the endorsement key changes + // every time we start the VM EVE, so we need to read it, create the enrollment, + // run the test and delete the enrollment. + t.Log("Creating a TPM enrollment in Azure IoT Hub") + endorsementKey, enrollmentID := "", "" + + // read the endorsement key from the EVE. + ek, id, err := readEveEndorsmentKey() + if err != nil { + t.Errorf("Failed to read endorsement key from EVE: %v", err) + } + endorsementKey, enrollmentID = ek, id + + // Get the provisioning service name from the connection string + provService, err := getProvisioningService(connectionString) + if err != nil { + t.Fatalf("Failed to get provisioning service: %v\n", err) + } + + // From the connection string generate a SAS token lasting for 1 hour + sasToken, err := getSasTokenFromConnectionString(connectionString, 1) + if err != nil { + t.Fatalf("Failed to generate a SAS token: %v\n", err) + } + + // Add the enrollment to azure iot hub portal + err = addTPMEnrollment(enrollmentID, endorsementKey, provService, sasToken) + if err != nil { + t.Fatalf("Failed to create a TPM enrollment on Azure: %v\n", err) + } + defer func() { + err = deleteEnrollment(enrollmentID, provService, sasToken) + if err != nil { + log.Printf("Failed to delete TPM enrollment, please remove it manually: %v\n", err) + } + }() + + // Prepare the test script for execution + command := fmt.Sprintf("chmod +x %s", testScriptPath) + _, err = eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed perpare the test script for execution: %v", err) + } + + // Execute the test script, this will configure the azure-iot in the VM + // and start the services. + command = fmt.Sprintf("ID_SCOPE=%s REGISTRATION_ID=%s %s", aziotIDScope, enrollmentID, testScriptPath) + _, err = eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed to execute test script in VM: %v", err) + } + + // Wait for the services to start + t.Logf("Waiting for services to start...") + time.Sleep(aziotwait * time.Second) + // this will check the status of the services and fail the test if any service is not running + checkAziotServices(t, appName) +} diff --git a/tests/aziot_legacy/aziotenroll.go b/tests/aziot_legacy/aziotenroll.go new file mode 100644 index 000000000..d0b7b640f --- /dev/null +++ b/tests/aziot_legacy/aziotenroll.go @@ -0,0 +1,257 @@ +package aziot + +import ( + "bufio" + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/google/go-tpm/tpmutil" + etpm "github.com/lf-edge/eve/pkg/pillar/evetpm" +) + +// Create, Update, Delete Enrollment API URL +const enrollmentAPIURL = "https://%s/enrollments/%s?api-version=2021-10-01" + +// getProvisioningService gets the provisioning service name from the IoT DPS connection string. +func getProvisioningService(connStr string) (hostName string, err error) { + parts := strings.Split(connStr, ";") + if len(parts) != 3 { + return "", fmt.Errorf("invalid connection string") + } + + for _, part := range parts { + if strings.HasPrefix(part, "HostName=") { + hostName = strings.TrimPrefix(part, "HostName=") + break + } + } + + if hostName == "" { + return "", fmt.Errorf("invalid connection string") + } + + return hostName, nil +} + +// parseConnectionString parses the IoT DPS connection string and returns its components. +func parseConnectionString(connStr string) (hostName, sharedAccessKeyName, sharedAccessKey string, err error) { + parts := strings.Split(connStr, ";") + if len(parts) != 3 { + return "", "", "", fmt.Errorf("invalid connection string") + } + + for _, part := range parts { + if strings.HasPrefix(part, "HostName=") { + hostName = strings.TrimPrefix(part, "HostName=") + } else if strings.HasPrefix(part, "SharedAccessKeyName=") { + sharedAccessKeyName = strings.TrimPrefix(part, "SharedAccessKeyName=") + } else if strings.HasPrefix(part, "SharedAccessKey=") { + sharedAccessKey = strings.TrimPrefix(part, "SharedAccessKey=") + } + } + + if hostName == "" || sharedAccessKeyName == "" || sharedAccessKey == "" { + return "", "", "", fmt.Errorf("invalid connection string") + } + + return hostName, sharedAccessKeyName, sharedAccessKey, nil +} + +// generateSasToken generates a SAS token for Azure IoT DPS +func generateSasToken(uri, keyName, key string, expiry int64) (string, error) { + encodedURI := url.QueryEscape(uri) + stringToSign := fmt.Sprintf("%s\n%d", encodedURI, expiry) + keyBytes, err := base64.StdEncoding.DecodeString(key) + if err != nil { + return "", err + } + + h := hmac.New(sha256.New, keyBytes) + h.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(h.Sum(nil)) + + token := fmt.Sprintf("SharedAccessSignature sr=%s&sig=%s&se=%d&skn=%s", encodedURI, url.QueryEscape(signature), expiry, keyName) + return token, nil +} + +func getSasTokenFromConnectionString(connectionString string, hours uint) (string, error) { + // Parse the connection string + hostName, sharedAccessKeyName, sharedAccessKey, err := parseConnectionString(connectionString) + if err != nil { + return "", fmt.Errorf("failed to parse connection string: %v", err) + } + + expiry := (time.Now().Unix() + 3600) * int64(hours) + sasToken, err := generateSasToken(hostName, sharedAccessKeyName, sharedAccessKey, expiry) + if err != nil { + return "", fmt.Errorf("failed to generate SAS token: %v", err) + } + + return sasToken, nil +} + +// addTPMEnrollment adds a new TPM enrollment to Azure IoT DPS. +func addTPMEnrollment(enrollmentID, endorsementKey, provService, sasToken string) error { + url := fmt.Sprintf(enrollmentAPIURL, provService, enrollmentID) + + // Prepare the enrollment body for TPM attestation + enrollment := map[string]interface{}{ + "registrationId": enrollmentID, + "attestation": map[string]interface{}{ + "type": "tpm", + "tpm": map[string]interface{}{ + "endorsementKey": endorsementKey, + }, + }, + "provisioningStatus": "enabled", + "allocationPolicy": "hashed", + "capabilities": map[string]bool{ + "iotEdge": false, + }, + "reprovisionPolicy": map[string]bool{ + "migrateDeviceData": true, + "updateHubAssignment": true, + }, + } + + enrollmentJSON, err := json.Marshal(enrollment) + if err != nil { + return fmt.Errorf("failed to marshal enrollment: %v", err) + } + + req, err := http.NewRequest("PUT", url, bytes.NewBuffer(enrollmentJSON)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %v", err) + } + req.Header.Set("Authorization", sasToken) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to create enrollment: %v", string(body)) + } + + return nil +} + +// deleteEnrollment deletes an enrollment from Azure IoT DPS. +func deleteEnrollment(enrollmentID, provService, sasToken string) error { + url := fmt.Sprintf(enrollmentAPIURL, provService, enrollmentID) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %v", err) + } + req.Header.Set("Authorization", sasToken) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to delete enrollment: %v", string(body)) + } + + return nil +} + +func readPublicKey(handle tpmutil.Handle) ([]byte, error) { + // unfortunately we can't used SWTPM socket directly, it is blocked because + // qemu is using it, so we have to use ssh and tpm2-tools + tpmToolsPath := "/containers/services/vtpm/lower/usr/bin/tpm2" + tpmToolsLibPath := "/containers/services/vtpm/lower/usr/lib" + command := fmt.Sprintf("LD_LIBRARY_PATH=%s %s readpublic -Q -c 0x%x -o pub.pub", tpmToolsLibPath, tpmToolsPath, handle) + _, err := eveNode.EveRunCommand(command) + if err != nil { + return nil, err + } + + out, err := eveNode.EveReadFile("pub.pub") + if err != nil { + return nil, err + } + + err = eveNode.EveDeleteFile("pub.pub") + if err != nil { + return nil, err + } + + return out, nil +} + +func readEveEndorsmentKey() (string, string, error) { + pub, err := readPublicKey(etpm.TpmEKHdl) + if err != nil { + return "", "", err + } + + hash := sha256.Sum256(pub) + hashHex := hex.EncodeToString(hash[:]) + + return base64.StdEncoding.EncodeToString(pub), hashHex, nil +} + +func getAzureIoTServicesStatus(output string) (map[string]string, error) { + // this what is being parse: + //$ sudo iotedge system status + //System services: + //aziot-edged Running + //aziot-identityd Down - activating + //aziot-keyd Ready + //aziot-certd Ready + //aziot-tpmd Running + + // Flag to indicate if we are in the "System services" section + inSystemServices := false + services := make(map[string]string, 0) + + scanner := bufio.NewScanner(strings.NewReader(output)) + + for scanner.Scan() { + line := scanner.Text() + + // Detect when we are in the "System services" section + if strings.Contains(line, "System services:") { + inSystemServices = true + continue + } + + // Exit the loop when we are out of the "System services" section + if inSystemServices && strings.TrimSpace(line) == "" { + break + } + + if inSystemServices { + parts := strings.Fields(line) + if len(parts) >= 2 { + services[parts[0]] = strings.Join(parts[1:], " ") + } + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return services, nil +} diff --git a/tests/aziot_legacy/eden-config.yml b/tests/aziot_legacy/eden-config.yml new file mode 100644 index 000000000..fed3110ef --- /dev/null +++ b/tests/aziot_legacy/eden-config.yml @@ -0,0 +1,7 @@ +--- +eden: + # test binary + test-bin: "eden.aziot.test" + + # test scenario + test-scenario: "eden.aziot.tests.txt" diff --git a/tests/aziot_legacy/eden.aziot.tests.txt b/tests/aziot_legacy/eden.aziot.tests.txt new file mode 100644 index 000000000..ba7df2404 --- /dev/null +++ b/tests/aziot_legacy/eden.aziot.tests.txt @@ -0,0 +1 @@ +eden.aziot.test diff --git a/tests/aziot_legacy/scripts/test_ubuntu20.04_aziot_1.4.0.sh b/tests/aziot_legacy/scripts/test_ubuntu20.04_aziot_1.4.0.sh new file mode 100644 index 000000000..2b8c604df --- /dev/null +++ b/tests/aziot_legacy/scripts/test_ubuntu20.04_aziot_1.4.0.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive + +MS_PROD="https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb" +AZIOT_IDENTITY_SERVICE="https://github.com/Azure/azure-iotedge/releases/download/1.4.0/aziot-identity-service_1.4.0-1_ubuntu20.04_amd64.deb" +AZIOT_EDGE="https://github.com/Azure/azure-iotedge/releases/download/1.4.0/aziot-edge_1.4.0-1_ubuntu20.04_amd64.deb" +EVE_TOOLS="https://github.com/shjala/eve-tools-deb/raw/main/lfedge-eve-tools-3.3-ubuntu20.04.deb" + +# install microsoft repository +wget $MS_PROD -O packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +rm packages-microsoft-prod.deb + +# install pre-requisites +sudo apt-get update +sudo apt-get install -y moby-engine tpm2-tools libprotobuf-dev libprotoc-dev net-tools libssl-dev +sudo apt-get purge -y aziot-identity-service aziot-edge + +# install aziot-identity-servic, this is needed by aziot-edge +wget $AZIOT_IDENTITY_SERVICE -O aziot-identity-service.deb +sudo dpkg -i aziot-identity-service.deb +rm aziot-identity-service.deb + +# install aziot-edge +wget $AZIOT_EDGE -O aziot-edge.deb +sudo dpkg -i aziot-edge.deb +rm aziot-edge.deb + +# install eve-tools, and patch aziot-idendity-service to use eve-tools +# for communication with TPM +wget $EVE_TOOLS -O eve-tools.deb +dpkg-deb -R eve-tools.deb . +sudo cp -r usr/ / +rm eve-tools.deb + +# generate certificates needed by aziot-certd and aziot-keyd +git clone https://github.com/Azure/iotedge.git +cd iotedge || exit +git checkout 1.4.0 +cd .. +mkdir test-certs +cd test-certs || exit +cp ../iotedge/tools/CACertificates/*.cnf . +cp ../iotedge/tools/CACertificates/certGen.sh . +./certGen.sh create_root_and_intermediate +./certGen.sh create_edge_device_ca_certificate test_aziot_eden_cert +cd .. + +>config.toml cat <<-EOF +## DPS provisioning with TPM +[provisioning] +source = "dps" +global_endpoint = "https://global.azure-devices-provisioning.net" +id_scope = "$ID_SCOPE" + +[provisioning.attestation] +method = "tpm" +registration_id = "$REGISTRATION_ID" + +[edge_ca] +cert = "file:///home/ubuntu/test-certs/certs/iot-edge-device-ca-test_aziot_eden_cert-full-chain.cert.pem" +pk = "file:///home/ubuntu/test-certs/private/iot-edge-device-ca-test_aziot_eden_cert.key.pem" +EOF + +sudo cp config.toml /etc/aziot/config.toml +sudo iotedge config apply + +rm config.toml +rm -rf usr/ DEBIAN/ iotedge/ diff --git a/tests/vtpm/Makefile b/tests/vtpm/Makefile new file mode 100644 index 000000000..90d5e311c --- /dev/null +++ b/tests/vtpm/Makefile @@ -0,0 +1,76 @@ +DEBUG ?= "debug" + +# HOSTARCH is the host architecture +# ARCH is the target architecture +# we need to keep track of them separately +HOSTARCH ?= $(shell uname -m) +HOSTOS ?= $(shell uname -s | tr A-Z a-z) + +# canonicalized names for host architecture +override HOSTARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(HOSTARCH))) + +# unless otherwise set, I am building for my own architecture, i.e. not cross-compiling +# and for my OS +ARCH ?= $(HOSTARCH) +OS ?= $(HOSTOS) + +# canonicalized names for target architecture +override ARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(ARCH))) + +WORKDIR ?= $(CURDIR)/../../dist +TESTDIR := tests/$(shell basename $(CURDIR)) +BINDIR := $(WORKDIR)/bin +DATADIR := $(WORKDIR)/$(TESTDIR)/ +BIN := eden +LOCALBIN := $(BINDIR)/$(BIN)-$(OS)-$(ARCH) +TESTNAME := eden.vtpm +TESTBIN := $(TESTNAME).test +TESTSCN := $(TESTNAME).tests.txt +LOCALTESTBIN := $(TESTBIN)-$(OS)-$(ARCH) +LINKDIR := ../../tests/vtpm + +.DEFAULT_GOAL := help + +clean: + rm -rf $(LOCALTESTBIN) $(BINDIR)/$(TESTBIN) $(WORKDIR)/$(TESTSCN) $(CURDIR)/$(TESTBIN) $(BINDIR)/$(TESTBIN) + +$(BINDIR): + mkdir -p $@ +$(DATADIR): + mkdir -p $@ + +test: + $(LOCALBIN) test $(CURDIR) -v $(DEBUG) + +build: setup + +testbin: $(TESTBIN) +$(LOCALTESTBIN): $(BINDIR) *.go + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go test -c -ldflags "-s -w" -o $@ *.go + +$(TESTBIN): $(LOCALTESTBIN) + ln -sf $(LOCALTESTBIN) $(CURDIR)/$(TESTBIN) + +setup: testbin $(BINDIR) $(DATADIR) + cp -a $(LOCALTESTBIN) $(CURDIR)/$(TESTBIN) $(BINDIR) + cp -a *.yml $(TESTSCN) $(DATADIR) + +debug: + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go test -c -gcflags "all=-N -l" -o $@ *.go + dlv dap --listen=:12345 --headless=true --api-version=2 exec ./debug -- -test.v + +.PHONY: test build setup clean all testbin debug + +help: + @echo "EDEN is the harness for testing EVE and ADAM" + @echo + @echo "This Makefile automates commons tasks of EDEN testing" + @echo + @echo "Commonly used maintenance and development targets:" + @echo " build build test-binary (OS and ARCH options supported, for ex. OS=linux ARCH=arm64)" + @echo " setup setup of test environment" + @echo " test run tests" + @echo " clean cleanup of test harness" + @echo + @echo "You need install requirements for EVE (look at https://github.com/lf-edge/eve#install-dependencies)." + @echo "You need access to docker socket and installed qemu packages." diff --git a/tests/vtpm/aziotenroll.go b/tests/vtpm/aziotenroll.go new file mode 100644 index 000000000..d0b7b640f --- /dev/null +++ b/tests/vtpm/aziotenroll.go @@ -0,0 +1,257 @@ +package aziot + +import ( + "bufio" + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/google/go-tpm/tpmutil" + etpm "github.com/lf-edge/eve/pkg/pillar/evetpm" +) + +// Create, Update, Delete Enrollment API URL +const enrollmentAPIURL = "https://%s/enrollments/%s?api-version=2021-10-01" + +// getProvisioningService gets the provisioning service name from the IoT DPS connection string. +func getProvisioningService(connStr string) (hostName string, err error) { + parts := strings.Split(connStr, ";") + if len(parts) != 3 { + return "", fmt.Errorf("invalid connection string") + } + + for _, part := range parts { + if strings.HasPrefix(part, "HostName=") { + hostName = strings.TrimPrefix(part, "HostName=") + break + } + } + + if hostName == "" { + return "", fmt.Errorf("invalid connection string") + } + + return hostName, nil +} + +// parseConnectionString parses the IoT DPS connection string and returns its components. +func parseConnectionString(connStr string) (hostName, sharedAccessKeyName, sharedAccessKey string, err error) { + parts := strings.Split(connStr, ";") + if len(parts) != 3 { + return "", "", "", fmt.Errorf("invalid connection string") + } + + for _, part := range parts { + if strings.HasPrefix(part, "HostName=") { + hostName = strings.TrimPrefix(part, "HostName=") + } else if strings.HasPrefix(part, "SharedAccessKeyName=") { + sharedAccessKeyName = strings.TrimPrefix(part, "SharedAccessKeyName=") + } else if strings.HasPrefix(part, "SharedAccessKey=") { + sharedAccessKey = strings.TrimPrefix(part, "SharedAccessKey=") + } + } + + if hostName == "" || sharedAccessKeyName == "" || sharedAccessKey == "" { + return "", "", "", fmt.Errorf("invalid connection string") + } + + return hostName, sharedAccessKeyName, sharedAccessKey, nil +} + +// generateSasToken generates a SAS token for Azure IoT DPS +func generateSasToken(uri, keyName, key string, expiry int64) (string, error) { + encodedURI := url.QueryEscape(uri) + stringToSign := fmt.Sprintf("%s\n%d", encodedURI, expiry) + keyBytes, err := base64.StdEncoding.DecodeString(key) + if err != nil { + return "", err + } + + h := hmac.New(sha256.New, keyBytes) + h.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(h.Sum(nil)) + + token := fmt.Sprintf("SharedAccessSignature sr=%s&sig=%s&se=%d&skn=%s", encodedURI, url.QueryEscape(signature), expiry, keyName) + return token, nil +} + +func getSasTokenFromConnectionString(connectionString string, hours uint) (string, error) { + // Parse the connection string + hostName, sharedAccessKeyName, sharedAccessKey, err := parseConnectionString(connectionString) + if err != nil { + return "", fmt.Errorf("failed to parse connection string: %v", err) + } + + expiry := (time.Now().Unix() + 3600) * int64(hours) + sasToken, err := generateSasToken(hostName, sharedAccessKeyName, sharedAccessKey, expiry) + if err != nil { + return "", fmt.Errorf("failed to generate SAS token: %v", err) + } + + return sasToken, nil +} + +// addTPMEnrollment adds a new TPM enrollment to Azure IoT DPS. +func addTPMEnrollment(enrollmentID, endorsementKey, provService, sasToken string) error { + url := fmt.Sprintf(enrollmentAPIURL, provService, enrollmentID) + + // Prepare the enrollment body for TPM attestation + enrollment := map[string]interface{}{ + "registrationId": enrollmentID, + "attestation": map[string]interface{}{ + "type": "tpm", + "tpm": map[string]interface{}{ + "endorsementKey": endorsementKey, + }, + }, + "provisioningStatus": "enabled", + "allocationPolicy": "hashed", + "capabilities": map[string]bool{ + "iotEdge": false, + }, + "reprovisionPolicy": map[string]bool{ + "migrateDeviceData": true, + "updateHubAssignment": true, + }, + } + + enrollmentJSON, err := json.Marshal(enrollment) + if err != nil { + return fmt.Errorf("failed to marshal enrollment: %v", err) + } + + req, err := http.NewRequest("PUT", url, bytes.NewBuffer(enrollmentJSON)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %v", err) + } + req.Header.Set("Authorization", sasToken) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to create enrollment: %v", string(body)) + } + + return nil +} + +// deleteEnrollment deletes an enrollment from Azure IoT DPS. +func deleteEnrollment(enrollmentID, provService, sasToken string) error { + url := fmt.Sprintf(enrollmentAPIURL, provService, enrollmentID) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %v", err) + } + req.Header.Set("Authorization", sasToken) + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send HTTP request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNoContent { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("failed to delete enrollment: %v", string(body)) + } + + return nil +} + +func readPublicKey(handle tpmutil.Handle) ([]byte, error) { + // unfortunately we can't used SWTPM socket directly, it is blocked because + // qemu is using it, so we have to use ssh and tpm2-tools + tpmToolsPath := "/containers/services/vtpm/lower/usr/bin/tpm2" + tpmToolsLibPath := "/containers/services/vtpm/lower/usr/lib" + command := fmt.Sprintf("LD_LIBRARY_PATH=%s %s readpublic -Q -c 0x%x -o pub.pub", tpmToolsLibPath, tpmToolsPath, handle) + _, err := eveNode.EveRunCommand(command) + if err != nil { + return nil, err + } + + out, err := eveNode.EveReadFile("pub.pub") + if err != nil { + return nil, err + } + + err = eveNode.EveDeleteFile("pub.pub") + if err != nil { + return nil, err + } + + return out, nil +} + +func readEveEndorsmentKey() (string, string, error) { + pub, err := readPublicKey(etpm.TpmEKHdl) + if err != nil { + return "", "", err + } + + hash := sha256.Sum256(pub) + hashHex := hex.EncodeToString(hash[:]) + + return base64.StdEncoding.EncodeToString(pub), hashHex, nil +} + +func getAzureIoTServicesStatus(output string) (map[string]string, error) { + // this what is being parse: + //$ sudo iotedge system status + //System services: + //aziot-edged Running + //aziot-identityd Down - activating + //aziot-keyd Ready + //aziot-certd Ready + //aziot-tpmd Running + + // Flag to indicate if we are in the "System services" section + inSystemServices := false + services := make(map[string]string, 0) + + scanner := bufio.NewScanner(strings.NewReader(output)) + + for scanner.Scan() { + line := scanner.Text() + + // Detect when we are in the "System services" section + if strings.Contains(line, "System services:") { + inSystemServices = true + continue + } + + // Exit the loop when we are out of the "System services" section + if inSystemServices && strings.TrimSpace(line) == "" { + break + } + + if inSystemServices { + parts := strings.Fields(line) + if len(parts) >= 2 { + services[parts[0]] = strings.Join(parts[1:], " ") + } + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return services, nil +} diff --git a/tests/vtpm/eden-config.yml b/tests/vtpm/eden-config.yml new file mode 100644 index 000000000..77a20c237 --- /dev/null +++ b/tests/vtpm/eden-config.yml @@ -0,0 +1,7 @@ +--- +eden: + # test binary + test-bin: "eden.vtpm.test" + + # test scenario + test-scenario: "eden.vtpm.tests.txt" diff --git a/tests/vtpm/eden.vtpm.tests.txt b/tests/vtpm/eden.vtpm.tests.txt new file mode 100644 index 000000000..db5e122ed --- /dev/null +++ b/tests/vtpm/eden.vtpm.tests.txt @@ -0,0 +1 @@ +eden.vtpm.test diff --git a/tests/vtpm/scripts/test_make_tpm_keys.sh b/tests/vtpm/scripts/test_make_tpm_keys.sh new file mode 100644 index 000000000..a6464a7f3 --- /dev/null +++ b/tests/vtpm/scripts/test_make_tpm_keys.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive + +sudo apt-get update +sudo apt-get install -y tpm2-tools + +# Make TPM devices accessible, 777 is OK for testing +sudo chmod 777 /dev/tpm* + +# Create the endorsement key (EK) and storage root key (SRK) +if ! tpm2_getcap handles-persistent | grep 0x81010001; then + tpm2_createek -c 0x81010001 -G rsa -u ek.pub + tpm2_createprimary -Q -C o -c srk.ctx > /dev/null + tpm2_evictcontrol -c srk.ctx 0x81000001 > /dev/null + tpm2_flushcontext -t > /dev/null +fi + +if [ ! -f ek.pub ]; then + echo "EK creation failed" + exit 1 +fi + +if [ ! -f srk.ctx ]; then + echo "SRK creation failed" + exit 1 +fi + +if ! tpm2_getcap handles-persistent | grep 0x81010001; then + echo "EK not found" + exit 1 +fi + +if ! tpm2_getcap handles-persistent | grep 0x81000001; then + echo "SRK not found" + exit 1 +fi + +exit 0 \ No newline at end of file diff --git a/tests/vtpm/scripts/test_ubuntu22.04_aziot_latest.sh b/tests/vtpm/scripts/test_ubuntu22.04_aziot_latest.sh new file mode 100644 index 000000000..0cdf0c970 --- /dev/null +++ b/tests/vtpm/scripts/test_ubuntu22.04_aziot_latest.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive + +# install microsoft repository +wget https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +rm packages-microsoft-prod.deb + +# install pre-requisites +sudo apt-get update +sudo apt-get install -y moby-engine +sudo apt-get install -y aziot-edge + +>config.toml cat <<-EOF +## DPS provisioning with TPM +[provisioning] +source = "dps" +global_endpoint = "https://global.azure-devices-provisioning.net" +id_scope = "$ID_SCOPE" + +[provisioning.attestation] +method = "tpm" +registration_id = "$REGISTRATION_ID" +EOF + +# Make TPM devices accessible, 777 is OK for testing +sudo chmod 777 /dev/tpm* + +sudo cp config.toml /etc/aziot/config.toml +sudo iotedge config apply +rm config.toml + diff --git a/tests/vtpm/scripts/test_vtpm_state_preservation.sh b/tests/vtpm/scripts/test_vtpm_state_preservation.sh new file mode 100644 index 000000000..9b75ff0ce --- /dev/null +++ b/tests/vtpm/scripts/test_vtpm_state_preservation.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Make TPM devices accessible, 777 is OK for testing +sudo chmod 777 /dev/tpm* + +if ! tpm2_getcap handles-persistent | grep 0x81010001; then + echo "EK not found" + exit 1 +fi + +if ! tpm2_getcap handles-persistent | grep 0x81000001; then + echo "SRK not found" + exit 1 +fi + +exit 0 diff --git a/tests/vtpm/vtpm_test.go b/tests/vtpm/vtpm_test.go new file mode 100644 index 000000000..bcb4e6abe --- /dev/null +++ b/tests/vtpm/vtpm_test.go @@ -0,0 +1,336 @@ +package aziot + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + tk "github.com/lf-edge/eden/pkg/evetestkit" + log "github.com/sirupsen/logrus" +) + +const ( + sshPort = "8027" + testScriptBasePath = "/home/ubuntu/" + projectName = "aziot-test" + aziotwait = 30 // seconds + appWait = 60 * 10 // 10 minutes + sshWait = 60 * 5 // 5 minutes + nodeRebootWait = 60 * 5 // 5 minutes +) + +var ( + appLink = "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img" + testScript = "scripts/test_ubuntu22.04_aziot_latest.sh" + eveNode *tk.EveNode + // We need a shared access policy with the following permissions: + // Registration Status Read, Registration Status Write, Enrollment Read, Enrollment Write + // We can create a new policy in the Azure portal by going to : + // IoT Hub -> Device Provisioning Service (DPS) -> Shared access policies -> Add + // and then copy the connection string. + connectionString = os.Getenv("AZIOT_CONNECTION_STRING") + // The ID Scope is required to configure azure-iot in the VM, + // we can get it from the Azure IoT Hub -> Device Provisioning Service -> Overview + // and copy the "ID Scope". + aziotIDScope = os.Getenv("AZIOT_ID_SCOPE") + appName = "" +) + +func deleteApp(appName string) { + err := eveNode.AppStopAndRemove(appName) + if err != nil { + log.Errorf("Failed to stop and remove app: %v", err) + } +} + +func setupApp() (string, error) { + appName := tk.GetRandomAppName(projectName + "-") + pubPorts := []string{sshPort + ":22"} + pc := tk.GetDefaultVMConfig(appName, tk.AppDefaultCloudConfig, pubPorts) + err := eveNode.EveDeployApp(appLink, pc, + tk.WithSSH(tk.AppDefaultSSHUser, tk.AppDefaultSSHPass, sshPort)) + if err != nil { + return "", fmt.Errorf("failed to deploy app: %v", err) + } + + return appName, nil +} + +func waitForApp(appName string) error { + // Wait for the app to start and ssh to be ready + log.Printf("Waiting for app %s to start...", appName) + err := eveNode.AppWaitForRunningState(appName, appWait) + if err != nil { + return fmt.Errorf("failed to wait for app to start: %v", err) + } + log.Printf("Waiting for ssh to be ready...") + err = eveNode.AppWaitForSSH(appName, sshWait) + if err != nil { + return fmt.Errorf("failed to wait for ssh: %v", err) + } + + log.Println("SSH connection established") + return nil +} + +func checkAziotServices(t *testing.T, appName string) { + // Check the status of the iotedge services + status, err := eveNode.AppSSHExec(appName, "sudo iotedge system status") + if err != nil { + t.Fatalf("Failed to get iotedge status: %v", err) + } + services, err := getAzureIoTServicesStatus(status) + if err != nil { + t.Fatalf("Failed to get Azure IoT services status: %v", err) + } + + // If all services are running we are good, otherwise fail the test + for service, status := range services { + if strings.ToLower(status) != "running" { + // Errorf calls Fail(), so we don't need to call it explicitly + t.Errorf("Service %s is not running", service) + } + } + t.Log("====================== SERVICES STATUS ======================") + for service, status := range services { + t.Logf("%s: \t\t%s\n", service, status) + } + + if t.Failed() { + // Get the aziot-tpmd logs, in one test we patch this service with eve-tools + // so good to have the logs for debugging. + command := "sudo iotedge system logs | grep aziot-tpmd" + tpmLog, err := eveNode.AppSSHExec(appName, command) + if err != nil { + t.Errorf("Failed to get aziot-tpmd logs: %v", err) + } else { + t.Log("====================== TPMD LOG ======================") + t.Log(tpmLog) + } + + // Get all the errors from the aziot logs + command = "sudo iotedge system logs | grep ERR | sed 's/.*ERR!] - //' | sort | uniq" + errors, err := eveNode.AppSSHExec(appName, command) + if err != nil { + t.Errorf("Failed to error logs: %v", err) + } else { + t.Log("====================== ERRORS ======================") + t.Log(errors) + } + } +} + +func TestMain(m *testing.M) { + log.Println("vTPM Test started") + defer log.Println("vTPM Test finished") + + node, err := tk.InitilizeTest(projectName, tk.WithControllerVerbosity("debug")) + if err != nil { + log.Fatalf("Failed to initialize test: %v", err) + } + + eveNode = node + name, err := setupApp() + if err != nil { + log.Fatalf("Failed to setup app: %v", err) + } + + // Wait for the deployed app to appear in the list + time.Sleep(30 * time.Second) + appName = name + + err = waitForApp(name) + if err != nil { + deleteApp(name) + log.Fatalf("Failed to wait for app: %v", err) + } + + res := m.Run() + deleteApp(name) + os.Exit(res) +} + +// TestVtpmIsRunningOnEVE checks if the vTPM process is running on the EVE node, +// it does this by checking if the vTPM control socket is open and the vTPM process +// is listening on it. +func TestVtpmIsRunningOnEVE(t *testing.T) { + t.Log("TestVtpmIsRunningOnEVE started") + defer t.Log("TestVtpmIsRunningOnEVE finished") + + // find the vTPM control socket and see if the vTPM process is listening on it. + command := "lsof -U | grep $(cat /proc/net/unix | grep vtpm | awk '{print $7}')" + out, err := eveNode.EveRunCommand(command) + if err != nil { + t.Fatalf("Failed to check if vTPM is running on EVE: %v", err) + } + + if len(out) == 0 || !strings.Contains(string(out), "vtpm") { + t.Fatalf("vTPM is not running on EVE : %s", out) + } +} + +// TestVtpmIsStatePreservation checks if the vTPM state is preserved after a reboot, +// it does this by creating a key in the vTPM (through a VM running on EVE) and +// then rebooting the EVE node, after the reboot it checks if the key is still +// present in the vTPM, by getting the list of vTPM persistent keys (through the +// the VM running on EVE). +func TestVtpmIsStatePreservation(t *testing.T) { + t.Log("TestVtpmIsStatePreservation started") + defer t.Log("TestVtpmIsStatePreservation finished") + + t.Log("Copying the key creation script to the VM") + createKeyScriptPath := testScriptBasePath + "test_make_tpm_keys.sh" + err := eveNode.AppSCPCopy(appName, "scripts/test_make_tpm_keys.sh", createKeyScriptPath) + if err != nil { + t.Fatalf("Failed to copy file to vm: %v", err) + } + + // Prepare the script for execution + command := fmt.Sprintf("chmod +x %s", createKeyScriptPath) + out, err := eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed perpare the TPM key creation script for execution \"%s\" : %v", out, err) + } + + // Execute the script to create the necessary TPM keys + _, err = eveNode.AppSSHExec(appName, createKeyScriptPath) + if err != nil { + t.Fatalf("Failed to execute TPM key creation script in VM: %v", err) + } + + // Reboot the EVE node and check the vTPM state is preserved + err = eveNode.EveRebootAndWait(nodeRebootWait) + if err != nil { + t.Fatalf("Failed to reboot EVE: %v", err) + } + + err = waitForApp(appName) + if err != nil { + t.Fatalf("Failed to wait for app: %v", err) + } + + createStatePresScriptPath := testScriptBasePath + "test_vtpm_state_preservation.sh" + err = eveNode.AppSCPCopy(appName, "scripts/test_vtpm_state_preservation.sh", createStatePresScriptPath) + if err != nil { + t.Fatalf("Failed to copy file to vm: %v", err) + } + + // Prepare the script for execution + command = fmt.Sprintf("chmod +x %s", createStatePresScriptPath) + _, err = eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed perpare the TPM state preservation test script for execution: %v", err) + } +} + +// TestAzureIotTPMEndrolmentWithVTPM tests the end-to-end scenario of enrolling a TPM device +// in Azure IoT Hub, this test will create a TPM enrollment in Azure IoT Hub, configure the +// Azure IoT Edge in a VM running on EVE, and check if the services are running. +func TestAzureIotTPMEndrolmentWithVTPM(t *testing.T) { + t.Log("TestAzureIotTPMEndrolmentWithTPM started") + t.Log("Setup :\n\tAziot (latest) on Ubuntu-22.04-amd64\n\twith VTPM") + defer t.Log("TestAzureIotTPMEndrolmentWithVTPM finished") + + if !eveNode.EveIsTpmEnabled() { + t.Skip("TPM is not enabled, skipping test") + } + + // Check for secrets, if not available don't bother running the tests. + if connectionString == "" { + log.Fatalf("AZIOT_CONNECTION_STRING environment variable is not set") + } + if aziotIDScope == "" { + log.Fatalf("AZIOT_ID_SCOPE environment variable is not set") + } + + // Copy the test script to the VM + testScriptPath := testScriptBasePath + filepath.Base(testScript) + err := eveNode.AppSCPCopy(appName, testScript, testScriptPath) + if err != nil { + t.Fatalf("Failed to copy file to vm: %v", err) + } + t.Log("Test script copied to VM") + + // for this to test to work, we need to create an enrollment in the Azure IoT Hub. + t.Log("Creating a TPM enrollment in Azure IoT Hub") + createKeyScriptPath := testScriptBasePath + "test_make_tpm_keys.sh" + err = eveNode.AppSCPCopy(appName, "scripts/test_make_tpm_keys.sh", createKeyScriptPath) + if err != nil { + t.Fatalf("Failed to copy file to vm: %v", err) + } + + // Prepare the script for execution + command := fmt.Sprintf("chmod +x %s", createKeyScriptPath) + _, err = eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed perpare the TPM key creation script for execution: %v", err) + } + + // Execute the script to create the necessary TPM keys + out, err := eveNode.AppSSHExec(appName, createKeyScriptPath) + if err != nil { + t.Fatalf("Failed to execute TPM key creation script in VM \"%s\": %v", out, err) + } + + // Get the endorsement key from the VM + ek, err := eveNode.AppSSHExec(appName, "base64 -w0 ek.pub") + if err != nil { + t.Fatalf("Failed to read endrosment key from VM: %v", err) + } + + // Get the enrollment ID from the VM + command = "sha256sum -b ek.pub | cut -d' ' -f1 | sed -e 's/[^[:alnum:]]//g'" + id, err := eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed to get enrollment ID from VM: %v", err) + } + endorsementKey, enrollmentID := strings.TrimSpace(ek), strings.TrimSpace(id) + + // Get the provisioning service name from the connection string + provService, err := getProvisioningService(connectionString) + if err != nil { + t.Fatalf("Failed to get provisioning service: %v\n", err) + } + + // From the connection string generate a SAS token lasting for 1 hour + sasToken, err := getSasTokenFromConnectionString(connectionString, 1) + if err != nil { + t.Fatalf("Failed to generate a SAS token: %v\n", err) + } + + // Add the enrollment to azure iot hub portal + err = addTPMEnrollment(enrollmentID, endorsementKey, provService, sasToken) + if err != nil { + t.Fatalf("Failed to create a TPM enrollment on Azure: %v\n", err) + } + defer func() { + err = deleteEnrollment(enrollmentID, provService, sasToken) + if err != nil { + log.Printf("Failed to delete TPM enrollment, please remove it manually: %v\n", err) + } + }() + + // Prepare the test script for execution + command = fmt.Sprintf("chmod +x %s", testScriptPath) + _, err = eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed perpare the test script for execution: %v", err) + } + + // Execute the test script, this will configure the azure-iot in the VM + // and start the services. + command = fmt.Sprintf("ID_SCOPE=%s REGISTRATION_ID=%s %s", aziotIDScope, enrollmentID, testScriptPath) + _, err = eveNode.AppSSHExec(appName, command) + if err != nil { + t.Fatalf("Failed to execute test script in VM: %v", err) + } + + // Wait for the services to start + t.Logf("Waiting for services to start...") + time.Sleep(aziotwait * time.Second) + // this will check the status of the services and fail the test if any service is not running + checkAziotServices(t, appName) +} diff --git a/tests/workflow/smoke.tests.txt b/tests/workflow/smoke.tests.txt index 4e7790b10..667ba0d2a 100644 --- a/tests/workflow/smoke.tests.txt +++ b/tests/workflow/smoke.tests.txt @@ -1,5 +1,5 @@ # Number of tests -{{$tests := 23}} +{{$tests := 24}} # EDEN_TEST_SETUP env. var. -- "y"(default) performs the EDEN setup steps {{$setup := "y"}} {{$setup_env := EdenGetEnv "EDEN_TEST_SETUP"}} @@ -40,6 +40,10 @@ eden.escript.test -test.run TestEdenScripts/eve_restart {{end}} {{end}} +# just for testing, will move it down +/bin/echo EVE VTPM tests (24/{{$tests}}) +eden.escript.test -test.run TestEdenScripts/eden_vtpm + /bin/echo Eden Log test (6/{{$tests}}) eden.escript.test -testdata ../lim/testdata/ -test.run TestEdenScripts/log_test /bin/echo Eden SSH test (7/{{$tests}}) diff --git a/tests/workflow/testdata/eden_vtpm.txt b/tests/workflow/testdata/eden_vtpm.txt new file mode 100644 index 000000000..e3c256e9b --- /dev/null +++ b/tests/workflow/testdata/eden_vtpm.txt @@ -0,0 +1,2 @@ +test eden.aziot.test +test eden.vtpm.test \ No newline at end of file