feat(experimental/gotls): support stripped and/or PIE enabled binary (#…

…147) * feat(experimental/gotls): support stripped and/or PIE enabled binary * fix ci
mozillazg · Sep 22, 2024 · c9099fb · c9099fb
1 parent bdb5f10
commit c9099fb
Show file tree

Hide file tree

Showing 116 changed files with 50,538 additions and 99 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -18,27 +18,11 @@ jobs:
 
     steps:
       - checkout
-      - restore_cache:
-          keys:
-            - go-mod-v4-{{ checksum "go.sum" }}
-      - run:
-          name: Install Dependencies
-          command: go mod download
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-      - run:
-          name: install deps
-          command: |
-            sudo apt-get update && sudo apt-get install -y gcc flex bison make libelf-dev autoconf
-            make libpcap
 
       - run:
           name: build
           command: |
-            make build
+            make build-via-docker
 
             echo '========== info =========='
             uname -a
@@ -98,7 +82,7 @@ jobs:
             exit 1
 
       - run:
-          name: e2e (test go tls keylog)
+          name: e2e (test go tls keylog, unstripped)
           command: |
             echo "wireshark-common wireshark-common/install-setuid boolean true" | sudo debconf-set-selections
             sudo add-apt-repository -y ppa:wireshark-dev/stable
@@ -111,31 +95,60 @@ jobs:
             done
             exit 1
 
+      - run:
+          name: e2e (test go tls keylog, PIE)
+          command: |
+            echo "wireshark-common wireshark-common/install-setuid boolean true" | sudo debconf-set-selections
+            sudo add-apt-repository -y ppa:wireshark-dev/stable
+            sudo apt update
+            sudo DEBIAN_FRONTEND=noninteractive apt install -y tshark
+            make -C testdata/gohttpapp build
+
+            for i in {1..10}; do
+              sudo bash testdata/test_gotls_keylog_pie.sh ./ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
+
+      - run:
+          name: e2e (test go tls keylog, stripped)
+          command: |
+            echo "wireshark-common wireshark-common/install-setuid boolean true" | sudo debconf-set-selections
+            sudo add-apt-repository -y ppa:wireshark-dev/stable
+            sudo apt update
+            sudo DEBIAN_FRONTEND=noninteractive apt install -y tshark
+            make -C testdata/gohttpapp build
+
+            for i in {1..10}; do
+              sudo bash testdata/test_gotls_keylog_stripped.sh ./ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
+
+      - run:
+          name: e2e (test go tls keylog, stripped + PIE)
+          command: |
+            echo "wireshark-common wireshark-common/install-setuid boolean true" | sudo debconf-set-selections
+            sudo add-apt-repository -y ppa:wireshark-dev/stable
+            sudo apt update
+            sudo DEBIAN_FRONTEND=noninteractive apt install -y tshark
+            make -C testdata/gohttpapp build
+
+            for i in {1..10}; do
+              sudo bash testdata/test_gotls_keylog_stripped_pie.sh ./ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
+
+
   docker-e2e:
     machine:
       image: ubuntu-2204:2024.04.4
     resource_class: medium
     steps:
       - checkout
-      - restore_cache:
-          keys:
-            - go-mod-v4-{{ checksum "go.sum" }}
-      - run:
-          name: Install Dependencies
-          command: go mod download
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-      - run:
-          name: install deps
-          command: |
-            sudo apt-get update && sudo apt-get install -y gcc flex bison make libelf-dev autoconf
-            make libpcap
+
       - run:
           name: build
           command: |
-            make build
+            make build-via-docker
 
             echo '========== info =========='
             uname -a
@@ -171,25 +184,11 @@ jobs:
             wget https://github.com/containernetworking/plugins/releases/download/v1.5.0/cni-plugins-linux-amd64-v1.5.0.tgz
             sudo mkdir -p /opt/cni/bin
             sudo tar Cxzvvf /opt/cni/bin cni-plugins-linux-amd64-v1.5.0.tgz
-      - restore_cache:
-          keys:
-            - go-mod-v4-{{ checksum "go.sum" }}
-      - run:
-          name: Install Dependencies
-          command: go mod download
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-      - run:
-          name: install deps
-          command: |
-            sudo apt-get update && sudo apt-get install -y gcc flex bison make libelf-dev autoconf
-            make libpcap
+
       - run:
           name: build
           command: |
-            make build
+            make build-via-docker
 
             echo '========== info =========='
             uname -a
@@ -232,25 +231,10 @@ jobs:
             sudo kind load docker-image busybox:1
             sudo kind export kubeconfig
 
-      - restore_cache:
-          keys:
-            - go-mod-v4-{{ checksum "go.sum" }}
-      - run:
-          name: Install Dependencies
-          command: go mod download
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-      - run:
-          name: install deps
-          command: |
-            sudo apt-get update && sudo apt-get install -y gcc flex bison make libelf-dev autoconf
-            make libpcap
       - run:
           name: build
           command: |
-            make build
+            make build-via-docker
 
             echo '========== info =========='
             uname -a

diff --git a/.github/workflows/docker-dev-image.yml b/.github/workflows/docker-dev-image.yml
@@ -67,8 +67,8 @@ jobs:
         with:
           context: .
           file: .github/build.Dockerfile
-          # platforms: linux/amd64,linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
+          # platforms: linux/amd64
           push: ${{ github.event_name != 'pull_request' }}
           tags: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.TAG_NAME }}'
           labels: ${{ steps.meta.outputs.labels }}
@@ -97,8 +97,8 @@ jobs:
         with:
           context: .
           file: .github/build.Dockerfile
-          # platforms: linux/amd64,linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
+          # platforms: linux/amd64
           push: ${{ github.event_name != 'pull_request' }}
           tags: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest'
           labels: ${{ steps.meta.outputs.labels }}

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -22,12 +22,8 @@ jobs:
       with:
         go-version: '1.22.4'
 
-    - name: Set up deps
-      run: |
-        sudo apt-get install -y gcc flex bison make libelf-dev autoconf
-
     - name: Build
-      run: make build
+      run: make build-via-docker
 
     - name: Test
       run: make test
@@ -255,7 +251,7 @@ jobs:
         run: |
           make -C testdata/gohttpapp build
 
-      - name: Test go tls keylog
+      - name: Test go tls keylog (unstripped)
         uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
         with:
           provision: 'false'
@@ -272,3 +268,57 @@ jobs:
               bash /host/testdata/test_gotls_keylog.sh /host/ptcpdump/ptcpdump && exit 0 || sleep 1
             done
             exit 1
+
+      - name: Test go tls keylog (PIE)
+        uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+            uname -a
+            cat /etc/issue
+
+            ls -lh /host/testdata/gohttpapp
+
+            apt update && yes | apt install -y tshark
+
+            for i in {1..10}; do
+              bash /host/testdata/test_gotls_keylog_pie.sh /host/ptcpdump/ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
+
+      - name: Test go tls keylog (stripped)
+        uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+            uname -a
+            cat /etc/issue
+
+            ls -lh /host/testdata/gohttpapp
+
+            apt update && yes | apt install -y tshark
+
+            for i in {1..10}; do
+              bash /host/testdata/test_gotls_keylog_stripped.sh /host/ptcpdump/ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
+
+      - name: Test go tls keylog (stripped + PIE)
+        uses: cilium/little-vm-helper@97c89f004bd0ab4caeacfe92ebc956e13e362e6b # v0.0.19
+        with:
+          provision: 'false'
+          cmd: |
+            set -ex
+            uname -a
+            cat /etc/issue
+
+            ls -lh /host/testdata/gohttpapp
+
+            apt update && yes | apt install -y tshark
+
+            for i in {1..10}; do
+              bash /host/testdata/test_gotls_keylog_stripped_pie.sh /host/ptcpdump/ptcpdump && exit 0 || sleep 1
+            done
+            exit 1
diff --git a/README.md b/README.md
@@ -334,7 +334,7 @@ Flags:
 
 ### Building
 
-1. Build eBPF programs:
+1. Build eBPF programs (optional):
 
     ```
     make build-bpf

diff --git a/README.zh-CN.md b/README.zh-CN.md
@@ -353,7 +353,7 @@ Flags:
 如何编程项目源代码。
 
 
-1. Build eBPF programs:
+1. Build eBPF programs (optional):
 
     ```
     make build-bpf

diff --git a/bpf/gotls.go b/bpf/gotls.go
@@ -3,6 +3,7 @@ package bpf
 import (
 	"fmt"
 	"github.com/cilium/ebpf/link"
+	"github.com/mozillazg/ptcpdump/internal/log"
 )
 
 func (b *BPF) AttachGoTLSUprobeHooks(exec *link.Executable, symbol string,
@@ -13,17 +14,22 @@ func (b *BPF) AttachGoTLSUprobeHooks(exec *link.Executable, symbol string,
 			Address: funcAddr,
 		})
 	if err != nil {
-		return fmt.Errorf("attach uprobe for %s: %w", symbol, err)
+		err = fmt.Errorf("attach uprobe for %s: %w", symbol, err)
+		log.Infof("%s", err)
+		return err
 	}
 	b.links = append(b.links, lk)
 
 	lk, err = exec.Uprobe(symbol, b.objs.UprobeGoBuiltinTlsWriteKeyLogRet,
 		&link.UprobeOptions{
-			PID:    pid,
-			Offset: retOffset, // if c.KeyLogWriter == nil { return nil }
+			PID:     pid,
+			Address: funcAddr,
+			Offset:  retOffset, // if c.KeyLogWriter == nil { return nil }
 		})
 	if err != nil {
-		return fmt.Errorf("attach uprobe for %s: %w", symbol, err)
+		err = fmt.Errorf("attach uprobe for %s: %w", symbol, err)
+		log.Infof("%s", err)
+		return err
 	}
 	b.links = append(b.links, lk)
 	return nil

diff --git a/cmd/gotls.go b/cmd/gotls.go
@@ -53,34 +53,47 @@ func attachGoTLSHooks(opts Options, bf *bpf.BPF) error {
 		return fmt.Errorf("could not find %s in PATH", opts.subProgArgs[0])
 	}
 	if _, err := buildinfo.ReadFile(path); err != nil {
-		log.Debugf("skip go TLS related logics due to %s", err)
+		log.Debugf("skip go TLS related logics due to %+v", err)
 		return nil
 	}
 	elff, err := elf.Open(path)
 	if err != nil {
-		log.Debugf("skip go TLS related logics due to %s", err)
+		log.Debugf("skip go TLS related logics due to %+v", err)
 		return nil
 	}
 
 	exc, err := link.OpenExecutable(path)
 	if err != nil {
-		log.Warnf("skip go TLS related logics due to %s", err)
+		log.DWarnf("skip go TLS related logics due to open elf file failed: %+v", err)
 		return nil
 	}
-	retOffsets, err := gosym.GetGoFuncRetOffsetsFromELF(elff, goTLSSymbolWriteKeyLog)
+
+	var symbolOffset uint64
+	var retOffsets []uint64
+	retOffsets, err = gosym.GetFuncRetOffsetsViaSymbolTable(elff, goTLSSymbolWriteKeyLog)
 	if err == nil && len(retOffsets) == 0 {
 		err = errors.New("not found any RET instruction")
 	}
 	if err != nil {
-		log.Warnf("skip go TLS related logics due to %s", err)
+		log.Infof("get offsets via symbol table failed: %+v", err)
+		symbolOffset, retOffsets, err = gosym.GetFuncRetOffsetsViaPclntab(path, elff, goTLSSymbolWriteKeyLog)
+	}
+	if err == nil && len(retOffsets) == 0 {
+		err = errors.New("not found any RET instruction")
+	}
+	if err != nil {
+		log.DWarnf("skip go TLS related logics due to parse elf failed: %s", err)
 		return nil
 	}
+
 	retOffset := retOffsets[len(retOffsets)-1]
-	log.Infof("got retOffsets: %v, will attach at ret offset: %d", retOffsets, retOffset)
+	log.Infof("got symbolOffset: %d, got retOffsets: %v, will attach at ret offset: %d",
+		symbolOffset, retOffsets, retOffset)
 
 	if err := bf.AttachGoTLSUprobeHooks(exc, goTLSSymbolWriteKeyLog,
-		0, retOffset, 0); err != nil {
-		log.Warnf("skip go TLS related logics due to could not attach go TLS hooks base on %s: %s", path, err)
+		symbolOffset, retOffset, 0); err != nil {
+		log.DWarnf(
+			"skip go TLS related logics due to could not attach go TLS hooks base on %s: %+v", path, err)
 	}
 	return nil
 }