Merge pull request #846 from openmeterio/generate-python-sdk

ci: generate python SDK using Dagger

.github/workflows/generate-client-python.yaml

@@ -1,4 +1,4 @@
-name: Python Client Generation
+name: Generate / Python SDK
 
 permissions:
   contents: write
@@ -17,31 +17,23 @@ jobs:
   generate:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/create-github-app-token@v1
-        id: app-token
+      - name: Checkout repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          app-id: ${{ vars.BOT_APP_ID }}
-          private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }}
-      - uses: actions/checkout@v4
-        with:
-          token: ${{ steps.app-token.outputs.token }}
           ref: ${{ github.head_ref }}
-          # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config
-          persist-credentials: false
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
+
       - name: Generate Client
-        run: |
-          make generate
-        working-directory: ./api/client/python
+        uses: dagger/dagger-for-github@eba69b4dddb54eddfdb51a88eb7fd86957137630 # v5.4.0
+        with:
+          verb: call
+          args: --source .:default generate python-sdk -o api/client/python
+          cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
+          version: "0.11.2"
+
       - name: Open Pull Request
         uses: peter-evans/create-pull-request@v6
         with:
-          branch: openapi/python-client
+          branch: openapi/python-sdk
           branch-suffix: short-commit-hash
           delete-branch: true
           commit-message: "chore(api): generate python client"
@@ -50,4 +42,4 @@ jobs:
           labels: |
             area/api
             release-note/misc
-          token: ${{ steps.app-token.outputs.token }}
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -41,8 +41,9 @@ jobs:
         with:
           verb: call
           module: github.com/${{ github.repository }}@${{ github.ref }}
-          args: --ref ${{ github.ref }} release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN
+          args: --ref ${{ github.ref }} release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN --pypi-token env:PYPI_TOKEN
           cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
           version: "0.11.2"
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

api/client/python/config.yaml

@@ -2,7 +2,7 @@ title: OpenMeterClient
 namespace: openmeter
 python: true
 black: true
-input-file: ./openapi.yaml
+input-file: ../../openapi.yaml
 output-folder: ./src
 clear-output-folder: true
 verbose: true

ci/generate.go

@@ -0,0 +1,32 @@
+package main
+
+// Generate various artifacts.
+func (m *Ci) Generate() *Generate {
+	return &Generate{
+		Source: m.Source,
+	}
+}
+
+type Generate struct {
+	// +private
+	Source *Directory
+}
+
+// Generate the Python SDK.
+func (m *Generate) PythonSdk() *Directory {
+	// We build our image as the official autorest Dockerfile is outdated
+	// and not compatible with the latest autorest.
+	// More specifically, the latest autorest npm package depends on
+	// other Azure packages that require a higher node version.
+	// Official image: https://github.com/Azure/autorest/blob/63ffe68961e24ed8aa59a2ca4c16a8019c271e45/docker/base/ubuntu/Dockerfile
+
+	// Autorest is incompatible with latest node version
+	return dag.Container().
+		From("node:20-alpine").
+		WithExec([]string{"npm", "install", "-g", "autorest"}).
+		WithExec([]string{"apk", "add", "python3", "py3-pip"}).
+		WithDirectory("/work", m.Source.Directory("api")).
+		WithWorkdir("/work/client/python").
+		WithExec([]string{"autorest", "config.yaml"}).
+		Directory("/work/client/python")
+}

ci/main.go

@@ -72,6 +72,8 @@ func (m *Ci) Ci(ctx context.Context) error {
 		wrapSyncable(m.Build().helmChart("benthos-collector", "0.0.0").File()),
 
 		wrapSyncables(m.releaseAssets("ci")),
+
+		wrapSyncable(m.Generate().PythonSdk()),
 	)
 
 	return p.wait()

ci/release.go

@@ -5,9 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 )
 
-func (m *Ci) Release(ctx context.Context, version string, githubActor string, githubToken *Secret) error {
+func (m *Ci) Release(ctx context.Context, version string, githubActor string, githubToken *Secret, pypiToken *Secret) error {
 	p := newPipeline(ctx)
 
 	p.addJobs(
@@ -38,6 +39,10 @@ func (m *Ci) Release(ctx context.Context, version string, githubActor string, gi
 
 			return err
 		},
+
+		func(ctx context.Context) error {
+			return m.publishPythonSdk(ctx, version, pypiToken)
+		},
 	)
 
 	return p.wait()
@@ -94,3 +99,23 @@ func (m *Ci) binaryArchive(version string, platform Platform) *File {
 			WithFile("", m.Source.File("LICENSE")),
 	)
 }
+
+func (m *Ci) publishPythonSdk(ctx context.Context, version string, pypiToken *Secret) error {
+	_, err := dag.Python(PythonOpts{
+		Container: dag.Python(PythonOpts{Container: dag.Container().From("pypy:3.10-slim")}).
+			WithPipCache(dag.CacheVolume("openmeter-pip")).
+			Container().
+			WithExec([]string{"pip", "--disable-pip-version-check", "install", "pipx"}).
+			WithExec([]string{"pipx", "install", "poetry"}),
+	}).
+		WithSource(m.Source.Directory("api/client/python")). // TODO: generate SDK on the fly?
+		Container().
+		WithExec([]string{"poetry", "install"}).
+		WithExec([]string{"poetry", "version", version}).
+		WithSecretVariable("POETRY_PYPI_TOKEN_PYPI", pypiToken).
+		WithEnvVariable("CACHE_BUSTER", time.Now().Format(time.RFC3339Nano)).
+		WithExec([]string{"poetry", "publish", "--build"}).
+		Sync(ctx)
+
+	return err
+}

dagger.json

@@ -40,6 +40,10 @@
       "name": "kafka",
       "source": "github.com/sagikazarmark/daggerverse/kafka@9a3074edb9cb21746ea2f9dcdd018d520b47f2e6"
     },
+    {
+      "name": "python",
+      "source": "github.com/sagikazarmark/daggerverse/python@d8b5409dc45cc2242792566059f18cb1e5488629"
+    },
     {
       "name": "spectral",
       "source": "github.com/sagikazarmark/daggerverse/spectral@9a3074edb9cb21746ea2f9dcdd018d520b47f2e6"