From 075eeac61b105ba94dffb6f551a106e0ed1dd7a1 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sat, 4 May 2024 22:29:26 +0200 Subject: [PATCH 1/2] ci: generate python SDK using Dagger Signed-off-by: Mark Sagi-Kazar --- .github/workflows/generate-client-python.yaml | 34 +++++++------------ api/client/python/Dockerfile | 16 --------- api/client/python/Makefile | 30 ---------------- api/client/python/config.yaml | 2 +- ci/generate.go | 32 +++++++++++++++++ ci/main.go | 2 ++ 6 files changed, 48 insertions(+), 68 deletions(-) delete mode 100644 api/client/python/Dockerfile delete mode 100644 api/client/python/Makefile create mode 100644 ci/generate.go diff --git a/.github/workflows/generate-client-python.yaml b/.github/workflows/generate-client-python.yaml index aa3a2c45a..56aae0eff 100644 --- a/.github/workflows/generate-client-python.yaml +++ b/.github/workflows/generate-client-python.yaml @@ -1,4 +1,4 @@ -name: Python Client Generation +name: Generate / Python SDK permissions: contents: write @@ -17,31 +17,23 @@ jobs: generate: runs-on: ubuntu-latest steps: - - uses: actions/create-github-app-token@v1 - id: app-token + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - app-id: ${{ vars.BOT_APP_ID }} - private-key: ${{ secrets.BOT_APP_PRIVATE_KEY }} - - uses: actions/checkout@v4 - with: - token: ${{ steps.app-token.outputs.token }} ref: ${{ github.head_ref }} - # Make sure the value of GITHUB_TOKEN will not be persisted in repo's config - persist-credentials: false - - name: Login to GitHub Container Registry - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} + - name: Generate Client - run: | - make generate - working-directory: ./api/client/python + uses: dagger/dagger-for-github@eba69b4dddb54eddfdb51a88eb7fd86957137630 # v5.4.0 + with: + verb: call + args: --source .:default generate python-sdk -o api/client/python + cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + version: "0.11.2" + - name: Open Pull Request uses: peter-evans/create-pull-request@v6 with: - branch: openapi/python-client + branch: openapi/python-sdk branch-suffix: short-commit-hash delete-branch: true commit-message: "chore(api): generate python client" @@ -50,4 +42,4 @@ jobs: labels: | area/api release-note/misc - token: ${{ steps.app-token.outputs.token }} + token: ${{ secrets.BOT_GITHUB_TOKEN }} diff --git a/api/client/python/Dockerfile b/api/client/python/Dockerfile deleted file mode 100644 index f0c455a1a..000000000 --- a/api/client/python/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -# We build our image as the official autorest Dockerfile is outdated -# and not compatible with the latest autorest. -# More specifically, the latest autorest npm package depends on -# other Azure packages that require a higher node version. -# Official image: https://github.com/Azure/autorest/blob/63ffe68961e24ed8aa59a2ca4c16a8019c271e45/docker/base/ubuntu/Dockerfile - -# Autorest is incompatible with latest node version -FROM node:20-alpine - -# Install autorest -RUN npm install -g autorest - -# Install python/pip -RUN apk add --no-cache python3 py3-pip - -ENTRYPOINT [ "autorest" ] diff --git a/api/client/python/Makefile b/api/client/python/Makefile deleted file mode 100644 index e246ed891..000000000 --- a/api/client/python/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -# A Self-Documenting Makefile: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html - -.PHONY: autorest-python -autorest-python: ## Build and publish autorest-python image - $(call print-target) - docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/openmeterio/autorest-python:latest --push . - -.PHONY: generate -generate: ## Generate code - $(call print-target) - cp ../../openapi.yaml . - docker run \ - --rm \ - -v "$$(pwd):/workdir" \ - -w /workdir \ - -t ghcr.io/openmeterio/autorest-python:latest config.yaml - rm openapi.yaml - -.PHONY: help -.DEFAULT_GOAL := help -help: - @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -# Variable outputting/exporting rules -var-%: ; @echo $($*) -varexport-%: ; @echo $*=$($*) - -define print-target - @printf "Executing target: \033[36m$@\033[0m\n" -endef diff --git a/api/client/python/config.yaml b/api/client/python/config.yaml index 1aae0bcf5..0b27c3cfb 100644 --- a/api/client/python/config.yaml +++ b/api/client/python/config.yaml @@ -2,7 +2,7 @@ title: OpenMeterClient namespace: openmeter python: true black: true -input-file: ./openapi.yaml +input-file: ../../openapi.yaml output-folder: ./src clear-output-folder: true verbose: true diff --git a/ci/generate.go b/ci/generate.go new file mode 100644 index 000000000..cd0157fdd --- /dev/null +++ b/ci/generate.go @@ -0,0 +1,32 @@ +package main + +// Generate various artifacts. +func (m *Ci) Generate() *Generate { + return &Generate{ + Source: m.Source, + } +} + +type Generate struct { + // +private + Source *Directory +} + +// Generate the Python SDK. +func (m *Generate) PythonSdk() *Directory { + // We build our image as the official autorest Dockerfile is outdated + // and not compatible with the latest autorest. + // More specifically, the latest autorest npm package depends on + // other Azure packages that require a higher node version. + // Official image: https://github.com/Azure/autorest/blob/63ffe68961e24ed8aa59a2ca4c16a8019c271e45/docker/base/ubuntu/Dockerfile + + // Autorest is incompatible with latest node version + return dag.Container(). + From("node:20-alpine"). + WithExec([]string{"npm", "install", "-g", "autorest"}). + WithExec([]string{"apk", "add", "python3", "py3-pip"}). + WithDirectory("/work", m.Source.Directory("api")). + WithWorkdir("/work/client/python"). + WithExec([]string{"autorest", "config.yaml"}). + Directory("/work/client/python") +} diff --git a/ci/main.go b/ci/main.go index f24f791f9..42b44ce8d 100644 --- a/ci/main.go +++ b/ci/main.go @@ -71,6 +71,8 @@ func (m *Ci) Ci(ctx context.Context) error { wrapSyncable(m.Build().helmChart("benthos-collector", "0.0.0").File()), wrapSyncables(m.releaseAssets("ci")), + + wrapSyncable(m.Generate().PythonSdk()), ) return p.wait() From e9ea5f7dfb5fdd416f2097fe137edff1b68dc9de Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Sun, 5 May 2024 12:19:11 +0200 Subject: [PATCH 2/2] ci: move pypi publishing to dagger Signed-off-by: Mark Sagi-Kazar --- .github/workflows/release.yaml | 3 ++- ci/release.go | 27 ++++++++++++++++++++++++++- dagger.json | 4 ++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9e15dce3b..dd081bb24 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,8 +41,9 @@ jobs: with: verb: call module: github.com/${{ github.repository }}@${{ github.ref }} - args: --ref ${{ github.ref }} release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN + args: --ref ${{ github.ref }} release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN --pypi-token env:PYPI_TOKEN cloud-token: ${{ secrets.DAGGER_CLOUD_TOKEN }} version: "0.11.2" env: GITHUB_TOKEN: ${{ github.token }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/ci/release.go b/ci/release.go index 7bbdfd7a8..9495dafc2 100644 --- a/ci/release.go +++ b/ci/release.go @@ -5,9 +5,10 @@ import ( "errors" "fmt" "strings" + "time" ) -func (m *Ci) Release(ctx context.Context, version string, githubActor string, githubToken *Secret) error { +func (m *Ci) Release(ctx context.Context, version string, githubActor string, githubToken *Secret, pypiToken *Secret) error { p := newPipeline(ctx) p.addJobs( @@ -38,6 +39,10 @@ func (m *Ci) Release(ctx context.Context, version string, githubActor string, gi return err }, + + func(ctx context.Context) error { + return m.publishPythonSdk(ctx, version, pypiToken) + }, ) return p.wait() @@ -94,3 +99,23 @@ func (m *Ci) binaryArchive(version string, platform Platform) *File { WithFile("", m.Source.File("LICENSE")), ) } + +func (m *Ci) publishPythonSdk(ctx context.Context, version string, pypiToken *Secret) error { + _, err := dag.Python(PythonOpts{ + Container: dag.Python(PythonOpts{Container: dag.Container().From("pypy:3.10-slim")}). + WithPipCache(dag.CacheVolume("openmeter-pip")). + Container(). + WithExec([]string{"pip", "--disable-pip-version-check", "install", "pipx"}). + WithExec([]string{"pipx", "install", "poetry"}), + }). + WithSource(m.Source.Directory("api/client/python")). // TODO: generate SDK on the fly? + Container(). + WithExec([]string{"poetry", "install"}). + WithExec([]string{"poetry", "version", version}). + WithSecretVariable("POETRY_PYPI_TOKEN_PYPI", pypiToken). + WithEnvVariable("CACHE_BUSTER", time.Now().Format(time.RFC3339Nano)). + WithExec([]string{"poetry", "publish", "--build"}). + Sync(ctx) + + return err +} diff --git a/dagger.json b/dagger.json index c4add0dbf..8590f88df 100644 --- a/dagger.json +++ b/dagger.json @@ -40,6 +40,10 @@ "name": "kafka", "source": "github.com/sagikazarmark/daggerverse/kafka@9a3074edb9cb21746ea2f9dcdd018d520b47f2e6" }, + { + "name": "python", + "source": "github.com/sagikazarmark/daggerverse/python@d8b5409dc45cc2242792566059f18cb1e5488629" + }, { "name": "spectral", "source": "github.com/sagikazarmark/daggerverse/spectral@9a3074edb9cb21746ea2f9dcdd018d520b47f2e6"