chore: support secretflow 1.5 (#44)

* chore: add 1.5 image * chore: support secretflow 1.5 * docs: update readme * chore: only check server in secretntoe project with pyrihgt
secretflow · Apr 20, 2024 · 46a3353 · 46a3353
1 parent 0b36189
commit 46a3353
Show file tree

Hide file tree

Showing 11 changed files with 41 additions and 163 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -17,9 +17,9 @@ jobs:
       tag:
         type: string
         default: latest
-      file:
+      version:
         type: string
-        default: docker/app/Dockerfile
+        default: 1.3.0b0
 
     steps:
       - checkout
@@ -32,7 +32,8 @@ jobs:
 
             docker buildx build \
               --platform=<< parameters.platform >> \
-              -f << parameters.file >> \
+              --build-arg version=<< parameters.version >> \
+              -f docker/app/Dockerfile \
               -t ${IMAGE_NAME} \
               --progress=plain \
               .
@@ -43,31 +44,33 @@ jobs:
 # Orchestrate jobs using workflows
 # See: https://circleci.com/docs/configuration-reference/#workflows
 workflows:
-  devcontainer-publish-amd64:
+  devcontainer-publish-1.3.0-amd64:
     jobs:
       - devcontainer-publish:
           filters:
             branches:
               only:
                 - main
           platform: linux/amd64
-          tag: unstable-amd64
-  devcontainer-publish-dev-1.5:
+          version: 1.3.0b0
+          tag: 1.3.0-amd64
+  devcontainer-publish-1.5.0-amd64:
     jobs:
       - devcontainer-publish:
           filters:
             branches:
               only:
                 - main
           platform: linux/amd64
-          tag: 1.5.0.dev
-          file: docker/app/Dockerfile.dev
-  # devcontainer-publish-arm64:
-  #   jobs:
-  #     - devcontainer-publish:
-  #         filters:
-  #           branches:
-  #             only:
-  #               - main
-  #         platform: linux/arm64
-  #         tag: unstable-arm64
+          version: 1.5.0b0
+          tag: 1.5.0-amd64
+  devcontainer-publish-1.5.0-arm64:
+    jobs:
+      - devcontainer-publish:
+          filters:
+            branches:
+              only:
+                - main
+          platform: linux/arm64
+          version: 1.5.0b0
+          tag: 1.5.0-arm64
diff --git a/README.md b/README.md
@@ -10,14 +10,14 @@ SecretNote 是专为隐语开发者打造的高级工具套件。以 notebook
 
 SecretNote 有单独的 [pip](https://pypi.org/project/secretnote/) 安装包，可以单独使用。为了避免安装、部署、启动等环境问题，推荐使用 docker 方式启动 SecretFlow 运行环境。
 
-**注意：当前 docker 镜像暂时不支持 Mac M1、M2 机器以及 ARM 架构机器**
+**注意：当前 docker 镜像提供 1.3.0-amd64 和 1.5.0-amd64 两个版本，和 secretflow 的版本是对应的，也就是说，如果你想使用 secretflow 1.5 版本，那就使用镜像 secretflow/secretnote:1.5.0-amd64**
 
 1. 启动两个容器，推荐使用 docker compose，这样方便管理容器之间的通信。新建文件夹，并新建文件 `docker-compose.yml`，内容如下：
 
 ```yml
 services:
   alice:
-    image: 'secretflow/secretnote:unstable-amd64'
+    image: 'secretflow/secretnote:1.3.0-amd64'
     platform: linux/amd64
     environment:
       - SELF_PARTY=alice
@@ -29,7 +29,7 @@ services:
       - /root/scripts
 
   bob:
-    image: 'secretflow/secretnote:unstable-amd64'
+    image: 'secretflow/secretnote:1.3.0-amd64'
     platform: linux/amd64
     environment:
       - SELF_PARTY=bob

diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile
@@ -1,4 +1,4 @@
-# doesn't support linux/arm64
+ARG version=1.5.0b0
 
 FROM alpine:3.14 as source-tree
 
@@ -9,6 +9,7 @@ RUN git init \
   && rm -rf .git \
   && rm -r /repo/docker
 
+
 FROM secretflow/devcontainer-web:latest as build
 
 ENV CI=true
@@ -23,14 +24,14 @@ RUN pnpm fetch
 COPY --from=source-tree /repo /repo
 
 RUN pnpm install --frozen-lockfile --offline
-# RUN rye sync --no-dev --no-lock --all-features
 
 RUN pnpm exec nx run-many -t build:js -t build:py
 
 RUN apt-get update && apt-get install -y rsync
 RUN rsync -avm --include='pyprojects/**/dist/*.whl' -f 'hide,! */' /repo/./ /dist
 
-FROM secretflow/secretflow-anolis8:1.3.0b0
+
+FROM secretflow/secretflow-anolis8:${version}
 
 COPY --from=build /dist /dist
 
@@ -42,13 +43,12 @@ RUN curl -fsSL https://rpm.nodesource.com/setup_18.x | bash -
 RUN yum install -y nodejs
 RUN npm install @difizen/libro-analyzer
 
-COPY ./docker/app/root/scripts /root/scripts
-COPY ./docker/app/root/.jupyter /root/.jupyter
+COPY ./docker/app/root/ /root/
 
 RUN mkdir workspace
 
 ENV SELF_PARTY=alice
-ENV ALL_PARTIES=alice
+ENV ALL_PARTIES=alice,bob
 
 EXPOSE 8888
 

diff --git a/docker/app/Dockerfile.dev b/docker/app/Dockerfile.dev
diff --git a/docker/sim/docker-compose.yml b/docker/sim/docker-compose.yml
@@ -1,6 +1,6 @@
 services:
   alice:
-    image: 'secretflow/secretnote:unstable-amd64'
+    image: 'secretflow/secretnote:1.3.0-amd64'
     platform: linux/amd64
     environment:
       - SELF_PARTY=alice
@@ -11,7 +11,7 @@ services:
     entrypoint: /root/scripts/start.sh
 
   bob:
-    image: 'secretflow/secretnote:unstable-amd64'
+    image: 'secretflow/secretnote:1.3.0-amd64'
     platform: linux/amd64
     environment:
       - SELF_PARTY=bob

diff --git a/docs/guide/secretnote-sf.md b/docs/guide/secretnote-sf.md
@@ -4,14 +4,14 @@
 
 在一台机器上使用 docker compose 启动两个容器，容器启动时会安装 SecretFLow、SecretNote，并且分别启动 Ray 服务以及 SecretNote 服务。
 
-**注意：当前 docker 镜像暂时不支持 Mac M1、M2 机器以及 ARM 架构机器**
+**注意：当前 docker 镜像提供 1.3.0-amd64 和 1.5.0-amd64 两个版本，和 secretflow 的版本是对应的，也就是说，如果你想使用 secretflow 1.5 版本，那就使用镜像 secretflow/secretnote:1.5.0-amd64**
 
 在一个文件夹下面新建文件 `docker-compose.yml`，内容如下：
 
 ```yml
 services:
   alice:
-    image: 'secretflow/secretnote:unstable-amd64'
+    image: 'secretflow/secretnote:1.3.0-amd64'
     platform: linux/amd64
     environment:
       - SELF_PARTY=alice
@@ -23,7 +23,7 @@ services:
       - /root/scripts
 
   bob:
-    image: 'secretflow/secretnote:unstable-amd64'
+    image: 'secretflow/secretnote:1.3.0-amd64'
     platform: linux/amd64
     environment:
       - SELF_PARTY=bob

diff --git a/package.json b/package.json
@@ -16,7 +16,9 @@
     "ci:setup": ":",
     "ci:check:javascript": "nx run-many -t format:prettier -t lint:eslint -t lint:stylelint -t test:vitest -t typecheck:tsc",
     "ci:check:python": "nx run-many -t format:black -t lint:ruff -t test:pytest -t typecheck:pyright",
-    "ci:build": "nx run-many -t build:js -t build:py"
+    "ci:build": "nx run-many -t build:js -t build:py",
+    "docker:build": "docker build --build-arg version=1.3.0b0 -t secretflow/secretnote:1.3.0-amd64 -f ./docker/app/Dockerfile .",
+    "docker:up": "docker-compose -f ./docker/sim/docker-compose.yml up"
   },
   "nx": {},
   "devDependencies": {

diff --git a/pyprojects/secretnote/package.json b/pyprojects/secretnote/package.json
@@ -4,13 +4,13 @@
   "files": [
     "static"
   ],
-  "version": "0.0.5",
+  "version": "0.0.6",
   "type": "module",
   "scripts": {
     "format:black": "python -m black --check src tests",
     "lint:ruff": "python -m ruff check src tests",
     "test:pytest": "python -m pytest",
-    "typecheck:pyright": "pyright --project ../.. src tests",
+    "typecheck:pyright": "pyright --project ../.. src/secretnote/server tests",
     "dev": "NODE_ENV=development python -m secretnote.server --config=./.jupyter/config_dev.py --debug --no-browser",
     "build:py": "rye build --out ./dist",
     "publish:py": "python -m hatch publish ./dist",

diff --git a/pyprojects/secretnote/pyproject.toml b/pyprojects/secretnote/pyproject.toml
@@ -24,14 +24,14 @@ dependencies = [
   "orjson>=3.9.9",
   "packaging>=23.1",
   "pydantic>=1.10, <2",
-  "ray[default]~=2.6.3",         # FIXME: ????????????
-  "secretflow-ray==2.2.0",
   "rich>=13.7.0",
   "stack_data>=0.6.3",
   "tqdm>=4.66.1",
   "dataset>=1.6.2",
   "ruff-lsp>=0.0.53",
   "python-lsp-server==1.8.2",
+  # https://github.com/python/typing_extensions/issues/369#issuecomment-2051331676
+  "typing_extensions>=4.6",
 ]
 description = "Notebook suite for SecretFlow"
 dynamic = ["version"]
@@ -58,7 +58,6 @@ dev-dependencies = [
   "matplotlib>=3.7.3",
   "opentelemetry-exporter-otlp-proto-grpc>=1.20.0",
   "pandas>=1.5.3, <2.0.0",
-  "secretflow-lite==1.3.0b0",
   "statsmodels>=0.14.0",
   "torch>=2.1.0",
   "torchmetrics>=1.2.0",