chore: optimize docker image build and libro's deps version control (#60

) * docs: update docs * chore: debug ci * chore: debug ci * chore: optimize docker build and version lock * refactor: handle ci errors * refactor: fix ci issues
secretflow · Sep 11, 2024 · 7af9f74 · 7af9f74
1 parent 880bbb2
commit 7af9f74
Show file tree

Hide file tree

Showing 25 changed files with 3,573 additions and 3,336 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -40,12 +40,12 @@ jobs:
 # Orchestrate jobs using workflows
 # See: https://circleci.com/docs/configuration-reference/#workflows
 workflows:
-  devcontainer-publish-1.6.1:
+  devcontainer-publish-1.9.0b0:
     jobs:
       - devcontainer-publish:
           filters:
             branches:
               only:
                 - main
           platform: linux/amd64
-          tag: 1.6.1
+          tag: 1.9.0b0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         node-version: ['18', '20']
-        python-version: ['3.8']
+        python-version: ['3.10']
 
   ci-python:
     name: 'CI: Python'
@@ -32,4 +32,4 @@ jobs:
       fail-fast: false
       matrix:
         node-version: ['18']
-        python-version: ['3.8']
+        python-version: ['3.10']
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -39,7 +39,7 @@ jobs:
 
     with:
       node-version: '18'
-      python-version: '3.8'
+      python-version: '3.10'
 
     permissions:
       contents: write

diff --git a/.npmrc b/.npmrc
@@ -1,4 +1,5 @@
 registry=https://registry.npmmirror.com
+disturl=https://npmmirror.com/mirrors/node
 
 # PNPM
 strict-peer-dependencies=false

diff --git a/.prettierignore b/.prettierignore
@@ -29,3 +29,6 @@ pnpm-lock.yaml
 
 # template
 **/template
+
+# JSON
+**/*.json
diff --git a/README.md b/README.md
@@ -1,11 +1,17 @@
-## 介绍
+<h1 align="center">
+SecretNote
+</h1>
 
-SecretNote 是专为隐语开发者打造的高级工具套件。以 notebook 的形式呈现，支持多节点代码执行和文件管理，同时提供运行状态追踪功能，能较大程度提升开发者的效率和工作体验。
+SecretNote 是为 SecretFlow (隐语) 学习者和开发者定制的高级工具套件，可帮助您快速开展隐语实验。它以 notebook 的形式呈现，支持多节点代码执行和文件管理，同时提供运行状态追踪功能，能较大程度提升开发者的效率和工作体验。
 
 **由于系统安全等问题，SecretNote 不是为生产设计的，请不要直接在生产环境中使用。**
 
 ## 使用
 
+### 隐语实训平台云 SecretNote
+
+SecretNote 现在隐语实训平台提供云上版本，开箱即用，无需环境配置直接拉起计算节点，欢迎 [立即体验](https://www.secret-flow.com/secretnote)。
+
 ### 与 SecretFlow 一起使用
 
 SecretNote 有单独的 [pip](https://pypi.org/project/secretnote/) 安装包，可以单独使用。为了避免安装、部署、启动等环境问题，推荐使用 docker 方式启动 SecretFlow 运行环境。

diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile
@@ -1,61 +1,50 @@
-FROM alpine:3.14 as source-tree
+# This Dockerfile will build the secretflow/secretnote image for depolying a node
+# for SecretNote.
 
+# Get the repository and remove unncessary files
+FROM alpine:3.14 AS source-tree
 RUN apk add --no-cache git
 COPY . /repo
 RUN git init \
   && git clean -fX . \
   && rm -rf .git \
   && rm -r /repo/docker
 
-
-FROM secretflow/devcontainer-web:amd64-v2 as build
-
+# Build the project
+FROM nikolaik/python-nodejs:python3.10-nodejs18 AS build
 ENV CI=true
-
 USER root
 WORKDIR /repo
-
+RUN npm config set registry https://registry.npmmirror.com
+RUN npm install -g pnpm@9.4.0
+RUN pip install build --break-system-packages
 COPY --from=source-tree /repo/pnpm-lock.yaml /repo/.npmrc /repo/
-
 RUN pnpm fetch
-
 COPY --from=source-tree /repo /repo
-
-RUN pnpm install --no-frozen-lockfile
-
-RUN cargo install gcc
-RUN pnpm exec nx run-many -t build:js -t build:py
-
+RUN pnpm install --frozen-lockfile
+RUN pnpm run ci:build
 RUN apt-get update && apt-get install -y rsync
 RUN rsync -avm --include='pyprojects/**/dist/*.whl' -f 'hide,! */' /repo/./ /dist
 
-
-FROM secretflow/secretflow-anolis8:1.6.1b0
-
+# Config the runtime
+# TODO Update the version here
+FROM secretflow/secretflow-anolis8:1.9.0b0
+RUN python --version
 RUN useradd -m secretnote
-
 COPY --from=build /dist /dist
-RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
 RUN pip install /dist/pyprojects/secretnote/dist/*.whl
-
 WORKDIR /home/secretnote
-
 COPY ./docker/app/root/ ./
-
 RUN curl -fsSL https://rpm.nodesource.com/setup_18.x | bash -
 RUN yum install -y nodejs
 RUN npm config set registry https://registry.npmmirror.com
-RUN npm install @difizen/libro-analyzer
-
+RUN npm install @difizen/libro-analyzer@0.1.2
 RUN mkdir workspace
-
 RUN chown -R secretnote:secretnote /home/secretnote
 USER secretnote
 
-
+# Start the project
 ENV SELF_PARTY=alice
 ENV ALL_PARTIES=alice,bob
-
 EXPOSE 8888
-
 ENTRYPOINT [ "/home/secretnote/scripts/start.sh" ]
diff --git a/nx.json b/nx.json
@@ -28,10 +28,6 @@
       ],
       "cache": true
     },
-    "lint:ruff": {
-      "inputs": [":default", ":python"],
-      "cache": true
-    },
     "format:prettier": {
       "inputs": ["{workspaceRoot}/**/*"],
       "cache": false

diff --git a/package.json b/package.json
@@ -15,9 +15,9 @@
     "format:prettier": "prettier --check --ignore-unknown .",
     "ci:setup": ":",
     "ci:check:javascript": "nx run-many -t format:prettier -t lint:eslint -t lint:stylelint -t test:vitest -t typecheck:tsc",
-    "ci:check:python": "nx run-many -t format:black -t lint:ruff -t test:pytest -t typecheck:pyright",
-    "ci:build": "nx run-many -t build:js -t build:py",
-    "docker:build": "docker build -t secretflow/secretnote:1.6.1-amd64 -f ./docker/app/Dockerfile .",
+    "ci:check:python": "nx run-many -t format:black -t test:pytest -t typecheck:pyright",
+    "ci:build": "nx run-many -t build:js && nx run-many -t build:py:ci",
+    "docker:build": "docker build -t secretflow/secretnote:1.9.0b0 -f ./docker/app/Dockerfile .",
     "docker:up": "docker-compose -f ./docker/sim/docker-compose.yml up"
   },
   "nx": {},
@@ -66,5 +66,16 @@
   "engines": {
     "node": ">=18.18.0"
   },
-  "packageManager": "pnpm@8.7.6"
+  "packageManager": "pnpm@9.4.0",
+  "pnpm": {
+    "overrides": {
+      "@difizen/mana-app": "0.1.23",
+      "@difizen/mana-core": "0.1.23",
+      "@difizen/mana-common": "0.1.23",
+      "@difizen/mana-l10n": "0.1.23",
+      "@difizen/mana-observable": "0.1.23",
+      "@difizen/mana-react": "0.1.23",
+      "@difizen/mana-syringe": "0.1.23"
+    }
+  }
 }
diff --git a/packages/secretnote-lite/package.json b/packages/secretnote-lite/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@alipay/secretnote-lite",
-  "version": "0.0.42",
+  "version": "0.0.43",
   "license": "Apache-2.0",
   "author": "vectorse@126.com",
   "repository": "https://github.com/secretflow/secretnote/tree/main/packages/secretnote",
@@ -22,12 +22,33 @@
   },
   "dependencies": {
     "@antv/s2": "^1.52.0",
-    "@difizen/libro-jupyter": "^0.2.20",
-    "@difizen/libro-language-client": "^0.2.20",
-    "@difizen/mana-app": "^0.1.19",
-    "@difizen/mana-common": "^0.1.19",
-    "@difizen/mana-l10n": "^0.1.19",
-    "@difizen/mana-react": "^0.1.19",
+    "@difizen/mana-app": "0.1.23",
+    "@difizen/mana-common": "0.1.23",
+    "@difizen/mana-l10n": "0.1.23",
+    "@difizen/mana-react": "0.1.23",
+    "@difizen/libro-jupyter": "0.2.24",
+    "@difizen/libro-language-client": "0.2.24",
+    "@difizen/libro-code-cell": "0.2.24",
+    "@difizen/libro-code-editor": "0.2.24",
+    "@difizen/libro-codemirror": "0.2.24",
+    "@difizen/libro-cofine-editor": "0.2.24",
+    "@difizen/libro-cofine-editor-core": "0.2.24",
+    "@difizen/libro-cofine-editor-contribution": "0.2.24",
+    "@difizen/libro-cofine-textmate": "0.2.24",
+    "@difizen/libro-common": "0.2.24",
+    "@difizen/libro-core": "0.2.24",
+    "@difizen/libro-kernel": "0.2.24",
+    "@difizen/libro-l10n": "0.2.24",
+    "@difizen/libro-lsp": "0.2.24",
+    "@difizen/libro-markdown": "0.2.24",
+    "@difizen/libro-markdown-cell": "0.2.24",
+    "@difizen/libro-output": "0.2.24",
+    "@difizen/libro-raw-cell": "0.2.24",
+    "@difizen/libro-rendermime": "0.2.24",
+    "@difizen/libro-search": "0.2.24",
+    "@difizen/libro-search-code-cell": "0.2.24",
+    "@difizen/libro-shared-model": "0.2.24",
+    "@difizen/libro-virtualized": "0.2.24",
     "@kanaries/web-data-loader": "^0.1.7",
     "@lumino/polling": "^2.1.3",
     "@tiptap/core": "^2.1.11",
@@ -54,6 +75,7 @@
     "markdown-it": "^14.1.0",
     "monaco-editor": "^0.45.0",
     "monaco-editor-webpack-plugin": "^7.1.0",
+    "plotly.js": "2.34.0",
     "styled-components": "^6.1.0",
     "tippy.js": "^6.3.7",
     "tiptap-markdown": "^0.8.2",

diff --git a/packages/secretnote-lite/src/modules/editor/contents/contents-contrib.ts b/packages/secretnote-lite/src/modules/editor/contents/contents-contrib.ts
@@ -28,6 +28,7 @@ export class SecretNoteContentContribution implements ContentContribution {
       secretNoteModel.currentFileContents = currentFileContents;
       secretNoteModel.filePath = currentFileContents.path;
 
+      /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */
       // @ts-ignore
       if (!secretNoteModel.quickEditMode && !secretNoteModel.readOnly) {
         secretNoteModel.startKernelConnection();

diff --git a/packages/secretnote-lite/src/modules/file/preview-view.tsx b/packages/secretnote-lite/src/modules/file/preview-view.tsx
@@ -44,6 +44,7 @@ export class FilePreviewView extends BaseView {
 
   constructor(
     @inject(FileService) fileService: FileService,
+    /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */
     // @ts-ignore
     @contrib(FilePreviewContribution)
     providers: Contribution.Provider<FilePreviewContribution>,

diff --git a/packages/secretnote-lite/src/modules/layout/index.less b/packages/secretnote-lite/src/modules/layout/index.less
@@ -47,8 +47,8 @@
 }
 
 .secretnote-header {
-  background: #fff;
   border-bottom: 1px solid var(--mana-secretnote-divider-color);
+  background: #fff;
 
   .logo {
     display: inline-flex;
@@ -76,15 +76,16 @@
     position: absolute;
     bottom: 0;
     width: 100%;
+    padding: 12px;
     border-top: 1px solid var(--mana-secretnote-divider-color);
     text-align: center;
-    padding: 12px;
   }
 
   .about-bar {
     a.ant-typography {
       color: var(--custom-blue-400) !important;
     }
+
     .icon-container {
       line-height: 0;
     }

diff --git a/packages/secretnote-lite/src/modules/layout/sidebar.tsx b/packages/secretnote-lite/src/modules/layout/sidebar.tsx
@@ -114,6 +114,7 @@ export class SideBarView extends DefaultSlotView {
   readonly providers: Contribution.Provider<SideBarContribution>;
 
   constructor(
+    /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */
     // @ts-ignore
     @contrib(SideBarContribution)
     providers: Contribution.Provider<SideBarContribution>,

diff --git a/packages/secretnote-lite/src/modules/metrics/index.less b/packages/secretnote-lite/src/modules/metrics/index.less
@@ -11,11 +11,11 @@
   }
 
   .server-name {
+    margin-bottom: 4px;
     color: var(--mana-secretnote-text-color);
     font-size: 14px;
     font-weight: 600;
     text-align: left;
-    margin-bottom: 4px;
   }
 
   .placeholder {

diff --git a/packages/secretnote-lite/src/modules/metrics/view.tsx b/packages/secretnote-lite/src/modules/metrics/view.tsx
@@ -27,6 +27,7 @@ const MetricsComponent = () => {
     return () => {
       service.disable();
     };
+    /* eslint-disable-next-line react-hooks/exhaustive-deps */
   }, []);
 
   const formatMetric = (v: (typeof metrics)[1]) =>

diff --git a/packages/secretnote-lite/src/modules/node/index.less b/packages/secretnote-lite/src/modules/node/index.less
@@ -51,10 +51,10 @@
   padding: 16px;
 
   .title {
+    margin-bottom: 20px;
+    color: rgb(45 55 72 / 88%);
     font-size: 16px;
     font-weight: 600;
-    color: rgba(45, 55, 72, 0.88);
-    margin-bottom: 20px;
   }
 
   label {

diff --git a/packages/secretnote-lite/src/modules/notebook/drive.ts b/packages/secretnote-lite/src/modules/notebook/drive.ts
@@ -25,7 +25,7 @@ export const DriveName = 'SecretNoteContentsDrive';
 @singleton()
 export class SecretNoteContentsDrive implements IContentsDrive {
   readonly name: string = DriveName; // Name of drive, used at the leading component of file paths.
-  protected apiEndpoint: string = 'api/contents';
+  protected apiEndpoint = 'api/contents';
   protected _isDisposed = false;
   protected fileChangedEmitter = new Emitter<IContentsChangedArgs>();
 
@@ -96,6 +96,7 @@ export class SecretNoteContentsDrive implements IContentsDrive {
    * @override Drive._getSettings
    */
   protected _getSettings(...args: any) /* :ISettings */ {
+    /* eslint-disable-next-line @typescript-eslint/ban-ts-comment */
     // @ts-ignore
     return void 0;
   }

diff --git a/packages/secretnote-lite/src/modules/server/server-manager.ts b/packages/secretnote-lite/src/modules/server/server-manager.ts
@@ -47,14 +47,17 @@ export class SecretNoteServerManager {
     this.defaultServerManager = defaultServerManager;
     this.updateServerConnectionSettings();
     this.getResourcesAndVersions();
-    this.getServerList().then((servers) => {
-      // The default ServerManager must be launched to get LSP to work.
-      // If there exists servers at the beginning, just launch directly.
-      // Otherwise postpone the launch until a server is added.
-      servers?.length &&
-        !this.defaultServerManager.loaded &&
-        this.defaultServerManager.launch();
-    });
+    this.getServerList()
+      .then((servers) => {
+        // The default ServerManager must be launched to get LSP to work.
+        // If there exists servers at the beginning, just launch directly.
+        // Otherwise postpone the launch until a server is added.
+        servers?.length &&
+          !this.defaultServerManager.loaded &&
+          this.defaultServerManager.launch();
+        return;
+      })
+      .catch((e) => genericErrorHandler(e, { silent: true }));
   }
 
   /**
@@ -228,6 +231,7 @@ export class SecretNoteServerManager {
     try {
       const data = await Promise.race([
         request('api/kernelspecs', {}, server.id),
+        /* eslint-disable-next-line promise/param-names */
         new Promise((_, reject) => {
           setTimeout(() => reject(new Error('timeout')), 5000);
         }),