Merge pull request #5684 from FederatedAI/dev-2.2.0

Merge FATE-v2.2.0's Upgrades
FederatedAI · Jul 31, 2024 · d4d8dd8 · d4d8dd8
2 parents 0e36edc + 64ad802
commit d4d8dd8
Show file tree

Hide file tree

Showing 18 changed files with 65 additions and 56 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -5,15 +5,15 @@
 [submodule "fate_client"]
  path = fate_client
  url = https://github.com/FederatedAI/FATE-Client
- branch = v2.1.1
+ branch = v2.2.0
 [submodule "fate_flow"]
  path = fate_flow
  url = https://github.com/FederatedAI/FATE-Flow
- branch = v2.1.0
+ branch = v2.2.0
 [submodule "fate_test"]
  path = fate_test
  url = https://github.com/FederatedAI/FATE-Test.git
- branch = v2.1.1
+ branch = v2.2.0
 [submodule "fate_board"]
  path = fate_board
  url = https://github.com/FederatedAI/FATE-Board.git

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,3 +1,10 @@
+## Release 2.2.0
+### Major Features and Improvements
+> Deployment
+* Upgrade from Python 3.8 to Python 3.10
+* Upgrade the PyTorch version to 2.x
+
+
 ## Release 2.1.1
 ### Major Features and Improvements
 > Component

diff --git a/bin/install_os_dependencies.sh b/bin/install_os_dependencies.sh
@@ -53,6 +53,9 @@ function ln_lib() {
 case "${system}" in
 "CentOS Linux")
  echo "CentOS System"
+ sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo
+ sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo
+ sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
  $command yum -y install gcc gcc-c++ make openssl-devel supervisor gmp-devel mpfr-devel libmpc-devel libaio numactl autoconf automake libtool libffi-devel snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan lsof
  ;;
 "Ubuntu")

diff --git a/deploy/standalone-deploy/README.md b/deploy/standalone-deploy/README.md
@@ -23,8 +23,8 @@ The single-node version provides 3 deployment methods, which can be selected bas
 - Prepare and install [conda](https://docs.conda.io/projects/miniconda/en/latest/) environment.
 - Create a virtual environment:
 ```shell
-# FATE requires Python >= 3.8
-conda create -n fate_env python=3.8
+# FATE requires Python >= 3.10
+conda create -n fate_env python=3.10
 conda activate fate_env
 ```
 
@@ -36,7 +36,7 @@ FATE-Flow provides federated job life cycle management, includes scheduling, dat
 
 ##### 2.2.1.1 Installing FATE, FATE-Flow, FATE-Client
 ```shell
-pip install fate_client[fate,fate_flow]==2.1.1
+pip install fate_client[fate,fate_flow]==2.2.0
 ```
 #### 2.2.1.2 Service Initialization
 ```shell
@@ -71,7 +71,7 @@ users can directly import fate and use built-in algorithms and secure protocols
 
 #### 2.2.2.1 Installing FATE
 ```shell
-pip install pyfate==2.1.1
+pip install pyfate==2.2.0
 ```
 #### 2.2.2.2 Using Guides
 Refer to [examples](../../doc/2.0/fate/ml)
@@ -90,13 +90,13 @@ Refer to [examples](../../doc/2.0/fate/ml)
 Set the necessary environment variables for deployment (note that environment variables set in this way are only valid for the current terminal session. If you open a new terminal session, such as logging in again or opening a new window, you will need to reset them).
 
 ```bash
-export version={FATE version number for this deployment, e.g., 2.1.1}
+export version={FATE version number for this deployment, e.g., 2.2.0}
 ```
 
 Example:
 
 ```bash
-export version=2.1.1
+export version=2.2.0
 ```
 
 ### 3.2 Pull Docker Images

diff --git a/deploy/standalone-deploy/README.zh.md b/deploy/standalone-deploy/README.zh.md
@@ -22,8 +22,8 @@
 - [conda](https://docs.conda.io/projects/miniconda/en/latest/) 环境准备及安装
 - 创建虚拟环境
 ```shell
-# fate的运行环境为python>=3.8
-conda create -n fate_env python=3.8
+# fate的运行环境为python>=3.10
+conda create -n fate_env python=3.10
 conda activate fate_env
 ```
 
@@ -35,7 +35,7 @@ FATE-Flow提供了联邦作业生命周期管理，包括调度、数据管理
 
 ##### 2.2.1.1 安装FATE、FATE-Flow、FATE-Client
 ```shell
-pip install fate_client[fate,fate_flow]==2.1.1
+pip install fate_client[fate,fate_flow]==2.2.0
 ```
 
 #### 2.2.1.2 服务初始化
@@ -70,7 +70,7 @@ FATE提供多种联邦算法和安全协议， 用户可以在安装 FATE 后直
 
 #### 2.2.1.1 安装 FATE
 ```shell
-pip install pyfate==2.1.1
+pip install pyfate==2.2.0
 ```
 
 #### 2.2.2.2 使用指引
@@ -92,13 +92,13 @@ pip install pyfate==2.1.1
 设置部署所需环境变量（注意，通过以下方式设置的环境变量仅在当前终端会话中有效。如果打开新的终端会话，例如重新登录或打开新窗口，请重新设置）。
 
 ```bash
-export version={本次部署的 FATE 版本号, 如 2.1.1}
+export version={本次部署的 FATE 版本号, 如 2.2.0}
 ```
 
 示例：
 
 ```bash
-export version=2.1.1
+export version=2.2.0
 ```
 
 ### 3.2 拉取镜像

diff --git a/doc/2.0/fate/dag.md b/doc/2.0/fate/dag.md
@@ -90,7 +90,7 @@ dag:
  max_depth: 3
  num_trees: 2
  ...
-schema_version: 2.1.1
+schema_version: 2.2.0
 kind: fate
 ```
 
@@ -309,7 +309,7 @@ dag:
  max_depth: 3
  num_trees: 2
  ...
-schema_version: 2.1.1
+schema_version: 2.2.0
 ```
 
 - Step1: Change the job-level stage in DAG to "predict"

diff --git a/doc/2.0/fate/dag.zh.md b/doc/2.0/fate/dag.zh.md
@@ -89,7 +89,7 @@ dag:
  max_depth: 3
  num_trees: 2
  ...
-schema_version: 2.1.1
+schema_version: 2.2.0
 kind: fate
 ```
 
@@ -302,7 +302,7 @@ dag:
  max_depth: 3
  num_trees: 2
  ...
-schema_version: 2.1.1
+schema_version: 2.2.0
 ```
 - Step1: 将dag下的全局job阶段的stage改成predict
 - Step2: 将用不到的组件从dag下的tasks，以及party_tasks进行删除，同时需要注意的是，删除组件可能会导致部分下游组件的dependent，以及输入发生改变，也需要对应修改。

diff --git a/doc/2.0/fate/homo_quick_start.md b/doc/2.0/fate/homo_quick_start.md
@@ -3,7 +3,7 @@
 1. install `fate_client` with extra package `fate` 
 
 ```sh
-python -m pip install -U pip && python -m pip install fate_client[fate,fate_flow]==2.1.1
+python -m pip install -U pip && python -m pip install fate_client[fate,fate_flow]==2.2.0
 ```
 after installing packages successfully, initialize fate_flow service and fate_client
 
@@ -141,4 +141,4 @@ predict_pipeline.predict()
 ```
 
 6. More tutorials
-More pipeline api guides can be found in this [link](https://github.com/FederatedAI/FATE-Client/blob/main/doc/pipeline.md)
+More pipeline api guides can be found in this [link](https://github.com/FederatedAI/FATE-Client/blob/main/doc/pipeline.md)
diff --git a/doc/2.0/fate/quick_start.md b/doc/2.0/fate/quick_start.md
@@ -3,7 +3,7 @@
 1. install `fate_client` with extra package `fate` 
 
 ```sh
-python -m pip install -U pip && python -m pip install fate_client[fate,fate_flow]==2.1.1
+python -m pip install -U pip && python -m pip install fate_client[fate,fate_flow]==2.2.0
 ```
 after installing packages successfully, initialize fate_flow service and fate_client
 
@@ -127,4 +127,4 @@ predict_pipeline.predict()
 ```
 
 6. More tutorials
-More pipeline api guides can be found in this [link](https://github.com/FederatedAI/FATE-Client/blob/main/doc/pipeline.md)
+More pipeline api guides can be found in this [link](https://github.com/FederatedAI/FATE-Client/blob/main/doc/pipeline.md)
diff --git a/doc/tutorial/README.md b/doc/tutorial/README.md
@@ -1 +1,4 @@
-aaa
+# Guide
+
+[Upload Data with FATE-Pipeline](./pipeline_tutorial_transform_local_file_to_dataframe.ipynb) 
+[Train & Predict Coordinated-LR with FATE-Pipeline](./pipeline_tutorial_hetero.ipynb)
diff --git a/fate_client b/fate_client
diff --git a/fate_flow b/fate_flow
diff --git a/fate_test b/fate_test
diff --git a/python/fate/_info.py b/python/fate/_info.py
@@ -12,5 +12,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "2.1.1"
+__version__ = "2.2.0"
 __provider__ = "fate"
diff --git a/python/requirements-eggroll.txt b/python/requirements-eggroll.txt
@@ -1,8 +1,8 @@
 opentelemetry-api
 opentelemetry-sdk
 protobuf==4.24.4
-grpcio==1.59.3
-grpcio-tools==1.59.3
+grpcio==1.62.1
+grpcio-tools==1.62.1
 cloudpickle==2.1.0
 psutil>=5.7.0
 pynvml==11.5.0
diff --git a/python/requirements-fate.txt b/python/requirements-fate.txt
@@ -1,26 +1,26 @@
 --extra-index-url https://download.pytorch.org/whl/cpu
 
-lmdb==1.3.0
-torch==1.13.1
+lmdb==1.4.1
+torch==2.3.1+cpu
 fate_utils
 pydantic==1.10.12
 cloudpickle==2.1.0
 click
 ruamel-yaml==0.16
-numpy
-pandas==2.0.3
+numpy<2.0.0
+pandas
 transformers==4.37.2
 accelerate==0.27.2
 beautifultable
 requests<2.26.0
-scikit-learn
+scikit-learn==1.4.1.post1
 omegaconf
-rich
+rich==13.7.1
 opentelemetry-api
 opentelemetry-sdk
 opentelemetry-exporter-otlp-proto-grpc
 mmh3==3.0.0
 protobuf==4.24.4
-grpcio==1.59.3
+grpcio==1.62.1
 safetensors==0.4.1
-scipy
+scipy
diff --git a/python/requirements-pulsar.txt b/python/requirements-pulsar.txt
@@ -1,3 +1 @@
-pulsar-client==3.3.0; sys_platform == 'darwin'
-pulsar-client==2.10.2; sys_platform != 'darwin'
-urllib3==1.26.18
+pulsar-client==3.4.0
diff --git a/python/setup.py b/python/setup.py
@@ -20,41 +20,39 @@
 
 # Base requirements
 install_requires = [
- "lmdb==1.3.0",
- "torch==1.13.1",
+ "lmdb==1.4.1",
+ "torch==2.3.1",
  "fate_utils",
  "pydantic==1.10.12",
  "cloudpickle==2.1.0",
  "click",
  "ruamel.yaml==0.16",
- "scikit-learn",
- "numpy",
+ "scikit-learn==1.4.1.post1",
+ "numpy<2.0.0",
  "pandas",
- "transformers",
- "accelerate",
+ "transformers==4.37.2",
+ "accelerate==0.27.2",
  "beautifultable",
- "requests",
- "grpcio",
- "protobuf",
- "rich",
+ "requests<2.26.0",
+ "grpcio==1.62.1",
+ "protobuf==4.24.4",
+ "rich==13.7.1",
  "omegaconf",
  "opentelemetry-api",
  "opentelemetry-sdk",
  "mmh3==3.0.0",
- "safetensors",
+ "safetensors==0.4.1",
 ]
 
 # Extra requirements
 extras_require = {
  "rabbitmq": ["pika"],
  "pulsar": [
- "pulsar-client==2.10.2; sys_platform != 'darwin'",
- "pulsar-client; sys_platform == 'darwin'",
- "urllib3",
+ "pulsar-client==3.4.0",
  ],
  "spark": ["pyspark"],
  "eggroll": [
- "grpcio-tools",
+ "grpcio-tools==1.62.1",
  "psutil>=5.7.0",
  ],
  "all": ["pyfate[rabbitmq,pulsar,spark,eggroll]"],