[feat] (#32)

1. local_data_statistic新增。
2. vertical_linear_regression新增。
3. horizontal_binning_woe_iv新增。
(包括单元测试,config,demo和算子代码)

Co-authored-by: chen.peng <chen.peng@basebit.ai>

demo/horizontal/binning_woe_iv/2party/config/fed_conf.json

@@ -0,0 +1,18 @@
+{
+ "fed_info": {
+ "scheduler": {
+ "scheduler": "localhost:55001"
+ },
+ "trainer": {
+ "node-1": "localhost:56001",
+ "node-2": "localhost:56002"
+ },
+ "assist_trainer": {
+ "assist_trainer": "localhost:55002"
+ }
+ },
+ "redis_server": "localhost:6379",
+ "grpc": {
+ "use_tls": false
+ } 
+}

demo/horizontal/binning_woe_iv/2party/config/trainer_config_assist_trainer.json

@@ -0,0 +1,33 @@
+[
+ {
+ "identity": "assist_trainer",
+ "model_info": {
+ "name": "horizontal_binning_woe_iv"
+ },
+ "output": {
+ "path": "/opt/checkpoints/[JOB_ID]/[NODE_ID]",
+ "result": {
+ "name": "woe_iv_result_[STAGE_ID].json"
+ }
+ },
+ "train_info": {
+ "train_params": {
+ "aggregation": {
+ "encryption": {
+ "method": "otp",
+ "key_bitlength": 64,
+ "data_type": "torch.Tensor",
+ "key_exchange": {
+ "key_bitlength": 3072,
+ "optimized": true
+ },
+ "csprng": {
+ "name": "hmac_drbg",
+ "method": "sha512"
+ }
+ }
+ }
+ }
+ }
+ }
+]

demo/horizontal/binning_woe_iv/2party/config/trainer_config_node-1.json

@@ -0,0 +1,43 @@
+[
+ {
+ "identity": "label_trainer",
+ "model_info": {
+ "name": "horizontal_binning_woe_iv"
+ },
+ "input": {
+ "trainset": [
+ {
+ "type": "csv",
+ "path": "./dataset/breast_cancer_wisconsin_horizontal/2party",
+ "name": "breast_cancer_wisconsin_horizontal_1.csv",
+ "has_id": true,
+ "has_label": true
+ }
+ ]
+ },
+ "train_info": {
+ "train_params": {
+ "aggregation": {
+ "encryption": {
+ "method": "otp",
+ "key_bitlength": 64,
+ "data_type": "torch.Tensor",
+ "key_exchange": {
+ "key_bitlength": 3072,
+ "optimized": true
+ },
+ "csprng": {
+ "name": "hmac_drbg",
+ "method": "sha512"
+ }
+ },
+ "weight": 1
+ },
+ "binning": {
+ "method": "equal_width",
+ "bins": 5
+ }
+ }
+ }
+ }
+]

demo/horizontal/binning_woe_iv/2party/config/trainer_config_node-2.json

@@ -0,0 +1,43 @@
+[
+ {
+ "identity": "label_trainer",
+ "model_info": {
+ "name": "horizontal_binning_woe_iv"
+ },
+ "input": {
+ "trainset": [
+ {
+ "type": "csv",
+ "path": "./dataset/breast_cancer_wisconsin_horizontal/2party",
+ "name": "breast_cancer_wisconsin_horizontal_2.csv",
+ "has_id": true,
+ "has_label": true
+ }
+ ]
+ },
+ "train_info": {
+ "train_params": {
+ "aggregation": {
+ "encryption": {
+ "method": "otp",
+ "key_bitlength": 64,
+ "data_type": "torch.Tensor",
+ "key_exchange": {
+ "key_bitlength": 3072,
+ "optimized": true
+ },
+ "csprng": {
+ "name": "hmac_drbg",
+ "method": "sha512"
+ }
+ },
+ "weight": 1
+ },
+ "binning": {
+ "method": "equal_width",
+ "bins": 5
+ }
+ }
+ }
+ }
+]

demo/horizontal/binning_woe_iv/2party/run.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+# Copyright 2022 The XFL Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "$(uname)" = "Darwin" ]; then
+ export PROJECT_HOME=$(greadlink -f ../../../../)
+ echo "PROJECT_HOME:""$PROJECT_HOME"
+elif [ "$(uname -s)" = "Linux" ]; then
+ export PROJECT_HOME=$(readlink -f ../../../../)
+ echo "PROJECT_HOME:""$PROJECT_HOME"
+fi
+
+export PYTHONPATH=$PYTHONPATH:$PROJECT_HOME/python:$PROJECT_HOME/python/common/communication/gRPC/python
+
+datapath="${PROJECT_HOME}/dataset"
+if [ ! -d "${PROJECT_HOME}/dataset/breast_cancer_wisconsin_horizontal/2party" ]; then
+ if [ ! -f "${PROJECT_HOME}/python/xfl.py" ]; then
+ python "${PROJECT_HOME}/common/dataset/breast_cancer_wisconsin.py" --mode "horizontal" --splits 2 --party "1" "2"
+ else
+ python "${PROJECT_HOME}/python/common/dataset/breast_cancer_wisconsin.py" --mode "horizontal" --splits 2 --party "1" "2"
+ fi
+fi
+
+type="horizontal"
+operator="binning_woe_iv"
+party="2party"
+code="${type}.${operator}.${party}"
+config_path="${PROJECT_HOME}/demo/${type}/${operator}/${party}/config"
+
+if [ ! -f "${PROJECT_HOME}/python/xfl.py" ]; then
+ EXECUTE_PATH=${PROJECT_HOME}/xfl.py
+else
+ EXECUTE_PATH=${PROJECT_HOME}/python/xfl.py
+fi
+
+cd $PROJECT_HOME
+python "$EXECUTE_PATH" -t node-1 --config_path ${config_path} &
+sleep 1
+python "$EXECUTE_PATH" -t node-2 --config_path ${config_path} &
+sleep 1
+python "$EXECUTE_PATH" -a --config_path ${config_path} &
+sleep 1
+python "$EXECUTE_PATH" -s --config_path ${config_path} &
+sleep 1
+python "$EXECUTE_PATH" -c start --config_path ${config_path} &

demo/horizontal/binning_woe_iv/2party/stop.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# Copyright 2022 The XFL Authors. All rights reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+pid=$(ps -ef | grep xfl | grep -v grep | awk '{print $2}')
+echo "XFL Process ${pid} killed"
+ps -ef | grep python | grep -v grep | awk '{print $2}' | xargs kill -9;

demo/local/data_statistic/1party/config/fed_conf.json

@@ -0,0 +1,14 @@
+{
+ "fed_info": {
+ "scheduler": {
+ "node-1": "localhost:55001"
+ },
+ "trainer": {
+ "node-1": "localhost:56001"
+ }
+ },
+ "redis_server": "localhost:6379",
+ "grpc": {
+ "use_tls": false
+ } 
+}

demo/local/data_statistic/1party/config/trainer_config_node-1.json

@@ -0,0 +1,34 @@
+[
+ {
+ "identity": "label_trainer",
+ "model_info": {
+ "name": "local_data_statistic"
+ },
+ "input": {
+ "dataset": [
+ {
+ "type": "csv",
+ "path": "./dataset/breast_cancer_wisconsin_vertical/2party",
+ "name": "breast_cancer_wisconsin_vertical_labeled_train.csv",
+ "has_label": true,
+ "has_id": true
+ }
+ ]
+ },
+ "output": {
+ "path": "/opt/checkpoints/[JOB_ID]/[NODE_ID]",
+ "summary": {
+ "name": "data_summary_[STAGE_ID].json"
+ }
+ },
+ "train_info": {
+ "train_params": {
+ "quantile": [
+ 0.5,
+ 0.8,
+ 0.9
+ ]
+ }
+ }
+ }
+]

demo/local/data_statistic/1party/run.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# Copyright 2022 The XFL Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "$(uname)" = "Darwin" ]; then
+ export PROJECT_HOME=$(greadlink -f ../../../../)
+ echo "PROJECT_HOME:""$PROJECT_HOME"
+elif [ "$(uname -s)" = "Linux" ]; then
+ export PROJECT_HOME=$(readlink -f ../../../../)
+ echo "PROJECT_HOME:""$PROJECT_HOME"
+fi
+
+export PYTHONPATH=$PYTHONPATH:$PROJECT_HOME/python:$PROJECT_HOME/python/common/communication/gRPC/python
+
+datapath="${PROJECT_HOME}/dataset"
+if [ ! -d "${PROJECT_HOME}/dataset/breast_cancer_wisconsin_vertical/2party" ]; then
+ if [ ! -f "${PROJECT_HOME}/python/xfl.py" ]; then
+ python "${PROJECT_HOME}/common/dataset/breast_cancer_wisconsin.py" --mode "vertical" --splits 2 --party "labeled" "1"
+ else
+ python "${PROJECT_HOME}/python/common/dataset/breast_cancer_wisconsin.py" --mode "vertical" --splits 2 --party "labeled" "1"
+ fi
+fi
+
+type="local"
+operator="data_statistic"
+party="1party"
+code="${type}.${operator}.${party}"
+config_path="${PROJECT_HOME}/demo/${type}/${operator}/${party}/config"
+
+if [ ! -f "${PROJECT_HOME}/python/xfl.py" ]; then
+ EXECUTE_PATH=${PROJECT_HOME}/xfl.py
+else
+ EXECUTE_PATH=${PROJECT_HOME}/python/xfl.py
+fi
+
+cd $PROJECT_HOME
+python "$EXECUTE_PATH" -s --config_path ${config_path} &
+sleep 1
+python "$EXECUTE_PATH" -t node-1 --config_path ${config_path} &
+sleep 1
+python "$EXECUTE_PATH" -c start --config_path ${config_path} &

demo/local/data_statistic/1party/stop.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# Copyright 2022 The XFL Authors. All rights reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+pid=$(ps -ef | grep xfl | grep -v grep | awk '{print $2}')
+echo "XFL Process ${pid} killed"
+ps -ef | grep xfl | grep -v grep | awk '{print $2}' | xargs kill -9

demo/vertical/linear_regression/2party/config/fed_conf.json

@@ -0,0 +1,18 @@
+{
+ "fed_info": {
+ "scheduler": {
+ "node-1": "localhost:55001"
+ },
+ "trainer": {
+ "node-1": "localhost:56001",
+ "node-2": "localhost:56002"
+ },
+ "assist_trainer": {
+ "assist_trainer": "localhost:55002"
+ }
+ },
+ "redis_server": "localhost:6379",
+ "grpc": {
+ "use_tls": false
+ } 
+}

demo/vertical/linear_regression/2party/config/trainer_config_assist_trainer.json

@@ -0,0 +1,8 @@
+[
+ {
+ "identity": "assist_trainer",
+ "model_info": {
+ "name": "vertical_linear_regression"
+ }
+ }
+]