Skip to content

Commit

Permalink
fix merge
Browse files Browse the repository at this point in the history 
Signed-off-by: weiwee <wbwmat@gmail.com>
  • Loading branch information
@sagewe
sagewe committed Jul 21, 2023
2 parents 2667650 + 70e11dc commit 7ef13b8
Show file tree
Hide file tree
Showing 355 changed files with 8,346 additions and 4,149 deletions.
63 changes: 39 additions & 24 deletions examples/pipeline/test_selection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@

pipeline = FateFlowPipeline().set_roles(guest="9999", host="9998", arbiter="9998")

intersection_0 = Intersection("intersection_0",
method="raw")
intersection_0.guest.component_setting(input_data=DataWarehouseChannel(name="breast_hetero_guest_sid",
namespace="experiment"))
intersection_0.hosts[0].component_setting(input_data=DataWarehouseChannel(name="breast_hetero_host_sid",
namespace="experiment"))
intersection_0 = Intersection("intersection_0", method="raw")
intersection_0.guest.component_setting(
input_data=DataWarehouseChannel(name="breast_hetero_guest_sid", namespace="experiment")
)
intersection_0.hosts[0].component_setting(
input_data=DataWarehouseChannel(name="breast_hetero_host_sid", namespace="experiment")
)

"""
intersection_1 = Intersection("intersection_1",
Expand All @@ -37,21 +38,35 @@
namespace="experiment_64"))
"""

statistics_0 = Statistics("statistics_0", input_data=intersection_0.outputs["output_data"],
metrics=["mean", "max", "std", "var", "kurtosis", "skewness"])
binning_0 = HeteroFeatureBinning("binning_0", train_data=intersection_0.outputs["output_data"],
method="quantile", n_bins=5, bin_col=None, category_col=None,
skip_metrics=False, transform_method="woe", local_only=False)
statistics_0 = Statistics(
"statistics_0",
input_data=intersection_0.outputs["output_data"],
metrics=["mean", "max", "std", "var", "kurtosis", "skewness"],
)
binning_0 = HeteroFeatureBinning(
"binning_0",
train_data=intersection_0.outputs["output_data"],
method="quantile",
n_bins=5,
bin_col=None,
category_col=None,
skip_metrics=False,
transform_method="woe",
local_only=False,
)

selection_0 = HeteroFeatureSelection("selection_0",
train_data=intersection_0.outputs["output_data"],
method=["iv"],
input_models=[binning_0.outputs["output_model"],
# statistics_0.outputs["output_model"]
],
iv_param={"select_federated": True, "threshold": 3},
statistic_param={"metrics": ["mean", "max", "kurtosis", "skewness"]},
manual_param={"filter_out_col": ["x0", "x3"]})
selection_0 = HeteroFeatureSelection(
"selection_0",
train_data=intersection_0.outputs["output_data"],
method=["iv"],
input_models=[
binning_0.outputs["output_model"],
# statistics_0.outputs["output_model"]
],
iv_param={"select_federated": True, "threshold": 3},
statistic_param={"metrics": ["mean", "max", "kurtosis", "skewness"]},
manual_param={"filter_out_col": ["x0", "x3"]},
)

pipeline.add_task(intersection_0)
pipeline.add_task(binning_0)
Expand All @@ -71,11 +86,11 @@
deployed_pipeline = pipeline.get_deployed_pipeline()

deployed_pipeline.intersection_0.guest.component_setting(
input_data=DataWarehouseChannel(name="breast_hetero_guest_sid",
namespace="experiment"))
input_data=DataWarehouseChannel(name="breast_hetero_guest_sid", namespace="experiment")
)
deployed_pipeline.intersection_0.hosts[0].component_setting(
input_data=DataWarehouseChannel(name="breast_hetero_host_sid",
namespace="experiment"))
input_data=DataWarehouseChannel(name="breast_hetero_host_sid", namespace="experiment")
)

predict_pipeline.add_task(deployed_pipeline)

Expand Down
8 changes: 1 addition & 7 deletions java/osx/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1 @@
# Release 1.0.0-alpha
## Major Features and Improvements
* Support grpc synchronous transmission and streaming transmission. Compatible with eggroll interface and can replace FATE1. x rollsite component
* Support asynchronous message transmission, which can replace rabbitmq and pulsar components in FATE1. x
* Support HTTP1. X protocol transmission
* Support cluster deployment and inter-site traffic control
* Support networking as an Exchange component
OSX: Open Site Exchange
7 changes: 7 additions & 0 deletions java/osx/RELEASE.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Release 1.0.0-alpha
## Major Features and Improvements
* Support grpc synchronous transmission and streaming transmission. Compatible with eggroll interface and can replace FATE1. x rollsite component
* Support asynchronous message transmission, which can replace rabbitmq and pulsar components in FATE1. x
* Support HTTP1. X protocol transmission
* Support cluster deployment and inter-site traffic control
* Support networking as an Exchange component
82 changes: 38 additions & 44 deletions java/osx/bin/common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow"
JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch"
JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=15g"
JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking"
#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n"
JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}"

set -e
Expand All @@ -93,7 +92,7 @@ getpid() {
pid=$(cat ./bin/broker.pid)
fi
if [[ -n ${pid} ]]; then
count=$(ps -ef | grep $pid | grep -v "grep" | wc -l)
count=$(ps -ef | grep $pid |grep 'com.osx' | grep -v "grep" | wc -l)
if [[ ${count} -eq 0 ]]; then
rm ./bin/broker.pid
unset pid
Expand All @@ -115,53 +114,50 @@ start() {
getpid $module
if [[ ! -n ${pid} ]]; then JAVA_OPT="${JAVA_OPT} "
mklogsdir
# if [[ -e "${module}.jar" ]]; then
# rm ${module}.jar
# fi
# ln -s ${module}-${module_version}.jar ${module}.jar
JAVA_OPT="${JAVA_OPT} -cp conf/broker/:lib/*"
# if [ ${module} = "transfer" ]; then
# echo "transfer"
# elif [ ${module} = "cluster-manager" ] || [ ${module} = "dashboard" ]; then
# JAVA_OPT="${JAVA_OPT} -Dspring.config.location=${configpath}/cluster-manager.properties"
# JAVA_OPT="${JAVA_OPT} -cp conf/:lib/*:${module}.jar"
# else
# echo "usage: ${module} {transfer|cluster-manager|dashboard}"
# fi

JAVA_OPT="${JAVA_OPT} -cp conf/broker/:lib/*:extension/*:${BASE_DIR}/${project_name}-${module}-${module_version}.jar"
JAVA_OPT="${JAVA_OPT} ${main_class}"
JAVA_OPT="${JAVA_OPT} -c ${configpath} "
echo $JAVA ${JAVA_OPT}
nohup $JAVA ${JAVA_OPT} >/dev/null 2>&1 &
inspect_pid 5 $!
if [[ "$exist" = 1 ]]; then
echo $! >./bin/${module}.pid
getpid ${module}
echo "service start sucessfully. pid: ${pid}"
else
echo "service start failed, "
fi
else
echo "service already started. pid: ${pid}"
fi
}

JAVA_OPT="${JAVA_OPT} -c ${configpath}/broker/broker.properties"
# if [ ${module} = "broker" -o ${module} = "cli" ]; then
# JAVA_OPT="${JAVA_OPT} -c ${configpath}/broker/broker.properties"
# elif [ ${module} = "cluster-manager" ]; then
# JAVA_OPT="${JAVA_OPT} -c ${configpath}/cluster-manager/cluster-manager.properties"
#
# elif [ ${module} = "dashboard" ]; then
# JAVA_OPT="-jar ${libpath}/dashboard-1.0.0.jar -spring.config.location=${configpath}/dashboard/application.properties"
# fi

if [ ${module} = "cli" ]; then
java ${JAVA_OPT}
else
nohup $JAVA ${JAVA_OPT} >/dev/null 2>&1 &
#sleep 5
#id=$(ps -p $! | awk '{print $1}' | sed -n '2p')
inspect_pid 5 $!

if [[ "$exist" = 1 ]]; then
echo $! >./bin/${module}.pid
getpid ${module}
echo "service start sucessfully. pid: ${pid}"
else
echo "service start failed"
fi
fi
debug() {
echo "try to start $1"
module=broker
main_class=com.osx.broker.Bootstrap
getpid $module
if [[ ! -n ${pid} ]]; then JAVA_OPT="${JAVA_OPT} "
mklogsdir
JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8008 -cp conf/broker/:lib/*:extension/*:${BASE_DIR}/${project_name}-${module}-${module_version}.jar"
JAVA_OPT="${JAVA_OPT} ${main_class}"
JAVA_OPT="${JAVA_OPT} -c ${configpath} "
echo $JAVA ${JAVA_OPT}
nohup $JAVA ${JAVA_OPT} >/dev/null 2>&1 &
inspect_pid 5 $!
if [[ "$exist" = 1 ]]; then
echo $! >./bin/${module}.pid
getpid ${module}
echo "service start sucessfully. pid: ${pid}"
else
echo "service start failed, "
fi
else
echo "service already started. pid: ${pid}"
fi
}


status() {
getpid $1
if [[ -n ${pid} ]]; then
Expand Down Expand Up @@ -195,11 +191,9 @@ stop() {
fi
}


inspect_pid() {
total=0
exist=0
#echo "inspect pid: $2,periods: $1"
if [[ -n $2 ]]; then
while [[ $total -le $1 ]]
do
Expand Down
14 changes: 12 additions & 2 deletions java/osx/bin/service.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ configpath=$(cd $basepath/conf;pwd)
libpath=$(cd $basepath/lib;pwd)
#module=transfer
#main_class=com.firework.transfer.Bootstrap
#module_version=1.0.0

module_version=1.0.0-alpha
project_name=osx



Expand All @@ -35,6 +35,10 @@ case "$1" in
start $2
status $2
;;
debug)
debug $2
status $2
;;
stop)
stop $2
;;
Expand All @@ -47,6 +51,12 @@ case "$1" in
start $2
status $2
;;
rebudeg)
stop $2
sleep 0.5
debug $2
status $2
;;
*)
echo "usage: $0 {start|stop|status|restart}"
exit 1
Expand Down
Loading

0 comments on commit 7ef13b8

Please sign in to comment.