diff --git a/doc/2.0/README.md b/doc/2.0/README.md deleted file mode 100644 index 5b0aa5341e..0000000000 --- a/doc/2.0/README.md +++ /dev/null @@ -1,5 +0,0 @@ -#### 文档索引 -- [FATE Flow V2.0 Quick Start](https://github.com/FederatedAI/FATE-Flow/blob/v2.0.0-beta/doc/quick_start.md) -- [FATE V2.0 Quick Start](./quick_start.md) -- [OSX方案](./osx/osx.md) -- [FATE Components](./fate/components) \ No newline at end of file diff --git a/doc/2.0/images/alllog.png b/doc/2.0/images/alllog.png new file mode 100644 index 0000000000..3fbd9bc23d Binary files /dev/null and b/doc/2.0/images/alllog.png differ diff --git a/doc/2.0/images/flow_log.png b/doc/2.0/images/flow_log.png new file mode 100644 index 0000000000..19217fd4be Binary files /dev/null and b/doc/2.0/images/flow_log.png differ diff --git a/doc/2.0/images/start.png b/doc/2.0/images/start.png new file mode 100644 index 0000000000..1e9758b39c Binary files /dev/null and b/doc/2.0/images/start.png differ diff --git a/doc/2.0/osx/osx-tls.md b/doc/2.0/osx/osx-tls.md new file mode 100644 index 0000000000..fe097a69fa --- /dev/null +++ b/doc/2.0/osx/osx-tls.md @@ -0,0 +1,140 @@ +# 证书生成： + +two-way TSL： + +## 1）方式一：使用keystore密码箱存储私钥、证书、信任证书方式 + +#### 生成client和server端的秘钥keystore文件、证书文件、信任证书链，具体命令步骤如下： + + 1. 创建一个包含服务器公钥和私钥的密钥库，并为其指定了一些属性： + + ``` + keytool -v -genkeypair -dname "CN=OSX,OU=Fate,O=WB,C=CN" -keystore server/identity.jks -storepass 123456 -keypass 123456 -keyalg RSA -keysize 2048 -alias server -validity 3650 -deststoretype pkcs12 -ext KeyUsage=digitalSignature,dataEncipherment,keyEncipherment,keyAgreement -ext ExtendedKeyUsage=serverAuth,clientAuth -ext SubjectAlternativeName:c=DNS:localhost,DNS:osx.local,IP:127.0.0.1 + ``` + + - `keytool`: Java密钥和证书管理工具。 + - `-v`: 详细输出。 + - `-genkeypair`: 生成密钥对。 + - `-dname "CN=OSX,OU=Fate,O=WB,C=CN"`: 设置证书主题（Distinguished Name）。 + - `-keystore server/identity.jks`: 设置密钥库的文件路径和名称。 + - `-storepass 123456`: 设置密钥库的密码。 + - `-keypass 123456`: 设置生成的密钥对的密码。 + - `-keyalg RSA`: 使用RSA算法生成密钥对。 + - `-keysize 2048`: 设置密钥的大小为2048位。 + - `-alias server`: 设置密钥对的别名。 + - `-validity 3650`: 设置证书的有效期为3650天。 + - `-deststoretype pkcs12`: 指定密钥库的类型为PKCS12。 + - `-ext KeyUsage=digitalSignature,dataEncipherment,keyEncipherment,keyAgreement`: 扩展密钥用途。 + - `-ext ExtendedKeyUsage=serverAuth,clientAuth`: 扩展密钥用途。 + - `-ext SubjectAlternativeName:c=DNS:localhost,DNS:osx.local,IP:127.0.0.1`: 设置主体备用名称，包括DNS和IP地址。(这些主体备用名称的添加允许证书在与这些名称关联的主机或IP地址上使用，而不仅限于使用通用名称（Common Name，CN）字段中指定的主机名。这对于在服务器证书中包含多个主机名或IP地址是很有用的，特别是在使用SSL/TLS进行多主机名（SAN）认证时。) + + 此命令用于生成包含服务器证书的密钥库，以便用于安全连接。请确保根据实际需求和环境进行适当的调整。 + + 2. 密钥库中导出证书，并将其保存为`.cer`文件: + + ``` + keytool -v -exportcert -file server/server.cer -alias server -keystore server/identity.jks -storepass 123456 -rfc + ``` + + - `keytool`: Java密钥和证书管理工具。 + - `-v`: 详细输出。 + - `-exportcert`: 导出证书。 + - `-file server/server.cer`: 指定导出证书的文件路径和名称。 + - `-alias server`: 指定要导出的证书条目的别名。 + - `-keystore server/identity.jks`: 指定密钥库的路径和名称。 + - `-storepass 123456`: 密钥库的密码。 + - `-rfc`: 以RFC 1421格式（Base64编码）输出证书。 + + 此命令用于从密钥库中导出服务器证书，并将其保存为`.cer`文件。请确保提供正确的密钥库路径、别名和密码，并根据需要更改导出证书的文件路径和名称。 + + 3. 从证书文件导入证书并将其添加到客户端的信任存储区: + + - ``` + keytool -v -importcert -file server/server.cer -alias server -keystore client/truststore.jks -storepass 123456 -noprompt + ``` + + `keytool`: Java密钥和证书管理工具。 + + - `-v`: 详细输出。 + + - `-importcert`: 导入证书。 + + - `-file server/server.cer`: 指定要导入的证书文件路径和名称。 + + - `-alias server`: 指定将证书存储在信任存储区时使用的别名。 + + - `-keystore client/truststore.jks`: 指定信任存储区的路径和名称。 + + - `-storepass 123456`: 信任存储区的密码。 + + - `-noprompt`: 在导入证书时不提示用户确认。 + + 此命令用于将服务器证书导入到客户端的信任存储区，以建立与服务器的安全连接。确保提供正确的证书文件路径、别名、信任存储区路径和密码，并根据需要更改相关参数。 `-noprompt` 标志确保在导入证书时不需要手动确认。 + + 4. 生成客户端的密钥对和自签名证书: + + ``` + keytool -v -genkeypair -dname "CN=Suleyman,OU=Altindag,O=Altindag,C=NL" -keystore client/identity.jks -storepass 123456 -keypass 123456 -keyalg RSA -keysize 2048 -alias client -validity 3650 -deststoretype pkcs12 -ext KeyUsage=digitalSignature,dataEncipherment,keyEncipherment,keyAgreement -ext ExtendedKeyUsage=serverAuth,clientAuth + ``` + + - `keytool`: Java密钥和证书管理工具。 + - `-v`: 详细输出。 + - `-genkeypair`: 生成密钥对。 + - `-dname "CN=Suleyman,OU=Altindag,O=Altindag,C=NL"`: 设置证书主题（Distinguished Name，DN）。 + - `-keystore client/identity.jks`: 设置密钥库的文件路径和名称。 + - `-storepass 123456`: 设置密钥库的密码。 + - `-keypass 123456`: 设置生成的密钥对的密码。 + - `-keyalg RSA`: 使用RSA算法生成密钥对。 + - `-keysize 2048`: 设置密钥的大小为2048位。 + - `-alias client`: 设置密钥对的别名。 + - `-validity 3650`: 设置证书的有效期为3650天。 + - `-deststoretype pkcs12`: 指定密钥库的类型为PKCS12。 + - `-ext KeyUsage=digitalSignature,dataEncipherment,keyEncipherment,keyAgreement`: 扩展密钥用途。 + - `-ext ExtendedKeyUsage=serverAuth,clientAuth`: 扩展密钥用途。 + + 此命令用于生成包含客户端证书的密钥库，以用于安全连接。请确保提供正确的密钥库路径、别名和密码，并根据需要更改其他参数。 + + 5. 从客户端的密钥库中导出证书，并将其保存为`.cer`文件: + + ``` + keytool -v -exportcert -file client/client.cer -alias client -keystore client/identity.jks -storepass 123456 -rfc + ``` + + - `keytool`: Java密钥和证书管理工具。 + - `-v`: 详细输出。 + - `-exportcert`: 导出证书。 + - `-file client/client.cer`: 指定导出证书的文件路径和名称。 + - `-alias client`: 指定要导出的证书条目的别名。 + - `-keystore client/identity.jks`: 指定密钥库的路径和名称。 + - `-storepass 123456`: 密钥库的密码。 + - `-rfc`: 以RFC 1421格式（Base64编码）输出证书。 + + 此命令用于从客户端的密钥库中导出客户端证书，并将其保存为`.cer`文件。请确保提供正确的密钥库路径、别名和密码，并根据需要更改导出证书的文件路径和名称。 + + 6. 从客户端的证书文件中导入证书，并将其添加到服务器的信任存储区: + + ``` + keytool -v -importcert -file client/client.cer -alias client -keystore server/truststore.jks -storepass 123456 -noprompt + ``` + + - `keytool`: Java密钥和证书管理工具。 + - `-v`: 详细输出。 + - `-importcert`: 导入证书。 + - `-file client/client.cer`: 指定要导入的证书文件路径和名称。 + - `-alias client`: 指定将证书存储在信任存储区时使用的别名。 + - `-keystore server/truststore.jks`: 指定信任存储区的路径和名称。 + - `-storepass 123456`: 信任存储区的密码。 + - `-noprompt`: 在导入证书时不提示用户确认。 + + 此命令用于将客户端证书导入到服务器的信任存储区，以建立与客户端的安全连接。确保提供正确的证书文件路径、别名、信任存储区路径和密码，并根据需要更改相关参数。 `-noprompt` 标志确保在导入证书时不需要手动确认。 + +#### 完成以上步骤您将生成如下证书： + +​ server文件夹包含： identity.jks 、server.cer、truststore.jks。 + +​ client文件夹包含： identity.jks 、client.cer、truststore.jks。 + +## 2）方式二：单独文件存储私钥、证书、信任证书方式 + +#### 生成ca.key、ca.crt、client.crt、client.csr、client.key、client.pem、server.crt、server.csr、server.key、server.pem 命令如下： + diff --git a/doc/2.0/osx/osx.md b/doc/2.0/osx/osx.md index 7bf73676ad..23bc9b6e9f 100644 --- a/doc/2.0/osx/osx.md +++ b/doc/2.0/osx/osx.md @@ -74,14 +74,27 @@ FATE1.X维护了多套通信架构，包括eggroll、spark+pulsar+nginx 、spark + + + + +#### 传输模式： + +osx 支持两种传输模式： + +- 流式（支持grpc，兼容FATE1.x版本传输，使用eggroll作为计算引擎） +- 队列（支持grpc、http1.x ，可以使用eggroll或者spark以及其他厂商计算引擎，按照《金融业隐私计算互联互通平台技术规范》实现的传输接口，推荐使用grpc协议，不同协议对应的路由表配置不一样，详情可见路由相关配置 ） + +不同的传输模式使用不同的传输接口 ， 不需要在 osx中配置中指定，而是由FATE的配置决定，有多个参与方的情况下，双方需要预先商定使用的传输模式，只有在双方都使用一种传输模式的情况下才能正常工作。 + + + ## 配置： 以下为osx最简配置,配置文件位于 {部署目录}/conf/broker/broker.properties ```properties grpc.port= 9377 （服务监听的grpc端口） -# eg: 9999,10000,10001 （本方partyId, 对应于互联互通协议中的nodeId） -self.party=10000 # （若使用eggroll作为计算引擎，此处填入eggroll cluster-manager 的ip） eggroll.cluster.manager.ip = localhost # （若使用eggroll作为计算引擎，此处填入eggroll cluster-manager 的端口） @@ -93,7 +106,7 @@ eggroll.cluster.manager.port = 4670 | 名称 | 含义 | 默认值 | 是否必须配置 | 说明 | | ---------------------------------------------- | ------------------------------------------------------------ | -------------------- | ----------------------------------- | ------------------------------------------------------------ | | grpc.port | 服务监听grpc端口（非TLS） | 9370 | 否 | 该端口用做默认的集群内部通信端口 ，若是用于非生产环境，出于方便测试以及调试考虑，可以将此端口作为集群间通信端口。若是生产环境使用，出于安全考虑，不应该将此对外暴露，而是将使用TLS 的端口对外暴露 ，参考配置open.grpc.tls.server grpc.tls.port | -| self.party | 本方partyId | 无 | 是 | **此配置非常重要，需要在部署集群前确定己方partyId，此处的配置会影响请求是否能正确路由** | +| | | | | | | eggroll.cluster.manager.ip | 若使用eggroll作为计算引擎，此处填入eggroll cluster-manager 的ip | 无 | 否 | | | eggroll.cluster.manager.port | 若使用eggroll作为计算引擎，此处填入eggroll cluster-manager 的端口 | 无 | 否 | | | open.grpc.tls.server | 是否开启使用TLS的grpc端口 | false | 否 | 开启之后，服务将会监听一个使用TLS的grpc端口 | @@ -122,17 +135,11 @@ eggroll.cluster.manager.port = 4670 ## 路由： -路由配置相关文件为{部署目录}/conf/broker/route_table.json ,与eggroll组件rollsite保持一致，下面介绍在不使用证书的情况下的操作步骤： - -先检查{部署目录}/conf/broker/broker.properties 中配置 self.party,如下所示，则代表本方partyId为9999，（若是与遵循互联互通协议的其他厂商隐私计算产品对接，此处对应于互联互通协议的nodeId） - -``` -self.party=9999 -``` +路由配置相关文件为{部署目录}/conf/broker/route_table.json ，下面介绍在不使用证书的情况下的操作步骤： +，self_party=[“9999”]则代表本方partyId为9999，（若是与遵循互联互通协议的其他厂商隐私计算产品对接，此处对应于互联互通协议的nodeId） - -**若发现该配置不符合预期，则需要修改成预期的partyId，并重启应用。** +**** 本方partyId ：9999 （若是与遵循互联互通协议的其他厂商隐私计算产品对接，此处对应于互联互通协议的nodeId） @@ -159,14 +166,14 @@ self.party=9999 } }, - "permission": - { - "default_allow": true - } + // self_party 为必填字段 + "self_party":[ + "9999" //本方partyId + ] } ``` -**路由表修改之后不需要重启应用，系统会自动读取。** +**路由表修改之后不需要重启应用，系统会自动读取，需要保证该文件内容为可解析的json格式，否则会解析失败。** ## @@ -189,21 +196,201 @@ self.party=9999 -### 日志分析： +### 日志分析 + +默认日志目录位于安装目录下 logs/broker 文件夹下，如下图所示 +![alllog](../images/alllog.png) + flow.log ： 记录请求日志 + broker.log ： 记录所有日志 + broker-error: 记录异常日志 + + broker-debug: debug日志默认不开启 + +正常启动后可以在broker.log中看到如下日志： + +![start](../images/start.png) + +在收到请求后可以在flow.log中看到访问日志： + +例如取其中几条日志分析 + +- 日志1 + +``` +2023-11-27 23:53:42,320|grpc|1701100422_1009|MSG_REDIRECT|session:202311271901517083180_lr_0_0|topic:202311271901517083180_lr_0_0-host-10000-guest-9999--|des:9999|code:E0000000000|cost:1|192.168.0.5:9370|size:224|msg:success +``` + +这条日志为使用队列模式传输时，osx收到了消息，且该osx不是该消息目的地时的日志，收到消息后osx采取的动作为MSG_REDIRECT，代表着将该消息传递至其他合作方。 + +它的格式为 时间戳|请求入口协议|流水号|处理动作|此次传输的sessionId|队列传输的topic|目的地partyId|返回码|耗时|目的地ip端口|传输数据大小|返回message + +- 日志2 + +``` +**2023-11-27 23:53:36,574|grpc|1701100416_9289|MSG_DOWNLOAD|session:202311271901517083180_lr_0_0|topic:202311271901517083180_lr_0_0-arbiter-10000-host-10000-g|offset-in-queue:100|des:10000|code:E0000000000|cost:0|size:2972| +``` + +这条日志为使用队列模式传输时，osx收到了消息，且该osx是该消息目的地时的日志，收到消息后osx采取的动作为MSG_DOWNLOAD，代表着将该消息本地持久化 + +它的格式为 时间戳|请求入口协议|流水号|处理动作|此次传输的sessionId|队列传输的topic|队列中消息数量|目的地partyId|返回码|耗时|数据大小| + +- 日志3 + +``` +2023-11-27 23:53:36,576|grpc|1701100416_7277|DEFUALT_CONSUME|session:202311271901517083180_lr_0_0|topic:202311271901517083180_lr_0_0-guest-9999-arbiter-10000-loss|req-offset:100|offset-in-queue:100|code:0|cost:0| +``` + +这条日志为使用队列模式传输时，osx收到了消费请求，osx采取的动作为DEFUALT_CONSUME，代表着从本地队列取出消息 + +它的格式为 时间戳|请求入口协议|流水号|处理动作|此次传输的sessionId|队列传输的topic|请求消费的序列号|队列中消息总数|返回码|耗时| + +- 日志4 + +``` +2023-11-27 23:54:23,318|grpc|UNARY_CALL|session:_fateflow_10000|topic:202311271901517083180|src:|des:10000|code:E0000000000|cost:18|172.16.153.111:9360|size:320| +``` + +这条日志为osx收到fateflow的请求，osx采取的动作为UNARY_CALL，代表着为fateflow所使用的接口 + +它的格式为 时间戳|请求入口协议|处理动作|此次传输的sessionId|topic|来源partid|目的partyId|返回码|目的地ip端口|数据大小| + +- 日志5 + +``` +2023-11-27 23:44:44,128|grpc|11|PUSH_REMOTE|session:202311271453035771030_sbt_0_0|topic:putBatch-__rsk#202311271453035771030_sbt_0_0#updated_data__table_persistent_0__#default.iterations-49.iterations-4#host#10000#guest#9999-12|src:10000|des:9999|code:E0000000000|cost:0|192.168.0.5:9370|size:90536| +``` + +这条日志为osx 在使用流式传输模式时，收到了流式传输请求，且osx不是该请求目的地，osx采取的动作为PUSH_REMOTE，代表着为将其请求转发至他处，其中的192.168.0.5:9370为目的地 + +- 日志6 + +``` +2023-11-27 23:44:47,703|grpc|3|PUSH_EGGPAIR|session:202311271453035771030_sbt_0_0_host_10000|topic:putBatch-__rsk#202311271453035771030_sbt_0_0#new_sample_pos__table_persistent_0__#default.iterations-49.iterations-4#guest#9999#host#10000-1|code:E0000000000|cost:152|192.168.0.5:36110|size:98080| +``` + +这条日志为osx 在使用流式传输模式时，收到了流式传输请求，且osx为该请求目的地，osx采取的动作为PUSH_EGGPAIR，代表着为将其请求转发至eggroll 所启动的进程 eggpair，其中的192.168.0.5:36110为目的地 ### 证书相关： +#### OSX相关配置： + +- grpcs： + + broker.property配置（使用keystore方式，即方式1）： + + ``` + # 打开grpcs server开关 + open.grpc.tls.server= true + # 是否使用keystore方式（默认为false） + open.grpc.tls.use.keystore= true + #server端密码箱路径以及密码 + server.keystore.file= + server.keystore.file.password= + #server端信任证书keystore路径及密码 + server.trust.keystore.file= + server.trust.keystore.file.password= + + ``` + + 相关client端路由表配置： + + ``` + "default": [ + { + "protocol": "grpc", + "keyStoreFile": "D:/webank/osx/test3/client/identity.jks", + "keyStorePassword": "123456", + "trustStoreFile": "D:/webank/osx/test3/client/truststore.jks", + "trustStorePassword": "123456", + "useSSL": true, + "port": 9885, + "ip": "127.0.0.1" + } + ] + ``` + + + +- https： + + broker.property配置（使用keystore方式，即方式1）： + + ``` + # grpcs端口 + https.port=8092 + # 打开grpcs server开关 + open.https.server= true + # server端密码箱路径以及密码 + server.keystore.file= + server.keystore.file.password= + # server端信任证书keystore路径及密码 + server.trust.keystore.file= + server.trust.keystore.file.password= + + ``` + + 相关client端路由表配置： + + ``` + + ``` + + + +2）方式二：单独文件存储私钥、证书、信任证书方式 + +​ 生成命令： + +​ + +#### OSX相关配置： + +- grpcs： + + broker.property配置（使用非keystore方式，即方式2）： + ``` + # 打开grpcs server开关 + open.grpc.tls.server= true + # 是否使用keystore方式 + open.grpc.tls.use.keystore= false + + + + + ``` + + 相关client端路由表配置： + ``` + + ``` + + + +- https： + + broker.property配置（使用非keystroke方式，即方式2）： + + ``` + + ``` + + 相关client端路由表配置： + + ``` + + ``` + + -### ### 常见问题： diff --git a/doc/2.0/quick_start.md b/doc/2.0/quick_start.md deleted file mode 100644 index 46863e57c1..0000000000 --- a/doc/2.0/quick_start.md +++ /dev/null @@ -1,119 +0,0 @@ -## Quick Start - -1. install `fate_client` with extra package `fate` - -```sh -python -m pip install -U pip && python -m pip install fate_client[fate,fate_flow]==2.0.0b0 -``` -after installing packages successfully, initialize fate_flow service and fate_client - -```sh -mkdir fate_workspace -fate_flow init --ip 127.0.0.1 --port 9380 --home $(pwd)/fate_workspace -pipeline init --ip 127.0.0.1 --port 9380 - -fate_flow start -fate_flow status # make sure fate_flow service is started -``` - - -2. download example data - -```sh -wget https://raw.githubusercontent.com/wiki/FederatedAI/FATE/example/data/breast_hetero_guest.csv && \ -wget https://raw.githubusercontent.com/wiki/FederatedAI/FATE/example/data/breast_hetero_host.csv -``` - -3. transform example data to dataframe using in fate -```python -import os -from fate_client.pipeline import FateFlowPipeline - - -base_path = os.path.abspath(os.path.join(__file__, os.path.pardir)) -guest_data_path = os.path.join(base_path, "breast_hetero_guest.csv") -host_data_path = os.path.join(base_path, "breast_hetero_host.csv") - -data_pipeline = FateFlowPipeline().set_parties(local="0") -guest_meta = { - "delimiter": ",", "dtype": "float64", "label_type": "int64","label_name": "y", "match_id_name": "id" -} -host_meta = { - "delimiter": ",", "input_format": "dense", "match_id_name": "id" -} -data_pipeline.transform_local_file_to_dataframe(file=guest_data_path, namespace="experiment", name="breast_hetero_guest", - meta=guest_meta, head=True, extend_sid=True) -data_pipeline.transform_local_file_to_dataframe(file=host_data_path, namespace="experiment", name="breast_hetero_host", - meta=host_meta, head=True, extend_sid=True) -``` -4. run example - -```python -from fate_client.pipeline.components.fate import ( - HeteroSecureBoost, - PSI, - Evaluation -) -from fate_client.pipeline import FateFlowPipeline -from fate_client.pipeline.interface import DataWarehouseChannel - - -# create pipeline for training -pipeline = FateFlowPipeline().set_parties(guest="9999", host="10000") - -# create psi component_desc -psi_0 = PSI("psi_0") -psi_0.guest.task_setting( - input_data=DataWarehouseChannel(name="breast_hetero_guest", namespace="experiment") -) -psi_0.hosts[0].task_setting( - input_data=DataWarehouseChannel(name="breast_hetero_host", namespace="experiment") -) - -# create hetero secure_boost component_desc -hetero_secureboost_0 = HeteroSecureBoost( - 'hetero_secureboost_0', num_trees=1, max_depth=5, - train_data=psi_0.outputs['output_data'], - validate_data=psi_0.outputs["output_data"] -) - -# create evaluation component_desc -evaluation_0 = Evaluation( - 'evaluation_0', runtime_roles=['guest'], metrics=['auc'], input_data=[hetero_secureboost_0.outputs['train_data_output']] -) - -# add training task -pipeline.add_task(psi_0) -pipeline.add_task(hetero_secureboost_0) -pipeline.add_task(evaluation_0) - -# compile and train -pipeline.compile() -pipeline.fit() - -# print metric and model info -print (pipeline.get_task_info("hetero_secureboost_0").get_output_model()) -print (pipeline.get_task_info("evaluation_0").get_output_metric()) - -# deploy task for inference -pipeline.deploy([psi_0, hetero_secureboost_0]) - -# create pipeline for predicting -predict_pipeline = FateFlowPipeline() - -# add input to deployed_pipeline -deployed_pipeline = pipeline.get_deployed_pipeline() -deployed_pipeline.psi_0.guest.task_setting( - input_data=DataWarehouseChannel(name="breast_hetero_guest", namespace=f"experiment") -) -deployed_pipeline.psi_0.hosts[0].task_setting( - input_data=DataWarehouseChannel(name="breast_hetero_host", namespace=f"experiment") -) - -# add task to predict pipeline -predict_pipeline.add_task(deployed_pipeline) - -# compile and predict -predict_pipeline.compile() -predict_pipeline.predict() -``` diff --git a/java/osx/bin/common.sh b/java/osx/bin/common.sh index abdea3b158..d0c382d3af 100644 --- a/java/osx/bin/common.sh +++ b/java/osx/bin/common.sh @@ -1,19 +1,3 @@ -#!/bin/bash - -# -# Copyright 2019 The FATE Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. error_exit (){ echo "ERROR: $1 !!" exit 1 @@ -92,7 +76,7 @@ getpid() { pid=$(cat ./bin/broker.pid) fi if [[ -n ${pid} ]]; then - count=$(ps -ef | grep $pid | grep -v "grep" | wc -l) + count=$(ps -c -ef | grep $pid | grep -v "grep" | wc -l) if [[ ${count} -eq 0 ]]; then if [ -e "./bin/broker.pid" ]; then rm ./bin/broker.pid @@ -166,6 +150,7 @@ status() { if [[ -n ${pid} ]]; then echo "status: $(ps -f -p ${pid})" exit 0 + else echo "service not running" exit 1 @@ -187,10 +172,10 @@ stop() { echo "please retry" fi else - echo "kill error" + echo "kill error " fi else - echo "service not running" + echo "service not running " fi } @@ -211,4 +196,4 @@ inspect_pid() { fi done fi -} +} \ No newline at end of file diff --git a/java/osx/bin/service.sh b/java/osx/bin/service.sh index 36f89e3e97..44da5df35c 100644 --- a/java/osx/bin/service.sh +++ b/java/osx/bin/service.sh @@ -1,4 +1,5 @@ #!/bin/bash + # # Copyright 2019 The FATE Authors. All Rights Reserved. # @@ -59,4 +60,5 @@ case "$1" in *) echo "usage: $0 {start|stop|status|restart}" exit 1 + esac diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/http/InnerServlet.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/http/InnerServlet.java index 9ee478f321..d30d7f3223 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/http/InnerServlet.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/http/InnerServlet.java @@ -79,7 +79,7 @@ protected void dispatch(HttpServletRequest req, HttpServletResponse resp) throws case HTTP_INVOKE: techProvider.processHttpInvoke(osxContext, req, resp); break; - + case HTTP_SET_SELF: case HTTP_SET_ROUTER: case HTTP_GET_ROUTER: case HTTP_ADD_ROUTER: diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterAddRequest.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterAddRequest.java index d4748cebc6..a52ebdb842 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterAddRequest.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterAddRequest.java @@ -8,7 +8,7 @@ import org.fedai.osx.core.router.RouterInfo; @Data -public class RouterAddRequest { +public class RouterAddRequest extends TokenHolder{ public static class BooleanFilter{ @Override diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableGetRequest.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableGetRequest.java new file mode 100644 index 0000000000..b2dddd8b91 --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableGetRequest.java @@ -0,0 +1,4 @@ +package org.fedai.osx.broker.pojo; + +public class RouterTableGetRequest extends TokenHolder{ +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableGetResponse.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableGetResponse.java new file mode 100644 index 0000000000..1b13824ee0 --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableGetResponse.java @@ -0,0 +1,10 @@ +package org.fedai.osx.broker.pojo; + +import lombok.Data; + +@Data +public class RouterTableGetResponse { + String content; + String code; + String msg; +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetRequest.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetRequest.java index 7d4bdfe34f..ed90b3a97d 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetRequest.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetRequest.java @@ -3,7 +3,7 @@ import lombok.Data; @Data -public class RouterTableSetRequest { +public class RouterTableSetRequest extends TokenHolder{ String data; } diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetResponse.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetResponse.java index f6f16f36e0..5cb877a844 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetResponse.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/RouterTableSetResponse.java @@ -4,4 +4,6 @@ @Data public class RouterTableSetResponse { + String code; + String msg; } diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SelfPartySetRequest.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SelfPartySetRequest.java deleted file mode 100644 index c6d6c36eca..0000000000 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SelfPartySetRequest.java +++ /dev/null @@ -1,7 +0,0 @@ -//package org.fedai.osx.broker.pojo; -// -//import lombok.Data; -// -//@Data -//public class SelfPartySetRequest { -//} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SetSelfPartyRequest.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SetSelfPartyRequest.java index 21de977b80..9c19e82310 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SetSelfPartyRequest.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/SetSelfPartyRequest.java @@ -5,8 +5,8 @@ import java.util.Set; @Data -public class SetSelfPartyRequest { +public class SetSelfPartyRequest extends TokenHolder{ - Set selfPartys; + Set selfParty; } diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/TokenHolder.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/TokenHolder.java new file mode 100644 index 0000000000..f8e6e99295 --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/pojo/TokenHolder.java @@ -0,0 +1,8 @@ +package org.fedai.osx.broker.pojo; + +import lombok.Data; + +@Data +public class TokenHolder { + String token; +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/provider/FateTechProvider.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/provider/FateTechProvider.java index 6a88e6caf7..040dcaa729 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/provider/FateTechProvider.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/provider/FateTechProvider.java @@ -25,6 +25,7 @@ import org.fedai.osx.broker.router.RouterServiceRegister; import org.fedai.osx.broker.service.ServiceRegisterInfo; import org.fedai.osx.broker.service.ServiceRegisterManager; +import org.fedai.osx.broker.token.TokenValidatorRegister; import org.fedai.osx.broker.util.TransferUtil; import org.fedai.osx.core.config.MetaInfo; import org.fedai.osx.core.constant.*; @@ -65,6 +66,8 @@ public class FateTechProvider implements TechProvider { ServiceRegisterManager serviceRegisterManager; @Inject RouterServiceRegister routerServiceRegister; + @Inject + TokenValidatorRegister tokenValidatorRegister; Base64.Encoder base64Encoder = Base64.getEncoder(); @@ -486,7 +489,9 @@ public void processRouterOperation(OsxContext context, HttpServletRequest httpS try { context.putData(Dict.HTTP_SERVLET_RESPONSE, httpServletResponse); byte[] payload = TransferUtil.read(httpServletRequest.getInputStream()); - String content = new String(payload); + String content=""; + if(payload!=null) + content = new String(payload); ServiceRegisterInfo serviceRegisterInfo = this.serviceRegisterManager.getServiceWithLoadBalance(context, "", context.getUri(), false); Preconditions.checkArgument(serviceRegisterInfo!=null,"uri : " +context.getUri()+" can not found service "); Object serviceAdaptorObject = serviceRegisterInfo.getServiceAdaptor(); diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/ProduceService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/ProduceService.java index 75d7699ac4..99ec029d32 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/ProduceService.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/ProduceService.java @@ -140,7 +140,7 @@ public ProduceRequest decode(Object object) { } if(produceRequest==null){ logger.error("invalid produce request {}",object.getClass()); - throw new InvalidRequestException(); + throw new InvalidRequestException("invalid request for produce msg"); } return produceRequest; } diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/RouterTableSetService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/RouterTableSetService.java deleted file mode 100644 index 5a9bff10d0..0000000000 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/RouterTableSetService.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.fedai.osx.broker.ptp; - -import com.google.common.base.Preconditions; -import com.google.inject.Inject; -import com.google.inject.Singleton; -import org.apache.commons.lang3.StringUtils; -import org.fedai.osx.broker.pojo.RouterAddRequest; -import org.fedai.osx.broker.pojo.RouterAddResponse; -import org.fedai.osx.broker.pojo.RouterTableSetRequest; -import org.fedai.osx.broker.pojo.RouterTableSetResponse; -import org.fedai.osx.broker.router.RouterService; -import org.fedai.osx.broker.router.RouterServiceRegister; -import org.fedai.osx.broker.service.Register; -import org.fedai.osx.core.config.MetaInfo; -import org.fedai.osx.core.constant.UriConstants; -import org.fedai.osx.core.context.OsxContext; -import org.fedai.osx.core.exceptions.ExceptionInfo; -import org.fedai.osx.core.service.AbstractServiceAdaptorNew; -import org.ppc.ptp.Osx; - -@Singleton -@Register(uris ={UriConstants.HTTP_SET_ROUTER ,},allowInterUse = false) -public class RouterTableSetService extends AbstractServiceAdaptorNew { - - @Inject - RouterServiceRegister routerServiceRegister; - - @Override - protected RouterTableSetResponse doService(OsxContext context, RouterTableSetRequest data) { - RouterService routerService = routerServiceRegister.select(MetaInfo.PROPERTY_FATE_TECH_PROVIDER); - Preconditions.checkArgument(data!=null&& StringUtils.isNotEmpty(data.getData())); - routerService.setRouterTable(data.getData()); - return new RouterTableSetResponse(); - } - - @Override - protected RouterTableSetResponse transformExceptionInfo(OsxContext context, ExceptionInfo exceptionInfo) { - return null; - } - - @Override - public RouterTableSetRequest decode(Object object) { - - - return null; - } - - -} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/SelfPartySetService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/SelfPartySetService.java deleted file mode 100644 index 8ddc1a565a..0000000000 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/SelfPartySetService.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.fedai.osx.broker.ptp; - -import com.google.inject.Singleton; -import org.fedai.osx.broker.pojo.RouterTableSetRequest; -import org.fedai.osx.broker.pojo.RouterTableSetResponse; -import org.fedai.osx.broker.pojo.SetSelfPartyRequest; -import org.fedai.osx.broker.pojo.SetSelfPartyResponse; -import org.fedai.osx.broker.router.RouterService; -import org.fedai.osx.broker.router.RouterServiceRegister; -import org.fedai.osx.broker.service.Register; -import org.fedai.osx.core.config.MetaInfo; -import org.fedai.osx.core.constant.UriConstants; -import org.fedai.osx.core.context.OsxContext; -import org.fedai.osx.core.exceptions.ExceptionInfo; -import org.fedai.osx.core.service.AbstractServiceAdaptorNew; - -import javax.inject.Inject; - -@Singleton -@Register(uris ={UriConstants.HTTP_SET_ROUTER },allowInterUse = false) -public class SelfPartySetService extends AbstractServiceAdaptorNew { - - @Inject - RouterServiceRegister routerServiceRegister; - - @Override - protected SetSelfPartyResponse doService(OsxContext context, SetSelfPartyRequest data) { - - RouterService routerService =routerServiceRegister.select(MetaInfo.PROPERTY_FATE_TECH_PROVIDER); - - routerService.setSelfPartyIds(data.getSelfPartys()); - return null; - } - - @Override - protected SetSelfPartyResponse transformExceptionInfo(OsxContext context, ExceptionInfo exceptionInfo) { - return null; - } - - @Override - public SetSelfPartyRequest decode(Object object) { - return null; - } -} - - - - - - diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/DefaultFateRouterServiceImpl.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/DefaultFateRouterServiceImpl.java index 9696036b94..2ca3b51c02 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/DefaultFateRouterServiceImpl.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/DefaultFateRouterServiceImpl.java @@ -20,9 +20,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.inject.Singleton; -import com.google.protobuf.InvalidProtocolBufferException; -import com.webank.ai.eggroll.api.networking.proxy.Proxy; -import com.webank.eggroll.core.transfer.Transfer; import org.apache.commons.lang3.StringUtils; import org.fedai.osx.broker.util.TelnetUtil; import org.fedai.osx.core.config.MetaInfo; @@ -81,17 +78,11 @@ private Map parseRouterInfoToMap(RouterInfo routerInfo){ Map content = JsonUtil.object2Objcet(routerInfo,Map.class); return content; } - private void validateRouterInfo(RouterInfo routerInfo){ - String desPartyId = routerInfo.getDesPartyId(); - Preconditions.checkArgument(StringUtils.isNotEmpty(desPartyId),"des party id is null"); - if(routerInfo.getProtocol()!=null||Protocol.grpc.equals(routerInfo.getProtocol())){ - Preconditions.checkArgument(StringUtils.isNotEmpty(routerInfo.getHost()), "host/ip is null"); - Preconditions.checkArgument(routerInfo.getPort()!=null, "port is null"); - } - } + @Override public synchronized String addRouterInfo(RouterInfo routerInfo) { + validateRouterInfo(routerInfo); String desPartyId = routerInfo.getDesPartyId(); String roleId = routerInfo.getDesRole(); Preconditions.checkArgument(StringUtils.isNotEmpty(desPartyId),"des party id is null"); @@ -119,7 +110,10 @@ public synchronized String addRouterInfo(RouterInfo routerInfo) { @Override public void setRouterTable(String content) { + if(JsonUtil.validateJson(content)){ + Map tempConf = JsonUtil.json2Object(content, Map.class); + validateAllRouterTable(tempConf); this.saveRouterTable(content); }else { throw new ParameterException("invalid json"); @@ -210,10 +204,7 @@ public RouterInfo route(String srcPartyId, String srcRole, String dstPartyId, St Map>> initRouteTable(Map confJson) { - // BasicMeta.Endpoint.Builder endpointBuilder = BasicMeta.Endpoint.newBuilder(); Map>> newRouteTable = new ConcurrentHashMap<>(); - // loop through coordinator - confJson.forEach((k, v) -> { String coordinatorKey = k.toString(); Map coordinatorValue = (Map) v; @@ -223,7 +214,6 @@ Map>> initRouteTable(Map confJson) { serviceTable = new ConcurrentHashMap<>(4); newRouteTable.put(coordinatorKey, serviceTable); } - // loop through role in coordinator for (Object roleEntryObject : coordinatorValue.entrySet()) { Map.Entry roleEntry = (Map.Entry) roleEntryObject; String roleKey = roleEntry.getKey().toString(); @@ -231,20 +221,16 @@ Map>> initRouteTable(Map confJson) { continue; } List roleValue = (List) roleEntry.getValue(); - List endpoints = serviceTable.get(roleKey); if (endpoints == null) { endpoints = new ArrayList<>(); serviceTable.put(roleKey, endpoints); } - // loop through endpoints for (Object endpointElement : roleValue) { Map element = Maps.newHashMap(); Map endpointJson = (Map) endpointElement; element.putAll(endpointJson); endpoints.add(element); - - } } @@ -390,7 +376,6 @@ private boolean checkCycle(String ip, int port) { } } } - return cycle; } @@ -408,9 +393,6 @@ public String getIpInfoFromUrl(String url) { return result; } - - - public synchronized boolean saveRouterTable( String content) { try { String routerTablePath = getRouterTablePath(); @@ -430,24 +412,73 @@ public synchronized boolean saveRouterTable( String content) { return FileUtils.writeStr2ReplaceFileSync(JsonUtil.formatJson(content), routerTablePath); } catch (Exception e) { logger.error("save router table failed ", e); - - return false; } } - private void loadSelfParty(Map totalConfig){ List selfParties = (List)totalConfig.get(SELF_PARTY); logger.info("load self party {}",selfParties); if(selfParties!=null){ - MetaInfo.PROPERTY_SELF_PARTY = new HashSet<>(selfParties); + Set partySet = new HashSet<>(); + selfParties.forEach(party->{ + partySet.add(party.toString()); + }); + MetaInfo.PROPERTY_SELF_PARTY = partySet; }else{ logger.error("self_party is not found in route_table.json"); } } + + private void validateRouterInfo(RouterInfo routerInfo){ + Preconditions.checkArgument(routerInfo!=null); + String desPartyId = routerInfo.getDesPartyId(); + Preconditions.checkArgument(StringUtils.isNotEmpty(desPartyId),"des party id is null"); + if(routerInfo.getProtocol()!=null||Protocol.grpc.equals(routerInfo.getProtocol())){ + Preconditions.checkArgument(StringUtils.isNotEmpty(routerInfo.getHost()), "route_table.json "+desPartyId+" host/ip is null"); + Preconditions.checkArgument(routerInfo.getPort()!=null, "route_table.json "+desPartyId+" port is null"); + } + } + + + private void validateAllRouterTable( Map tempConf){ + if(tempConf==null){ + throw new SysException("please check route_table.json, it is not a valid json or file is not found"); + } + Object selfPartyObject = tempConf.get(SELF_PARTY); + if(selfPartyObject==null){ + logger.error("{} is not found in route_table.json",SELF_PARTY); + throw new SysException("self_party is not found in route_table.json"); + } + if(!(selfPartyObject instanceof List)){ + throw new SysException("self_party in route_table.json is invalid, it should be an array"); + } + Map content = (Map) tempConf.get(ROUTE_TABLE); + Map>> temp = initRouteTable(content); + + temp.forEach((k,v)->{ + if(StringUtils.isEmpty(k)){ + throw new SysException(""); + } + if(!(v instanceof Map)){ + throw new SysException(""); + } + v.forEach((role,routerMaps)->{ + for (Map routerMap : routerMaps) { + RouterInfo routerInfo = buildRouterInfo(routerMap, "", "", k, role); + validateRouterInfo(routerInfo); + } + }); + }); + + + + } + private void loadRouterTable(String conf){ Map tempConf = JsonUtil.json2Object(conf, Map.class); + + validateAllRouterTable(tempConf); if (tempConf != null) { loadSelfParty(tempConf); Map content = (Map) tempConf.get(ROUTE_TABLE); diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/RouterTableAddService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableAddService.java similarity index 61% rename from java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/RouterTableAddService.java rename to java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableAddService.java index e33003d5f5..89975a51d5 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/ptp/RouterTableAddService.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableAddService.java @@ -1,35 +1,54 @@ -package org.fedai.osx.broker.ptp; +package org.fedai.osx.broker.router; import com.google.inject.Inject; import com.google.inject.Singleton; -import org.fedai.osx.broker.pojo.*; -import org.fedai.osx.broker.router.RouterService; -import org.fedai.osx.broker.router.RouterServiceRegister; +import org.fedai.osx.broker.pojo.RouterAddRequest; +import org.fedai.osx.broker.pojo.RouterAddResponse; import org.fedai.osx.broker.service.Register; -import org.fedai.osx.core.constant.*; +import org.fedai.osx.broker.token.TokenValidator; +import org.fedai.osx.broker.token.TokenValidatorRegister; +import org.fedai.osx.core.constant.ActionType; +import org.fedai.osx.core.constant.StatusCode; +import org.fedai.osx.core.constant.UriConstants; import org.fedai.osx.core.context.OsxContext; import org.fedai.osx.core.exceptions.ExceptionInfo; +import org.fedai.osx.core.exceptions.ParameterException; import org.fedai.osx.core.router.RouterInfo; import org.fedai.osx.core.service.AbstractServiceAdaptorNew; -import org.ppc.ptp.Osx; +import org.fedai.osx.core.utils.JsonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_NEED_TOKEN; +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR; + @Singleton @Register(uris ={UriConstants.HTTP_ADD_ROUTER ,},allowInterUse = false) public class RouterTableAddService extends AbstractServiceAdaptorNew{ + Logger logger = LoggerFactory.getLogger(RouterTableAddService.class); @Inject RouterServiceRegister routerServiceRegister; + @Inject + TokenValidatorRegister tokenValidatorRegister; @Override protected RouterAddResponse doService(OsxContext context, RouterAddRequest data) { - context.setActionType(ActionType.SET_ROUTER.name()); + context.setActionType(ActionType.ADD_ROUTER.name()); + if(PROPERTY_ROUTER_CHANGE_NEED_TOKEN){ + TokenValidator tokenValidator = tokenValidatorRegister.select(PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + if(tokenValidator!=null) { + tokenValidator.validate(data.getToken()); + }else { + logger.error("token validator {} is not found",PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + } + } RouterAddResponse response = new RouterAddResponse(); RouterService routerService = routerServiceRegister.getTechProvider(context); String allRouterInfo = routerService.addRouterInfo(buildRouterInfo(data)); response.setData(allRouterInfo); + response.setCode(StatusCode.PTP_SUCCESS); return response; } private RouterInfo buildRouterInfo(RouterAddRequest data){ @@ -64,11 +83,14 @@ protected RouterAddResponse transformExceptionInfo(OsxContext context, Exceptio @Override public RouterAddRequest decode(Object object) { - return null; + if(object instanceof String){ + RouterAddRequest result =JsonUtil.json2Object(object.toString(),RouterAddRequest.class); + if(result==null){ + throw new ParameterException("invalid param for router operation"); + } + } + throw new ParameterException("invalid param for router operation"); } - @Override - public Osx.Outbound toOutbound(RouterAddResponse response) { - return null; - } + } diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableGetService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableGetService.java new file mode 100644 index 0000000000..a810cf13f3 --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableGetService.java @@ -0,0 +1,72 @@ +package org.fedai.osx.broker.router; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.fedai.osx.broker.pojo.RouterTableGetRequest; +import org.fedai.osx.broker.pojo.RouterTableGetResponse; +import org.fedai.osx.broker.service.Register; +import org.fedai.osx.broker.token.TokenValidator; +import org.fedai.osx.broker.token.TokenValidatorRegister; +import org.fedai.osx.core.config.MetaInfo; +import org.fedai.osx.core.constant.ActionType; +import org.fedai.osx.core.constant.StatusCode; +import org.fedai.osx.core.constant.UriConstants; +import org.fedai.osx.core.context.OsxContext; +import org.fedai.osx.core.exceptions.ExceptionInfo; +import org.fedai.osx.core.exceptions.ParameterException; +import org.fedai.osx.core.service.AbstractServiceAdaptorNew; +import org.fedai.osx.core.utils.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_NEED_TOKEN; +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR; + +@Singleton +@Register(uris ={UriConstants.HTTP_GET_ROUTER ,},allowInterUse = false) +public class RouterTableGetService extends AbstractServiceAdaptorNew + { + Logger logger = LoggerFactory.getLogger(RouterTableGetService.class); + @Inject + RouterServiceRegister routerServiceRegister; + @Inject + TokenValidatorRegister tokenValidatorRegister; + + @Override + protected RouterTableGetResponse doService(OsxContext context, RouterTableGetRequest data) { + context.setActionType(ActionType.GET_ROUTER.name()); + if(PROPERTY_ROUTER_CHANGE_NEED_TOKEN){ + TokenValidator tokenValidator = tokenValidatorRegister.select(PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + if(tokenValidator!=null) { + tokenValidator.validate(data.getToken()); + }else { + logger.error("token validator {} is not found",PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + } + } + RouterService routerService = routerServiceRegister.select(MetaInfo.PROPERTY_FATE_TECH_PROVIDER); + RouterTableGetResponse response = new RouterTableGetResponse(); + response.setContent(routerService.getRouterTable()); + response.setCode(StatusCode.PTP_SUCCESS); + return response; + } + + @Override + protected RouterTableGetResponse transformExceptionInfo(OsxContext context, ExceptionInfo exceptionInfo) { + RouterTableGetResponse response = new RouterTableGetResponse(); + response.setCode(exceptionInfo.getCode()); + response.setMsg(exceptionInfo.getMessage()); + return response; + } + @Override + public RouterTableGetRequest decode(Object object) { + if(object instanceof String){ + RouterTableGetRequest result = JsonUtil.json2Object(object.toString(),RouterTableGetRequest.class); + if(result==null){ + throw new ParameterException("invalid param for router operation"); + } + return result; + } + throw new ParameterException("invalid param"); + } + + } diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableSetService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableSetService.java new file mode 100644 index 0000000000..487448841b --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/RouterTableSetService.java @@ -0,0 +1,76 @@ +package org.fedai.osx.broker.router; + +import com.google.common.base.Preconditions; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.commons.lang3.StringUtils; +import org.fedai.osx.broker.pojo.RouterTableSetRequest; +import org.fedai.osx.broker.pojo.RouterTableSetResponse; +import org.fedai.osx.broker.service.Register; +import org.fedai.osx.broker.token.TokenValidator; +import org.fedai.osx.broker.token.TokenValidatorRegister; +import org.fedai.osx.core.config.MetaInfo; +import org.fedai.osx.core.constant.ActionType; +import org.fedai.osx.core.constant.StatusCode; +import org.fedai.osx.core.constant.UriConstants; +import org.fedai.osx.core.context.OsxContext; +import org.fedai.osx.core.exceptions.ExceptionInfo; +import org.fedai.osx.core.exceptions.ParameterException; +import org.fedai.osx.core.service.AbstractServiceAdaptorNew; +import org.fedai.osx.core.utils.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_NEED_TOKEN; +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR; + +@Singleton +@Register(uris ={UriConstants.HTTP_SET_ROUTER ,},allowInterUse = false) +public class RouterTableSetService extends AbstractServiceAdaptorNew { + + Logger logger = LoggerFactory.getLogger(RouterTableSetService.class); + @Inject + RouterServiceRegister routerServiceRegister; + + @Inject + TokenValidatorRegister tokenValidatorRegister; + + @Override + protected RouterTableSetResponse doService(OsxContext context, RouterTableSetRequest data) { + context.setActionType(ActionType.SET_ROUTER.name()); + if(PROPERTY_ROUTER_CHANGE_NEED_TOKEN){ + TokenValidator tokenValidator = tokenValidatorRegister.select(PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + if(tokenValidator!=null) { + tokenValidator.validate(data.getToken()); + }else { + logger.error("token validator {} is not found",PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + } + } + RouterService routerService = routerServiceRegister.select(MetaInfo.PROPERTY_FATE_TECH_PROVIDER); + Preconditions.checkArgument(data!=null&& StringUtils.isNotEmpty(data.getData())); + routerService.setRouterTable(data.getData()); + RouterTableSetResponse response = new RouterTableSetResponse(); + response.setCode(StatusCode.PTP_SUCCESS); + return response; + } + + @Override + protected RouterTableSetResponse transformExceptionInfo(OsxContext context, ExceptionInfo exceptionInfo) { + RouterTableSetResponse routerTableSetResponse = new RouterTableSetResponse(); + routerTableSetResponse.setCode(exceptionInfo.getCode()); + routerTableSetResponse.setMsg(exceptionInfo.getMessage()); + return routerTableSetResponse; + } + @Override + public RouterTableSetRequest decode(Object object) { + if(object instanceof String){ + RouterTableSetRequest result = JsonUtil.json2Object(object.toString(),RouterTableSetRequest.class); + if(result==null){ + throw new ParameterException("invalid param for router operation"); + } + return result; + } + throw new ParameterException("invalid param"); + } + +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/SelfPartySetService.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/SelfPartySetService.java new file mode 100644 index 0000000000..ee5a7debd0 --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/router/SelfPartySetService.java @@ -0,0 +1,81 @@ +package org.fedai.osx.broker.router; + +import com.google.common.base.Preconditions; +import com.google.inject.Singleton; +import org.fedai.osx.broker.pojo.SetSelfPartyRequest; +import org.fedai.osx.broker.pojo.SetSelfPartyResponse; +import org.fedai.osx.broker.service.Register; +import org.fedai.osx.broker.token.TokenValidator; +import org.fedai.osx.broker.token.TokenValidatorRegister; +import org.fedai.osx.core.config.MetaInfo; +import org.fedai.osx.core.constant.ActionType; +import org.fedai.osx.core.constant.StatusCode; +import org.fedai.osx.core.constant.UriConstants; +import org.fedai.osx.core.context.OsxContext; +import org.fedai.osx.core.exceptions.ExceptionInfo; +import org.fedai.osx.core.exceptions.ParameterException; +import org.fedai.osx.core.service.AbstractServiceAdaptorNew; +import org.fedai.osx.core.utils.JsonUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_NEED_TOKEN; +import static org.fedai.osx.core.config.MetaInfo.PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR; + +@Singleton +@Register(uris ={UriConstants.HTTP_SET_SELF },allowInterUse = false) +public class SelfPartySetService extends AbstractServiceAdaptorNew { + + Logger logger = LoggerFactory.getLogger(SelfPartySetService.class); + @Inject + RouterServiceRegister routerServiceRegister; + @Inject + TokenValidatorRegister tokenValidatorRegister; + + @Override + protected SetSelfPartyResponse doService(OsxContext context, SetSelfPartyRequest data) { + context.setActionType(ActionType.SET_SELF_PARTY.name()); + if(PROPERTY_ROUTER_CHANGE_NEED_TOKEN){ + TokenValidator tokenValidator = tokenValidatorRegister.select(PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + if(tokenValidator!=null) { + tokenValidator.validate(data.getToken()); + }else { + logger.error("token validator {} is not found",PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR); + } + } + Preconditions.checkArgument(data!=null&&data.getSelfParty()!=null,"self_party is null"); + RouterService routerService =routerServiceRegister.select(MetaInfo.PROPERTY_FATE_TECH_PROVIDER); + routerService.setSelfPartyIds(data.getSelfParty()); + SetSelfPartyResponse response = new SetSelfPartyResponse(); + response.setCode(StatusCode.PTP_SUCCESS); + return response; + } + + @Override + protected SetSelfPartyResponse transformExceptionInfo(OsxContext context, ExceptionInfo exceptionInfo) { + SetSelfPartyResponse response = new SetSelfPartyResponse(); + response.setCode(exceptionInfo.getCode()); + response.setMsg(exceptionInfo.getMessage()); + return response; + } + + @Override + public SetSelfPartyRequest decode(Object object) { + if(object instanceof String){ + SetSelfPartyRequest result = JsonUtil.json2Object(object.toString(),SetSelfPartyRequest.class); + if(result==null||result.getSelfParty()==null||result.getSelfParty().size()==0){ + throw new ParameterException("invalid param for router operation"); + } + return result; + } + throw new ParameterException("invalid param for set self party"); + } +} + + + + + + diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/security/TokenValidatorRegister.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/security/TokenValidatorRegister.java index dbd8798da9..bcfafe37fe 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/security/TokenValidatorRegister.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/security/TokenValidatorRegister.java @@ -1,75 +1,75 @@ -package org.fedai.osx.broker.security; - -import org.fedai.osx.core.config.MetaInfo; -import org.fedai.osx.core.constant.Dict; -import org.fedai.osx.core.frame.Lifecycle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; - -public class TokenValidatorRegister implements Lifecycle { - - final String DEFAULT_KEY = "default"; - final String TOKEY_VALIDATOR_CONFIG_FILE = "token_validator.properties"; - Logger logger = LoggerFactory.getLogger(TokenValidatorRegister.class); - private Map tokenValidatorMap = new ConcurrentHashMap<>(); - - public TokenValidator getTokenValidator(String key, String defaultKey) { - TokenValidator result = tokenValidatorMap.get(key); - if (result == null) { - result = tokenValidatorMap.get(defaultKey); - } - ; - return result; - } - - @Override - public void init() { - if (MetaInfo.PROPERTY_OPEN_TOKEN_GENERATOR) { - String configDir = MetaInfo.PROPERTY_CONFIG_DIR; - String fileName = configDir + Dict.SLASH + TOKEY_VALIDATOR_CONFIG_FILE; - File file = new File(fileName); - Properties config = new Properties(); - try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) { - config.load(inputStream); - } catch (Exception e) { - logger.error("parse file {} error", fileName); - } - - config.forEach((k, v) -> { - if (v != null) { - try { - Class genClass = Class.forName(v.toString()); - Object rawObject = genClass.getConstructor().newInstance(); - if (!(rawObject instanceof TokenValidator)) { - logger.error("parse token validator err , {} ", v); - return; - } - tokenValidatorMap.put(k.toString(), (TokenValidator) rawObject); - } catch (Exception e) { - logger.error("register token validator error {} : {}", k, v); - } - } - }); - } - } - - @Override - public void start() { - logger.info("register token validator : {}", this.tokenValidatorMap); - } - - @Override - public void destroy() { - this.tokenValidatorMap.clear(); - } - - -} +//package org.fedai.osx.broker.security; +// +//import org.fedai.osx.core.config.MetaInfo; +//import org.fedai.osx.core.constant.Dict; +//import org.fedai.osx.core.frame.Lifecycle; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +//import java.io.BufferedInputStream; +//import java.io.File; +//import java.io.FileInputStream; +//import java.io.InputStream; +//import java.util.Map; +//import java.util.Properties; +//import java.util.concurrent.ConcurrentHashMap; +// +//public class TokenValidatorRegister implements Lifecycle { +// +// final String DEFAULT_KEY = "default"; +// final String TOKEY_VALIDATOR_CONFIG_FILE = "token_validator.properties"; +// Logger logger = LoggerFactory.getLogger(TokenValidatorRegister.class); +// private Map tokenValidatorMap = new ConcurrentHashMap<>(); +// +// public TokenValidator getTokenValidator(String key, String defaultKey) { +// TokenValidator result = tokenValidatorMap.get(key); +// if (result == null) { +// result = tokenValidatorMap.get(defaultKey); +// } +// ; +// return result; +// } +// +// @Override +// public void init() { +// if (MetaInfo.PROPERTY_OPEN_TOKEN_GENERATOR) { +// String configDir = MetaInfo.PROPERTY_CONFIG_DIR; +// String fileName = configDir + Dict.SLASH + TOKEY_VALIDATOR_CONFIG_FILE; +// File file = new File(fileName); +// Properties config = new Properties(); +// try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) { +// config.load(inputStream); +// } catch (Exception e) { +// logger.error("parse file {} error", fileName); +// } +// +// config.forEach((k, v) -> { +// if (v != null) { +// try { +// Class genClass = Class.forName(v.toString()); +// Object rawObject = genClass.getConstructor().newInstance(); +// if (!(rawObject instanceof TokenValidator)) { +// logger.error("parse token validator err , {} ", v); +// return; +// } +// tokenValidatorMap.put(k.toString(), (TokenValidator) rawObject); +// } catch (Exception e) { +// logger.error("register token validator error {} : {}", k, v); +// } +// } +// }); +// } +// } +// +// @Override +// public void start() { +// logger.info("register token validator : {}", this.tokenValidatorMap); +// } +// +// @Override +// public void destroy() { +// this.tokenValidatorMap.clear(); +// } +// +// +//} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/server/OsxServer.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/server/OsxServer.java index 0c626503a5..24887dd732 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/server/OsxServer.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/server/OsxServer.java @@ -270,7 +270,7 @@ private io.grpc.Server buildTlsServer() { NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forAddress(address); SslContextBuilder sslContextBuilder = null; - if (PROPERTY_OPEN_TLS_USE_KEYSTORE) { + if (StringUtils.isNotBlank(PROPERTY_SERVER_KEYSTORE_FILE)) { // Load the truststore file KeyStore trustStore = loadKeyStore(trustFilePath, trustJksPassword); // Create a TrustManagerFactory and initialize it with the truststore diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/service/ServiceRegisterManager.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/service/ServiceRegisterManager.java index 08c6067db0..2e0a36aab3 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/service/ServiceRegisterManager.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/service/ServiceRegisterManager.java @@ -59,7 +59,7 @@ public ServiceRegisterInfo getServiceWithLoadBalance(OsxContext osxContext, Stri if (services != null && services.size() > 0) { result = services.get((int) (now % services.size())); if (interInvoke && result.isAllowInterUse()) { - throw new InvalidRequestException(); + throw new InvalidRequestException("invalid request"); } } return result; diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/SimpleTokenValidator.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/SimpleTokenValidator.java new file mode 100644 index 0000000000..c2e3dd042f --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/SimpleTokenValidator.java @@ -0,0 +1,22 @@ +package org.fedai.osx.broker.token; + +import org.fedai.osx.core.exceptions.InvalidRequestException; + +import java.util.Properties; + +public class SimpleTokenValidator implements TokenValidator{ + + String rightToken = ""; + + @Override + public void init(String name ,Properties properties) { + rightToken= properties.getProperty(name+".token"); + } + + @Override + public void validate(String token) { + if(token==null||!token.equals(rightToken)){ + throw new InvalidRequestException("token is invalid"); + } + } +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/TokenValidator.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/TokenValidator.java new file mode 100644 index 0000000000..90c860aead --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/TokenValidator.java @@ -0,0 +1,8 @@ +package org.fedai.osx.broker.token; + +import java.util.Properties; + +public interface TokenValidator { + public void init(String name ,Properties properties); + public void validate(String token); +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/TokenValidatorRegister.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/TokenValidatorRegister.java new file mode 100644 index 0000000000..c43cbb5884 --- /dev/null +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/token/TokenValidatorRegister.java @@ -0,0 +1,71 @@ +package org.fedai.osx.broker.token; + +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Singleton; +import org.apache.commons.lang3.StringUtils; +import org.fedai.osx.broker.router.RouterService; +import org.fedai.osx.broker.router.RouterServiceRegister; +import org.fedai.osx.core.config.MetaInfo; +import org.fedai.osx.core.constant.Dict; +import org.fedai.osx.core.context.OsxContext; +import org.fedai.osx.core.frame.Lifecycle; +import org.fedai.osx.core.service.ApplicationStartedRunner; +import org.fedai.osx.core.utils.PropertiesUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +@Singleton +public class TokenValidatorRegister implements Lifecycle , ApplicationStartedRunner { + + final String configFileName = "components/token_validator.properties"; + + Logger logger = LoggerFactory.getLogger(TokenValidatorRegister.class); + + @Inject + Injector injector; + + ConcurrentMap registerMap = new ConcurrentHashMap<>(); + + final public TokenValidator select(String name) { + if (this.registerMap.containsKey(name)) { + return this.registerMap.get(name); + } + return null; + } + + public void init() { + Properties properties = PropertiesUtil.getProperties(MetaInfo.PROPERTY_CONFIG_DIR + Dict.SLASH + Dict.SLASH + configFileName); + properties.forEach((k, v) -> { + try { + if(k.toString().contains(".impl")){ + TokenValidator tokenValidator = (TokenValidator) injector.getInstance(Class.forName(v.toString())); + String name = k.toString().split("\\.")[0]; + tokenValidator.init(name,properties); + this.registerMap.put(name, tokenValidator); + } + } catch (Exception e) { + logger.error("token validator {} class {} init error", k, v, e); + } + }); + logger.info("token validator register : {}", this.registerMap); + } + + @Override + public void start() { + init(); + } + + @Override + public void destroy() { + this.registerMap.clear(); + } + + @Override + public void run(String[] args) throws Exception { + start(); + } +} diff --git a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/util/TransferUtil.java b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/util/TransferUtil.java index 2a7bbb7ed5..20a16c266a 100644 --- a/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/util/TransferUtil.java +++ b/java/osx/osx-broker/src/main/java/org/fedai/osx/broker/util/TransferUtil.java @@ -407,7 +407,7 @@ static public Object redirect(OsxContext context, Object inbound = inboundBuilder.build(); } if(inbound==null){ - throw new InvalidRequestException(); + throw new InvalidRequestException("invalid request"); } context.setDataSize(inbound.getSerializedSize()); diff --git a/java/osx/osx-broker/src/main/resources/broker/route_table.json b/java/osx/osx-broker/src/main/resources/broker/route_table.json index 6bed834ece..d76fa4ab14 100644 --- a/java/osx/osx-broker/src/main/resources/broker/route_table.json +++ b/java/osx/osx-broker/src/main/resources/broker/route_table.json @@ -1,15 +1,6 @@ { "route_table":{ - "3488":{ - "default":[ - { - - "port":8888, - "ip":"localhost" - } - ] - }, - "3477":{ + "3479":{ "default":[ { "port":8889, @@ -17,37 +8,16 @@ } ] }, - "10000":{ + "1234":{ "default":[ { - "protocol":"grpc", - "ip":"localhost", - "port":9366, - "url":"http://127.0.0.1:8122/v1/interconn/chan/invoke" - } - ] - }, - "9999":{ - "default":[ - { - "protocol":"grpc", - "port":7304, - "ip":"127.0.0.1" - } - ] - }, - "10001":{ - "default":[ - { - "protocol":"grpc", - "port":9375, - "url":"http://127.0.0.1:8087/osx/inbound", - "ip":"127.0.0.1" + "port":9999, + "ip":"localhost" } ] } }, "self_party":[ - "9999" + "xxxxx" ] } \ No newline at end of file diff --git a/java/osx/osx-broker/src/main/resources/components/token_validator.properties b/java/osx/osx-broker/src/main/resources/components/token_validator.properties new file mode 100644 index 0000000000..d5d6e53bc9 --- /dev/null +++ b/java/osx/osx-broker/src/main/resources/components/token_validator.properties @@ -0,0 +1,2 @@ +default.impl=org.fedai.osx.broker.token.SimpleTokenValidator +default.token=1234 \ No newline at end of file diff --git a/java/osx/osx-broker/src/test/java/org/fedai/osx/broker/test/router/RouterSetTest.java b/java/osx/osx-broker/src/test/java/org/fedai/osx/broker/test/router/RouterSetTest.java index db80e332ce..f5fdfc012f 100644 --- a/java/osx/osx-broker/src/test/java/org/fedai/osx/broker/test/router/RouterSetTest.java +++ b/java/osx/osx-broker/src/test/java/org/fedai/osx/broker/test/router/RouterSetTest.java @@ -3,6 +3,8 @@ import com.google.common.collect.Maps; import okhttp3.*; import org.fedai.osx.broker.pojo.RouterAddRequest; +import org.fedai.osx.broker.pojo.RouterTableGetRequest; +import org.fedai.osx.broker.pojo.SetSelfPartyRequest; import org.fedai.osx.core.config.MetaInfo; import org.fedai.osx.core.constant.PtpHttpHeader; import org.fedai.osx.core.constant.UriConstants; @@ -12,17 +14,81 @@ import java.io.IOException; import java.util.Base64; +import java.util.HashSet; import java.util.Map; public class RouterSetTest { String url = "http://localhost:8807"; + @Test - public void testSetRouter() { + public void testGetRouteTable() { + RouterTableGetRequest getRequest = new RouterTableGetRequest(); + MediaType jsonMime = MediaType.parse("application/json"); + OkHttpClient okHttpClient = new OkHttpClient(); + Request.Builder builder = new Request.Builder(); + getRequest.setToken("1234"); + String requestContent = JsonUtil.object2Json(getRequest); + System.err.println("request content "+requestContent); + RequestBody requestBody = RequestBody.create(requestContent, jsonMime); + Request request = builder.url(url + UriConstants.HTTP_GET_ROUTER).post(requestBody) + .build(); + try { + Response response = okHttpClient.newCall(request).execute(); + System.err.println(response); + System.err.println("body " + new String(response.body().bytes())); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Test + public void testSetSelfParty() { + SetSelfPartyRequest routerInfo = new SetSelfPartyRequest(); + HashSet set = new HashSet<>(); + set.add("77377"); + routerInfo.setSelfParty(set); + MediaType jsonMime = MediaType.parse("application/json"); + OkHttpClient okHttpClient = new OkHttpClient(); + Request.Builder builder = new Request.Builder(); + String requestContent = JsonUtil.object2Json(routerInfo); + System.err.println("request content "+requestContent); + RequestBody requestBody = RequestBody.create(requestContent, jsonMime); + Request request = builder.url(url + UriConstants.HTTP_SET_SELF).post(requestBody) + .build(); + try { + Response response = okHttpClient.newCall(request).execute(); + System.err.println(response); + System.err.println("body " + new String(response.body().bytes())); + } catch (IOException e) { + e.printStackTrace(); + } + } + +// curl --location 'http://localhost:8807/v1/inner/router/add' --header 'Content-Type: application/json' --data '{ +// "desPartyId": "1234", +// "ip": "localhost", +// "port": "9999" +// +//}' +// +// curl --location 'http://localhost:8807/v1/inner/router/get' --header 'Content-Type: application/json' --data '{ +//}' +// +// curl --location 'http://localhost:8807/v1/inner/router/self' --header 'Content-Type: application/json' --data '{ +// "selfParty": ["xxxxx"] +//}' + + + + + + @Test + public void testAddRouter() { RouterAddRequest routerInfo = new RouterAddRequest(); - routerInfo.setDesPartyId("3477"); + routerInfo.setDesPartyId("3479"); routerInfo.setIp("localhost"); routerInfo.setPort(8889); MediaType jsonMime = MediaType.parse("application/json"); diff --git a/java/osx/osx-core/src/main/java/org/fedai/osx/core/config/MetaInfo.java b/java/osx/osx-core/src/main/java/org/fedai/osx/core/config/MetaInfo.java index f600501140..23a53bc5e9 100644 --- a/java/osx/osx-core/src/main/java/org/fedai/osx/core/config/MetaInfo.java +++ b/java/osx/osx-core/src/main/java/org/fedai/osx/core/config/MetaInfo.java @@ -140,7 +140,7 @@ public class MetaInfo { @Config(confKey = "https.port", pattern = Dict.POSITIVE_INTEGER_PATTERN) public static Integer PROPERTY_HTTPS_PORT=8809; @Config(confKey = "open.http.server", pattern = Dict.BOOLEAN_PATTERN) - public static Boolean PROPERTY_OPEN_HTTP_SERVER = false; + public static Boolean PROPERTY_OPEN_HTTP_SERVER = true; @Config(confKey = "open.https.server", pattern = Dict.BOOLEAN_PATTERN) public static Boolean PROPERTY_HTTP_USE_TLS = false; @Config(confKey = "http.server.acceptor.num", pattern = Dict.POSITIVE_INTEGER_PATTERN) @@ -249,6 +249,11 @@ public class MetaInfo { public static Integer PROPERTY_ROUTER_CHECK_INTERVAL= 300000; @Config(confKey = "channel.pool.info") public static Integer PROPERTY_CHANNEL_POOL_INFO = 30000; + @Config(confKey = "router.change.need.token") + public static Boolean PROPERTY_ROUTER_CHANGE_NEED_TOKEN= false; + @Config(confKey = "router.change.token.validator") + public static String PROPERTY_ROUTER_CHANGE_TOKEN_VALIDATOR= Dict.DEFAULT; + public static boolean isCluster() { return PROPERTY_DEPLOY_MODE.equals(DeployMode.cluster.name()); diff --git a/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/ActionType.java b/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/ActionType.java index c48fa1d6a9..a08c6828f2 100644 --- a/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/ActionType.java +++ b/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/ActionType.java @@ -31,5 +31,9 @@ public enum ActionType { UNARY_CALL_NEW, CLUSTER_TOKEN_APPLY, TOPIC_APPLY, - SET_ROUTER; + SET_ROUTER, + GET_ROUTER, + ADD_ROUTER, + SET_SELF_PARTY + } diff --git a/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/UriConstants.java b/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/UriConstants.java index 153fabaee6..300e163f95 100644 --- a/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/UriConstants.java +++ b/java/osx/osx-core/src/main/java/org/fedai/osx/core/constant/UriConstants.java @@ -1,42 +1,20 @@ package org.fedai.osx.core.constant; public class UriConstants { - - // "/v1/interconn/chan/pop": -// "/v1/interconn/chan/push": -// "/v1/interconn/chan/peek": -// "/v1/interconn/chan/release": -// "/v1/interconn/net/weave": - //"/org.ppc.ptp.PrivateTransferTransport/peek": - //"/org.ppc.ptp.PrivateTransferTransport/pop": - // "/org.ppc.ptp.PrivateTransferTransport/push": - //"/org.ppc.ptp.PrivateTransferTransport/release": - - public static final String POP = "/org.ppc.ptp.PrivateTransferTransport/pop"; public static final String PEEK = "/org.ppc.ptp.PrivateTransferTransport/peek"; public static final String PUSH = "/org.ppc.ptp.PrivateTransferTransport/push"; public static final String RELEASE = "/org.ppc.ptp.PrivateTransferTransport/release"; public static final String UNARYCALL = "/org.fedai.osx.proxy/unary"; - - public static final String EGGROLL_PUSH = "/org.fedai.osx.proxy/push"; - public static final String HTTP_PUSH = "/v1/interconn/chan/push"; public static final String HTTP_POP = "/v1/interconn/chan/pop"; public static final String HTTP_PEEK = "/v1/interconn/chan/peek"; - public static final String HTTP_RELEASE = "/v1/interconn/chan/release"; - public static final String HTTP_INVOKE = "/v1/interconn/chan/invoke"; - public static final String HTTP_SET_SELF = "/v1/inner/router/self"; public static final String HTTP_ADD_ROUTER = "/v1/inner/router/add"; public static final String HTTP_SET_ROUTER = "/v1/inner/router/set"; public static final String HTTP_GET_ROUTER = "/v1/inner/router/get"; - public static final String CLUSTER_TOKEN_APPLY = "/org.fedai.osx.proxy/tokenApply"; - public static final String CLUSTER_TOPIC_APPLY = "/org.fedai.osx.proxy/topicApply"; - - } diff --git a/java/osx/osx-core/src/main/java/org/fedai/osx/core/datasource/AutoRefreshDataSource.java b/java/osx/osx-core/src/main/java/org/fedai/osx/core/datasource/AutoRefreshDataSource.java index 67e1735da5..9975a51234 100644 --- a/java/osx/osx-core/src/main/java/org/fedai/osx/core/datasource/AutoRefreshDataSource.java +++ b/java/osx/osx-core/src/main/java/org/fedai/osx/core/datasource/AutoRefreshDataSource.java @@ -57,7 +57,7 @@ public void run() { T newValue = loadConfig(); getProperty().updateValue(newValue); } catch (Throwable e) { - logger.info("loadConfig exception", e); + logger.error("load route_table.json exception", e); } } }, recommendRefreshMs, recommendRefreshMs, TimeUnit.MILLISECONDS); diff --git a/java/osx/osx-core/src/main/java/org/fedai/osx/core/exceptions/InvalidRequestException.java b/java/osx/osx-core/src/main/java/org/fedai/osx/core/exceptions/InvalidRequestException.java index 96303a7b58..f3715a675b 100644 --- a/java/osx/osx-core/src/main/java/org/fedai/osx/core/exceptions/InvalidRequestException.java +++ b/java/osx/osx-core/src/main/java/org/fedai/osx/core/exceptions/InvalidRequestException.java @@ -1,4 +1,9 @@ package org.fedai.osx.core.exceptions; +import org.fedai.osx.core.constant.StatusCode; + public class InvalidRequestException extends BaseException { + public InvalidRequestException(String msg){ + super(StatusCode.PTP_INVALID_REQUEST,msg); + } } diff --git a/java/osx/osx-register/pom.xml b/java/osx/osx-register/pom.xml deleted file mode 100644 index 44ad0d3999..0000000000 --- a/java/osx/osx-register/pom.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - osx - osx - 1.0.0 - - 4.0.0 - - osx-register - - - 8 - 8 - - - \ No newline at end of file