diff --git a/README.md b/README.md index 94d76b8..e6c5822 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,13 @@ _聚合阿里、百度等开放平台，提供与`openai`请求格式对齐的 ## 功能 ### 支持内容审核渠道 + - [x] 阿里云 - [x] 百度智能云 -- [ ] 七牛云 +- [x] 七牛云 ### 支持`openai`对齐的接口 (支持的接口会先内容审核再转发，不支持的接口会直接转发) + - [x] `v1/chat/completions` 用户输入的内容审查。 - [x] `v1/images/generations`用户输入的内容审查。 @@ -116,38 +118,46 @@ Render 可以直接部署 docker 镜像,不需要 fork 仓库：[Render](https:/ - ## 配置 ### 环境变量 #### 通用 -| 变量参数 | 变量描述 | 是否必填 | -|:------------------:|:------------------------------------------------:|:----:| -| AUDIT_CHANNEL_TYPE | 审核渠道类型[ali:阿里、baidu:百度] | Y | +| 变量参数 | 变量描述 | 是否必填 | +|:------------------:|:---------------------------------------------------:|:----:| +| AUDIT_CHANNEL_TYPE | 审核渠道类型[ali:阿里、baidu:百度] | Y | | BASE_URL | 审核通过后的转发接口请求地址域名或IP:端口（例如: https://api.openai.com ） | Y | -| AUTHORIZATION | 鉴权密钥，与转发接口的API-Key保持一致，多个以`,`分隔 | Y | -| ENABLE | 审核启用开关[0:关闭、1:打开]（默认:1） | N | +| AUTHORIZATION | 鉴权密钥，与转发接口的API-Key保持一致，多个以`,`分隔 | Y | +| ENABLE | 审核启用开关[0:关闭、1:打开]（默认:1） | N | -#### 审核渠道-阿里 [阿里云-内容审核](https://vision.console.aliyun.com/cn-shanghai/detail/imageaudit) +#### 审核渠道-阿里云 [阿里云-内容审核](https://vision.console.aliyun.com/cn-shanghai/detail/imageaudit) | 变量参数 | 变量描述 | 是否必填 | |:------------------------:|:--------------------------------------------------------------------------------------------------:|:----:| -| ALI_ACCESS_KEY_ID | 阿里云开放平台AccessKeyId | Y | -| ALI_ACCESS_KEY_SECRET | 阿里云开放平台AccessKeySecret | Y | -| ALI_ENDPOINT | 阿里云开放平台Endpoint | Y | +| ALI_ACCESS_KEY_ID | 阿里云AccessKeyId | Y | +| ALI_ACCESS_KEY_SECRET | 阿里云AccessKeySecret | Y | +| ALI_ENDPOINT | 阿里云Endpoint | Y | | ALI_LABEL | 内容审核类型[spam:垃圾、politics:敏感、abuse:辱骂、terrorism:暴恐、porn:鉴黄、flood:灌水、contraband:违禁、ad:广告] （多个以`,`分隔 ） | Y | | ALI_AUDIT_CONTENT_LENGTH | 审核文本切割字节长度[默认:4000] | N | -#### 审核渠道-百度 [百度智能云-内容审核平台](https://ai.baidu.com/censoring#/strategylist) +#### 审核渠道-百度智能云 [百度智能云-内容审核平台](https://ai.baidu.com/censoring#/strategylist) + +| 变量参数 | 变量描述 | 是否必填 | +|:--------------------------:|:-------------------------------------------------------------------------------------------------------------------:|:----:| +| BAIDU_API_KEY | 百度智能云APIKey | Y | +| BAIDU_SECRET_KEY | 百度智能云SecretKey | Y | +| BAIDU_LABEL | 内容审核类型[default:默认违禁词库、politics:政治敏感、abuse:低俗辱骂、terrorism:暴恐违禁、porn:文本色情、flood:低质灌水、ad:恶意推广、black:自定义黑名单]（多个以`,`分隔 ） | Y | +| BAIDU_AUDIT_CONTENT_LENGTH | 审核文本切割字节长度（默认:4000） | N | + +#### 审核渠道-七牛云 [七牛云-内容审核](https://portal.qiniu.com/censor/main/overview) -| 变量参数 | 变量描述 | 是否必填 | -|:--------------------------:|:--------------------------------------------------------------------------------------------------------------------------------:|:----:| -| BAIDU_API_KEY | 百度开放平台APIKey | Y | -| BAIDU_SECRET_KEY | 百度开放平台SecretKey | Y | -| BAIDU_LABEL | 内容审核类型[default:默认违禁词库、politics:政治敏感、abuse:低俗辱骂、terrorism:暴恐违禁、porn:文本色情、flood:低质灌水、ad:恶意推广、black:自定义黑名单、white:自定义白名单]（多个以`,`分隔 ） | Y | -| BAIDU_AUDIT_CONTENT_LENGTH | 审核文本切割字节长度（默认:4000） | N | +| 变量参数 | 变量描述 | 是否必填 | +|:--------------------------:|:--------------------------------------------------------------------------------------------------------------------:|:----:| +| QINIU_ACCESS_KEY | 七牛云APIKey | Y | +| QINIU_SECRET_KEY | 七牛云SecretKey | Y | +| QINIU_LABEL | 内容审核类型[spam:含垃圾信息、politics:涉政、abuse:辱骂、terrorism:暴恐、porn:色情、flood:灌水、ad:广告、contraband:违禁、meaningless:无意义]（多个以`,`分隔 ） | Y | +| QINIU_AUDIT_CONTENT_LENGTH | 审核文本切割字节长度（默认:4000） | N | diff --git a/check/check.go b/check/check.go index 00cf2b7..7789d15 100644 --- a/check/check.go +++ b/check/check.go @@ -23,6 +23,16 @@ func CheckEnvVariable() { if config.AliLabel == "" { logger.FatalLog("环境变量 ALI_LABEL 未设置") } + } else if strings.ToLower(config.AuditChannelType) == "qiniu" { + if config.QiNiuAccessKey == "" { + logger.FatalLog("环境变量 QINIU_ACCESS_KEY 未设置") + } + if config.QiNiuSecretKey == "" { + logger.FatalLog("环境变量 QINIU_SECRET_KEY 未设置") + } + if config.QiNiuLabel == "" { + logger.FatalLog("环境变量 QINIU_LABEL 未设置") + } } else if strings.ToLower(config.AuditChannelType) == "baidu" { if config.BaiduApiKey == "" { logger.FatalLog("环境变量 BAIDU_API_KEY 未设置") @@ -43,7 +53,7 @@ func CheckEnvVariable() { logger.FatalLog("环境变量 BASE_URL 未设置") } - if config.Authorization == "" { + if config.ApiKey == "" { logger.FatalLog("环境变量 AUTHORIZATION 未设置") } diff --git a/common/config/config.go b/common/config/config.go index 42a8e8e..9215db8 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -13,8 +13,8 @@ import ( var AuditChannelType = os.Getenv("AUDIT_CHANNEL_TYPE") var Enable = env.Int("ENABLE", 1) var BaseUrl = os.Getenv("BASE_URL") -var Authorization = os.Getenv("AUTHORIZATION") -var Authorizations = strings.Split(os.Getenv("AUTHORIZATION"), ",") +var ApiKey = os.Getenv("API_KEY") +var ApiKeys = strings.Split(os.Getenv("API_KEY"), ",") /* spam：文字垃圾内容识别 @@ -48,6 +48,24 @@ var BaiduSecretKey = os.Getenv("BAIDU_SECRET_KEY") var BaiduLabel = os.Getenv("BAIDU_LABEL") var BaiduAuditContentLength = env.Int("BAIDU_AUDIT_CONTENT_LENGTH", 4000) +/* +normal：正常文本 +spam：含垃圾信息 +ad：广告 +politics：涉政 +terrorism：暴恐 +abuse：辱骂 +porn：色情 +flood：灌水 +contraband：违禁 +meaningless：无意义 +*/ + +var QiNiuAccessKey = os.Getenv("QINIU_ACCESS_KEY") +var QiNiuSecretKey = os.Getenv("QINIU_SECRET_KEY") +var QiNiuLabel = os.Getenv("QINIU_LABEL") +var QiNiuAuditContentLength = env.Int("QINIU_AUDIT_CONTENT_LENGTH", 4000) + var DebugEnabled = strings.ToLower(os.Getenv("DEBUG")) == "true" var SessionSecret = uuid.New().String() diff --git a/common/constants.go b/common/constants.go index 87221b6..6dddfe0 100644 --- a/common/constants.go +++ b/common/constants.go @@ -3,4 +3,4 @@ package common import "time" var StartTime = time.Now().Unix() // unit: second -var Version = "v0.0.0" // this hard coding will be replaced automatically when building, no need to manually change +var Version = "v0.1.0" // this hard coding will be replaced automatically when building, no need to manually change diff --git a/controller/chat.go b/controller/chat.go index f16ea23..10c959e 100644 --- a/controller/chat.go +++ b/controller/chat.go @@ -79,6 +79,18 @@ func ChatForOpenAI(c *gin.Context) { }) return } + } else if strings.ToLower(config.AuditChannelType) == "qiniu" { + response, err = utils.QiNiuAudit(request) + if err != nil { + c.JSON(http.StatusInternalServerError, model.OpenAIErrorResponse{ + OpenAIError: model.OpenAIError{ + Message: "Unknown audit channel", + Type: "request_error", + Code: "AUDIT_CHANNEL_ERROR", + }, + }) + return + } } else { c.JSON(http.StatusInternalServerError, model.OpenAIErrorResponse{ OpenAIError: model.OpenAIError{ @@ -174,6 +186,18 @@ func ImagesForOpenAI(c *gin.Context) { }) return } + } else if strings.ToLower(config.AuditChannelType) == "qiniu" { + response, err = utils.QiNiuAudit(request) + if err != nil { + c.JSON(http.StatusInternalServerError, model.OpenAIErrorResponse{ + OpenAIError: model.OpenAIError{ + Message: "Unknown audit channel", + Type: "request_error", + Code: "AUDIT_CHANNEL_ERROR", + }, + }) + return + } } else { c.JSON(http.StatusInternalServerError, model.OpenAIErrorResponse{ OpenAIError: model.OpenAIError{ diff --git a/go.mod b/go.mod index 26b9502..0cbc297 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,13 @@ require ( github.com/gin-contrib/sessions v1.0.1 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 + github.com/qiniu/go-sdk/v7 v7.21.0 github.com/samber/lo v1.39.0 ) require ( + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect github.com/alibabacloud-go/darabonba-number v1.0.4 // indirect github.com/alibabacloud-go/debug v1.0.0 // indirect @@ -39,13 +42,14 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/gorilla/context v1.1.2 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -57,6 +61,7 @@ require ( golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect diff --git a/go.sum b/go.sum index 9a662bf..c93a7b8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= +github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 h1:iC9YFYKDGEy3n/FtqJnOkZsene9olVspKmkX5A2YBEo= github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc= github.com/alibabacloud-go/darabonba-number v1.0.4 h1:aTY1TanasI0A1AYT3Co+PLttFSW0qzUz9wFLIpG0tqI= @@ -59,6 +63,7 @@ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -74,16 +79,24 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -100,6 +113,7 @@ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kX github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gorilla/sessions v1.2.2 h1:lqzMYz6bOfvn2WriPUjNByzeXIlVzURcPmgMczkmTjY= github.com/gorilla/sessions v1.2.2/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ= +github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -109,14 +123,19 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f h1:B0OD7nYl2FPQEVrw8g2uyc1lGEzNbvrKh7fspGZcbvY= +github.com/matishsiao/goInfo v0.0.0-20210923090445-da2e3fa8d45f/go.mod h1:aEt7p9Rvh67BYApmZwNDPpgircTO2kgdmDUoF/1QmwA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -129,8 +148,14 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= +github.com/qiniu/go-sdk/v7 v7.21.0 h1:2Ghl5swQ1PJgfKHf8BzCCAOAxcdshGP/Hfuluv9VC18= +github.com/qiniu/go-sdk/v7 v7.21.0/go.mod h1:8EM2awITynlem2VML2dXGHkMYP2UyECsGLOdp6yMpco= +github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= @@ -147,6 +172,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -170,7 +196,9 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= @@ -188,6 +216,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -201,24 +230,30 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= @@ -227,7 +262,9 @@ golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -251,14 +288,17 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.56.0 h1:DPMeDvGTM54DXbPkVIZsp19fp/I2K7zwA/itHYHKo8Y= gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/middleware/auth.go b/middleware/auth.go index 1ba1975..53597d9 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -10,7 +10,7 @@ import ( ) func isValidSecret(secret string) bool { - return config.Authorization != "" && !lo.Contains(config.Authorizations, secret) + return config.ApiKey != "" && !lo.Contains(config.ApiKeys, secret) } func authHelper(c *gin.Context) { diff --git a/model/qiniu.go b/model/qiniu.go new file mode 100644 index 0000000..a20fdf3 --- /dev/null +++ b/model/qiniu.go @@ -0,0 +1,46 @@ +package model + +type QiNiuRequest struct { + Data QiNiuTextData `json:"data"` // 文本内容 + Params QiNiuParamData `json:"params"` // 请求参数，包括审核类型等 +} + +type QiNiuTextData struct { + Text string `json:"text"` // 文本内容 +} + +type QiNiuParamData struct { + Scenes []string `json:"scenes"` // 审核类型 +} + +type QiNiuResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Result QiNiuResult `json:"result"` +} + +type QiNiuResult struct { + Suggestion string `json:"suggestion"` + Scenes map[string]QiNiuSceneDetail `json:"scenes"` +} + +type QiNiuSceneDetail struct { + Suggestion string `json:"suggestion"` + Details []QiNiuDetails `json:"details"` +} + +type QiNiuDetails struct { + Label string `json:"label"` + Score float64 `json:"score"` + Contexts []QiNiuContexts `json:"contexts"` +} + +type QiNiuContexts struct { + Context string `json:"context"` + Positions []QiNiuPosition `json:"positions"` +} + +type QiNiuPosition struct { + StartPos int `json:"startPos"` + EndPos int `json:"endPos"` +} diff --git a/utils/qiniu.go b/utils/qiniu.go new file mode 100644 index 0000000..0f208ba --- /dev/null +++ b/utils/qiniu.go @@ -0,0 +1,127 @@ +package utils + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "fmt" + "github.com/samber/lo" + "gpt-content-audit/common" + "gpt-content-audit/common/config" + "gpt-content-audit/model" + "io" + "net/http" + "strings" +) + +const ( + apiURL = "https://ai.qiniuapi.com/v3/text/censor" + contentType = "application/json" +) + +func generateQiNiuToken(method, path, rawQuery, host, contentType, bodyStr, accessKey, secretKey string) string { + var buffer bytes.Buffer + buffer.WriteString(method + " " + path) + if rawQuery != "" { + buffer.WriteString("?" + rawQuery) + } + buffer.WriteString("\nHost: " + host) + if contentType != "" { + buffer.WriteString("\nContent-Type: " + contentType) + } + buffer.WriteString("\n\n") + if bodyStr != "" && contentType != "" && contentType != "application/octet-stream" { + buffer.WriteString(bodyStr) + } + + h := hmac.New(sha1.New, []byte(secretKey)) + h.Write(buffer.Bytes()) + sign := h.Sum(nil) + encodedSign := base64.URLEncoding.EncodeToString(sign) + return "Qiniu " + accessKey + ":" + encodedSign +} + +func QiNiuAudit[T model.GetUserContent](t T) (model.AuditResponse, error) { + + labels := strings.Split(config.QiNiuLabel, ",") + + var response model.AuditResponse + response.Channel = "qiniu" + + for _, totalContent := range t.GetUserContent() { + for _, content := range common.SplitStringByBytes(totalContent, config.QiNiuAuditContentLength) { + request := model.QiNiuRequest{ + Data: model.QiNiuTextData{ + Text: content, + }, + Params: model.QiNiuParamData{ + Scenes: []string{"antispam"}, + }, + } + + jsonData, err := json.Marshal(request) + if err != nil { + return model.AuditResponse{}, err + } + + token := generateQiNiuToken("POST", "/v3/text/censor", "", "ai.qiniuapi.com", contentType, string(jsonData), config.QiNiuAccessKey, config.QiNiuSecretKey) + + req, err := http.NewRequest("POST", apiURL, bytes.NewBuffer(jsonData)) + if err != nil { + return model.AuditResponse{}, err + } + + req.Header.Set("Content-Type", contentType) + req.Header.Set("Authorization", token) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return model.AuditResponse{}, err + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return model.AuditResponse{}, err + } + + var qiNiuResp model.QiNiuResponse + err = json.Unmarshal(bodyBytes, &qiNiuResp) + if err != nil { + return model.AuditResponse{}, err + } + + if qiNiuResp.Code != 200 && qiNiuResp.Message != "" { + return model.AuditResponse{}, fmt.Errorf(qiNiuResp.Message) + } + + //qiNiuResp.Result.Scenes["antispam"]. + + if qiNiuResp.Result.Suggestion != "pass" || qiNiuResp.Result.Scenes["antispam"].Suggestion != "review" { + if qiNiuResp.Result.Scenes["antispam"].Suggestion != "pass" || qiNiuResp.Result.Scenes["antispam"].Suggestion != "review" { + for _, detail := range qiNiuResp.Result.Scenes["antispam"].Details { + if !lo.Contains(labels, detail.Label) { + continue + } + for _, context := range detail.Contexts { + resultRes := &model.AuditResultResponse{ + //MessageIndex: i, + Context: context.Context, + Label: detail.Label, + Suggestion: strings.ToUpper(qiNiuResp.Result.Scenes["antispam"].Suggestion), + } + if !resultRes.ContainsDuplicate(response.Results) { + response.Results = append(response.Results, resultRes) + } + } + } + } + } + } + } + + return response, nil +}