add toml config option to cmd

.gitignore

@@ -19,6 +19,7 @@ run_server_local.sh
 run_user_local.sh
 cmd/csghub-server/__debug_*
 run_dataviewer_local.sh
+local.toml
 
 .data
 /data

README.md

@@ -32,6 +32,18 @@ curl -L https://raw.githubusercontent.com/OpenCSGs/csghub-server/main/docker-com
 docker-compose -f docker-compose.yml up -d
 ```
 
+## Start CSGHub Server Services Locally
+
+CSGHub supports TOML format for config files. When starting any service from the command line, you can specify the config file with the `--config` option:
+
+```
+go run cmd/csghub-server/main.go start server --config local.toml
+go run cmd/csghub-server/main.go deploy runner --config local.toml
+...
+```
+
+We provide an [example config file](common/config/config.toml.example), you can rename it, modify as needed and use. All available configurations are defined in [this Go file](common/config/config.go). The TOML configuration uses snake_case naming convention, and names automatically map to corresponding struct field names.
+
 ## Technical Architecture
 <div align=center>
   <img src="docs/csghub_server-arch.png" alt="csghub-server architecture" width="800px">

builder/git/gitserver/gitea/client.go

@@ -38,7 +38,7 @@ type TokenResponse struct {
 
 func NewClient(config *config.Config) (client *Client, err error) {
 	httpClient := &http.Client{
-		Timeout: time.Duration(config.GitServer.TimtoutSEC) * time.Second,
+		Timeout: time.Duration(config.GitServer.TimeoutSEC) * time.Second,
 	}
 	token, err := findOrCreateAccessToken(context.Background(), config)
 	if err != nil {

cmd/csghub-server/cmd/root.go

@@ -18,11 +18,13 @@ import (
 	"opencsg.com/csghub-server/cmd/csghub-server/cmd/sync"
 	"opencsg.com/csghub-server/cmd/csghub-server/cmd/trigger"
 	"opencsg.com/csghub-server/cmd/csghub-server/cmd/user"
+	"opencsg.com/csghub-server/common/config"
 )
 
 var (
-	logLevel  string
-	logFormat string
+	logLevel   string
+	logFormat  string
+	configFile string
 )
 
 var RootCmd = &cobra.Command{
@@ -41,10 +43,12 @@ func init() {
 
 	RootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "set log level to debug, info, warn, error or fatal (case-insensitive). default is INFO")
 	RootCmd.PersistentFlags().StringVarP(&logFormat, "log-format", "f", "json", "set log format to json or text. default is json")
+	RootCmd.PersistentFlags().StringVarP(&configFile, "config", "", "", "set config file path.")
 	RootCmd.DisableAutoGenTag = true
 
 	cobra.OnInitialize(func() {
 		setupLog(logLevel, logFormat)
+		config.SetConfigFile(configFile)
 	})
 
 	RootCmd.AddCommand(

common/config/README.md

@@ -0,0 +1,22 @@
+CSGHub server supports two configuration methods: using environment variables or configuration files. You can also combine them, for example, by placing non-sensitive configurations in a config file and overriding sensitive configurations with environment variables.
+
+### Using a Config File
+
+CSGHub supports TOML format for config files. When starting any service from the command line, you can specify the config file with the `--config` option:
+
+```
+go run cmd/csghub-server/main.go start server --config local.toml
+go run cmd/csghub-server/main.go deploy runner --config local.toml
+```
+
+We provide an [example config file](common/config/config.toml.example), you can rename it, modify as needed and use. All available configurations are defined in [this Go file](common/config/config.go). The TOML configuration uses snake_case naming convention, and names automatically map to corresponding struct field names.
+
+If a config value is missing in the config file, the `default` tag value specified in the Go struct file will be used.
+
+### Using Environment Variables
+
+CSGHub also supports configuration through environment variables. The relevant environment variable names are defined in [this Go file](common/config/config.go) under the `env` tag. If an environment variable is absent, the value in the `default` tag will be used.
+
+### Combining Config File with Environment Variables
+
+You can use config file together with environment variables. When both are used, environment variables take **higher priority** than the config file. For example, if you have a `Port` setting, and you specify it in the TOML file as `port=1234` and in an environment variable as `export PORT=5678`, the environment variable value (5678) will be used for the port configuration.

common/config/config.toml.example

@@ -0,0 +1,163 @@
+saas = false
+instance_id = ""
+enable_swagger = false
+enable_https = false
+api_token = "f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf"
+docs_host = "http://localhost:6636"
+
+[api_server]
+port = 8080
+public_domain = "http://localhost:8080"
+ssh_domain = "git@localhost:2222"
+
+[mirror]
+url = "http://localhost:8085"
+token = ""
+port = 8085
+session_secret_key = "mirror"
+worker_number = 5
+
+[database]
+driver = "pg"
+dsn = "postgresql://postgres:postgres@localhost:5432/starhub_server?sslmode=disable"
+timezone = "Asia/Shanghai"
+
+[redis]
+endpoint = "localhost:6379"
+max_retries = 3
+min_idle_connections = 0
+user = ""
+password = ""
+sentinel_mode = false
+sentinel_master = ""
+sentinel_endpoint = ""
+
+[git_server]
+url = "http://localhost:3000"
+type = "gitea"
+host = "http://localhost:3000"
+secret_key = "619c849c49e03754454ccd4cda79a209ce0b30b3"
+username = "root"
+password = "password123"
+timeout_sec = 5
+
+[gitaly_server]
+address = "tcp://localhost:9999"
+storge = "default"
+token = "abc123secret"
+jwt_secret = "signing-key"
+
+[mirror_server]
+enable = false
+url = "http://localhost:3001"
+type = "gitea"
+host = "http://localhost:3001"
+secret_key = "619c849c49e03754454ccd4cda79a209ce0b30b3"
+username = "root"
+password = "password123"
+
+[frontend]
+url = "https://opencsg.com"
+
+[s3]
+ssl = false
+access_key_id = ""
+access_key_secret = ""
+region = ""
+endpoint = "localhost:9000"
+internal_endpoint = ""
+bucket = "opencsg-test"
+enable_ssl = false
+
+[sensitive_check]
+enable = false
+access_key_id = ""
+access_key_secret = ""
+region = ""
+endpoint = "oss-cn-beijing.aliyuncs.com"
+enable_ssl = true
+
+[jwt]
+signing_key = "signing-key"
+valid_hour = 24
+
+[inference]
+server_addr = "http://localhost:8000"
+
+[space]
+builder_endpoint = "http://localhost:8081"
+runner_endpoint = "http://localhost:8082"
+runner_server_port = 8082
+internal_root_domain = "internal.example.com"
+public_root_domain = "public.example.com"
+docker_reg_base = "registry.cn-beijing.aliyuncs.com/opencsg_public/"
+image_pull_secret = "opencsg-pull-secret"
+rproxy_server_port = 8083
+session_secret_key = "secret"
+deploy_timeout_in_min = 30
+gpu_model_label = "aliyun.accelerator/nvidia_name"
+readness_delay_seconds = 120
+readness_period_seconds = 10
+readness_failure_threshold = 3
+
+[model]
+deploy_timeout_in_min = 60
+download_endpoint = "https://hub.opencsg.com"
+docker_reg_base = "opencsg-registry.cn-beijing.cr.aliyuncs.com/public/"
+nim_docker_secret_name = "ngc-secret"
+nim_ngc_secret_name = "nvidia-nim-secrets"
+
+[event]
+sync_interval = 1
+
+[casdoor]
+client_id = "client_id"
+client_secret = "client_secret"
+endpoint = "http://localhost:80"
+certificate = "/etc/casdoor/certificate.pem"
+organization_name = "opencsg"
+application_name = "opencsg"
+
+[nats]
+url = "nats://account:g98dc5FA8v4J7ck90w@natsmaster:4222"
+msg_fetch_timeout_in_sec = 5
+meter_request_subject = "accounting.metering.>"
+meter_duration_send_subject = "accounting.metering.duration"
+meter_token_send_subject = "accounting.metering.token"
+meter_quota_send_subject = "accounting.metering.quota"
+
+[accounting]
+host = "http://localhost"
+port = 8086
+
+[user]
+host = "http://localhost"
+port = 8088
+signin_success_redirect_url = "http://localhost:3000/server/callback"
+
+[multi_sync]
+saas_api_domain = "https://hub.opencsg.com"
+saas_sync_domain = "https://sync.opencsg.com"
+enabled = true
+
+[telemetry]
+enable = true
+report_url = "http://hub.opencsg.com/api/v1/telemetry"
+
+[auto_clean]
+instance = false
+
+[dataset]
+prompt_max_jsonl_file_size = 1048576
+
+[dataflow]
+host = "http://127.0.0.1"
+port = 8000
+
+[moderation]
+host = "http://localhost"
+port = 8089
+encoded_sensitive_words = "5Lmg6L+R5bmzLHhpamlucGluZw=="
+
+[workflow]
+endpoint = "localhost:7233"

common/config/config_test.go

@@ -0,0 +1,41 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfig_LoadConfig(t *testing.T) {
+	t.Run("config env", func(t *testing.T) {
+		t.Setenv("STARHUB_SERVER_INSTANCE_ID", "foo")
+		t.Setenv("STARHUB_SERVER_SERVER_PORT", "6789")
+		cfg, err := LoadConfig()
+		require.Nil(t, err)
+
+		require.Equal(t, "foo", cfg.InstanceID)
+		require.Equal(t, 6789, cfg.APIServer.Port)
+		require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain)
+	})
+
+	t.Run("config file", func(t *testing.T) {
+		SetConfigFile("test.toml")
+		cfg, err := LoadConfig()
+		require.Nil(t, err)
+
+		require.Equal(t, "bar", cfg.InstanceID)
+		require.Equal(t, 4321, cfg.APIServer.Port)
+		require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain)
+	})
+
+	t.Run("file and env", func(t *testing.T) {
+		SetConfigFile("test.toml")
+		t.Setenv("STARHUB_SERVER_INSTANCE_ID", "foobar")
+		cfg, err := LoadConfig()
+		require.Nil(t, err)
+
+		require.Equal(t, "foobar", cfg.InstanceID)
+		require.Equal(t, 4321, cfg.APIServer.Port)
+		require.Equal(t, "git@localhost:2222", cfg.APIServer.SSHDomain)
+	})
+}

common/config/test.toml

@@ -0,0 +1,4 @@
+instance_id = "bar"
+
+[api_server]
+port = 4321

go.mod

@@ -18,12 +18,14 @@ require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
-	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/marcboeker/go-duckdb v1.5.6
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/minio/sha256-simd v1.0.1
+	github.com/naoina/toml v0.1.1
 	github.com/redis/go-redis/v9 v9.3.0
+	github.com/sethvargo/go-envconfig v1.1.0
 	github.com/spf13/cobra v1.8.0
+	github.com/stretchr/testify v1.9.0
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.2
@@ -73,6 +75,7 @@ require (
 	github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect
 	github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/nexus-rpc/sdk-go v0.0.11 // indirect
@@ -90,7 +93,6 @@ require (
 	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect
 	github.com/shirou/gopsutil/v3 v3.21.12 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/tinylib/msgp v1.1.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
 	github.com/tklauser/numcpus v0.3.0 // indirect

go.sum

@@ -328,8 +328,6 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
-github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
@@ -348,6 +346,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
@@ -392,6 +392,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nats-io/nats.go v1.35.0 h1:XFNqNM7v5B+MQMKqVGAyHwYhyKb48jrenXNxIU20ULk=
 github.com/nats-io/nats.go v1.35.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
@@ -470,6 +474,8 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a h1:iLcLb5Fwwz7g/DLK89F+uQBDeAhHhwdzB5fSlVdhGcM=
 github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a/go.mod h1:wozgYq9WEBQBaIJe4YZ0qTSFAMxmcwBhQH0fO0R34Z0=
+github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog=
+github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw=
 github.com/shirou/gopsutil/v3 v3.21.2/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
 github.com/shirou/gopsutil/v3 v3.21.12 h1:VoGxEW2hpmz0Vt3wUvHIl9fquzYLNpVpgNNB7pGJimA=
 github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=