Merge remote-tracking branch 'origin/zhenghaoz/nn' into zhenghaoz/nn

gorse-io · Dec 8, 2024 · 9ff4807 · 9ff4807
2 parents 42cd63c + e7fe64a
commit 9ff4807
Show file tree

Hide file tree

Showing 65 changed files with 11,888 additions and 1,693 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -61,89 +61,7 @@ jobs:
           no_output_timeout: 20m
           command: go test -timeout 20m -v ./...
 
-  unit-test-macos:
-    macos:
-      xcode: 14.2.0
-    working_directory: ~/repo
-    steps:
-      - checkout
-      - run:
-          name: Set up dataset
-          command: |
-            mkdir -p ~/.gorse/dataset
-            mkdir -p ~/.gorse/download
-            wget https://cdn.gorse.io/datasets/ml-100k.zip -P ~/.gorse/download
-            wget https://cdn.gorse.io/datasets/ml-1m.zip -P ~/.gorse/download
-            wget https://cdn.gorse.io/datasets/pinterest-20.zip -P ~/.gorse/download
-            wget https://cdn.gorse.io/datasets/frappe.zip -P ~/.gorse/download
-            wget https://cdn.gorse.io/datasets/ml-tag.zip -P ~/.gorse/download
-            wget https://cdn.gorse.io/datasets/criteo.zip -P ~/.gorse/download
-            unzip ~/.gorse/download/ml-100k.zip -d ~/.gorse/dataset
-            unzip ~/.gorse/download/ml-1m.zip -d ~/.gorse/dataset
-            unzip ~/.gorse/download/pinterest-20.zip -d ~/.gorse/dataset
-            unzip ~/.gorse/download/frappe.zip -d ~/.gorse/dataset
-            unzip ~/.gorse/download/ml-tag.zip -d ~/.gorse/dataset
-            unzip ~/.gorse/download/criteo.zip -d ~/.gorse/dataset
-      - run:
-          name: Install Go
-          command: brew install go
-      - restore_cache:
-          keys:
-            - go-mod-v4-{{ checksum "go.sum" }}
-      - run:
-          name: Install Dependencies
-          command: go mod download
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-      - run:
-          name: Run tests
-          no_output_timeout: 20m
-          command: go test -timeout 20m -v ./... -skip "TestPostgres|TestMySQL|TestMongo|TestRedis"
-
-  unit-test-windows:
-    executor: win/server-2022
-    steps:
-      - checkout
-      - run:
-          name: Set up dataset
-          command: |
-            New-Item -Type Directory -Path ~/.gorse/dataset
-            New-Item -Type Directory -Path ~/.gorse/download
-            Invoke-WebRequest https://cdn.gorse.io/datasets/ml-100k.zip -OutFile ~/.gorse/download/ml-100k.zip
-            Invoke-WebRequest https://cdn.gorse.io/datasets/ml-1m.zip -OutFile ~/.gorse/download/ml-1m.zip
-            Invoke-WebRequest https://cdn.gorse.io/datasets/pinterest-20.zip -OutFile ~/.gorse/download/pinterest-20.zip
-            Invoke-WebRequest https://cdn.gorse.io/datasets/frappe.zip -OutFile ~/.gorse/download/frappe.zip
-            Invoke-WebRequest https://cdn.gorse.io/datasets/ml-tag.zip -OutFile ~/.gorse/download/ml-tag.zip
-            Invoke-WebRequest https://cdn.gorse.io/datasets/criteo.zip -OutFile ~/.gorse/download/criteo.zip
-            Expand-Archive ~/.gorse/download/ml-100k.zip -DestinationPath ~/.gorse/dataset
-            Expand-Archive ~/.gorse/download/ml-1m.zip -DestinationPath ~/.gorse/dataset
-            Expand-Archive ~/.gorse/download/pinterest-20.zip -DestinationPath ~/.gorse/dataset
-            Expand-Archive ~/.gorse/download/frappe.zip -DestinationPath ~/.gorse/dataset
-            Expand-Archive ~/.gorse/download/ml-tag.zip -DestinationPath ~/.gorse/dataset
-            Expand-Archive ~/.gorse/download/criteo.zip -DestinationPath ~/.gorse/dataset
-      - run:
-          name: Upgrade Go
-          command: choco upgrade golang -y
-      - restore_cache:
-          keys:
-            - go-mod-v4-{{ checksum "go.sum" }}
-      - run:
-          name: Install Dependencies
-          command: go mod download
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-      - run:
-          name: Run tests
-          no_output_timeout: 20m
-          command: go test -timeout 20m -v ./... -skip "TestPostgres|TestMySQL|TestMongo|TestRedis"
-
 workflows:
   unit-test:
     jobs:
       - unit-test-arm64
-      - unit-test-macos
-      - unit-test-windows
diff --git a/.github/workflows/build_release.yml b/.github/workflows/build_release.yml
@@ -24,7 +24,7 @@ jobs:
       - name: Build release
         run: >
           gox  -output="{{.OS}}/{{.Arch}}/{{.Dir}}" \
-            -osarch='darwin/arm64 darwin/amd64 windows/arm64 windows/amd64 linux/arm64 linux/amd64' -ldflags="
+            -osarch='darwin/arm64 windows/arm64 windows/amd64 linux/arm64 linux/amd64' -ldflags="
             -X 'github.com/zhenghaoz/gorse/cmd/version.Version=$(git describe --tags $(git rev-parse HEAD))'
             -X 'github.com/zhenghaoz/gorse/cmd/version.GitCommit=$(git rev-parse HEAD)'
             -X 'github.com/zhenghaoz/gorse/cmd/version.BuildTime=$(date)'" ./...
@@ -40,7 +40,6 @@ jobs:
           zip -j gorse_linux_arm64.zip linux/arm64/gorse-*
           zip -j gorse_windows_amd64.zip windows/amd64/gorse-*
           zip -j gorse_windows_arm64.zip windows/arm64/gorse-*
-          zip -j gorse_darwin_amd64.zip darwin/amd64/gorse-*
           zip -j gorse_darwin_arm64.zip darwin/arm64/gorse-*
           
       - name: Upload release

diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -50,6 +50,16 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+      clickhouse:
+        image: clickhouse/clickhouse-server:22
+        ports:
+          - 8123
+        options: >-
+          --health-cmd="clickhouse-client --query 'SELECT 1'" 
+          --health-interval=10s 
+          --health-timeout=5s 
+          --health-retries=5
+
       redis:
         image: redis/redis-stack:6.2.6-v9
         ports:
@@ -82,17 +92,6 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
-    - name: Get dependencies
-      run: |
-        go get -v -t -d ./...
-        if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-        fi
-        
-    - name: Build
-      run: go build -v ./...
-
     - name: Test
       run: go test -timeout 20m -v ./... -coverprofile=coverage.txt -covermode=atomic
       env:
@@ -102,18 +101,86 @@ jobs:
         POSTGRES_URI: postgres://gorse:gorse_pass@localhost:${{ job.services.postgres.ports[5432] }}/
         # MongoDB
         MONGO_URI: mongodb://root:password@localhost:${{ job.services.mongo.ports[27017] }}/
+        # ClickHouse
+        CLICKHOUSE_URI: clickhouse://localhost:${{ job.services.clickhouse.ports[8123] }}/
         # Redis
         REDIS_URI: redis://localhost:${{ job.services.redis.ports[6379] }}/
 
     - name: Upload 
       run: bash <(curl -s https://codecov.io/bash)
 
+  unit_test_macos:
+    name: unit tests (macOS)
+    runs-on: macos-latest
+    steps:
+    - name: Set up dataset
+      run: |
+        mkdir -p ~/.gorse/dataset
+        mkdir -p ~/.gorse/download
+        wget https://cdn.gorse.io/datasets/ml-100k.zip -P ~/.gorse/download
+        wget https://cdn.gorse.io/datasets/ml-1m.zip -P ~/.gorse/download
+        wget https://cdn.gorse.io/datasets/pinterest-20.zip -P ~/.gorse/download
+        wget https://cdn.gorse.io/datasets/frappe.zip -P ~/.gorse/download
+        wget https://cdn.gorse.io/datasets/ml-tag.zip -P ~/.gorse/download
+        wget https://cdn.gorse.io/datasets/criteo.zip -P ~/.gorse/download
+        unzip ~/.gorse/download/ml-100k.zip -d ~/.gorse/dataset
+        unzip ~/.gorse/download/ml-1m.zip -d ~/.gorse/dataset
+        unzip ~/.gorse/download/pinterest-20.zip -d ~/.gorse/dataset
+        unzip ~/.gorse/download/frappe.zip -d ~/.gorse/dataset
+        unzip ~/.gorse/download/ml-tag.zip -d ~/.gorse/dataset
+        unzip ~/.gorse/download/criteo.zip -d ~/.gorse/dataset
+
+    - name: Set up Go 1.23.x
+      uses: actions/setup-go@v4
+      with:
+        go-version: 1.23.x
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Test
+      run: go test -timeout 20m -v ./... -skip "TestPostgres|TestMySQL|TestMongo|TestRedis|TestClickHouse"
+
+  unit_test_windows:
+    name: unit tests (Windows)
+    runs-on: windows-latest
+    steps:
+    - name: Set up dataset
+      run: |
+        New-Item -Type Directory -Path ~/.gorse/dataset
+        New-Item -Type Directory -Path ~/.gorse/download
+        Invoke-WebRequest https://cdn.gorse.io/datasets/ml-100k.zip -OutFile ~/.gorse/download/ml-100k.zip
+        Invoke-WebRequest https://cdn.gorse.io/datasets/ml-1m.zip -OutFile ~/.gorse/download/ml-1m.zip
+        Invoke-WebRequest https://cdn.gorse.io/datasets/pinterest-20.zip -OutFile ~/.gorse/download/pinterest-20.zip
+        Invoke-WebRequest https://cdn.gorse.io/datasets/frappe.zip -OutFile ~/.gorse/download/frappe.zip
+        Invoke-WebRequest https://cdn.gorse.io/datasets/ml-tag.zip -OutFile ~/.gorse/download/ml-tag.zip
+        Invoke-WebRequest https://cdn.gorse.io/datasets/criteo.zip -OutFile ~/.gorse/download/criteo.zip
+        Expand-Archive ~/.gorse/download/ml-100k.zip -DestinationPath ~/.gorse/dataset
+        Expand-Archive ~/.gorse/download/ml-1m.zip -DestinationPath ~/.gorse/dataset
+        Expand-Archive ~/.gorse/download/pinterest-20.zip -DestinationPath ~/.gorse/dataset
+        Expand-Archive ~/.gorse/download/frappe.zip -DestinationPath ~/.gorse/dataset
+        Expand-Archive ~/.gorse/download/ml-tag.zip -DestinationPath ~/.gorse/dataset
+        Expand-Archive ~/.gorse/download/criteo.zip -DestinationPath ~/.gorse/dataset
+
+    - name: Set up Go 1.23.x
+      uses: actions/setup-go@v4
+      with:
+        go-version: 1.23.x
+      id: go
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+
+    - name: Test
+      run: go test -timeout 20m -v ./... -skip "TestPostgres|TestMySQL|TestMongo|TestRedis|TestClickHouse"
+
   integrate_test:
     name: integrate tests
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        database: [mysql, postgres, mongo]
+        database: [mysql, postgres, mongo, clickhouse]
     steps:
     - uses: actions/checkout@v1
 
@@ -182,7 +249,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Test MariaDB
-        run: go test ./storage/data -run ^TestMySQL_
+        run: go test ./storage/data -run TestMySQL
         env:
           MYSQL_URI: mysql://root:password@tcp(localhost:${{ job.services.mariadb.ports[3306] }})/
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -80,6 +80,7 @@ The default database URLs are directed to these databases in `storage/docker-com
 | `MYSQL_URI`       | `mysql://root:password@tcp(127.0.0.1:3306)/` |
 | `POSTGRES_URI`    | `postgres://gorse:gorse_pass@127.0.0.1/`     |
 | `MONGO_URI`       | `mongodb://root:password@127.0.0.1:27017/`   |
+| `CLICKHOUSE_URI`  | `clickhouse://127.0.0.1:8123/`               |
 | `REDIS_URI`       | `redis://127.0.0.1:6379/`                    |
 
 For example, use TiDB as a test database by:

diff --git a/README.md b/README.md
@@ -9,6 +9,7 @@
 [![GoDoc](https://godoc.org/github.com/zhenghaoz/gorse?status.svg)](https://godoc.org/github.com/zhenghaoz/gorse)
 [![Discord](https://img.shields.io/discord/830635934210588743)](https://discord.gg/x6gAtNNkAE)
 [![Twitter Follow](https://img.shields.io/twitter/follow/gorse_io?label=Follow&style=social)](https://twitter.com/gorse_io)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20Gorse%20Guru-006BFF)](https://gurubase.io/g/gorse)
 
 Gorse is an open-source recommendation system written in Go. Gorse aims to be a universal open-source recommender system that can be quickly introduced into a wide variety of online services. By importing items, users, and interaction data into Gorse, the system will automatically train models to generate recommendations for each user. Project features are as follows.
 
@@ -95,7 +96,7 @@ For more information：
 
 ## Architecture
 
-Gorse is a single-node training and distributed prediction recommender system. Gorse stores data in MySQL, MongoDB, or Postgres, with intermediate results cached in Redis, MySQL, MongoDB and Postgres.
+Gorse is a single-node training and distributed prediction recommender system. Gorse stores data in MySQL, MongoDB, Postgres, or ClickHouse, with intermediate results cached in Redis, MySQL, MongoDB and Postgres.
 
 1. The cluster consists of a master node, multiple worker nodes, and server nodes.
 1. The master node is responsible for model training, non-personalized item recommendation, configuration management, and membership management.

diff --git a/client/docker-compose.yml.j2 b/client/docker-compose.yml.j2
@@ -63,6 +63,22 @@ services:
       timeout: 5s
       retries: 5
 
+  {% elif database == 'clickhouse' %}
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:22
+    ports:
+      - 8123:8123
+    environment:
+      CLICKHOUSE_DB: gorse
+      CLICKHOUSE_USER: gorse
+      CLICKHOUSE_PASSWORD: gorse_pass
+    healthcheck:
+      test: clickhouse-client --user $$CLICKHOUSE_USER --password $$CLICKHOUSE_PASSWORD --query "SELECT 1"
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   {% endif %}
 
   worker:
@@ -117,6 +133,8 @@ services:
       GORSE_DATA_STORE: postgres://gorse:gorse_pass@postgres/gorse?sslmode=disable
       {% elif database == 'mongo' %}
       GORSE_DATA_STORE: mongodb://root:password@mongo:27017/gorse?authSource=admin&connect=direct
+      {% elif database == 'clickhouse' %}
+      GORSE_DATA_STORE: clickhouse://gorse:gorse_pass@clickhouse:8123/gorse?mutations_sync=2
       {% endif %}
     command: >
       -c /etc/gorse/config.toml 
@@ -138,6 +156,9 @@ services:
       {% elif database == 'mongo' %}
       mongo:
         condition: service_healthy
+      {% elif database == 'clickhouse' %}
+      clickhouse:
+        condition: service_healthy
       {% endif %}
 
 volumes:

diff --git a/cmd/gorse-in-one/main.go b/cmd/gorse-in-one/main.go
@@ -103,7 +103,7 @@ var oneCommand = &cobra.Command{
 		// Start worker
 		workerJobs, _ := cmd.PersistentFlags().GetInt("recommend-jobs")
 		w := worker.NewWorker(conf.Master.Host, conf.Master.Port, conf.Master.Host,
-			0, workerJobs, "", managedMode)
+			0, workerJobs, "", managedMode, nil)
 		go func() {
 			w.SetOneMode(m.Settings)
 			w.Serve()

diff --git a/cmd/gorse-server/main.go b/cmd/gorse-server/main.go
@@ -21,6 +21,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/zhenghaoz/gorse/base/log"
 	"github.com/zhenghaoz/gorse/cmd/version"
+	"github.com/zhenghaoz/gorse/protocol"
 	"github.com/zhenghaoz/gorse/server"
 	"go.uber.org/zap"
 )
@@ -46,7 +47,25 @@ var serverCommand = &cobra.Command{
 		httpPort, _ := cmd.PersistentFlags().GetInt("http-port")
 		httpHost, _ := cmd.PersistentFlags().GetString("http-host")
 		cachePath, _ := cmd.PersistentFlags().GetString("cache-path")
-		s := server.NewServer(masterHost, masterPort, httpHost, httpPort, cachePath)
+		caFile, _ := cmd.PersistentFlags().GetString("ssl-ca")
+		certFile, _ := cmd.PersistentFlags().GetString("ssl-cert")
+		keyFile, _ := cmd.PersistentFlags().GetString("ssl-key")
+		var tlsConfig *protocol.TLSConfig
+		if caFile != "" && certFile != "" && keyFile != "" {
+			tlsConfig = &protocol.TLSConfig{
+				SSLCA:   caFile,
+				SSLCert: certFile,
+				SSLKey:  keyFile,
+			}
+		} else if caFile == "" && certFile == "" && keyFile == "" {
+			tlsConfig = nil
+		} else {
+			log.Logger().Fatal("incomplete SSL configuration",
+				zap.String("ssl_ca", caFile),
+				zap.String("ssl_cert", certFile),
+				zap.String("ssl_key", keyFile))
+		}
+		s := server.NewServer(masterHost, masterPort, httpHost, httpPort, cachePath, tlsConfig)
 
 		// stop server
 		done := make(chan struct{})
@@ -74,6 +93,9 @@ func init() {
 	serverCommand.PersistentFlags().String("http-host", "127.0.0.1", "port for RESTful APIs and Prometheus metrics export")
 	serverCommand.PersistentFlags().Bool("debug", false, "use debug log mode")
 	serverCommand.PersistentFlags().String("cache-path", "server_cache.data", "path of cache file")
+	serverCommand.PersistentFlags().String("ssl-ca", "", "path of SSL CA")
+	serverCommand.PersistentFlags().String("ssl-cert", "", "path of SSL certificate")
+	serverCommand.PersistentFlags().String("ssl-key", "", "path of SSL key")
 }
 
 func main() {