imager: seed database from csv files

.gitignore

@@ -1,2 +1,22 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# Misc
 .envrc
 .sqlfluff
+data/
+seed.zip
+tmp/

Taskfile.yaml

@@ -10,12 +10,10 @@ tasks:
       - cmd: echo "Codegen complete"
         silent: true
 
-  db:dump:
-    dir: "{{.TASKFILE_DIR}}/sql/schema"
+  db:down:
+    dir: "{{.TASKFILE_DIR}}/docker"
     cmds:
-      - cmd: rm schema.sql
-        silent: true
-      - cmd: pg_dump -s -n f1db -d f1db -U $DATABASE_USER > schema.sql
+      - cmd: docker compose -f compose-dev.yaml down
 
   db:migrate:
     dir: "{{.TASKFILE_DIR}}/sql/schema"
@@ -29,49 +27,43 @@ tasks:
       - cmd: goose create {{.CLI_ARGS}} sql
         silent: true
 
-  db:push:
-    cmds:
-      - cmd: echo $CR_PAT | docker login ghcr.io -u jsec --password-stdin
-        silent: true
-      - cmd: docker push ghcr.io/jsec/f1db:latest
-
-  db:rollback:
+  db:migrate:down:
     dir: "{{.TASKFILE_DIR}}/sql/schema"
     cmds:
       - cmd: goose down
         silent: true
 
-  db:seed:
-    dir: "{{.TASKFILE_DIR}}/docker"
+  db:push:
     cmds:
-      - task: mysql:build
-      - cmd: docker compose -f compose-seed.yaml up --force-recreate --wait
+      - cmd: echo $CR_PAT | docker login ghcr.io -u jsec --password-stdin
         silent: true
-      - cmd: docker run --rm -it --net=host --name pgloader dimitri/pgloader:latest pgloader mysql://seed:seed@host.docker.internal:3306/f1db postgresql://$DATABASE_USER:$DATABASE_PASSWORD@host.docker.internal:5432/f1db
-
-  db:seed:build:
-    dir: "{{.TASKFILE_DIR}}/docker"
-    cmds:
-      - cmd: docker build -f Dockerfile.mysql -t f1db/seed:latest --no-cache .
+      - cmd: docker push ghcr.io/jsec/f1db:latest
 
-  db:seed:down:
+  db:up:
     dir: "{{.TASKFILE_DIR}}/docker"
     cmds:
-      - cmd: docker compose -f compose-seed.yaml down -v
-        silent: true
+      - cmd: docker compose -f compose-dev.yaml up --wait
 
   db:update:
     cmds:
       - task: db:migrate
       - task: db:codegen
 
-  dev:
+  image:up:
     dir: "{{.TASKFILE_DIR}}/docker"
     cmds:
-      - cmd: docker compose -f compose-dev.yaml up --wait
+      - cmd: docker compose -f compose-seed.yaml up --wait
+        silent: true
+      - task: db:migrate
 
-  dev:down:
+  image:down:
     dir: "{{.TASKFILE_DIR}}/docker"
     cmds:
-      - cmd: docker compose -f compose-dev.yaml down
+      - cmd: docker compose -f compose-seed.yaml down
+        silent: true
 
+  image:new:
+    cmds:
+      - task: image:up
+      - cmd: go run ./cmd/imager
+        silent: true

cmd/imager/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/jsec/f1-data-hub/internal/imager"
+)
+
+func main() {
+	if err := imager.Run(context.Background()); err != nil {
+		log.Fatal(err)
+	}
+}

docker/compose-seed.yaml

@@ -2,34 +2,17 @@ services:
   postgres:
     build:
       context: ./
-      dockerfile: ./Dockerfile.postgres
     restart: always
     shm_size: 128mb
+    container_name: f1db-seed
     environment:
       POSTGRES_USER: ${DATABASE_USER}
       POSTGRES_PASSWORD: ${DATABASE_PASSWORD}
-      POSTGRES_DB: f1db
+      POSTGRES_DB: ${DATABASE_NAME}
     ports:
       - 5432:5432
     healthcheck:
       test: [ "CMD", "pg_isready" ]
       interval: 10s
       timeout: 5s
       retries: 5
-
-  mysql:
-    image: f1db/seed
-    command: "--default-authentication-plugin=mysql_native_password"
-    restart: always
-    environment:
-      MYSQL_USER: seed
-      MYSQL_PASSWORD: seed
-      MYSQL_DATABASE: f1db
-      MYSQL_ROOT_PASSWORD: root
-    ports:
-      - 3306:3306
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
-      interval: 10s
-      timeout: 5s
-      retries: 10

go.mod

@@ -3,8 +3,11 @@ module github.com/jsec/f1-data-hub
 go 1.23.1
 
 require (
+	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
+	github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e
 	github.com/jackc/pgx/v5 v5.7.1
 	github.com/labstack/echo v3.3.10+incompatible
+	github.com/shopspring/decimal v1.4.0
 )
 
 require (

go.sum

@@ -3,10 +3,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e h1:i3gQ/Zo7sk4LUVbsAjTNeC4gIjoPNIZVzs4EXstssV4=
+github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e/go.mod h1:zUHglCZ4mpDUPgIwqEKoba6+tcUQzRdb1+DPTuYe9pI=
 github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
 github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
@@ -22,6 +26,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/database/connection.go

@@ -0,0 +1,32 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	pgxdecimal "github.com/jackc/pgx-shopspring-decimal"
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func Connect(ctx context.Context) (*pgxpool.Pool, error) {
+	connString := os.Getenv("DATABASE_URL")
+
+	config, err := pgxpool.ParseConfig(connString)
+	if err != nil {
+		return nil, fmt.Errorf("Error parsing connection string: %w", err)
+	}
+
+	config.AfterConnect = func(ctx context.Context, c *pgx.Conn) error {
+		pgxdecimal.Register(c.TypeMap())
+		return nil
+	}
+
+	pool, err := pgxpool.NewWithConfig(ctx, config)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating connection: %w", err)
+	}
+
+	return pool, nil
+}