feat: typescript and kysely

.github/workflows/release.yml

@@ -16,6 +16,14 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: Installing
+        run: yarn --frozen-lockfile --perfer-offline --link-duplicates
+
+      - name: Build and test
+        run: |
+          yarn build
+          yarn test
+
       - name: Semantic Release
         uses: cycjimmy/semantic-release-action@v2
         with:
@@ -31,3 +39,4 @@ jobs:
           GIT_COMMITTER_EMAIL: ${{ secrets.SOCIALGROOVYBOT_EMAIL }}
           GIT_COMMITTER_NAME: ${{ secrets.SOCIALGROOVYBOT_NAME }}
           NPM_TOKEN: ${{ secrets.SOCIALGROOVYBOT_NPM_TOKEN }}
+

.github/workflows/test.yml

@@ -24,4 +24,6 @@ jobs:
       #   run: yarn lint
 
       - name: Unit tests
-        run: yarn test
+        run: |
+          yarn build
+          yarn test

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .env
-.env*
+.env*
+dist

.sonarcloud.properties

@@ -0,0 +1 @@
+sonar.exclusions=**/src/__tests__/**,**/__snapshots__/**,**/src/migrations/**

README.md

@@ -4,8 +4,6 @@
 
 Extract matomo data from [`Live.getLastVisitsDetails`](https://developer.matomo.org/api-reference/reporting-api) API and push events and visits informations to Postgres.
 
-Use [pg_partman](https://github.com/pgpartman/pg_partman) to partition data by month.
-
 ## Usage
 
 Create the [initial table](./initial.sql) database table then run the following job with correct environment variables.
@@ -16,16 +14,16 @@ npx @socialgouv/matomo-postgres
 
 ### Environment variables Deployment
 
-| name              | value                                          |
-| ----------------- | ---------------------------------------------- |
-| MATOMO_KEY\*      | matomo api token                               |
-| MATOMO_SITE\*     | matomo site id                                 |
-| MATOMO_URL\*      | matomo url                                     |
-| PGDATABASE\*      | Postgres connection string                     |
-| DESTINATION_TABLE | `matomo`                                       |
-| STARTDATE         | default to today()                             |
-| RESULTPERPAGE     | matomo pagination : `100`                      |
-| OFFSET            | default days to check in the past; default = 3 |
+| name              | value                                                    |
+| ----------------- | -------------------------------------------------------- |
+| MATOMO_KEY\*      | matomo api token                                         |
+| MATOMO_SITE\*     | matomo site id                                           |
+| MATOMO_URL\*      | matomo url                                               |
+| PGDATABASE\*      | Postgres connection string                               |
+| DESTINATION_TABLE | `matomo`                                                 |
+| STARTDATE         | default to today()                                       |
+| RESULTPERPAGE     | matomo pagination (defaults to 500)                      |
+| INITIAL_OFFSET    | How many days to fetch on initialisation (defaults to 3) |
 
 ## Dev
 
@@ -43,3 +41,6 @@ yarn start
 
 Use `yarn test -u` to update the snapshots
 
+## Database migrations
+
+`yarn migrate` is run on each `yarn start` with Kysely migrations at [./src/migrations](./src/migrations/)

bin/index.js

@@ -1,11 +1,22 @@
 #!/usr/bin/env node
 
-const run = require("../src/index");
+const { db } = require("../dist/db");
+
+const { default: run } = require("../dist/index");
+const { default: migrate } = require("../dist/migrate-latest");
+
+async function start(date) {
+  console.log(`\nRunning migrations\n`);
+  await migrate();
+  console.log(`\nStarting import\n`);
+  await run(date);
+  db.destroy();
+}
 
 if (require.main === module) {
   const date =
     (process.argv[process.argv.length - 1].match(/^\d\d\d\d-\d\d-\d\d$/) && process.argv[process.argv.length - 1]) ||
     "";
   console.log(`\nRunning @socialgouv/matomo-postgres ${date}\n`);
-  run(date);
+  start(date);
 }

docker-compose.yml

@@ -1,9 +1,10 @@
 version: "3.0"
 services:
   postgres:
-    build:
-      context: ./docker
-      dockerfile: ./Dockerfile
+    image: postgres:15
+    # build:
+    #   context: ./docker
+    #   dockerfile: ./Dockerfile
     volumes:
       - postgres_data:/var/lib/postgresql/data
     environment:

jest.config.js

@@ -0,0 +1,16 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  testMatch: ["**/*.test.ts"],
+  transform: {
+    // '^.+\\.[tj]sx?$' to process js/ts with `ts-jest`
+    // '^.+\\.m?[tj]sx?$' to process js/ts/mjs/mts with `ts-jest`
+    "^.+\\.tsx?$": [
+      "ts-jest",
+      {
+        isolatedModules: true,
+      },
+    ],
+  },
+};

package.json

@@ -4,33 +4,45 @@
   "version": "0.0.1",
   "types": "types/index.d.ts",
   "license": "Apache-2.0",
-  "main": "src/index.js",
+  "main": "dist/index.js",
   "preferGlobal": true,
   "publishConfig": {
     "access": "public"
   },
   "bin": {
     "matomo-postgres": "./bin/index.js"
   },
+  "files": [
+    "bin",
+    "dist"
+  ],
   "scripts": {
-    "start": "node ./src/index.js",
+    "start": "yarn migrate && node ./dist/index.js",
+    "build": "tsc",
+    "prepublish": "yarn build",
+    "migrate": "node ./dist/migrate-latest.js",
     "test": "jest --verbose"
   },
   "prettier": {
     "printWidth": 120
   },
   "dependencies": {
-    "date-fns": "^2.23.0",
-    "debug": "^4.3.3",
-    "dotenv": "^10.0.0",
-    "p-all": "^3.0.0",
-    "pg": "^8.7.1",
+    "date-fns": "^2.29.3",
+    "debug": "^4.3.4",
+    "dotenv": "^16.0.3",
+    "kysely": "^0.23.4",
+    "p-all": "^3",
+    "pg": "^8.9.0",
     "piwik-client": "^0.2.2"
   },
   "devDependencies": {
-    "@types/jest": "^27.0.3",
-    "@types/pg": "^8.6.1",
-    "jest": "^27.1.1",
-    "typescript": "^4.4.4"
+    "@types/debug": "^4.1.7",
+    "@types/jest": "^29.4.0",
+    "@types/node": "^18.14.4",
+    "@types/pg": "^8.6.6",
+    "jest": "^29.4.3",
+    "ts-jest": "^29.0.5",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.5"
   }
 }