Track packages views

Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com>
Signed-off-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>
Co-authored-by: Sergio Castaño Arteaga <tegioz@icloud.com>
Co-authored-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
       image: artifacthub/db-tests
     services:
       postgres:
-        image: artifacthub/postgres-pgtap
+        image: artifacthub/postgres
         env:
           POSTGRES_USER: tests
           POSTGRES_PASSWORD: tests

charts/artifact-hub/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: artifact-hub
 description: Artifact Hub is a web-based application that enables finding, installing, and publishing Kubernetes packages.
 type: application
-version: 1.4.1-3
+version: 1.4.1-4
 appVersion: 1.4.0
 kubeVersion: ">= 1.19.0-0"
 home: https://artifacthub.io

charts/artifact-hub/values.yaml

@@ -273,8 +273,8 @@ trivy:
 postgresql:
   enabled: true
   image:
-    repository: postgres
-    tag: 12
+    repository: artifacthub/postgres
+    tag: latest
   persistence:
     mountPath: /data
   postgresqlUsername: postgres

cmd/hub/main.go

@@ -56,6 +56,7 @@ func main() {
 		log.Fatal().Err(err).Msg("authorizer setup failed")
 	}
 	hc := util.SetupHTTPClient(cfg.GetBool("restrictedHTTPClient"))
+	vt := pkg.NewViewsTracker(db)
 
 	// Setup and launch http server
 	ctx, stop := context.WithCancel(context.Background())
@@ -72,6 +73,7 @@ func main() {
 		Authorizer:          az,
 		HTTPClient:          hc,
 		OCIPuller:           &oci.Puller{},
+		ViewsTracker:        vt,
 	}
 	h, err := handlers.Setup(ctx, cfg, hSvc)
 	if err != nil {
@@ -101,8 +103,12 @@ func main() {
 		}
 	}()
 
-	// Setup and launch events dispatcher
+	// Launch views tracker flusher
 	var wg sync.WaitGroup
+	wg.Add(1)
+	go vt.Flusher(ctx, &wg)
+
+	// Setup and launch events dispatcher
 	eSvc := &event.Services{
 		DB:                  db,
 		EventManager:        event.NewManager(),

database/migrations/functions/001_load_functions.sql

@@ -52,8 +52,9 @@
 {{ template "packages/semver_gt.sql" }}
 {{ template "packages/semver_gte.sql" }}
 {{ template "packages/toggle_star.sql" }}
-{{ template "packages/update_snapshot_security_report.sql" }}
 {{ template "packages/unregister_package.sql" }}
+{{ template "packages/update_packages_views.sql" }}
+{{ template "packages/update_snapshot_security_report.sql" }}
 
 {{ template "repositories/add_repository.sql" }}
 {{ template "repositories/delete_repository.sql" }}

database/migrations/functions/packages/update_packages_views.sql

@@ -0,0 +1,17 @@
+-- update_packages_views updates the views of the packages provided.
+create or replace function update_packages_views(p_lock_key bigint, p_data jsonb)
+returns void as $$
+    -- Make sure only one batch of updates is processed at a time
+    select pg_advisory_xact_lock(p_lock_key);
+
+    -- Insert or update the corresponding views counters as needed
+    insert into package_views (package_id, version, day, total)
+    select
+        (value->>0)::uuid as package_id,
+        (value->>1)::text as version,
+        (value->>2)::date as day,
+        (value->>3)::integer as total
+    from jsonb_array_elements(p_data)
+    on conflict (package_id, version, day) do
+    update set total = package_views.total + excluded.total;
+$$ language sql;

database/migrations/schema/032_packages_views.sql

@@ -0,0 +1,17 @@
+create extension pg_partman;
+
+create table if not exists package_views (
+    package_id uuid not null references package on delete cascade,
+    version text not null,
+    day date not null,
+    total integer not null,
+    unique (package_id, version, day)
+) partition by range (day);
+
+select create_parent('public.package_views', 'day', 'native', 'monthly', p_start_partition := current_date::text);
+
+---- create above / drop below ----
+
+drop table if exists package_views;
+drop table template_public_package_views;
+delete from part_config where parent_table = 'public.package_views';

database/tests/Dockerfile-postgres-pgtap → database/tests/Dockerfile-postgres

@@ -1,10 +1,13 @@
-# Build pgtap extension
+# Build extensions
 FROM postgres:13 AS builder
 RUN apt-get update
 RUN apt-get install -y git make postgresql-server-dev-13
 RUN git clone https://github.com/theory/pgtap
-RUN cd pgtap && make && make install
+RUN cd pgtap && make && make install && cd ..
+RUN git clone https://github.com/pgpartman/pg_partman
+RUN cd pg_partman && make NO_BGW=1 install
 
 # Build final image
 FROM postgres:13
 COPY --from=builder /usr/share/postgresql/13/extension/pgtap* /usr/share/postgresql/13/extension/
+COPY --from=builder /usr/share/postgresql/13/extension/pg_partman* /usr/share/postgresql/13/extension/

database/tests/functions/packages/update_packages_views.sql

@@ -0,0 +1,81 @@
+-- Start transaction and plan tests
+begin;
+select plan(3);
+
+-- Declare some variables
+\set lockKey 1
+\set org1ID '00000000-0000-0000-0000-000000000001'
+\set repo1ID '00000000-0000-0000-0000-000000000001'
+\set package1ID '00000000-0000-0000-0000-000000000001'
+\set package2ID '00000000-0000-0000-0000-000000000002'
+
+-- Seed some data
+insert into organization (organization_id, name, display_name, description, home_url)
+values (:'org1ID', 'org1', 'Organization 1', 'Description 1', 'https://org1.com');
+insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id)
+values (:'repo1ID', 'repo1', 'Repo 1', 'https://repo1.com', 0, :'org1ID');
+insert into package (
+    package_id,
+    name,
+    latest_version,
+    repository_id
+) values (
+    :'package1ID',
+    'pkg1',
+    '1.0.0',
+    :'repo1ID'
+);
+insert into package (
+    package_id,
+    name,
+    latest_version,
+    repository_id
+) values (
+    :'package2ID',
+    'pkg2',
+    '1.0.0',
+    :'repo1ID'
+);
+
+-- Run some tests
+select update_packages_views(:lockKey, '[
+    ["00000000-0000-0000-0000-000000000001", "1.0.0", "2021-12-3", 10]
+]');
+select results_eq(
+    'select * from package_views',
+    $$ values
+        ('00000000-0000-0000-0000-000000000001'::uuid, '1.0.0', '2021-12-3'::date, 10)
+    $$,
+    'First run: one insert'
+);
+select update_packages_views(:lockKey, '[
+    ["00000000-0000-0000-0000-000000000001", "1.0.0", "2021-12-3", 10]
+]');
+select results_eq(
+    'select * from package_views',
+    $$ values
+        ('00000000-0000-0000-0000-000000000001'::uuid, '1.0.0', '2021-12-3'::date, 20)
+    $$,
+    'Second run: one update'
+);
+select update_packages_views(:lockKey, '[
+    ["00000000-0000-0000-0000-000000000001", "1.0.0", "2021-12-5", 10],
+    ["00000000-0000-0000-0000-000000000001", "1.0.1", "2021-12-5", 10],
+    ["00000000-0000-0000-0000-000000000002", "1.0.0", "2021-12-5", 10],
+    ["00000000-0000-0000-0000-000000000002", "1.0.0", "2021-12-6", 5]
+]');
+select results_eq(
+    'select * from package_views',
+    $$ values
+        ('00000000-0000-0000-0000-000000000001'::uuid, '1.0.0', '2021-12-3'::date, 20),
+        ('00000000-0000-0000-0000-000000000001'::uuid, '1.0.0', '2021-12-5'::date, 10),
+        ('00000000-0000-0000-0000-000000000001'::uuid, '1.0.1', '2021-12-5'::date, 10),
+        ('00000000-0000-0000-0000-000000000002'::uuid, '1.0.0', '2021-12-5'::date, 10),
+        ('00000000-0000-0000-0000-000000000002'::uuid, '1.0.0', '2021-12-6'::date, 5)
+    $$,
+    'Third run: one update and four inserts'
+);
+
+-- Finish tests and rollback transaction
+select * from finish();
+rollback;

database/tests/schema/schema.sql

@@ -1,6 +1,6 @@
 -- Start transaction and plan tests
 begin;
-select plan(153);
+select plan(184);
 
 -- Check default_text_search_config is correct
 select results_eq(
@@ -13,38 +13,38 @@ select results_eq(
 select has_extension('pgcrypto');
 select has_extension('pg_trgm');
 select has_extension('tsm_system_rows');
+select has_extension('pg_partman');
 
 -- Check expected tables exist
-select tables_are(array[
-    'api_key',
-    'delete_user_code',
-    'email_verification_code',
-    'event',
-    'event_kind',
-    'image',
-    'image_version',
-    'maintainer',
-    'notification',
-    'opt_out',
-    'organization',
-    'package',
-    'package__maintainer',
-    'password_reset_code',
-    'production_usage',
-    'repository',
-    'repository_kind',
-    'session',
-    'snapshot',
-    'subscription',
-    'user',
-    'user_starred_package',
-    'user__organization',
-    'version_functions',
-    'version_schema',
-    'webhook',
-    'webhook__event_kind',
-    'webhook__package'
-]);
+select has_table('api_key');
+select has_table('delete_user_code');
+select has_table('email_verification_code');
+select has_table('event');
+select has_table('event_kind');
+select has_table('image');
+select has_table('image_version');
+select has_table('maintainer');
+select has_table('notification');
+select has_table('opt_out');
+select has_table('organization');
+select has_table('package');
+select has_table('package_views');
+select has_table('package__maintainer');
+select has_table('password_reset_code');
+select has_table('production_usage');
+select has_table('repository');
+select has_table('repository_kind');
+select has_table('session');
+select has_table('snapshot');
+select has_table('subscription');
+select has_table('user');
+select has_table('user_starred_package');
+select has_table('user__organization');
+select has_table('version_functions');
+select has_table('version_schema');
+select has_table('webhook');
+select has_table('webhook__event_kind');
+select has_table('webhook__package');
 
 -- Check tables have expected columns
 select columns_are('api_key', array[
@@ -140,6 +140,12 @@ select columns_are('package', array[
     'created_at',
     'repository_id'
 ]);
+select columns_are('package_views', array[
+    'package_id',
+    'version',
+    'day',
+    'total'
+]);
 select columns_are('package__maintainer', array[
     'package_id',
     'maintainer_id'
@@ -330,6 +336,9 @@ select indexes_are('package', array[
     'package_repository_id_idx',
     'package_repository_id_name_key'
 ]);
+select indexes_are('package_views', array[
+    'package_views_package_id_version_day_key'
+]);
 select indexes_are('package__maintainer', array[
     'package__maintainer_pkey'
 ]);

docs/dev.md

@@ -8,7 +8,7 @@ To start, please clone the [Artifact Hub repository](https://github.com/artifact
 
 ## Database
 
-The datastore used by Artifact Hub is PostgreSQL. You can install it locally using your favorite OS package manager.
+The datastore used by Artifact Hub is PostgreSQL. You can install it locally using your favorite OS package manager. The [pg_partman](https://github.com/pgpartman/pg_partman) extension must be installed as well.
 
 Once PostgreSQL is installed and its binaries are available in your `PATH`, we can initialize the database cluster and launch the database server:
 

internal/handlers/handlers.go

@@ -50,6 +50,7 @@ type Services struct {
 	Authorizer          hub.Authorizer
 	HTTPClient          hub.HTTPClient
 	OCIPuller           hub.OCIPuller
+	ViewsTracker        hub.ViewsTracker
 }
 
 // Metrics groups some metrics collected from a Handlers instance.
@@ -92,7 +93,14 @@ func Setup(ctx context.Context, cfg *viper.Viper, svc *Services) (*Handlers, err
 		Organizations: org.NewHandlers(svc.OrganizationManager, svc.Authorizer, cfg),
 		Users:         userHandlers,
 		Repositories:  repo.NewHandlers(cfg, svc.RepositoryManager),
-		Packages:      pkg.NewHandlers(svc.PackageManager, svc.RepositoryManager, cfg, svc.HTTPClient, svc.OCIPuller),
+		Packages: pkg.NewHandlers(
+			svc.PackageManager,
+			svc.RepositoryManager,
+			cfg,
+			svc.HTTPClient,
+			svc.OCIPuller,
+			svc.ViewsTracker,
+		),
 		Subscriptions: subscription.NewHandlers(svc.SubscriptionManager),
 		Webhooks:      webhook.NewHandlers(svc.WebhookManager, svc.HTTPClient),
 		APIKeys:       apikey.NewHandlers(svc.APIKeyManager),
@@ -267,6 +275,7 @@ func (h *Handlers) setupRouter() {
 			r.Get("/{packageID}/{version}/values", h.Packages.GetChartValues)
 			r.Get("/{packageID}/{version}/values-schema", h.Packages.GetValuesSchema)
 			r.Get("/{packageID}/{version}/templates", h.Packages.GetChartTemplates)
+			r.Post("/{packageID}/{version}/views", h.Packages.TrackView)
 			r.Get("/{packageID}/changelog", h.Packages.GetChangelog)
 		})
 

internal/handlers/pkg/handlers.go

@@ -37,6 +37,7 @@ type Handlers struct {
 	logger          zerolog.Logger
 	hc              hub.HTTPClient
 	op              hub.OCIPuller
+	vt              hub.ViewsTracker
 	tmplChangelogMD *template.Template
 }
 
@@ -47,6 +48,7 @@ func NewHandlers(
 	cfg *viper.Viper,
 	hc hub.HTTPClient,
 	op hub.OCIPuller,
+	vt hub.ViewsTracker,
 ) *Handlers {
 	return &Handlers{
 		pkgManager:      pkgManager,
@@ -55,6 +57,7 @@ func NewHandlers(
 		logger:          log.With().Str("handlers", "pkg").Logger(),
 		hc:              hc,
 		op:              op,
+		vt:              vt,
 		tmplChangelogMD: setupChangelogMDTmpl(),
 	}
 }
@@ -532,6 +535,17 @@ func (h *Handlers) ToggleStar(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+// TrackView is an http handler used to track a view of a given package version.
+func (h *Handlers) TrackView(w http.ResponseWriter, r *http.Request) {
+	packageID := chi.URLParam(r, "packageID")
+	version := chi.URLParam(r, "version")
+	if err := h.vt.TrackView(packageID, version); err != nil {
+		helpers.RenderErrorJSON(w, err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
 // getChartArchive is a helper function used to download a chart's archive from
 // the original source.
 func (h *Handlers) getChartArchive(ctx context.Context, packageID, version string) (*chart.Chart, error) {