Add experimental support for containers images (#1777)

Closes #1685 Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com> Signed-off-by: Cintia Sanchez Garcia <cynthiasg@icloud.com> Co-authored-by: Sergio Castaño Arteaga <tegioz@icloud.com> Co-authored-by: Cintia Sanchez Garcia <cynthiasg@icloud.com>
artifacthub · Jan 12, 2022 · 14b5d2b · 14b5d2b
1 parent b21846c
commit 14b5d2b
Show file tree

Hide file tree

Showing 126 changed files with 2,934 additions and 273 deletions.
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -49,6 +49,18 @@ dockers:
     build_flag_templates:
       - "--build-arg=VERSION={{ .Version }}"
       - "--build-arg=GIT_COMMIT={{ .FullCommit }}"
+      - "--label=org.opencontainers.image.description='Artifact Hub command line tool'"
+      - "--label=org.opencontainers.image.version='{{ .Version }}'"
+      - "--label=org.opencontainers.image.created='{{ .CommitDate }}'"
+      - "--label=org.opencontainers.image.documentation='https://artifacthub.io/docs/topics/cli'"
+      - "--label=org.opencontainers.image.source='https://github.com/artifacthub/hub/tree/{{ .FullCommit }}/cmd/ah'"
+      - "--label=org.opencontainers.image.vendor='Artifact Hub'"
+      - "--label=io.artifacthub.package.readmeURL='https://raw.githubusercontent.com/artifacthub/hub/{{ .FullCommit }}/docs/cli.md'"
+      - '--label=io.artifacthub.package.maintainers=''[{"name":"Artifact Hub maintainers","email":"cncf-artifacthub-maintainers@lists.cncf.io"}]'''
+      - "--label=io.artifacthub.package.logoURL='https://raw.githubusercontent.com/artifacthub/hub/master/docs/logo/logo.svg'"
+      - "--label=io.artifacthub.package.keywords='artifact hub,cli,lint'"
+      - "--label=io.artifacthub.package.license='Apache-2.0'"
+      - "--label=io.artifacthub.package.alternativeLocations='public.ecr.aws/artifacthub/ah:{{ .Tag }}'"
     extra_files:
       - go.mod
       - go.sum

diff --git a/README.md b/README.md
@@ -12,6 +12,7 @@ Discovering artifacts to use with CNCF projects can be difficult. If every CNCF
 
 At the moment, the following artifacts kinds are supported *(with plans to support more projects to follow)*:
 
+- [Containers images](https://opencontainers.org)
 - [CoreDNS plugins](https://coredns.io/)
 - [Falco configurations](https://falco.org/)
 - [Helm charts](https://helm.sh/)

diff --git a/charts/artifact-hub/Chart.yaml b/charts/artifact-hub/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: artifact-hub
 description: Artifact Hub is a web-based application that enables finding, installing, and publishing Kubernetes packages.
 type: application
-version: 1.5.1-1
+version: 1.5.1-3
 appVersion: 1.5.0
 kubeVersion: ">= 1.19.0-0"
 home: https://artifacthub.io
@@ -19,6 +19,7 @@ keywords:
   - keda scalers
   - coredns
   - keptn
+  - containers images
 maintainers:
   - name: Sergio
     email: tegioz@icloud.com

diff --git a/charts/artifact-hub/templates/tracker_secret.yaml b/charts/artifact-hub/templates/tracker_secret.yaml
@@ -15,6 +15,9 @@ stringData:
       database: {{ .Values.db.database }}
       user: {{ .Values.db.user }}
       password: {{ .Values.db.password }}
+    creds:
+      dockerUsername: {{ .Values.creds.dockerUsername }}
+      dockerPassword: {{ .Values.creds.dockerPassword }}
     images:
       store: {{ .Values.images.store }}
     events:

diff --git a/charts/artifact-hub/values.schema.json b/charts/artifact-hub/values.schema.json
@@ -775,7 +775,7 @@
                 },
                 "repositoriesKinds": {
                     "title": "Repositories kinds to process ([] = all)",
-                    "description": "The following kinds are supported at the moment: falco, helm, olm, opa, tbaction, krew, helm-plugin, tekton-task, keda-scaler, coredns, keptn, tekton-pipeline",
+                    "description": "The following kinds are supported at the moment: falco, helm, olm, opa, tbaction, krew, helm-plugin, tekton-task, keda-scaler, coredns, keptn, tekton-pipeline, container",
                     "type": "array",
                     "items": {
                         "type": "string"

diff --git a/configs/tracker.yaml b/configs/tracker.yaml
@@ -7,6 +7,9 @@ db:
   port: "5432"
   database: hub
   user: postgres
+creds:
+  dockerUsername: ""
+  dockerPassword: ""
 tracker:
   concurrency: 10
   repositoriesNames: []

diff --git a/database/migrations/functions/001_load_functions.sql b/database/migrations/functions/001_load_functions.sql
@@ -47,6 +47,7 @@
 {{ template "packages/get_production_usage.sql" }}
 {{ template "packages/get_random_packages.sql" }}
 {{ template "packages/get_snapshots_to_scan.sql" }}
+{{ template "packages/is_latest.sql" }}
 {{ template "packages/register_package.sql" }}
 {{ template "packages/search_packages.sql" }}
 {{ template "packages/search_packages_monocular.sql" }}

diff --git a/database/migrations/functions/packages/get_package.sql b/database/migrations/functions/packages/get_package.sql
@@ -55,8 +55,12 @@ begin
                 'prerelease', prerelease,
                 'ts', floor(extract(epoch from ts))
             ))
-            from snapshot
-            where package_id = v_package_id
+            from (
+                select *
+                from snapshot
+                where package_id = v_package_id
+                order by ts desc
+            ) s
         ),
         'app_version', s.app_version,
         'digest', s.digest,

diff --git a/database/migrations/functions/packages/is_latest.sql b/database/migrations/functions/packages/is_latest.sql
@@ -0,0 +1,21 @@
+-- is_latest checks if the package version we are trying to register is the
+-- latest or not. For repositories of container image kind, we check the latest
+-- version timestamp. For the other kinds, we check the latest version, which
+-- must be a valid semver.
+create or replace function is_latest(
+    p_kind integer,
+    p_version text,
+    p_previous_latest_version text,
+    p_ts timestamptz,
+    p_previous_latest_version_ts timestamptz
+)
+returns boolean as $$
+begin
+    case p_kind
+        when 12 then -- Container image
+            return p_ts >= p_previous_latest_version_ts;
+        else         -- Any other kind
+            return semver_gte(p_version, p_previous_latest_version);
+    end case;
+end
+$$ language plpgsql;
diff --git a/database/migrations/functions/packages/register_package.sql b/database/migrations/functions/packages/register_package.sql
@@ -6,26 +6,36 @@
 create or replace function register_package(p_pkg jsonb)
 returns void as $$
 declare
-    v_previous_latest_version text;
-    v_package_id uuid;
     v_name text := p_pkg->>'name';
     v_display_name text := nullif(p_pkg->>'display_name', '');
     v_description text := nullif(p_pkg->>'description', '');
     v_keywords text[] := (select nullif(array(select jsonb_array_elements_text(nullif(p_pkg->'keywords', 'null'::jsonb))), '{}'));
     v_version text := p_pkg->>'version';
     v_repository_id uuid := ((p_pkg->'repository')->>'repository_id')::uuid;
+    v_provider text := nullif(p_pkg->>'provider', '');
+    v_signatures text[] := (select nullif(array(select jsonb_array_elements_text(nullif(p_pkg->'signatures', 'null'::jsonb))), '{}'));
+
+    v_latest_version_updated boolean;
     v_maintainer jsonb;
     v_maintainer_id uuid;
+    v_package_id uuid;
+    v_previous_latest_version text;
+    v_previous_latest_version_ts timestamptz;
+    v_repository_disabled boolean;
+    v_repository_kind_id integer;
     v_ts timestamptz;
-    v_provider text := nullif(p_pkg->>'provider', '');
-    v_ts_repository text[];
     v_ts_publisher text[];
-    v_repository_disabled boolean;
-    v_signatures text[] := (select nullif(array(select jsonb_array_elements_text(nullif(p_pkg->'signatures', 'null'::jsonb))), '{}'));
+    v_ts_repository text[];
 begin
+    -- Convert package version ts to timestamptz when available, otherwise use current
+    v_ts := to_timestamp((p_pkg->>'ts')::int);
+    if v_ts is null then
+        v_ts = current_timestamp;
+    end if;
+
     -- Get some repository information (some of it for tsdoc)
-    select r.disabled, array[r.name, r.display_name], array[u.alias, o.name, o.display_name, v_provider]
-    into v_repository_disabled, v_ts_repository, v_ts_publisher
+    select r.disabled, array[r.name, r.display_name], array[u.alias, o.name, o.display_name, v_provider], repository_kind_id
+    into v_repository_disabled, v_ts_repository, v_ts_publisher, v_repository_kind_id
     from repository r
     left join "user" u using (user_id)
     left join organization o using (organization_id)
@@ -38,13 +48,15 @@ begin
         raise 'repository is disabled';
     end if;
 
-    -- Get package's latest version before registration, if available
-    select latest_version into v_previous_latest_version
-    from package
-    where name = v_name
-    and repository_id = v_repository_id;
+    -- Get package's latest version info before registration, if available
+    select p.latest_version, s.ts into v_previous_latest_version, v_previous_latest_version_ts
+    from package p
+    join snapshot s using (package_id)
+    where p.name = v_name
+    and p.repository_id = v_repository_id
+    and s.version = p.latest_version;
 
-    -- Package
+    -- Package (insert or update if latest has changed)
     insert into package (
         name,
         latest_version,
@@ -70,7 +82,13 @@ begin
         is_operator = excluded.is_operator,
         channels = excluded.channels,
         default_channel = excluded.default_channel
-    where semver_gte(v_version, package.latest_version) = true
+    where is_latest(
+        v_repository_kind_id,
+        v_version,
+        v_previous_latest_version,
+        v_ts,
+        v_previous_latest_version_ts
+    ) = true
     returning package_id into v_package_id;
 
     -- If package record has been created or updated
@@ -122,10 +140,6 @@ begin
     end if;
 
     -- Package snapshot
-    v_ts := to_timestamp((p_pkg->>'ts')::int);
-    if v_ts is null then
-        v_ts = current_timestamp;
-    end if;
     insert into snapshot (
         package_id,
         version,
@@ -227,7 +241,18 @@ begin
         ts = v_ts;
 
     -- Register new release event if package's latest version has been updated
-    if semver_gt(v_version, v_previous_latest_version) then
+    v_latest_version_updated := false;
+    case v_repository_kind_id
+        when 12 then -- Container image
+            if v_ts > v_previous_latest_version_ts then
+                v_latest_version_updated := true;
+            end if;
+        else         -- Any other kind
+            if semver_gt(v_version, v_previous_latest_version) then
+                v_latest_version_updated := true;
+            end if;
+    end case;
+    if v_latest_version_updated then
         insert into event (package_id, package_version, event_kind_id)
         values (v_package_id, v_version, 0);
     end if;

diff --git a/database/migrations/functions/repositories/add_repository.sql b/database/migrations/functions/repositories/add_repository.sql
@@ -26,6 +26,7 @@ begin
         auth_pass,
         disabled,
         scanner_disabled,
+        data,
         repository_kind_id,
         user_id,
         organization_id
@@ -38,6 +39,7 @@ begin
         nullif(p_repository->>'auth_pass', ''),
         (p_repository->>'disabled')::boolean,
         (p_repository->>'scanner_disabled')::boolean,
+        nullif(p_repository->'data', 'null'),
         (p_repository->>'kind')::int,
         v_owner_user_id,
         v_owner_organization_id

diff --git a/database/migrations/functions/repositories/get_repository_by_id.sql b/database/migrations/functions/repositories/get_repository_by_id.sql
@@ -21,6 +21,7 @@ returns setof json as $$
         'last_scanning_errors', r.last_scanning_errors,
         'last_tracking_ts', floor(extract(epoch from last_tracking_ts)),
         'last_tracking_errors', r.last_tracking_errors,
+        'data', r.data,
         'user_alias', u.alias,
         'organization_name', o.name,
         'organization_display_name', o.display_name

diff --git a/database/migrations/functions/repositories/update_repository.sql b/database/migrations/functions/repositories/update_repository.sql
@@ -61,7 +61,8 @@ begin
             end
         ),
         disabled = (p_repository->>'disabled')::boolean,
-        scanner_disabled = (p_repository->>'scanner_disabled')::boolean
+        scanner_disabled = (p_repository->>'scanner_disabled')::boolean,
+        data = nullif(p_repository->'data', 'null')
     where repository_id = v_repository_id;
 
     -- If the repository has been disabled, remove packages belonging to it and

diff --git a/database/migrations/schema/035_containers_images.sql b/database/migrations/schema/035_containers_images.sql
@@ -0,0 +1,9 @@
+insert into repository_kind values (12, 'Containers images');
+alter table snapshot drop constraint snapshot_package_id_digest_key;
+alter table repository add column data jsonb;
+
+---- create above / drop below ----
+
+delete from repository_kind where repository_kind_id = 12;
+alter table snapshot add constraint snapshot_package_id_digest_key unique (package_id, digest);
+alter table repository drop column data;
diff --git a/database/tests/functions/packages/get_package.sql b/database/tests/functions/packages/get_package.sql
@@ -291,17 +291,17 @@ select is(
         },
         "version": "1.0.0",
         "available_versions": [
-            {
-                "version": "0.0.9",
-                "contains_security_updates": false,
-                "prerelease": false,
-                "ts": 1592299233
-            },
             {
                 "version": "1.0.0",
                 "contains_security_updates": true,
                 "prerelease": true,
                 "ts": 1592299234
+            },
+            {
+                "version": "0.0.9",
+                "contains_security_updates": false,
+                "prerelease": false,
+                "ts": 1592299233
             }
         ],
         "app_version": "12.1.0",
@@ -444,17 +444,17 @@ select is(
         },
         "version": "1.0.0",
         "available_versions": [
-            {
-                "version": "0.0.9",
-                "contains_security_updates": false,
-                "prerelease": false,
-                "ts": 1592299233
-            },
             {
                 "version": "1.0.0",
                 "contains_security_updates": true,
                 "prerelease": true,
                 "ts": 1592299234
+            },
+            {
+                "version": "0.0.9",
+                "contains_security_updates": false,
+                "prerelease": false,
+                "ts": 1592299233
             }
         ],
         "app_version": "12.1.0",
@@ -587,17 +587,17 @@ select is(
         },
         "version": "0.0.9",
         "available_versions": [
-            {
-                "version": "0.0.9",
-                "contains_security_updates": false,
-                "prerelease": false,
-                "ts": 1592299233
-            },
             {
                 "version": "1.0.0",
                 "contains_security_updates": true,
                 "prerelease": true,
                 "ts": 1592299234
+            },
+            {
+                "version": "0.0.9",
+                "contains_security_updates": false,
+                "prerelease": false,
+                "ts": 1592299233
             }
         ],
         "app_version": "12.0.0",

diff --git a/database/tests/functions/packages/is_latest.sql b/database/tests/functions/packages/is_latest.sql
@@ -0,0 +1,29 @@
+-- Start transaction and plan tests
+begin;
+select plan(2);
+
+-- Test function
+select is(
+    is_latest(
+        0,
+        '1.0.1',
+        '1.0.0',
+        current_timestamp - '2 days'::interval,
+        current_timestamp - '1 day'::interval
+    ),
+    true
+);
+select is(
+    is_latest(
+        12,
+        '1.0.1',
+        '1.0.0',
+        current_timestamp - '2 days'::interval,
+        current_timestamp - '1 day'::interval
+    ),
+    false
+);
+
+-- Finish tests and rollback transaction
+select * from finish();
+rollback;