background: moved crono job

The crono job has been moved into the background framework. This way:

- We avoid concurrency problems with registry events and the crono
  synchronization process.
- We can remove a dependency.
- We use only a single process for all background jobs.

This should hopefully also polish some rough edges of this feature.

Signed-off-by: Miquel Sabaté Solà <msabate@suse.com>

Gemfile

@@ -5,7 +5,6 @@ source "https://rubygems.org"
 gem "active_record_union"
 gem "base32"
 gem "bootstrap-sass", "~> 3.3.4"
-gem "crono"
 gem "devise"
 gem "font-awesome-rails", "= 4.7.0.1"
 gem "grape"

Gemfile.lock

@@ -82,9 +82,6 @@ GEM
     concurrent-ruby (1.0.4)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crono (0.9.0)
-      activerecord (~> 4.0)
-      activesupport (~> 4.0)
     database_cleaner (1.4.1)
     debug_inspector (0.0.2)
     descendants_tracker (0.0.4)
@@ -449,7 +446,6 @@ DEPENDENCIES
   cconfig (~> 1.2.0)
   codeclimate-test-reporter
   concurrent-ruby (= 1.0.4)
-  crono
   database_cleaner
   devise
   docker-api (~> 1.28.0)

bin/background.rb

@@ -29,8 +29,13 @@
 
 require "portus/background/registry"
 require "portus/background/security_scanning"
+require "portus/background/sync"
 
-they = [::Portus::Background::Registry.new, ::Portus::Background::SecurityScanning.new]
+they = [
+  ::Portus::Background::Registry.new,
+  ::Portus::Background::SecurityScanning.new,
+  ::Portus::Background::Sync.new
+]
 values = they.map { |v| "'#{v}'" }.join(", ")
 Rails.logger.tagged("Initialization") { Rails.logger.info "Running: #{values}" }
 

db/migrate/20171220095321_drop_crono_jobs_table.rb

@@ -0,0 +1,5 @@
+class DropCronoJobsTable < ActiveRecord::Migration
+  def change
+    drop_table :crono_jobs
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171213094038) do
+ActiveRecord::Schema.define(version: 20171220095321) do
 
   create_table "activities", force: :cascade do |t|
     t.integer  "trackable_id",   limit: 4
@@ -52,16 +52,6 @@
   add_index "comments", ["repository_id"], name: "index_comments_on_repository_id", using: :btree
   add_index "comments", ["user_id"], name: "index_comments_on_user_id", using: :btree
 
-  create_table "crono_jobs", force: :cascade do |t|
-    t.string   "job_id",            limit: 255, null: false
-    t.datetime "last_performed_at"
-    t.boolean  "healthy"
-    t.datetime "created_at",                    null: false
-    t.datetime "updated_at",                    null: false
-  end
-
-  add_index "crono_jobs", ["job_id"], name: "index_crono_jobs_on_job_id", unique: true, using: :btree
-
   create_table "namespaces", force: :cascade do |t|
     t.string   "name",        limit: 255
     t.datetime "created_at",                                null: false

docker-compose.yml

@@ -24,26 +24,6 @@ services:
     volumes:
       - .:/srv/Portus
 
-  crono:
-    image: opensuse/portus:development
-    command: ./bin/crono
-    depends_on:
-      - portus
-      - db
-    environment:
-      - PORTUS_MACHINE_FQDN_VALUE=${MACHINE_FQDN}
-
-      # Theoretically not needed, but cconfig's been buggy on this...
-      - CCONFIG_PREFIX=PORTUS
-
-      - PORTUS_DB_HOST=db
-      - PORTUS_DB_PASSWORD=portus
-      - PORTUS_DB_POOL=5
-    volumes:
-      - .:/srv/Portus
-    links:
-      - db
-
   background:
     image: opensuse/portus:development
     entrypoint: bundle exec rails runner /srv/Portus/bin/background.rb

examples/compose/docker-compose.insecure.yml

@@ -31,8 +31,12 @@ services:
     volumes:
       - ./secrets:/certificates:ro
 
-  crono:
+  background:
     image: opensuse/portus:head
+    entrypoint: bundle exec rails runner /srv/Portus/bin/background.rb
+    depends_on:
+      - portus
+      - db
     environment:
       # Theoretically not needed, but cconfig's been buggy on this...
       - CCONFIG_PREFIX=PORTUS
@@ -49,9 +53,6 @@ services:
       - PORTUS_SECRET_KEY_BASE=${SECRET_KEY_BASE}
       - PORTUS_KEY_PATH=/certificates/portus.key
       - PORTUS_PASSWORD=${PORTUS_PASSWORD}
-
-      # We want to execute crono, not portus
-      - PORTUS_INIT_COMMAND=bin/crono
     links:
       - db
     volumes:

examples/compose/docker-compose.yml

@@ -33,8 +33,12 @@ services:
       - ./secrets:/certificates:ro
       - static:/srv/Portus/public
 
-  crono:
+  background:
     image: opensuse/portus:head
+    entrypoint: bundle exec rails runner /srv/Portus/bin/background.rb
+    depends_on:
+      - portus
+      - db
     environment:
       # Theoretically not needed, but cconfig's been buggy on this...
       - CCONFIG_PREFIX=PORTUS
@@ -51,9 +55,6 @@ services:
       - PORTUS_SECRET_KEY_BASE=${SECRET_KEY_BASE}
       - PORTUS_KEY_PATH=/certificates/portus.key
       - PORTUS_PASSWORD=${PORTUS_PASSWORD}
-
-      # We want to execute crono, not portus
-      - PORTUS_INIT_COMMAND=bin/crono
     links:
       - db
     volumes:

lib/portus/background/sync.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "portus/errors"
+
+module Portus
+  module Background
+    # Sync synchronizes the contents from the registry into the DB.
+    class Sync
+      # Returns how many seconds has to pass between each loop for this
+      # synchronization to happen.
+      def sleep_value
+        20
+      end
+
+      # Returns always true because this does not depend on some configuration
+      # options.
+      def work?
+        true
+      end
+
+      # Simply fetches the catalog from the registry and calls
+      # `update_registry!`.
+      def execute!
+        ::Registry.find_each do |registry|
+          begin
+            cat = registry.client.catalog
+            Rails.logger.debug "Catalog:\n #{cat}"
+
+            # Update the registry in a transaction, since we don't want to leave
+            # the DB in an unknown state because of an update failure.
+            ActiveRecord::Base.transaction { update_registry!(cat) }
+          rescue *::Portus::Errors::NET,
+                 ::Portus::Errors::NoBearerRealmException, ::Portus::Errors::AuthorizationError,
+                 ::Portus::Errors::NotFoundError, ::Portus::Errors::CredentialsMissingError => e
+            Rails.logger.warn "Exception: #{e.message}"
+          end
+        end
+      end
+
+      def to_s
+        "Registry synchronization"
+      end
+
+      protected
+
+      # This method updates the database of this application with the given
+      # registry contents.
+      def update_registry!(catalog)
+        dangling_repos = Repository.all.pluck(:id)
+
+        # In this loop we will create/update all the repos from the catalog.
+        # Created/updated repos will be removed from the "repos" array.
+        catalog.each do |r|
+          if r["tags"].blank?
+            Rails.logger.debug "skip upload not finished repo #{r["name"]}"
+          else
+            repository = Repository.create_or_update!(r)
+            dangling_repos.delete repository.id unless repository.nil?
+          end
+        end
+
+        # At this point, the remaining items in the "repos" array are repos that
+        # exist in the DB but not in the catalog. Remove all of them.
+        portus = User.find_by(username: "portus")
+        Tag.where(repository_id: dangling_repos).find_each { |t| t.delete_and_update!(portus) }
+        Repository.where(id: dangling_repos).find_each { |r| r.delete_and_update!(portus) }
+      end
+    end
+  end
+end

packaging/suse/portus.spec.in

@@ -128,9 +128,7 @@ rm -rf %{buildroot}/%{portusdir}/tmp
 mkdir %{buildroot}/%{portusdir}/tmp
 
 install -d %{buildroot}/%{_sbindir}
-cp packaging/suse/bin/portus_crono %{buildroot}/%{_sbindir}/
 cp packaging/suse/bin/portusctl %{buildroot}/%{_sbindir}/
-%__install -D -m 444 packaging/suse/conf/portus_crono.service %{buildroot}%{_unitdir}/portus_crono.service
 
 # Man pages
 install -d %{buildroot}%{_mandir}/man1
@@ -139,24 +137,20 @@ install -p -m 644 packaging/suse/portusctl/man/man1/*.1 %{buildroot}%{_mandir}/m
 %fdupes %{buildroot}/%{portusdir}
 
 %pre
-%service_add_pre portus_crono.service
 
 %post
 if [ -d /srv/Portus/tmp ];then
   chown -R wwwrun:www /srv/Portus/tmp
 fi
-%service_add_post portus_crono.service
 
 if [ \! -e "/srv/Portus/config/config-local.yml" ]; then
   # First installation of Portus
   cat %{portusdir}/packaging/suse/README.install
 fi
 
 %preun
-%service_del_preun portus_crono.service
 
 %postun
-%service_del_postun portus_crono.service
 
 %files
 %defattr(-,root,root)
@@ -184,9 +178,7 @@ fi
 %{portusdir}/log/
 %{portusdir}/tmp/
 %{portusdir}/db/
-%{_sbindir}/portus_crono
 %{_sbindir}/portusctl
-%{_unitdir}/portus_crono.service
 %{_mandir}/man1/portusctl-*.1.gz
 %{_mandir}/man1/portusctl.1.gz
 

packaging/suse/portusctl/lib/cli.rb

@@ -259,9 +259,8 @@ def logs(*args)
     ensure_root
 
     Runner.produce_versions_file!
-    Runner.produce_crono_log_file!
     Runner.exec("cp", ["/var/log/apache2/error_log", File.join(PORTUS_ROOT, "log/production.log")])
-    Runner.tar_files("log/production.log", "log/crono.log", "log/versions.log")
+    Runner.tar_files("log/production.log", "log/versions.log")
   end
 end