ISSUE-199: Added sidekiq for background work like registry synchroniz…

…ation. Added synchronization endpoint and worker

Gemfile

@@ -29,6 +29,12 @@ gem 'kaminari'
 # Memoization
 gem 'memoist'
 
+# Background Jobs
+gem 'sidekiq'
+# sinatra for sidekiq web ui
+# if you require 'sinatra' you get the DSL extended to Object
+gem 'sinatra', require: nil
+
 # In order to create the Gemfile.lock required for packaging
 # meaning that it should contain only the production packages
 # run:
@@ -58,6 +64,7 @@ unless ENV['PACKAGING'] && ENV['PACKAGING'] == 'yes'
   end
 
   group :test do
+    gem 'rspec-sidekiq'
     gem 'shoulda'
     gem 'rspec-rails'
     gem 'vcr'

Gemfile.lock

@@ -65,6 +65,8 @@ GEM
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       xpath (~> 2.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
     cliver (0.3.2)
     codeclimate-test-reporter (0.4.7)
       simplecov (>= 0.7.1, < 1.0.0)
@@ -77,6 +79,7 @@ GEM
       execjs
     coffee-script-source (1.9.1.1)
     columnize (0.9.0)
+    connection_pool (2.2.0)
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     daemons (1.2.2)
@@ -115,6 +118,7 @@ GEM
     hashie (3.4.2)
     hike (1.2.3)
     hirb (0.7.3)
+    hitimes (1.2.2)
     i18n (0.7.0)
     jquery-rails (4.0.4)
       rails-dom-testing (~> 1.0)
@@ -177,6 +181,8 @@ GEM
     rack (1.6.2)
     rack-mini-profiler (0.9.3)
       rack (>= 1.1.3)
+    rack-protection (1.5.3)
+      rack
     rack-test (0.6.3)
       rack (>= 1.0)
     rails (4.2.2)
@@ -207,9 +213,16 @@ GEM
       thor (>= 0.18.1, < 2.0)
     rainbow (2.0.0)
     rake (10.4.2)
+    redis (3.2.1)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
     responders (2.1.0)
       railties (>= 4.2.0, < 5)
     rotp (2.1.1)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
     rspec-core (3.3.0)
       rspec-support (~> 3.3.0)
     rspec-expectations (3.3.0)
@@ -226,6 +239,9 @@ GEM
       rspec-expectations (~> 3.3.0)
       rspec-mocks (~> 3.3.0)
       rspec-support (~> 3.3.0)
+    rspec-sidekiq (2.0.0)
+      rspec (~> 3.0, >= 3.0.0)
+      sidekiq (>= 2.4.0)
     rspec-support (3.3.0)
     rubocop (0.32.0)
       astrolabe (~> 1.3)
@@ -253,11 +269,21 @@ GEM
     shoulda-context (1.2.1)
     shoulda-matchers (2.8.0)
       activesupport (>= 3.0.0)
+    sidekiq (3.4.2)
+      celluloid (~> 0.16.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      json (~> 1.0)
+      redis (~> 3.2, >= 3.2.1)
+      redis-namespace (~> 1.5, >= 1.5.2)
     simplecov (0.10.0)
       docile (~> 1.1.0)
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
     slim (3.0.6)
       temple (~> 0.7.3)
       tilt (>= 1.3.3, < 2.1)
@@ -280,6 +306,8 @@ GEM
     thread_safe (0.3.5)
     tilt (1.4.1)
     timecop (0.7.4)
+    timers (4.0.1)
+      hitimes
     treetop (1.6.2)
       polyglot (~> 0.3)
     turbolinks (2.5.3)
@@ -348,11 +376,14 @@ DEPENDENCIES
   rails-observers
   rotp
   rspec-rails
+  rspec-sidekiq
   rubocop
   sass-rails (>= 3.2)
   search_cop
   shoulda
+  sidekiq
   simplecov
+  sinatra
   slim
   sprockets (~> 2.12.3)
   thin

app/controllers/admin/registries_controller.rb

@@ -1,4 +1,5 @@
 class Admin::RegistriesController < Admin::BaseController
+  before_action :set_registry, only: :synchronize
   # GET /admin/registries/
   def index
     @registries = Registry.all
@@ -27,8 +28,19 @@ def create
     end
   end
 
+  # POST /admin/registries/:id/synchronize
+  def synchronize
+    # TODO: Authorize user to queue registry synchronization
+    RegistrySynchronizationWorker.perform_async(@registry.id)
+    render nothing: true, status: :accepted
+  end
+
   private
 
+  def set_registry
+    @registry = Registry.find(params[:id])
+  end
+
   def registries_params
     params.require(:registry).permit(:name, :hostname, :use_ssl)
   end

app/models/registry.rb

@@ -35,15 +35,16 @@ def synchronize
   end
 
   def synchronize_from_catalog(catalog)
+    # 1. Figure out namespace/repo K/V
+    repository_addresses = catalog['repositories']
+      .reject(&:nil?)
+      .map do |repository_fullname|
+        logger.debug "Detected repo for synchronization: #{repository_fullname}"
+        next { 'repository_name' => repository_fullname } unless repository_fullname.include? '/'
+        EXPLODE_REPO_NAME_REGEXP.match(repository_fullname)
+      end
 
-    # 2. Figure out namespace/repo K/V
-    repository_addresses = catalog['repositories'].reject(&:nil?).map do |repository_fullname|
-      logger.debug "Detected repo for synchronization: #{repository_fullname}"
-      next { 'repository_name' => repository_fullname } unless repository_fullname.include? '/'
-      EXPLODE_REPO_NAME_REGEXP.match(repository_fullname)
-    end
-
-    # 3. Iterate over each namespace/repo K/V
+    # 2. Iterate over each namespace/repo K/V
     repositories = repository_addresses
       .reject { |address| address.nil? || address['repository_name'].nil? }
       .map do |address|
@@ -54,10 +55,13 @@ def synchronize_from_catalog(catalog)
           .repositories.find_or_create_by!(name: address['repository_name'])
       end
 
-    # 4. Synchronize each repository in catalog
-    repositories.flat_map do |repo|
-      { repository: repo, status: repo.synchronize }
-    end
+    # 3. Synchronize each repository in catalog
+    synchronization_result = repositories.flat_map { |repo| { repository: repo, status: repo.synchronize } }
+
+    # 4. Update last manual synchronization
+    self.last_manual_synchronization = Time.zone.now
+    save
+    synchronization_result
   end
 
   # Find the registry for the given push event.

app/models/repository.rb

@@ -3,6 +3,8 @@ class Repository < ActiveRecord::Base
   include PublicActivity::Common
   include SearchCop
 
+  scope :global_namespaces, -> { joins(:namespaces).where('namespaces.global = ?', true) }
+
   belongs_to :namespace
   has_many :tags
 
@@ -35,6 +37,18 @@ def full_name
     "#{namespace.name}/#{name}"
   end
 
+  EXPLODE_REPO_NAME_REGEXP = %r{(?<namespace_name>.*)\/(?<repository_name>.*)}
+
+  def self.find_from_manifest(manifest)
+    if !manifest['name'].include? '/'
+      Repository.global_namespaces.where(name: manifest['name'])
+    else
+      match = EXPLODE_REPO_NAME_REGEXP.match(manifest['name'])
+      namespace = Namespace.find_by(name: match['namespace_name'])
+      namespace.repositories.find_or_create_by(name: match['repository_name'])
+    end
+  end
+
   # Handle a push event from the registry.
   def self.handle_push_event(event)
     registry = Registry.find_from_event(event)

app/models/tag.rb

@@ -20,6 +20,12 @@ def synchronize_from_manifest(manifest)
     self.architecture = manifest['architecture']
     self.schema_version = manifest['schemaVersion']
     manifest['fsLayers'].map { |layer|  fs_layers.find_or_create_by!(blob_sum: layer['blobSum']) }
-    self.save
+    save
+  end
+
+  def self.create_or_update_from_manifest(manifest)
+    repository = Repository.find_from_manifest(manifest)
+    tag = repository.tags.find_or_create_by!(name: manifest['tag'])
+    tag.synchronize_from_manifest(manifest)
   end
 end

app/views/admin/registries/index.html.slim

@@ -20,11 +20,13 @@
             tr
               th Name
               th Hostname
+              th Synchronize
           tbody
             - @registries.each do |registry|
               tr
                 td= registry.name
                 td= registry.hostname
+                td= link_to 'Synchronize', synchronize_admin_registry_path(registry), class: 'btn btn-primary', method: :post
 
   p
     strong Note well:

app/workers/registry_synchronization_worker.rb

@@ -0,0 +1,7 @@
+class RegistrySynchronizationWorker
+  include Sidekiq::Worker
+
+  def perform(registry_id)
+    Registry.find(registry_id).synchronize
+  end
+end

config/routes.rb

@@ -1,4 +1,8 @@
+require 'sidekiq/web'
 Rails.application.routes.draw do
+  authenticate :user, -> (u) { u.admin? } do
+    mount Sidekiq::Web => '/sidekiq'
+  end
 
   resources :teams, only: [:index, :show, :create]
   resources :team_users, only: [:create, :destroy, :update]
@@ -29,7 +33,9 @@
   namespace :admin do
     resources :activities, only: [:index]
     resources :dashboard, only: [:index]
-    resources :registries, only: [:index, :create, :new]
+    resources :registries, only: [:index, :create, :new] do
+      post 'synchronize', on: :member
+    end
     resources :namespaces, only: [:index]
     resources :teams, only: [:index]
     resources :users, only: [:index] do

db/migrate/20150731025822_add_last_manual_synchronization_to_registries.rb

@@ -0,0 +1,5 @@
+class AddLastManualSynchronizationToRegistries < ActiveRecord::Migration
+  def change
+    add_column :registries, :last_manual_synchronization, :datetime
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150731011042) do
+ActiveRecord::Schema.define(version: 20150731025822) do
 
   create_table "activities", force: :cascade do |t|
     t.integer  "trackable_id",   limit: 4
@@ -55,11 +55,12 @@
   add_index "namespaces", ["team_id"], name: "index_namespaces_on_team_id", using: :btree
 
   create_table "registries", force: :cascade do |t|
-    t.string   "name",       limit: 255,                 null: false
-    t.string   "hostname",   limit: 255,                 null: false
-    t.datetime "created_at",                             null: false
-    t.datetime "updated_at",                             null: false
-    t.boolean  "use_ssl",    limit: 1,   default: false
+    t.string   "name",                        limit: 255,                 null: false
+    t.string   "hostname",                    limit: 255,                 null: false
+    t.datetime "created_at",                                              null: false
+    t.datetime "updated_at",                                              null: false
+    t.boolean  "use_ssl",                     limit: 1,   default: false
+    t.datetime "last_manual_synchronization"
   end
 
   add_index "registries", ["hostname"], name: "index_registries_on_hostname", unique: true, using: :btree

spec/controllers/admin/registries_controller_spec.rb

@@ -21,6 +21,17 @@
     end
   end
 
+  describe 'POST #synchronize' do
+    context 'as an admin' do
+      # TODO: Add admin login logic
+      it 'creates a new registry' do
+        registry = create :registry
+        post :synchronize, id: registry.id
+        expect(RegistrySynchronizationWorker).to have_enqueued_job(registry.id)
+      end
+    end
+  end
+
   describe 'POST #create' do
     context 'no registry' do
 

spec/models/tag_spec.rb

@@ -5,9 +5,9 @@
   it { should belong_to(:repository) }
   it { should have_many(:fs_layers) }
 
-  describe "update from manifest api" do
+  describe 'update from manifest api response' do
     let(:repository) { create(:repository, name: 'busybox') }
-    let(:tag){ create(:tag, name: '0.1', repository: repository) }
+    let(:tag) { create(:tag, name: '0.1', repository: repository) }
 
     it 'should update a manifest from a manifest response' do
 

spec/spec_helper.rb

@@ -5,6 +5,7 @@
 require 'simplecov'
 require 'webmock/rspec'
 require 'vcr'
+require 'rspec-sidekiq'
 
 SimpleCov.minimum_coverage 100
 SimpleCov.start 'rails'
@@ -32,6 +33,15 @@
   config.before :each do
     Timecop.return
   end
+end
+
+RSpec::Sidekiq.configure do |config|
+  # Clears all job queues before each example
+  config.clear_all_enqueued_jobs = true # default => true
+
+  # Whether to use terminal colours when outputting messages
+  config.enable_terminal_colours = true # default => true
 
-  config.order = :random
+  # Warn when jobs are not enqueued to Redis but to a job array
+  config.warn_when_jobs_not_processed_by_sidekiq = true # default => true
 end