Update pluginsync to create PR for documents

This creates a fork and updates documentation, and generates a pr
against the upstream repository.

Gemfile

@@ -4,7 +4,6 @@ gem "hashie"
 gem "modulesync"
 gem "octokit"
 gem "rake"
-gem "semantic"
 gem "slack-ruby-client"
 gem "travis"
 

PLUGIN_CATALOG.md

@@ -6,18 +6,18 @@ This file is automatically generated. If you would like to add to the plugin lis
 
 <%-
 metadata = Pluginsync::Plugins.metadata
-%w[ collector processor publisher ].each do |type| 
+%w[ collector processor publisher ].each do |type|
 -%>
 <%= "### #{Pluginsync::Util.plugin_capitalize(type)}s" %>
 
 | Name | Maintainer | Description | CI | Download |
 |------|------------|-------------|----|----------|
 <%-
   metadata.find_all{|p| p['type'] == type }.sort_by{|p| p['name']}.each do |p|
-    maintainer = "[#{Pluginsync::Util.org_capitalize(p["maintainer"])}](#{p['repo_url']})"
+    maintainer = "[#{Pluginsync::Util.org_capitalize(p["maintainer"])}](#{p['maintainer_url']})"
     downloads = []
     downloads += ["[release](#{p['github_release']})"] if p.include? "github_release"
-    downloads += p['download']['s3_latest'].collect{|h| '[#{h.keys.first}](#{h[h.keys.first]})' } if p['download'] and p['download']['s3_latest']
+    downloads += p['download']['s3_latest'].collect{|h| "[#{h.keys.first}](#{h[h.keys.first]})" } if p['download'] and p['download']['s3_latest']
 -%>
 | [<%= p['name'] %>](<%= p['repo_url'] %>) | <%= maintainer %> | <%= p['description'] %> | <%= Pluginsync::Util.html_list(p['badge']) -%> | <%= Pluginsync::Util.html_list(downloads) %> |
   <%- end -%>
@@ -30,7 +30,7 @@ There will always be more plugins we wish we had. To make sure others can contri
 | Issue | Description |
 |-------|-------------|
 <%-
-wishlist = Pluginsync::Plugins.wishlist 
+wishlist = Pluginsync::Plugins.wishlist
 wishlist.each do |i|
 -%>
 | [#<%= i["number"] %>](<%= i["url"] %>) | <%= i["description"] %> |

Rakefile

@@ -2,7 +2,6 @@
 # unavoidable between FPM and homebrew.
 
 require "rake"
-require "rake/testtask"
 require_relative "lib/pluginsync"
 
 begin
@@ -31,6 +30,11 @@ namespace :plugin do
   task :wishlist do
     puts Pluginsync::Plugins.wishlist
   end
+
+  desc "generate plugin wishlist"
+  task :pull_request do
+    Pluginsync::Plugins.pull_request
+  end
 end
 
 namespace :notify do

lib/pluginsync.rb

@@ -5,15 +5,22 @@ module Pluginsync
   $:.unshift(LIBDIR) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(LIBDIR)
 
+  require 'logger'
   require 'pluginsync/util'
   require 'pluginsync/config'
 
   @@config = Pluginsync::Config.new
+  @@log = Logger.new(STDOUT)
+  @@log.level = @@config.log_level
 
   def self.config
     @@config
   end
 
+  def self.log
+    @@log
+  end
+
   require 'pluginsync/github'
   require 'pluginsync/plugins'
   require 'pluginsync/notify'

lib/pluginsync/config.rb

@@ -1,6 +1,6 @@
 module Pluginsync
   class Config
-    attr_reader :plugins_yml, :org, :path
+    attr_reader :plugins_yml, :plugin_catalog_md, :org, :path, :branch, :log_level
 
     def initialize
       @path = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
@@ -14,7 +14,10 @@ def initialize
       end
 
       @plugins_yml = settings["plugins.yml"]
+      @plugin_catalog_md = settings["plugin_catalog.md"]
       @org = settings["namespace"]
+      @branch = settings["branch"]
+      @log_level = settings["log_level"] || Logger::INFO
     end
 
     def default
@@ -23,7 +26,12 @@ def default
           "repo" =>  "intelsdi-x/snap",
           "path" => "docs/plugins.yml",
         },
+        "plugin_catalog.md" => {
+          "repo" =>  "intelsdi-x/snap",
+          "path" => "docs/PLUGIN_CATALOG.md",
+        },
         "org" => "intelsdi-x",
+        "fork" => ENV["GITHUB_USERNAME"] || ENV["USERNAME"],
       }
     end
   end

lib/pluginsync/github.rb

@@ -32,18 +32,99 @@ def self.repo name
     end
 
     class Repo
-      attr_reader :name 
+      @log = Pluginsync.log
+
+      attr_reader :name
 
       def initialize(name)
         @name = name
         @gh = Pluginsync::Github.client
+        raise(ArgumentError, "#{name} is not a valid github repository (or your account does not have access to this private repo)") unless @gh.repository? name
         @repo = @gh.repo name
         @owner = @repo.owner.login
       end
 
       def content(path, default=nil)
         file = @gh.contents(@name, :path=>path)
-        YAML.load(Base64.decode64(file.content))
+        Base64.decode64 file.content
+      rescue
+        nil
+      end
+
+      def upstream
+        if @repo.fork?
+          @repo.parent.full_name
+        else
+          nil
+        end
+      end
+
+      def ref_sha(ref, repo=@name)
+        refs = @gh.refs repo
+        if result = refs.find{ |r| r.ref == ref }
+          result.object.sha
+        else
+          nil
+        end
+      end
+
+      def sync_branch(branch, opt={})
+        parent = opt[:origin] || upstream || raise(ArgumentError, "Repo #{@name} is not a fork and no origin specified for syncing.")
+        origin_branch = opt[:branch] || 'master'
+
+        origin_sha = ref_sha("refs/heads/#{origin_branch}", parent)
+
+        fork_ref = "heads/#{branch}"
+        fork_sha = ref_sha("refs/heads/#{branch}")
+
+        if ! fork_sha
+          @gh.create_ref(@name, fork_ref, origin_sha)
+        elsif origin_sha != fork_sha
+          begin
+            @gh.update_ref(@name, fork_ref, origin_sha)
+          rescue Octokit::UnprocessableEntity
+            @log.warn "Fork #{name} is out of sync with #{parent}, syncing to #{name} #{origin_branch}"
+            origin_sha = ref_sha("refs/heads/#{origin_branch}")
+            @gh.update_ref(@name, fork_ref, origin_sha)
+          end
+        end
+      end
+
+      def update_content(path, content, opt={})
+        branch = opt[:branch] || "master"
+
+        raise(Argument::Error, "This tool cannot directly commit to #{INTEL_ORG} repos") if @name =~ /^#{INTEL_ORG}/
+        raise(Argument::Error, "This tool cannot directly commit to master branch") if branch == 'master'
+
+        message = "update #{path} by pluginsync tool"
+        content = Base64.encode64 content
+
+        ref = "heads/#{branch}"
+        latest_commit = @gh.ref(@name, ref).object.sha
+        base_tree = @gh.commit(@name, latest_commit).commit.tree.sha
+
+        sha = @gh.create_blob(@name, content, "base64")
+        new_tree = @gh.create_tree(
+          @name,
+          [ {
+            :path => path,
+            :mode => "100644",
+            :type => "blob",
+            :sha => sha
+          } ],
+          { :base_tree => base_tree }
+        ).sha
+
+        new_commit = @gh.create_commit(@name, message, new_tree, latest_commit).sha
+        @gh.update_ref(@name, ref, new_commit) if branch
+      end
+
+      def create_pull_request(branch, message)
+        @gh.create_pull_request(upstream, "master", "#{@repo.owner.login}:#{branch}", message)
+      end
+
+      def yml_content(path, default={})
+        YAML.load(content(path))
       rescue
         default
       end
@@ -78,7 +159,7 @@ def fetch_sync_yml
           path = File.join(Pluginsync::PROJECT_PATH, 'config_defaults.yml')
           config = Pluginsync::Util.load_yaml(path)
           config.extend Hashie::Extensions::DeepMerge
-          config.deep_merge(content('.sync.yml', {}))
+          config.deep_merge(yml_content('.sync.yml'))
         else
           {}
         end
@@ -90,11 +171,12 @@ def metadata
           "type" =>  plugin_type,
           "description" => @repo.description || 'No description available.',
           "maintainer" => @owner,
+          "maintainer_url" => @repo.owner.html_url,
           "repo_name" => @repo.name,
           "repo_url" => @repo.html_url,
         }
 
-        metadata = content('metadata.yml', {})
+        metadata = yml_content('metadata.yml')
 
         if @owner == Pluginsync::Github::INTEL_ORG
           metadata["download"] = {
@@ -103,6 +185,7 @@ def metadata
           }
         end
 
+        metadata["name"] = Pluginsync::Util.plugin_capitalize metadata["name"] if metadata["name"]
         metadata["github_release"] = @repo.html_url + "/releases/latest" if @gh.releases(@name).size > 0
         metadata["maintainer"] = "intelsdi-x" if metadata["maintainer"] == "core"
 
@@ -117,7 +200,7 @@ def s3_url(build)
                  else
                    go["GOARCH"]
                  end
-          { "#{go['GOOS']}/#{arch}" => "http://snap.ci.snap-telemetry.io/plugins/#{@repo.name}/#{build}/#{go['GOOS']}/#{arch}/#{@repo.name}" }
+          { "#{go['GOOS']}/#{arch}" => "https://s3-us-west-2.amazonaws.com/snap.ci.snap-telemetry.io/plugins/#{@repo.name}/#{build}/#{go['GOOS']}/#{arch}/#{@repo.name}" }
         end
       end
     end

lib/pluginsync/plugins.rb

@@ -7,31 +7,40 @@ module Pluginsync
   module Plugins
     @github = Pluginsync::Github
     @config = Pluginsync.config
+    @log = Pluginsync.log
 
     def self.repos
-      @repos ||= Set.new plugins.collect{ |p| Pluginsync::Github::Repo.new(p) }
+      @repos ||= Set.new plugins.collect do |p|
+        begin
+          Pluginsync::Github::Repo.new p
+        rescue ArgumentError => e
+          @log.error e.message
+          nil
+        end
+      end
+      @repos.reject{|p| p.nil?}
     end
 
     def self.plugins
       plugin_repo = Pluginsync::Github::Repo.new @config.plugins_yml["repo"]
-      result = plugin_repo.content @config.plugins_yml["path"]
+      plugin_repo.yml_content @config.plugins_yml["path"]
     end
 
     def self.metadata
       repos.collect{|r| r.metadata}
     end
 
     def self.catalog
-      template = File.read(File.join(@config.path, "PLUGINS_CATALOG.md"))
-      ERB.new(template, nil, '-').result
+      template = File.read(File.join(@config.path, "PLUGIN_CATALOG.md"))
+      @catalog ||= ERB.new(template, nil, '-').result
     end
 
     def self.wishlist
       data = []
 
       snap_issues = @github.issues 'intelsdi-x/snap'
 
-      wishlist = snap_issues.find_all{ |issue| issue.labels.find{|label| label.name=='plugin-wishlist'} }
+      wishlist = snap_issues.find_all{|issue| issue.labels.find{|label| label.name=='plugin-wishlist'}}
       wishlist.each do |issue|
         data << {
           "number" => issue.number,
@@ -43,5 +52,28 @@ def self.wishlist
       data
     end
 
+    def self.pull_request
+      catalog_repo = @config.plugin_catalog_md["repo"]
+      catalog_path = @config.plugin_catalog_md["path"]
+
+      fork_name = @config.plugin_catalog_md["fork"] || raise("Please configure plugin_catalog.md['fork'] in configuration.")
+
+      origin_repo = Pluginsync::Github::Repo.new catalog_repo
+      fork_repo = Pluginsync::Github::Repo.new fork_name
+
+      fork_repo.sync_branch(@config.branch)
+
+      current_catalog = origin_repo.content catalog_path
+
+      if catalog != current_catalog
+        @log.info "Updating plugins_catalog.md in #{fork_name} branch #{@config.branch}"
+        fork_repo.update_content(catalog_path, catalog, :branch => @config.branch)
+
+        pr = fork_repo.create_pull_request(@config.branch, "Updating plugins_catalog.md by pluginsync. [ci skip]")
+        @log.info "Creating pull request: #{pr.html_url}"
+      else
+        puts "No new updates to plugin_catalog.md."
+      end
+    end
   end
 end

lib/pluginsync/util.rb

@@ -1,6 +1,5 @@
 require 'fileutils'
 require 'json'
-require 'semantic'
 require 'yaml'
 
 module Pluginsync
@@ -68,16 +67,6 @@ def self.os_family
       family ||= "Unknown"
     end
 
-    ##
-    # return semver from malform git tags (e.g. v0.13.0-beta) so they can be sorted.
-
-    def self.semver version
-      ::Semantic::Version.new version
-    rescue ArgumentError
-      clean_version = version.match(/(\d*\.\d*\.\d*)/)[0]
-      ::Semantic::Version.new clean_version
-    end
-
     ##
     # cd into working directory temporarily
 

managed_modules.yml

@@ -35,6 +35,7 @@
 - snap-plugin-collector-psutil
 - snap-plugin-collector-rabbitmq
 - snap-plugin-collector-scaleio
+- snap-plugin-collector-snmp
 - snap-plugin-collector-smart
 - snap-plugin-collector-swap
 - snap-plugin-collector-users

modulesync.yml

@@ -6,3 +6,6 @@ message: "Updates to repository from pluginsync utility"
 plugins.yml:
   repo: intelsdi-x/snap
   path: docs/plugins.yml
+plugin_catalog.md:
+  repo: intelsdi-x/snap
+  path: docs/PLUGIN_CATALOG.md