Merge pull request #26 from nanliu/metadata

(SDI-2052) add task for metadata generation for plugins

Gemfile

@@ -2,4 +2,11 @@ source "https://rubygems.org"
 
 gem "hashie"
 gem "modulesync"
+gem "octokit"
+gem "rake"
+gem "slack-ruby-client"
 gem "travis"
+
+group :development do
+  gem "pry"
+end

PLUGIN_CATALOG.md

@@ -0,0 +1,37 @@
+# Plugin Catalog
+This is the master catalog of plugins for Snap. The plugins in this list may be written by multiple sources. Please examine the license and documentation of each plugin for more information.
+
+## All Plugins
+This file is automatically generated. If you would like to add to the plugin list, [add your plugin to this list](docs/plugins.yml) and it will be added (usually within 24 hours).
+
+<%-
+metadata = Pluginsync::Plugins.metadata
+%w[ collector processor publisher ].each do |type|
+-%>
+<%= "### #{Pluginsync::Util.plugin_capitalize(type)}s" %>
+
+| Name | Maintainer | Description | CI | Download |
+|------|------------|-------------|----|----------|
+<%-
+  metadata.find_all{|p| p['type'] == type }.sort_by{|p| p['name']}.each do |p|
+    maintainer = "[#{Pluginsync::Util.org_capitalize(p["maintainer"])}](#{p['maintainer_url']})"
+    downloads = []
+    downloads += ["[release](#{p['github_release']})"] if p.include? "github_release"
+    downloads += p['download']['s3_latest'].collect{|h| "[#{h.keys.first}](#{h[h.keys.first]})" } if p['download'] and p['download']['s3_latest']
+-%>
+| [<%= p['name'] %>](<%= p['repo_url'] %>) | <%= maintainer %> | <%= p['description'] %> | <%= Pluginsync::Util.html_list(p['badge']) -%> | <%= Pluginsync::Util.html_list(downloads) %> |
+  <%- end -%>
+<%- end -%>
+
+### Wishlist
+
+There will always be more plugins we wish we had. To make sure others can contribute to our community goals, we keep a wish list of what people would like to see. If you see one here and want to start on it please let us know by commenting on the corresponding issue!
+
+| Issue | Type | Description |
+|-------|------|-------------|
+<%-
+wishlist = Pluginsync::Plugins.wishlist
+wishlist.each do |i|
+-%>
+| [#<%= i["number"] %>](<%= i["url"] %>) | <%= i["type"] %> | <%= i["description"] %> |
+<%- end -%>

Rakefile

@@ -0,0 +1,60 @@
+# NOTE: Using rake instead of writing a shell script because Ruby seems
+# unavoidable between FPM and homebrew.
+
+require "rake"
+require_relative "lib/pluginsync"
+
+begin
+  require "pry"
+rescue LoadError
+end
+
+desc "Show the list of Rake tasks (rake -T)"
+task :help do
+  sh "rake -T"
+end
+task :default => :help
+
+namespace :plugin do
+  desc "generate plugin catalog"
+  task :catalog do
+    puts Pluginsync::Plugins.catalog
+  end
+
+  desc "generate plugin metadata"
+  task :metadata do
+    puts JSON.pretty_generate Pluginsync::Plugins.metadata
+  end
+
+  desc "generate plugin wishlist"
+  task :wishlist do
+    puts Pluginsync::Plugins.wishlist
+  end
+
+  desc "generate plugin json for github.io page"
+  task :github_io do
+    data = Pluginsync::Plugins.metadata
+    result = data.collect do |i|
+      {
+        name: i["name"],
+        type: i["type"].slice(0,1).capitalize + i["type"].slice(1..-1),
+        description: i["description"],
+        url: i["repo_url"],
+      }
+    end
+
+    puts "myfcn(\n" + JSON.pretty_generate(result) + "\n)"
+  end
+
+  desc "generate pull request for plugin_metadata.json"
+  task :pull_request do
+    Pluginsync::Plugins.pull_request
+  end
+end
+
+namespace :notify do
+  desc "send a slack notification"
+  task :slack do
+    Pluginsync::Notify::Slack.message "#build-snap", "Snap packages version <https://packagecloud.io/nanliu/snap|#{@snap.pkgversion} now available.>"
+  end
+end

lib/pluginsync.rb

@@ -0,0 +1,28 @@
+module Pluginsync
+  LIBDIR = File.expand_path(File.dirname(__FILE__))
+  PROJECT_PATH = File.join(File.expand_path(File.dirname(__FILE__)), "..")
+
+  $:.unshift(LIBDIR) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(LIBDIR)
+
+  require 'logger'
+  require 'pluginsync/util'
+  require 'pluginsync/config'
+
+  @@config = Pluginsync::Config.new
+  @@log = Logger.new(STDOUT)
+  @@log.level = @@config.log_level
+
+  def self.config
+    @@config
+  end
+
+  def self.log
+    @@log
+  end
+
+  require 'pluginsync/github'
+  require 'pluginsync/plugins'
+  require 'pluginsync/notify'
+end
+

lib/pluginsync/config.rb

@@ -0,0 +1,38 @@
+module Pluginsync
+  class Config
+    attr_reader :plugins_yml, :plugin_catalog_md, :org, :path, :branch, :log_level
+
+    def initialize
+      @path = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+      config = File.join @path, 'modulesync.yml'
+
+      if File.exists? config
+        settings = Pluginsync::Util.load_yaml(config)
+        settings = default.merge settings
+      else
+        settings = default
+      end
+
+      @plugins_yml = settings["plugins.yml"]
+      @plugin_catalog_md = settings["plugin_catalog.md"]
+      @org = settings["namespace"]
+      @branch = settings["branch"]
+      @log_level = settings["log_level"] || Logger::INFO
+    end
+
+    def default
+      {
+        "plugins.yml" => {
+          "repo" =>  "intelsdi-x/snap",
+          "path" => "docs/plugins.yml",
+        },
+        "plugin_catalog.md" => {
+          "repo" =>  "intelsdi-x/snap",
+          "path" => "docs/PLUGIN_CATALOG.md",
+        },
+        "org" => "intelsdi-x",
+        "fork" => ENV["GITHUB_USERNAME"] || ENV["USERNAME"],
+      }
+    end
+  end
+end

lib/pluginsync/github.rb

@@ -0,0 +1,208 @@
+require 'netrc'
+require 'octokit'
+
+module Pluginsync
+  module Github
+    INTEL_ORG = Pluginsync.config.org
+
+    Octokit.auto_paginate = true
+    @@client = Octokit::Client.new(:netrc => true) if File.exists? File.join(ENV["HOME"], ".netrc")
+
+    begin
+      require 'faraday-http-cache'
+      stack = Faraday::RackBuilder.new do |builder|
+        builder.use Faraday::HttpCache, :serializer => Marshal
+        builder.use Octokit::Response::RaiseError
+        builder.adapter Faraday.default_adapter
+      end
+      Octokit.middleware = stack
+    rescue LoadError
+    end
+
+    def self.client
+      @@client || Octokit
+    end
+
+    def self.issues name
+      client.issues name
+    end
+
+    def self.repo name
+      client.repo name
+    end
+
+    class Repo
+      @log = Pluginsync.log
+
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+        @gh = Pluginsync::Github.client
+        raise(ArgumentError, "#{name} is not a valid github repository (or your account does not have access to this private repo)") unless @gh.repository? name
+        @repo = @gh.repo name
+        @owner = @repo.owner.login
+      end
+
+      def content(path, default=nil)
+        file = @gh.contents(@name, :path=>path)
+        Base64.decode64 file.content
+      rescue
+        nil
+      end
+
+      def upstream
+        if @repo.fork?
+          @repo.parent.full_name
+        else
+          nil
+        end
+      end
+
+      def ref_sha(ref, repo=@name)
+        refs = @gh.refs repo
+        if result = refs.find{ |r| r.ref == ref }
+          result.object.sha
+        else
+          nil
+        end
+      end
+
+      def sync_branch(branch, opt={})
+        parent = opt[:origin] || upstream || raise(ArgumentError, "Repo #{@name} is not a fork and no origin specified for syncing.")
+        origin_branch = opt[:branch] || 'master'
+
+        origin_sha = ref_sha("refs/heads/#{origin_branch}", parent)
+
+        fork_ref = "heads/#{branch}"
+        fork_sha = ref_sha("refs/heads/#{branch}")
+
+        if ! fork_sha
+          @gh.create_ref(@name, fork_ref, origin_sha)
+        elsif origin_sha != fork_sha
+          begin
+            @gh.update_ref(@name, fork_ref, origin_sha)
+          rescue Octokit::UnprocessableEntity
+            @log.warn "Fork #{name} is out of sync with #{parent}, syncing to #{name} #{origin_branch}"
+            origin_sha = ref_sha("refs/heads/#{origin_branch}")
+            @gh.update_ref(@name, fork_ref, origin_sha)
+          end
+        end
+      end
+
+      def update_content(path, content, opt={})
+        branch = opt[:branch] || "master"
+
+        raise(Argument::Error, "This tool cannot directly commit to #{INTEL_ORG} repos") if @name =~ /^#{INTEL_ORG}/
+        raise(Argument::Error, "This tool cannot directly commit to master branch") if branch == 'master'
+
+        message = "update #{path} by pluginsync tool"
+        content = Base64.encode64 content
+
+        ref = "heads/#{branch}"
+        latest_commit = @gh.ref(@name, ref).object.sha
+        base_tree = @gh.commit(@name, latest_commit).commit.tree.sha
+
+        sha = @gh.create_blob(@name, content, "base64")
+        new_tree = @gh.create_tree(
+          @name,
+          [ {
+            :path => path,
+            :mode => "100644",
+            :type => "blob",
+            :sha => sha
+          } ],
+          { :base_tree => base_tree }
+        ).sha
+
+        new_commit = @gh.create_commit(@name, message, new_tree, latest_commit).sha
+        @gh.update_ref(@name, ref, new_commit) if branch
+      end
+
+      def create_pull_request(branch, message)
+        @gh.create_pull_request(upstream, "master", "#{@repo.owner.login}:#{branch}", message)
+      end
+
+      def yml_content(path, default={})
+        YAML.load(content(path))
+      rescue
+        default
+      end
+
+      def plugin_name
+        @name.match(/snap-plugin-(collector|processor|publisher)-(.*)$/)
+        @plugin_name = Pluginsync::Util.plugin_capitalize($2) || raise(ArgumentError, "Unable to parse plugin name from repo: #{@name}")
+      end
+
+      def plugin_type
+        @plugin_type ||= case @name
+          when /collector/
+            "collector"
+          when /processor/
+            "processor"
+          when /publisher/
+            "publisher"
+          else
+            "unknown"
+          end
+      end
+
+      def sync_yml
+        @sync_yml ||= fetch_sync_yml.extend Hashie::Extensions::DeepFetch
+      end
+
+      ##
+      # For intelsdi-x plugins merge pluginsync config_defaults with repo .sync.yml
+      #
+      def fetch_sync_yml
+        if @owner == Pluginsync::Github::INTEL_ORG
+          path = File.join(Pluginsync::PROJECT_PATH, 'config_defaults.yml')
+          config = Pluginsync::Util.load_yaml(path)
+          config.extend Hashie::Extensions::DeepMerge
+          config.deep_merge(yml_content('.sync.yml'))
+        else
+          {}
+        end
+      end
+
+      def metadata
+        result = {
+          "name" => plugin_name,
+          "type" =>  plugin_type,
+          "description" => @repo.description || 'No description available.',
+          "maintainer" => @owner,
+          "maintainer_url" => @repo.owner.html_url,
+          "repo_name" => @repo.name,
+          "repo_url" => @repo.html_url,
+        }
+
+        metadata = yml_content('metadata.yml')
+
+        if @owner == Pluginsync::Github::INTEL_ORG
+          metadata["download"] = {
+            "s3_latest"       => s3_url('latest'),
+            "s3_latest_build" => s3_url('latest_build'),
+          }
+        end
+
+        metadata["name"] = Pluginsync::Util.plugin_capitalize metadata["name"] if metadata["name"]
+        metadata["github_release"] = @repo.html_url + "/releases/latest" if @gh.releases(@name).size > 0
+        metadata["maintainer"] = "intelsdi-x" if metadata["maintainer"] == "core"
+
+        result.merge(metadata)
+      end
+
+      def s3_url(build)
+        matrix = sync_yml.deep_fetch :global, "build", "matrix"
+        matrix.collect do |go|
+          arch = if go["GOARCH"] == "amd64"
+                   "x86_64"
+                 else
+                   go["GOARCH"]
+                 end
+          { "#{go['GOOS']}/#{arch}" => "https://s3-us-west-2.amazonaws.com/snap.ci.snap-telemetry.io/plugins/#{@repo.name}/#{build}/#{go['GOOS']}/#{arch}/#{@repo.name}" }
+        end
+      end
+    end
+  end
+end