diff --git a/Gemfile b/Gemfile index f3022ea..c5c5db8 100644 --- a/Gemfile +++ b/Gemfile @@ -2,4 +2,11 @@ source "https://rubygems.org" gem "hashie" gem "modulesync" +gem "octokit" +gem "rake" +gem "slack-ruby-client" gem "travis" + +group :development do + gem "pry" +end diff --git a/PLUGIN_CATALOG.md b/PLUGIN_CATALOG.md new file mode 100644 index 0000000..6d918ed --- /dev/null +++ b/PLUGIN_CATALOG.md @@ -0,0 +1,37 @@ +# Plugin Catalog +This is the master catalog of plugins for Snap. The plugins in this list may be written by multiple sources. Please examine the license and documentation of each plugin for more information. + +## All Plugins +This file is automatically generated. If you would like to add to the plugin list, [add your plugin to this list](docs/plugins.yml) and it will be added (usually within 24 hours). + +<%- +metadata = Pluginsync::Plugins.metadata +%w[ collector processor publisher ].each do |type| +-%> +<%= "### #{Pluginsync::Util.plugin_capitalize(type)}s" %> + +| Name | Maintainer | Description | CI | Download | +|------|------------|-------------|----|----------| +<%- + metadata.find_all{|p| p['type'] == type }.sort_by{|p| p['name']}.each do |p| + maintainer = "[#{Pluginsync::Util.org_capitalize(p["maintainer"])}](#{p['maintainer_url']})" + downloads = [] + downloads += ["[release](#{p['github_release']})"] if p.include? "github_release" + downloads += p['download']['s3_latest'].collect{|h| "[#{h.keys.first}](#{h[h.keys.first]})" } if p['download'] and p['download']['s3_latest'] +-%> +| [<%= p['name'] %>](<%= p['repo_url'] %>) | <%= maintainer %> | <%= p['description'] %> | <%= Pluginsync::Util.html_list(p['badge']) -%> | <%= Pluginsync::Util.html_list(downloads) %> | + <%- end -%> +<%- end -%> + +### Wishlist + +There will always be more plugins we wish we had. To make sure others can contribute to our community goals, we keep a wish list of what people would like to see. If you see one here and want to start on it please let us know by commenting on the corresponding issue! + +| Issue | Type | Description | +|-------|------|-------------| +<%- +wishlist = Pluginsync::Plugins.wishlist +wishlist.each do |i| +-%> +| [#<%= i["number"] %>](<%= i["url"] %>) | <%= i["type"] %> | <%= i["description"] %> | +<%- end -%> diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..0d5c8f1 --- /dev/null +++ b/Rakefile @@ -0,0 +1,60 @@ +# NOTE: Using rake instead of writing a shell script because Ruby seems +# unavoidable between FPM and homebrew. + +require "rake" +require_relative "lib/pluginsync" + +begin + require "pry" +rescue LoadError +end + +desc "Show the list of Rake tasks (rake -T)" +task :help do + sh "rake -T" +end +task :default => :help + +namespace :plugin do + desc "generate plugin catalog" + task :catalog do + puts Pluginsync::Plugins.catalog + end + + desc "generate plugin metadata" + task :metadata do + puts JSON.pretty_generate Pluginsync::Plugins.metadata + end + + desc "generate plugin wishlist" + task :wishlist do + puts Pluginsync::Plugins.wishlist + end + + desc "generate plugin json for github.io page" + task :github_io do + data = Pluginsync::Plugins.metadata + result = data.collect do |i| + { + name: i["name"], + type: i["type"].slice(0,1).capitalize + i["type"].slice(1..-1), + description: i["description"], + url: i["repo_url"], + } + end + + puts "myfcn(\n" + JSON.pretty_generate(result) + "\n)" + end + + desc "generate pull request for plugin_metadata.json" + task :pull_request do + Pluginsync::Plugins.pull_request + end +end + +namespace :notify do + desc "send a slack notification" + task :slack do + Pluginsync::Notify::Slack.message "#build-snap", "Snap packages version " + end +end diff --git a/lib/pluginsync.rb b/lib/pluginsync.rb new file mode 100644 index 0000000..db4e37a --- /dev/null +++ b/lib/pluginsync.rb @@ -0,0 +1,28 @@ +module Pluginsync + LIBDIR = File.expand_path(File.dirname(__FILE__)) + PROJECT_PATH = File.join(File.expand_path(File.dirname(__FILE__)), "..") + + $:.unshift(LIBDIR) unless + $:.include?(File.dirname(__FILE__)) || $:.include?(LIBDIR) + + require 'logger' + require 'pluginsync/util' + require 'pluginsync/config' + + @@config = Pluginsync::Config.new + @@log = Logger.new(STDOUT) + @@log.level = @@config.log_level + + def self.config + @@config + end + + def self.log + @@log + end + + require 'pluginsync/github' + require 'pluginsync/plugins' + require 'pluginsync/notify' +end + diff --git a/lib/pluginsync/config.rb b/lib/pluginsync/config.rb new file mode 100644 index 0000000..f692280 --- /dev/null +++ b/lib/pluginsync/config.rb @@ -0,0 +1,38 @@ +module Pluginsync + class Config + attr_reader :plugins_yml, :plugin_catalog_md, :org, :path, :branch, :log_level + + def initialize + @path = File.expand_path(File.join(File.dirname(__FILE__), "../..")) + config = File.join @path, 'modulesync.yml' + + if File.exists? config + settings = Pluginsync::Util.load_yaml(config) + settings = default.merge settings + else + settings = default + end + + @plugins_yml = settings["plugins.yml"] + @plugin_catalog_md = settings["plugin_catalog.md"] + @org = settings["namespace"] + @branch = settings["branch"] + @log_level = settings["log_level"] || Logger::INFO + end + + def default + { + "plugins.yml" => { + "repo" => "intelsdi-x/snap", + "path" => "docs/plugins.yml", + }, + "plugin_catalog.md" => { + "repo" => "intelsdi-x/snap", + "path" => "docs/PLUGIN_CATALOG.md", + }, + "org" => "intelsdi-x", + "fork" => ENV["GITHUB_USERNAME"] || ENV["USERNAME"], + } + end + end +end diff --git a/lib/pluginsync/github.rb b/lib/pluginsync/github.rb new file mode 100644 index 0000000..624176f --- /dev/null +++ b/lib/pluginsync/github.rb @@ -0,0 +1,208 @@ +require 'netrc' +require 'octokit' + +module Pluginsync + module Github + INTEL_ORG = Pluginsync.config.org + + Octokit.auto_paginate = true + @@client = Octokit::Client.new(:netrc => true) if File.exists? File.join(ENV["HOME"], ".netrc") + + begin + require 'faraday-http-cache' + stack = Faraday::RackBuilder.new do |builder| + builder.use Faraday::HttpCache, :serializer => Marshal + builder.use Octokit::Response::RaiseError + builder.adapter Faraday.default_adapter + end + Octokit.middleware = stack + rescue LoadError + end + + def self.client + @@client || Octokit + end + + def self.issues name + client.issues name + end + + def self.repo name + client.repo name + end + + class Repo + @log = Pluginsync.log + + attr_reader :name + + def initialize(name) + @name = name + @gh = Pluginsync::Github.client + raise(ArgumentError, "#{name} is not a valid github repository (or your account does not have access to this private repo)") unless @gh.repository? name + @repo = @gh.repo name + @owner = @repo.owner.login + end + + def content(path, default=nil) + file = @gh.contents(@name, :path=>path) + Base64.decode64 file.content + rescue + nil + end + + def upstream + if @repo.fork? + @repo.parent.full_name + else + nil + end + end + + def ref_sha(ref, repo=@name) + refs = @gh.refs repo + if result = refs.find{ |r| r.ref == ref } + result.object.sha + else + nil + end + end + + def sync_branch(branch, opt={}) + parent = opt[:origin] || upstream || raise(ArgumentError, "Repo #{@name} is not a fork and no origin specified for syncing.") + origin_branch = opt[:branch] || 'master' + + origin_sha = ref_sha("refs/heads/#{origin_branch}", parent) + + fork_ref = "heads/#{branch}" + fork_sha = ref_sha("refs/heads/#{branch}") + + if ! fork_sha + @gh.create_ref(@name, fork_ref, origin_sha) + elsif origin_sha != fork_sha + begin + @gh.update_ref(@name, fork_ref, origin_sha) + rescue Octokit::UnprocessableEntity + @log.warn "Fork #{name} is out of sync with #{parent}, syncing to #{name} #{origin_branch}" + origin_sha = ref_sha("refs/heads/#{origin_branch}") + @gh.update_ref(@name, fork_ref, origin_sha) + end + end + end + + def update_content(path, content, opt={}) + branch = opt[:branch] || "master" + + raise(Argument::Error, "This tool cannot directly commit to #{INTEL_ORG} repos") if @name =~ /^#{INTEL_ORG}/ + raise(Argument::Error, "This tool cannot directly commit to master branch") if branch == 'master' + + message = "update #{path} by pluginsync tool" + content = Base64.encode64 content + + ref = "heads/#{branch}" + latest_commit = @gh.ref(@name, ref).object.sha + base_tree = @gh.commit(@name, latest_commit).commit.tree.sha + + sha = @gh.create_blob(@name, content, "base64") + new_tree = @gh.create_tree( + @name, + [ { + :path => path, + :mode => "100644", + :type => "blob", + :sha => sha + } ], + { :base_tree => base_tree } + ).sha + + new_commit = @gh.create_commit(@name, message, new_tree, latest_commit).sha + @gh.update_ref(@name, ref, new_commit) if branch + end + + def create_pull_request(branch, message) + @gh.create_pull_request(upstream, "master", "#{@repo.owner.login}:#{branch}", message) + end + + def yml_content(path, default={}) + YAML.load(content(path)) + rescue + default + end + + def plugin_name + @name.match(/snap-plugin-(collector|processor|publisher)-(.*)$/) + @plugin_name = Pluginsync::Util.plugin_capitalize($2) || raise(ArgumentError, "Unable to parse plugin name from repo: #{@name}") + end + + def plugin_type + @plugin_type ||= case @name + when /collector/ + "collector" + when /processor/ + "processor" + when /publisher/ + "publisher" + else + "unknown" + end + end + + def sync_yml + @sync_yml ||= fetch_sync_yml.extend Hashie::Extensions::DeepFetch + end + + ## + # For intelsdi-x plugins merge pluginsync config_defaults with repo .sync.yml + # + def fetch_sync_yml + if @owner == Pluginsync::Github::INTEL_ORG + path = File.join(Pluginsync::PROJECT_PATH, 'config_defaults.yml') + config = Pluginsync::Util.load_yaml(path) + config.extend Hashie::Extensions::DeepMerge + config.deep_merge(yml_content('.sync.yml')) + else + {} + end + end + + def metadata + result = { + "name" => plugin_name, + "type" => plugin_type, + "description" => @repo.description || 'No description available.', + "maintainer" => @owner, + "maintainer_url" => @repo.owner.html_url, + "repo_name" => @repo.name, + "repo_url" => @repo.html_url, + } + + metadata = yml_content('metadata.yml') + + if @owner == Pluginsync::Github::INTEL_ORG + metadata["download"] = { + "s3_latest" => s3_url('latest'), + "s3_latest_build" => s3_url('latest_build'), + } + end + + metadata["name"] = Pluginsync::Util.plugin_capitalize metadata["name"] if metadata["name"] + metadata["github_release"] = @repo.html_url + "/releases/latest" if @gh.releases(@name).size > 0 + metadata["maintainer"] = "intelsdi-x" if metadata["maintainer"] == "core" + + result.merge(metadata) + end + + def s3_url(build) + matrix = sync_yml.deep_fetch :global, "build", "matrix" + matrix.collect do |go| + arch = if go["GOARCH"] == "amd64" + "x86_64" + else + go["GOARCH"] + end + { "#{go['GOOS']}/#{arch}" => "https://s3-us-west-2.amazonaws.com/snap.ci.snap-telemetry.io/plugins/#{@repo.name}/#{build}/#{go['GOOS']}/#{arch}/#{@repo.name}" } + end + end + end + end +end diff --git a/lib/pluginsync/notify.rb b/lib/pluginsync/notify.rb new file mode 100644 index 0000000..e53b143 --- /dev/null +++ b/lib/pluginsync/notify.rb @@ -0,0 +1,27 @@ +module Pluginsync + module Notify + module Slack + require "slack-ruby-client" + + def self.client + @client ||= ( + ::Slack.configure do |config| + file = File.join ENV["HOME"], ".slack" + conf = YAML.load_file file rescue conf = {} + config.token = ENV["SLACK_API_TOKEN"] || conf["API_TOKEN"] || raise(ArgumentError, "Missing slack api token in config: #{file}.") + end + + ::Slack::Web::Client.new + ) + end + + def self.message(channel, text) + client.chat_postMessage( + channel: channel, + as_user: true, + text: text, + ) + end + end + end +end diff --git a/lib/pluginsync/plugins.rb b/lib/pluginsync/plugins.rb new file mode 100644 index 0000000..c54deea --- /dev/null +++ b/lib/pluginsync/plugins.rb @@ -0,0 +1,82 @@ +require 'erb' +require 'set' +require 'pluginsync' +require 'pluginsync/github' + +module Pluginsync + module Plugins + @github = Pluginsync::Github + @config = Pluginsync.config + @log = Pluginsync.log + + def self.repos + @repos ||= Set.new plugins.collect do |p| + begin + Pluginsync::Github::Repo.new p + rescue ArgumentError => e + @log.error e.message + nil + end + end + @repos.reject{|p| p.nil?} + end + + def self.plugins + plugin_repo = Pluginsync::Github::Repo.new @config.plugins_yml["repo"] + plugin_repo.yml_content @config.plugins_yml["path"] + end + + def self.metadata + repos.collect{|r| r.metadata} + end + + def self.catalog + template = File.read(File.join(@config.path, "PLUGIN_CATALOG.md")) + @catalog ||= ERB.new(template, nil, '-').result + end + + def self.wishlist + data = [] + + snap_issues = @github.issues 'intelsdi-x/snap' + + wishlist = snap_issues.find_all{|issue| issue.labels.find{|label| label.name =~ /^plugin-wishlist/}} + wishlist.each do |issue| + wish_label = issue.labels.find{|l| l.name =~ /^plugin-wishlist/} + type = wish_label['name'].split('/').last + data << { + "number" => issue.number, + "url" => "https://github.com/intelsdi-x/snap/issues/#{issue.number}", + "description" => issue.title, + "body" => issue.body, + "type" => type, + } + end + data + end + + def self.pull_request + catalog_repo = @config.plugin_catalog_md["repo"] + catalog_path = @config.plugin_catalog_md["path"] + + fork_name = @config.plugin_catalog_md["fork"] || raise("Please configure plugin_catalog.md['fork'] in configuration.") + + origin_repo = Pluginsync::Github::Repo.new catalog_repo + fork_repo = Pluginsync::Github::Repo.new fork_name + + fork_repo.sync_branch(@config.branch) + + current_catalog = origin_repo.content catalog_path + + if catalog != current_catalog + @log.info "Updating plugins_catalog.md in #{fork_name} branch #{@config.branch}" + fork_repo.update_content(catalog_path, catalog, :branch => @config.branch) + + pr = fork_repo.create_pull_request(@config.branch, "Updating plugins_catalog.md by pluginsync. [ci skip]") + @log.info "Creating pull request: #{pr.html_url}" + else + puts "No new updates to plugin_catalog.md." + end + end + end +end diff --git a/lib/pluginsync/util.rb b/lib/pluginsync/util.rb new file mode 100644 index 0000000..d52b169 --- /dev/null +++ b/lib/pluginsync/util.rb @@ -0,0 +1,150 @@ +require 'fileutils' +require 'json' +require 'yaml' + +module Pluginsync + module Util + + ## + # make directory only if it does not exists + + def self.mkdir_p *dirs + dirs.each do |dir| + FileUtils.mkdir_p dir unless File.directory? dir + end + end + + ## + # safer symlink that creates the parent directory with support for force option + + def self.ln_s link, target, options = { :force => false } + return if valid_symlink? link, target + FileUtils.mkdir_p Pathname.new(link).parent + FileUtils.ln_s target, link, options + end + + ## + # verify symlink, we are not concerned about non existing target files + + def self.valid_symlink? source, target + File.symlink?(source) && File.readlink(source) == target + end + + ## + # load json configuration file + + def self.load_json file + file = File.expand_path file + raise ArgumentError, "Invalid json file path: #{file}" unless File.exist? file + JSON.parse File.read file + end + + ## + # load yaml configuration file + + def self.load_yaml file + file = File.expand_path file + raise ArgumentError, "Invalid yaml file path: #{file}" unless File.exist? file + YAML.load_file file + end + + ## + # return current system osfamily + + def self.os_family + output = %x{uname -a} + case output + when /^Darwin/ + family = "MacOS" + when /^Linux/ + if File.exists? "/etc/redhat-release" + family = "RedHat" + elsif File.exists? "/etc/lsb-release" + family = File.read("/etc/lsb-release").match(/^DISTRIB_ID=(.*)/)[1] + end + end + + family ||= "Unknown" + end + + ## + # cd into working directory temporarily + + def self.working_dir path = Dir.getwd + raise ArgumentError, "invalid working directory: #{path}" unless File.directory? path + + current_path = Dir.getwd + Dir.chdir path + Bundler.with_clean_env do + yield + end + ensure + Dir.chdir current_path + end + + ## + # cd into working directory with go environment + + def self.go_build path = Dir.getwd + go_path = Pluginsync.config.artifacts_path + working_dir path do + ENV["GOPATH"] = go_path + ENV["PATH"] += ":#{File.join go_path, 'bin'}" + # NOTE: hide source file path on panic along with -trimpath + # https://github.com/golang/go/issues/13809 + ENV["GOROOT_FINAL"] = "/usr/local/go" + + yield + end + end + + ## + # only capitalize orgs we recognize + + def self.org_capitalize(word) + { + 'intelsdi-x': 'Intel', + 'Staples-Inc': 'Staples Inc', + }[word.to_sym] || word + end + + ## + # custom capitalize for plugin names + + def self.plugin_capitalize(word) + { + 'ceph': 'CEPH', + 'cpu': 'CPU', + 'dbi': 'DBI', + 'heka': 'HEKA', + 'hana': 'HANA', + 'haproxy': 'HAproxy', + 'iostat': 'IOstat', + 'influxdb': 'InfluxDB', + 'jmx': 'Java JMX', + 'mysql': 'MySQL', + 'nfs-client': 'NFS Client', + 'opentsdb': 'OpenTSDB', + 'osv': 'OSv', + 'postgresql': 'PostgreSQL', + 'pcm': 'PCM', + 'psutil': 'PSUtil', + 'rabbitmq': 'RabbitMQ', + }[word.to_sym] || word.slice(0,1).capitalize + word.slice(1..-1) + end + + def self.html_list(list) + case list + when ::Array + if list.size > 0 + items = list.collect{|i| "
  • #{i}
    • "}.join('') + "" + else + "" + end + else + list + end + end + end +end diff --git a/managed_modules.yml b/managed_modules.yml index 45ad69f..7af8b9c 100644 --- a/managed_modules.yml +++ b/managed_modules.yml @@ -35,6 +35,7 @@ - snap-plugin-collector-psutil - snap-plugin-collector-rabbitmq - snap-plugin-collector-scaleio +- snap-plugin-collector-snmp - snap-plugin-collector-smart - snap-plugin-collector-swap - snap-plugin-collector-users diff --git a/modulesync.yml b/modulesync.yml index 3c12701..9b9cd56 100644 --- a/modulesync.yml +++ b/modulesync.yml @@ -3,3 +3,9 @@ git_base: 'git@github.com:' namespace: intelsdi-x branch: pluginsync message: "Updates to repository from pluginsync utility" +plugins.yml: + repo: intelsdi-x/snap + path: docs/plugins.yml +plugin_catalog.md: + repo: intelsdi-x/snap + path: docs/PLUGIN_CATALOG.md