Use Bundler to manage gem dependencies

* Makes builds more deterministic
* Avoids issues with plugin incompatibility with fluentd version
* Vulnerability scanning from github
* ~ moves a little closer to solving fluent#43 perhaps
* Fixes fluent#27

Signed-off-by: Ed Robinson <edward-robinson@cookpad.com>

Makefile

@@ -127,7 +127,7 @@ release-all:
 # Usage:
 #	make src [DOCKERFILE=] [VERSION=] [TAGS=t1,t2,...]
 
-src: dockerfile fluent.conf systemd.conf kubernetes.conf plugins post-push-hook entrypoint.sh
+src: dockerfile gemfile fluent.conf systemd.conf kubernetes.conf plugins post-push-hook entrypoint.sh
 
 # Generate sources for all supported Docker images.
 #
@@ -158,6 +158,20 @@ dockerfile:
 			version='$(VERSION)' \
 		/Dockerfile.erb > docker-image/$(DOCKERFILE)/Dockerfile
 
+# Generate Gemfile and Gemfile.lock from template.
+#
+# Usage:
+#	make gemfile [DOCKERFILE=] [VERSION=]
+gemfile:
+	mkdir -p docker-image/$(DOCKERFILE)
+	docker run --rm -i -v $(PWD)/templates/Gemfile.erb:/Gemfile.erb:ro \
+		ruby:alpine erb -U -T 1 \
+			dockerfile='$(DOCKERFILE)' \
+			version='$(VERSION)' \
+		/Gemfile.erb > docker-image/$(DOCKERFILE)/Gemfile
+	docker run --rm -i -v $(PWD)/docker-image/$(DOCKERFILE)/Gemfile:/Gemfile:ro \
+		ruby:alpine sh -c "apk add --no-cache --quiet git && bundle lock --print" > docker-image/${DOCKERFILE}/Gemfile.lock
+
 # Generate entrypoint.sh from template.
 #
 # Usage:
@@ -243,6 +257,19 @@ dockerfile-all:
 			                 $(word 2,$(subst :, ,$(img))))) ; \
 	))
 
+# Generate Gemfile and Gemfile.lock from template for all supported Docker images.
+#
+# Usage:
+#	make gemfile-all
+
+gemfile-all:
+	(set -e ; $(foreach img,$(ALL_IMAGES), \
+		make gemfile \
+			DOCKERFILE=$(word 1,$(subst :, ,$(img))) \
+			VERSION=$(word 1,$(subst $(comma), ,\
+			                 $(word 2,$(subst :, ,$(img))))) ; \
+	))
+
 # Generate entrypoint.sh from template for all supported Docker images.
 #
 # Usage:
@@ -310,6 +337,7 @@ post-push-hook-all:
         release release-all \
         src src-all \
         dockerfile dockerfile-all \
+        gemfile gemfile-all \
         entrypoint.sh entrypoint.sh-all \
         fluent.conf fluent.conf-all \
         kubernetes.conf kubernetes.conf-all\

templates/Dockerfile.erb

@@ -1,5 +1,6 @@
 <% fluentd_ver = version.split("-").first %>
 <% target = (dockerfile.split("/").last.split("-").last) %>
+<% requires_git = %w(graylog syslog).include? target %>
 <% is_alpine = (dockerfile.split("/").last.split("-").first == "alpine") %>
 # AUTOMATICALLY GENERATED
 # DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb
@@ -15,65 +16,32 @@ USER root
 WORKDIR /home/fluent
 ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH
 
+COPY Gemfile* /fluentd/
 <% if is_alpine %>
 RUN set -ex \
-    && apk update && apk upgrade \
+    && apk upgrade --no-cache \
+    && apk add --no-cache ruby-bundler \
     && apk add --no-cache --virtual .build-deps \
         build-base \
         ruby-dev \
         libffi-dev \
-<% case target when "graylog"%>
+<% if target == "graylog"%>
         zlib-dev \
+<% end %>
+<% if requires_git %>
         git \
 <% end %>
 <% else %>
-RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev<% case target when "graylog" %> git build-essential patch zlib1g-dev liblzma-dev<% end %>" \
+  RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev<% if target == "graylog" %> build-essential patch zlib1g-dev liblzma-dev<% end %><% if requires_git %> git<% end %>" \
      && apt-get update \
      && apt-get upgrade -y \
      && apt-get install \
      -y --no-install-recommends \
      $buildDeps \
+     ruby-bundler \
 <% end %>
-    && echo 'gem: --no-document' >> /etc/gemrc \
-    && gem install fluent-plugin-secure-forward \
-    && gem install fluent-plugin-record-reformer \
-    && gem install fluent-plugin-detect-exceptions -v 0.0.9 \
-<% case target when "elasticsearch"%>
-    && gem install fluent-plugin-elasticsearch -v "<2" \
-<% when "logentries"%>
-    # && gem install fluent-plugin-logentries \
-<% when "loggly"%>
-    && gem install fluent-plugin-loggly \
-<% when "cloudwatch"%>
-    && gem install aws-sdk-core -v 2.10.50 \
-    && gem install fluent-plugin-cloudwatch-logs -v 0.4.0 \
-<% when "stackdriver" %>
-    && gem install fluent-plugin-google-cloud \
-<% when "s3" %>
-    && gem install fluent-plugin-s3 -v "~> 0.8" \
-<% when "gcs" %>
-    && gem install fluent-plugin-gcs -v "~> 0.3"  --no-document \
-<% when "graylog" %>
-    && gem install jeweler \
-    && git clone https://github.com/graylog-labs/gelf-rb.git \
-    && cd gelf-rb \
-    && rake install \
-    && gem install fluent-plugin-gelf-hs \
-<% when "logzio" %>
-    && gem install fluent-plugin-logzio \
-<% when "papertrail" %>
-    && gem install fluent-plugin-papertrail -v 0.1.1 \
-<% when "syslog" %>
-    && gem install specific_install \
-    && gem specific_install -l https://github.com/georgegoh/fluent-plugin-kubernetes_remote_syslog.git \
-<% when "kafka" %>
-    && gem install fluent-plugin-kafka \
-<% end%>
-    && gem install fluent-plugin-kubernetes_metadata_filter \
-<% if !is_alpine %>
-    && gem install ffi \
-    && gem install fluent-plugin-systemd -v 0.0.8 \
-<% end %>
+    && bundle config silence_root_warning true \
+    && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
 <% if is_alpine %>
     && apk del .build-deps \
 <% else %>

templates/Gemfile.erb

@@ -0,0 +1,43 @@
+<% fluentd_ver = version.split("-").first[1..-1] %>
+<% target = (dockerfile.split("/").last.split("-").last) %>
+<% is_alpine = (dockerfile.split("/").last.split("-").first == "alpine") %>
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "fluentd", "<%= fluentd_ver %>"
+gem "fluent-plugin-secure-forward"
+gem "fluent-plugin-record-reformer"
+<% case target when "elasticsearch"%>
+gem "fluent-plugin-elasticsearch"
+<% when "logentries"%>
+#gem "fluent-plugin-logentries"
+<% when "loggly"%>
+gem "fluent-plugin-loggly"
+<% when "cloudwatch"%>
+gem "aws-sdk-core", "2.10.50"
+gem "fluent-plugin-cloudwatch-logs", "0.4.0"
+<% when "stackdriver" %>
+gem "fluent-plugin-google-cloud"
+<% when "s3" %>
+gem "fluent-plugin-s3", "~>0.8"
+<% when "gcs" %>
+gem "fluent-plugin-gcs", "~>0.3"
+<% when "graylog" %>
+gem "gelf", git: "https://github.com/graylog-labs/gelf-rb.git"
+gem "fluent-plugin-gelf-hs"
+<% when "logzio" %>
+gem "fluent-plugin-logzio"
+<% when "papertrail" %>
+gem "fluent-plugin-papertrail"
+<% when "syslog" %>
+gem "fluent-plugin-kubernetes_remote_syslog", git: "https://github.com/georgegoh/fluent-plugin-kubernetes_remote_syslog.git"
+<% when "kafka" %>
+gem "fluent-plugin-kafka"
+<% end %>
+gem "fluent-plugin-kubernetes_metadata_filter"
+<% if !is_alpine %>
+gem "ffi"
+gem "fluent-plugin-systemd"
+<% end %>

templates/entrypoint.sh

@@ -10,4 +10,4 @@ if [ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ] ; then
    sed -i  '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
 fi
 
-exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
+exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}