Use Bundler to manage gem dependencies

* Makes builds more deterministic
* Avoids issues with plugin incompatibility with fluentd version
* Vulnerability scanning from github
* ~ moves a little closer to solving fluent#43 perhaps
* Fixes fluent#27

Makefile

@@ -117,7 +117,7 @@ release-all:
 # Usage:
 #	make src [DOCKERFILE=] [VERSION=] [TAGS=t1,t2,...]
 
-src: dockerfile fluent.conf systemd.conf kubernetes.conf plugins post-push-hook entrypoint.sh
+src: dockerfile gemfile fluent.conf systemd.conf kubernetes.conf plugins post-push-hook entrypoint.sh
 
 # Generate sources for all supported Docker images.
 #
@@ -148,6 +148,20 @@ dockerfile:
 			version='$(VERSION)' \
 		/Dockerfile.erb > docker-image/$(DOCKERFILE)/Dockerfile
 
+# Generate Gemfile and Gemfile.lock from template.
+#
+# Usage:
+#	make gemfile [DOCKERFILE=] [VERSION=]
+gemfile:
+	mkdir -p docker-image/$(DOCKERFILE)
+	docker run --rm -i -v $(PWD)/templates/Gemfile.erb:/Gemfile.erb:ro \
+		ruby:alpine erb -U -T 1 \
+			dockerfile='$(DOCKERFILE)' \
+			version='$(VERSION)' \
+		/Gemfile.erb > docker-image/$(DOCKERFILE)/Gemfile
+	docker run --rm -i -v $(PWD)/docker-image/$(DOCKERFILE)/Gemfile:/Gemfile:ro \
+		ruby:alpine bundle lock --print > docker-image/${DOCKERFILE}/Gemfile.lock
+
 # Generate entrypoint.sh from template.
 #
 # Usage:
@@ -233,6 +247,19 @@ dockerfile-all:
 			                 $(word 2,$(subst :, ,$(img))))) ; \
 	))
 
+# Generate Gemfile and Gemfile.lock from template for all supported Docker images.
+#
+# Usage:
+#	make gemfile-all
+
+gemfile-all:
+	(set -e ; $(foreach img,$(ALL_IMAGES), \
+		make gemfile \
+			DOCKERFILE=$(word 1,$(subst :, ,$(img))) \
+			VERSION=$(word 1,$(subst $(comma), ,\
+			                 $(word 2,$(subst :, ,$(img))))) ; \
+	))
+
 # Generate entrypoint.sh from template for all supported Docker images.
 #
 # Usage:
@@ -300,6 +327,7 @@ post-push-hook-all:
         release release-all \
         src src-all \
         dockerfile dockerfile-all \
+        gemfile gemfile-all \
         entrypoint.sh entrypoint.sh-all \
         fluent.conf fluent.conf-all \
         kubernetes.conf kubernetes.conf-all\

templates/Dockerfile.erb

@@ -1,5 +1,4 @@
 <% fluentd_ver = version.split("-").first %>
-<% target = (dockerfile.split("/").last.split("-").last) %>
 <% is_alpine = (dockerfile.split("/").last.split("-").first == "alpine") %>
 # AUTOMATICALLY GENERATED
 # DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Dockerfile.erb
@@ -15,9 +14,11 @@ USER root
 WORKDIR /home/fluent
 ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH
 
+COPY Gemfile* /fluentd/
 <% if is_alpine %>
 RUN set -ex \
-    && apk update && apk upgrade \
+    && apk upgrade --no-cache \
+    && apk add --no-cache ruby-bundler \
     && apk add --no-cache --virtual .build-deps \
         build-base \
         ruby-dev \
@@ -29,33 +30,10 @@ RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \
      && apt-get install \
      -y --no-install-recommends \
      $buildDeps \
+     ruby-bundler \
 <% end %>
-    && echo 'gem: --no-document' >> /etc/gemrc \
-    && gem install fluent-plugin-secure-forward \
-    && gem install fluent-plugin-record-reformer \
-<% case target when "elasticsearch"%>
-    && gem install fluent-plugin-elasticsearch \
-<% when "logentries"%>
-    # && gem install fluent-plugin-logentries \
-<% when "loggly"%>
-    && gem install fluent-plugin-loggly \
-<% when "cloudwatch"%>
-    && gem install aws-sdk-core -v 2.10.50 \
-    && gem install fluent-plugin-cloudwatch-logs -v 0.4.0 \
-<% when "stackdriver" %>
-    && gem install fluent-plugin-google-cloud \
-<% when "s3" %>
-    && gem install fluent-plugin-s3 -v "~> 0.8" \
-<% when "papertrail" %>
-    && gem install fluent-plugin-papertrail -v 0.1.1 \
-<% when "kafka" %>
-    && gem install fluent-plugin-kafka \
-<% end%>
-    && gem install fluent-plugin-kubernetes_metadata_filter \
-<% if !is_alpine %>
-    && gem install ffi \
-    && gem install fluent-plugin-systemd -v 0.0.8 \
-<% end %>
+    && bundle config silence_root_warning true \
+    && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
 <% if is_alpine %>
     && apk del .build-deps \
 <% else %>

templates/Gemfile.erb

@@ -0,0 +1,34 @@
+<% fluentd_ver = version.split("-").first[1..-1] %>
+<% target = (dockerfile.split("/").last.split("-").last) %>
+<% is_alpine = (dockerfile.split("/").last.split("-").first == "alpine") %>
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "fluentd", "<%= fluentd_ver %>"
+gem "fluent-plugin-secure-forward"
+gem "fluent-plugin-record-reformer"
+<% case target when "elasticsearch"%>
+gem "fluent-plugin-elasticsearch"
+<% when "logentries"%>
+#gem "fluent-plugin-logentries"
+<% when "loggly"%>
+gem "fluent-plugin-loggly"
+<% when "cloudwatch"%>
+gem "aws-sdk-core", "2.10.50"
+gem "fluent-plugin-cloudwatch-logs", "0.4.0"
+<% when "stackdriver" %>
+gem "fluent-plugin-google-cloud"
+<% when "s3" %>
+gem "fluent-plugin-s3"
+<% when "papertrail" %>
+gem "fluent-plugin-papertrail"
+<% when "kafka" %>
+gem "fluent-plugin-kafka"
+<% end %>
+gem "fluent-plugin-kubernetes_metadata_filter"
+<% if !is_alpine %>
+gem "ffi"
+gem "fluent-plugin-systemd"
+<% end %>

templates/entrypoint.sh

@@ -10,4 +10,4 @@ if [[ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ]] ; then
    sed -i  '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
 fi
 
-exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
+exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}