Skip to content

Commit

Permalink
Use Bundler to manage gem dependencies
Browse files Browse the repository at this point in the history
* Makes builds more deterministic
* Avoids issues with plugin incompatibility with fluentd version
* Vulnerability scanning from github
* ~ moves a little closer to solving fluent#43 perhaps
* Fixes fluent#27
  • Loading branch information
errm committed Mar 19, 2018
1 parent 4491d44 commit 9fc6df3
Show file tree
Hide file tree
Showing 70 changed files with 2,029 additions and 146 deletions.
30 changes: 29 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ release-all:
# Usage:
# make src [DOCKERFILE=] [VERSION=] [TAGS=t1,t2,...]

src: dockerfile fluent.conf systemd.conf kubernetes.conf plugins post-push-hook entrypoint.sh
src: dockerfile gemfile fluent.conf systemd.conf kubernetes.conf plugins post-push-hook entrypoint.sh

# Generate sources for all supported Docker images.
#
Expand Down Expand Up @@ -148,6 +148,20 @@ dockerfile:
version='$(VERSION)' \
/Dockerfile.erb > docker-image/$(DOCKERFILE)/Dockerfile

# Generate Gemfile and Gemfile.lock from template.
#
# Usage:
# make gemfile [DOCKERFILE=] [VERSION=]
gemfile:
mkdir -p docker-image/$(DOCKERFILE)
docker run --rm -i -v $(PWD)/templates/Gemfile.erb:/Gemfile.erb:ro \
ruby:alpine erb -U -T 1 \
dockerfile='$(DOCKERFILE)' \
version='$(VERSION)' \
/Gemfile.erb > docker-image/$(DOCKERFILE)/Gemfile
docker run --rm -i -v $(PWD)/docker-image/$(DOCKERFILE)/Gemfile:/Gemfile:ro \
ruby:alpine bundle lock --print > docker-image/${DOCKERFILE}/Gemfile.lock

# Generate entrypoint.sh from template.
#
# Usage:
Expand Down Expand Up @@ -233,6 +247,19 @@ dockerfile-all:
$(word 2,$(subst :, ,$(img))))) ; \
))

# Generate Gemfile and Gemfile.lock from template for all supported Docker images.
#
# Usage:
# make gemfile-all

gemfile-all:
(set -e ; $(foreach img,$(ALL_IMAGES), \
make gemfile \
DOCKERFILE=$(word 1,$(subst :, ,$(img))) \
VERSION=$(word 1,$(subst $(comma), ,\
$(word 2,$(subst :, ,$(img))))) ; \
))

# Generate entrypoint.sh from template for all supported Docker images.
#
# Usage:
Expand Down Expand Up @@ -300,6 +327,7 @@ post-push-hook-all:
release release-all \
src src-all \
dockerfile dockerfile-all \
gemfile gemfile-all \
entrypoint.sh entrypoint.sh-all \
fluent.conf fluent.conf-all \
kubernetes.conf kubernetes.conf-all\
Expand Down
12 changes: 5 additions & 7 deletions docker-image/v0.12/alpine-cloudwatch/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,16 @@ USER root
WORKDIR /home/fluent
ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH

COPY Gemfile* /home/fluent/
RUN set -ex \
&& apk update && apk upgrade \
&& apk upgrade --no-cache \
&& apk add --no-cache ruby-bundler \
&& apk add --no-cache --virtual .build-deps \
build-base \
ruby-dev \
libffi-dev \
&& echo 'gem: --no-document' >> /etc/gemrc \
&& gem install fluent-plugin-secure-forward \
&& gem install fluent-plugin-record-reformer \
&& gem install aws-sdk-core -v 2.10.50 \
&& gem install fluent-plugin-cloudwatch-logs -v 0.4.0 \
&& gem install fluent-plugin-kubernetes_metadata_filter \
&& bundle config silence_root_warning true \
&& bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
&& apk del .build-deps \
&& gem sources --clear-all \
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
Expand Down
11 changes: 11 additions & 0 deletions docker-image/v0.12/alpine-cloudwatch/Gemfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# AUTOMATICALLY GENERATED
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb

source "https://rubygems.org"

gem "fluentd", "0.12.33"
gem "fluent-plugin-secure-forward"
gem "fluent-plugin-record-reformer"
gem "aws-sdk-core", "2.10.50"
gem "fluent-plugin-cloudwatch-logs", "0.4.0"
gem "fluent-plugin-kubernetes_metadata_filter"
104 changes: 104 additions & 0 deletions docker-image/v0.12/alpine-cloudwatch/Gemfile.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (5.1.5)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (~> 0.7)
minitest (~> 5.1)
tzinfo (~> 1.1)
addressable (2.5.2)
public_suffix (>= 2.0.2, < 4.0)
aws-sdk-core (2.10.50)
aws-sigv4 (~> 1.0)
jmespath (~> 1.0)
aws-sigv4 (1.0.2)
concurrent-ruby (1.0.5)
cool.io (1.5.3)
domain_name (0.5.20170404)
unf (>= 0.0.5, < 1.0.0)
fluent-mixin-config-placeholders (0.4.0)
fluentd
uuidtools (>= 2.1.5)
fluent-plugin-cloudwatch-logs (0.4.0)
aws-sdk-core (>= 2.0.7)
fluent-mixin-config-placeholders (>= 0.2.0)
fluentd
fluent-plugin-kubernetes_metadata_filter (1.0.1)
fluentd (>= 0.12.0)
kubeclient (~> 1.1.4)
lru_redux
fluent-plugin-record-reformer (0.9.1)
fluentd
fluent-plugin-secure-forward (0.4.5)
fluentd (>= 0.10.46)
proxifier
resolve-hostname
fluentd (0.12.33)
cool.io (>= 1.2.2, < 2.0.0)
http_parser.rb (>= 0.5.1, < 0.7.0)
json (>= 1.4.3)
msgpack (>= 0.5.11, < 2)
sigdump (~> 0.2.2)
string-scrub (>= 0.0.3, <= 0.0.5)
tzinfo (>= 1.0.0)
tzinfo-data (>= 1.0.0)
yajl-ruby (~> 1.0)
http (0.9.8)
addressable (~> 2.3)
http-cookie (~> 1.0)
http-form_data (~> 1.0.1)
http_parser.rb (~> 0.6.0)
http-cookie (1.0.3)
domain_name (~> 0.5)
http-form_data (1.0.3)
http_parser.rb (0.6.0)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
jmespath (1.3.1)
json (2.1.0)
kubeclient (1.1.4)
activesupport
http (= 0.9.8)
recursive-open-struct (= 1.0.0)
rest-client
lru_redux (1.1.0)
mime-types (3.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2016.0521)
minitest (5.11.3)
msgpack (1.2.4)
netrc (0.11.0)
proxifier (1.0.3)
public_suffix (3.0.2)
recursive-open-struct (1.0.0)
resolve-hostname (0.1.0)
rest-client (2.0.2)
http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 4.0)
netrc (~> 0.8)
sigdump (0.2.4)
string-scrub (0.0.5)
thread_safe (0.3.6)
tzinfo (1.2.5)
thread_safe (~> 0.1)
tzinfo-data (1.2018.3)
tzinfo (>= 1.0.0)
unf (0.1.4)
unf_ext
unf_ext (0.0.7.5)
uuidtools (2.1.5)
yajl-ruby (1.3.1)

PLATFORMS
ruby

DEPENDENCIES
aws-sdk-core (= 2.10.50)
fluent-plugin-cloudwatch-logs (= 0.4.0)
fluent-plugin-kubernetes_metadata_filter
fluent-plugin-record-reformer
fluent-plugin-secure-forward
fluentd (= 0.12.33)

BUNDLED WITH
1.16.1
2 changes: 1 addition & 1 deletion docker-image/v0.12/alpine-cloudwatch/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ if [[ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ]] ; then
sed -i '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
fi

exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}
11 changes: 5 additions & 6 deletions docker-image/v0.12/alpine-elasticsearch/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,16 @@ USER root
WORKDIR /home/fluent
ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH

COPY Gemfile* /home/fluent/
RUN set -ex \
&& apk update && apk upgrade \
&& apk upgrade --no-cache \
&& apk add --no-cache ruby-bundler \
&& apk add --no-cache --virtual .build-deps \
build-base \
ruby-dev \
libffi-dev \
&& echo 'gem: --no-document' >> /etc/gemrc \
&& gem install fluent-plugin-secure-forward \
&& gem install fluent-plugin-record-reformer \
&& gem install fluent-plugin-elasticsearch \
&& gem install fluent-plugin-kubernetes_metadata_filter \
&& bundle config silence_root_warning true \
&& bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
&& apk del .build-deps \
&& gem sources --clear-all \
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
Expand Down
10 changes: 10 additions & 0 deletions docker-image/v0.12/alpine-elasticsearch/Gemfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# AUTOMATICALLY GENERATED
# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb

source "https://rubygems.org"

gem "fluentd", "0.12.33"
gem "fluent-plugin-secure-forward"
gem "fluent-plugin-record-reformer"
gem "fluent-plugin-elasticsearch"
gem "fluent-plugin-kubernetes_metadata_filter"
107 changes: 107 additions & 0 deletions docker-image/v0.12/alpine-elasticsearch/Gemfile.lock
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (5.1.5)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (~> 0.7)
minitest (~> 5.1)
tzinfo (~> 1.1)
addressable (2.5.2)
public_suffix (>= 2.0.2, < 4.0)
concurrent-ruby (1.0.5)
cool.io (1.5.3)
domain_name (0.5.20170404)
unf (>= 0.0.5, < 1.0.0)
elasticsearch (6.0.1)
elasticsearch-api (= 6.0.1)
elasticsearch-transport (= 6.0.1)
elasticsearch-api (6.0.1)
multi_json
elasticsearch-transport (6.0.1)
faraday
multi_json
excon (0.61.0)
faraday (0.14.0)
multipart-post (>= 1.2, < 3)
fluent-plugin-elasticsearch (1.13.3)
elasticsearch
excon
fluentd (>= 0.12.10)
fluent-plugin-kubernetes_metadata_filter (1.0.1)
fluentd (>= 0.12.0)
kubeclient (~> 1.1.4)
lru_redux
fluent-plugin-record-reformer (0.9.1)
fluentd
fluent-plugin-secure-forward (0.4.5)
fluentd (>= 0.10.46)
proxifier
resolve-hostname
fluentd (0.12.33)
cool.io (>= 1.2.2, < 2.0.0)
http_parser.rb (>= 0.5.1, < 0.7.0)
json (>= 1.4.3)
msgpack (>= 0.5.11, < 2)
sigdump (~> 0.2.2)
string-scrub (>= 0.0.3, <= 0.0.5)
tzinfo (>= 1.0.0)
tzinfo-data (>= 1.0.0)
yajl-ruby (~> 1.0)
http (0.9.8)
addressable (~> 2.3)
http-cookie (~> 1.0)
http-form_data (~> 1.0.1)
http_parser.rb (~> 0.6.0)
http-cookie (1.0.3)
domain_name (~> 0.5)
http-form_data (1.0.3)
http_parser.rb (0.6.0)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
json (2.1.0)
kubeclient (1.1.4)
activesupport
http (= 0.9.8)
recursive-open-struct (= 1.0.0)
rest-client
lru_redux (1.1.0)
mime-types (3.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2016.0521)
minitest (5.11.3)
msgpack (1.2.4)
multi_json (1.13.1)
multipart-post (2.0.0)
netrc (0.11.0)
proxifier (1.0.3)
public_suffix (3.0.2)
recursive-open-struct (1.0.0)
resolve-hostname (0.1.0)
rest-client (2.0.2)
http-cookie (>= 1.0.2, < 2.0)
mime-types (>= 1.16, < 4.0)
netrc (~> 0.8)
sigdump (0.2.4)
string-scrub (0.0.5)
thread_safe (0.3.6)
tzinfo (1.2.5)
thread_safe (~> 0.1)
tzinfo-data (1.2018.3)
tzinfo (>= 1.0.0)
unf (0.1.4)
unf_ext
unf_ext (0.0.7.5)
yajl-ruby (1.3.1)

PLATFORMS
ruby

DEPENDENCIES
fluent-plugin-elasticsearch
fluent-plugin-kubernetes_metadata_filter
fluent-plugin-record-reformer
fluent-plugin-secure-forward
fluentd (= 0.12.33)

BUNDLED WITH
1.16.1
2 changes: 1 addition & 1 deletion docker-image/v0.12/alpine-elasticsearch/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ if [[ -z ${FLUENT_ELASTICSEARCH_PASSWORD} ]] ; then
sed -i '/FLUENT_ELASTICSEARCH_PASSWORD/d' /fluentd/etc/${FLUENTD_CONF}
fi

exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins ${FLUENTD_OPT}
exec fluentd -c /fluentd/etc/${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}
13 changes: 7 additions & 6 deletions docker-image/v0.12/alpine-kafka/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ USER root
WORKDIR /home/fluent
ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH

COPY Gemfile* /home/fluent/
RUN set -ex \
&& apk upgrade --no-cache \
&& apk add --no-cache ruby-bundler \
&& apk add --no-cache --virtual .build-deps \
build-base \
ruby-dev \
libffi-dev \
&& echo 'gem: --no-document' >> /etc/gemrc \
&& gem install fluent-plugin-secure-forward \
&& gem install fluent-plugin-record-reformer \
&& gem install fluent-plugin-kafka \
&& gem install fluent-plugin-kubernetes_metadata_filter \
&& bundle config silence_root_warning true \
&& bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \
&& apk del .build-deps \
&& gem sources --clear-all \
&& rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
Expand All @@ -27,6 +27,7 @@ COPY ./conf/kubernetes.conf /fluentd/etc/

# Copy plugins
COPY plugins /fluentd/plugins/
COPY entrypoint.sh /fluentd/entrypoint.sh

# Environment variables
ENV FLUENTD_OPT=""
Expand All @@ -38,4 +39,4 @@ ENV FLUENTD_CONF="fluent.conf"
#ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"

# Run Fluentd
CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
CMD ["/fluentd/entrypoint.sh"]
Loading

0 comments on commit 9fc6df3

Please sign in to comment.