diff --git a/Dockerfile b/Dockerfile index eb9fe93..404ee92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,33 @@ FROM ubuntu:14.04 -MAINTAINER Michal Raczka me@michaloo.net +MAINTAINER Nelson Hernandez nelson@meetearnest.com -# install curl and fluentd deps -RUN apt-get update \ - && apt-get install -y curl libcurl4-openssl-dev ruby ruby-dev make +#install apt tools for adding repositories +RUN apt-get update +RUN apt-get install -y --force-yes curl software-properties-common python-software-properties + +#install apt repository for easily installing native ruby's +RUN apt-add-repository ppa:brightbox/ruby-ng +RUN apt-get update + +#fluent plugsins below depened on ruby2.0 and ruby2.0-dev specifically. +RUN apt-get -y install ruby2.0 ruby2.0-dev ruby-switch make g++ +RUN ruby-switch --set ruby2.0 # install fluentd with plugins -RUN gem install fluentd --no-ri --no-rdoc \ - && fluent-gem install fluent-plugin-elasticsearch \ - fluent-plugin-record-modifier fluent-plugin-exclude-filter \ +RUN gem install fluentd fluent-plugin-cloudwatch-logs fluent-plugin-kubernetes_metadata_filter --no-ri --no-rdoc \ && mkdir /etc/fluentd/ -# install docker-gen -RUN cd /usr/local/bin \ - && curl -L https://github.com/jwilder/docker-gen/releases/download/0.4.0/docker-gen-linux-amd64-0.4.0.tar.gz \ - | tar -xzv - # add startup scripts and config files ADD ./bin /app/bin ADD ./config /app/config -ADD config/fluentd-custom.tmpl /app/config/fluentd.tmpl - WORKDIR /app + +RUN cd /usr/local/bin \ + && curl -L https://github.com/jwilder/docker-gen/releases/download/0.4.0/docker-gen-linux-amd64-0.4.0.tar.gz \ + | tar -xzv + ENV ES_HOST localhost ENV ES_PORT 9200 ENV LOG_ENV production diff --git a/README.md b/README.md index 5f38518..f239703 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,17 @@ # Docker fluentd +## servers require the following files to deploy with fleetctl +``` +/etc/sysconfig/aws_credentials +/home/core/.docker/config.json +``` + +## to deploy using Fleet, the FLEETCTL_TUNNEL must be set locally. +``` +export FLEETCLT_TUNNEL= +``` + + Docker image with: - [docker-gen](https://github.com/jwilder/docker-gen) diff --git a/config/fluentd-custom.tmpl b/config/fluentd-custom.tmpl deleted file mode 100644 index edc7bbd..0000000 --- a/config/fluentd-custom.tmpl +++ /dev/null @@ -1,25 +0,0 @@ -## File input -## read docker logs with tag=docker.container - - type tail - format json - time_key time - path /var/lib/docker/containers/*/*-json.log - pos_file /var/lib/docker/containers/containers.log.pos - time_format %Y-%m-%dT%H:%M:%S - tag docker.container.* - - - - type elasticsearch - log_level info - include_tag_key true - host {{ .Env.ES_HOST }} - port {{ .Env.ES_PORT }} - logstash_format true - flush_interval 5s - # Never wait longer than 5 minutes between retries. - max_retry_wait 300 - # Disable the limit on the number of retries (retry forever). - disable_retry_limit - diff --git a/config/fluentd.tmpl b/config/fluentd.tmpl index fffb823..9a8aba3 100644 --- a/config/fluentd.tmpl +++ b/config/fluentd.tmpl @@ -1,42 +1,21 @@ -## File input -## read docker logs with tag=docker.container - -{{range $key, $value := .}}{{ $logs := $value.Env.LOG }}{{ if $logs }} - type tail format json time_key time - time_format %Y-%m-%dT%H:%M:%S.%N%Z - path /var/lib/docker/containers/{{ $value.ID }}/{{ $value.ID }}-json.log - pos_file /var/lib/docker/containers/{{ $value.ID }}/{{ $value.ID }}-json.log.pos - tag docker.container.{{printf "%.*s" 12 $value.ID}} - rotate_wait 5 + path /var/lib/docker/containers/*/*-json.log + pos_file /var/lib/docker/containers/containers.log.pos + time_format %Y-%m-%dT%H:%M:%S + tag docker.container.* - - type record_modifier - tag docker.filtered.{{printf "%.*s" 12 $value.ID}} - image {{ $value.Image }} - host ${hostname} - name {{ $logs }} - env {{LOG_ENV}} - -{{end}}{{ end }} - - - type exclude_filter - key stream - value stdout - regexp false # default false, string comparison - add_tag_prefix stderr - + + @type kubernetes_metadata + - - type elasticsearch - host {{ .Env.ES_HOST }} - port {{ .Env.ES_PORT }} - index_name fluentd - type_name fluentd - logstash_format true + + type cloudwatch_logs + log_group_name kubernetes + log_stream_name "#{ENV['HOSTNAME']}" + auto_create_stream true + include_time_key true diff --git a/crane.yml b/crane.yml deleted file mode 100644 index 06ffaeb..0000000 --- a/crane.yml +++ /dev/null @@ -1,25 +0,0 @@ -containers: - fluentd_it: - image: michaloo/fluentd - dockerfile: . - run: - volume: - - ".:/app" - - "/var/run/docker.sock:/tmp/docker.sock" - - "/var/lib/docker:/var/lib/docker" - env-file: [".env"] - interactive: true - tty: true - entrypoint: /bin/bash - cmd: - - -c - - "bash" - rm: true - fluentd_test: - image: michaloo/fluentd - dockerfile: . - run: - volume: - - "/var/run/docker.sock:/tmp/docker.sock" - - "/var/lib/docker:/var/lib/docker" - detach: true diff --git a/fluentd.service b/fluentd.service new file mode 100644 index 0000000..7b9189e --- /dev/null +++ b/fluentd.service @@ -0,0 +1,30 @@ +## config file used for fleetctl +[Unit] +Description=Ship container logs to elasticsearch. +After=docker.service +Requires=docker.service + +[Service] +TimeoutStartSec=10m +User=core +EnvironmentFile=/etc/sysconfig/aws_credentials + +ExecStartPre=-/usr/bin/docker kill fluentd +ExecStartPre=-/usr/bin/docker rm fluentd +ExecStartPre=/usr/bin/docker pull earnest/fluentd:latest + +ExecStart=/usr/bin/docker run --name fluentd \ + -e AWS_REGION=${AWS_REGION} \ + -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \ + -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/run/docker.sock:/tmp/docker.sock \ + earnest/fluentd + +ExecStop=/usr/bin/docker stop -t 30 fluentd + +[Install] +WantedBy=multi-user.target + +[X-Fleet] +Global=true