Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

log-pilot 支持 Flume #167

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions Dockerfile.flume
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
FROM golang:1.9-alpine3.6 as builder

ENV PILOT_DIR /go/src/github.com/AliyunContainerService/log-pilot
ARG GOOS=linux
ARG GOARCH=amd64
RUN set -ex && apk add --no-cache make git
WORKDIR $PILOT_DIR
COPY . $PILOT_DIR
RUN go install

FROM alpine:3.6
COPY assets/glibc/glibc-2.26-r0.apk /tmp/

RUN apk update && \
apk add tzdata && \
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
echo "Asia/Shanghai" > /etc/timezone && \
apk add python && \
apk add ca-certificates wget && \
update-ca-certificates && \
wget wget --no-cookie --header "Cookie: oraclelicense=accept-securebackup-cookie" https://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.tar.gz && \
tar -xvzf jdk-8u191-linux-x64.tar.gz -C /usr/local/ && ln -s /usr/local/jdk/jdk1.8.0_191 /usr/local/java && \
rm -f jdk-8u191-linux-x64.tar.gz && \
apk add curl openssl && \
apk add --allow-untrusted /tmp/glibc-2.26-r0.apk && \
rm -rf /tmp/glibc-2.26-r0.apk && \
apk add --no-cache bash bash-doc bash-completion && \
rm -rf /var/cache/apk/*

ENV JAVA_HOME="/usr/local/java" \
JRE_HOME="/usr/local/java/jre" \
CLASSPATH=".:/usr/local/java/lib:/opt/soft/java/jre/lib:" \
PATH="/usr/local/java/bin:/usr/local/java/jre/bin:$PATH"

COPY --from=builder /go/bin/log-pilot /pilot/pilot
COPY assets/entrypoint assets/flume/ assets/healthz /pilot/
RUN cd /pilot && \
wget http://mirrors.hust.edu.cn/apache/flume/1.8.0/apache-flume-1.8.0-bin.tar.gz && \
tar -zvxf apache-flume-1.8.0-bin.tar.gz && ln -s apache-flume-1.8.0-bin flume && mkdir -p /pilot/flume/conf/tmp && \
chmod +x /pilot/pilot /pilot/entrypoint /pilot/healthz

HEALTHCHECK CMD /pilot/healthz

VOLUME /opt/log
VOLUME /flume/log_meta
WORKDIR /pilot/
ENV PILOT_TYPE=flume
ENTRYPOINT ["/pilot/entrypoint"]
8 changes: 5 additions & 3 deletions assets/entrypoint
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,10 @@ def run():
pilot_type = os.environ.get(ENV_PILOT_TYPE)
if pilot_filebeat == pilot_type:
tpl_config = "/pilot/filebeat.tpl"
else:
elif pilot_fluentd == pilot_type:
tpl_config = "/pilot/fluentd.tpl"
else:
tpl_config = "/pilot/flume.tpl"

os.execve('/pilot/pilot', ['/pilot/pilot', '-template', tpl_config, '-base', base, '-log-level', 'debug'],
os.environ)
Expand All @@ -49,8 +51,8 @@ def config():
if pilot_filebeat == pilot_type:
print "start log-pilot:", pilot_filebeat
subprocess.check_call(['/pilot/config.filebeat'])
else:
print "start log-pilot:", pilot_fluentd
elif pilot_fluentd == pilot_type:
print "enable pilot:", pilot_fluentd
subprocess.check_call(['/pilot/config.fluentd'])


Expand Down
20 changes: 20 additions & 0 deletions assets/flume/flume.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{{range .configList}}

a1.sources.{{ $.containerId }}_{{ .Name }}_source.type = TAILDIR
a1.sources.{{ $.containerId }}_{{ .Name }}_source.channels = {{ $.containerId }}_{{ .Name }}_channel
a1.sources.{{ $.containerId }}_{{ .Name }}_source.positionFile = /flume/log_meta/source/{{ $.containerId }}/{{ .Name }}/taildir_position.json
a1.sources.{{ $.containerId }}_{{ .Name }}_source.filegroups = f1
a1.sources.{{ $.containerId }}_{{ .Name }}_source.filegroups.f1 = {{ .HostDir }}/{{ .File }}

a1.channels.{{ $.containerId }}_{{ .Name }}_channel.type = file
a1.channels.{{ $.containerId }}_{{ .Name }}_channel.checkpointDir = /flume/log_meta/channel/{{ $.containerId }}/{{ .Name }}/checkpoint
a1.channels.{{ $.containerId }}_{{ .Name }}_channel.dataDirs = /flume/log_meta/channel/{{ $.containerId }}/{{ .Name }}/buffer

a1.sinks.{{ $.containerId }}_{{ .Name }}_sink.type = file_roll
a1.sinks.{{ $.containerId }}_{{ .Name }}_sink.channel = {{ $.containerId }}_{{ .Name }}_channel
a1.sinks.{{ $.containerId }}_{{ .Name }}_sink.sink.directory = {{ $.output }}/{{ index $.container "docker_container" }}
a1.sinks.{{ $.containerId }}_{{ .Name }}_sink.sink.rollInterval = 3600000
a1.sinks.{{ $.containerId }}_{{ .Name }}_sink.sink.pathManager.prefix = {{ .Name }}
a1.sinks.{{ $.containerId }}_{{ .Name }}_sink.sink.pathManager.extension = log

{{end}}
3 changes: 3 additions & 0 deletions build-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ case $1 in
fluentd)
build fluentd
;;
flume)
build flume
;;
*)
build filebeat
;;
Expand Down
60 changes: 60 additions & 0 deletions examples/pilot-flume.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: log-pilot
labels:
k8s-app: log-pilot
spec:
template:
metadata:
labels:
k8s-app: log-pilot
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: log-pilot
image: registry.cn-hangzhou.aliyuncs.com/acs/log-pilot:0.9.5-flume
imagePullPolicy: Always
env:
- name: "PILOT_TYPE"
value: "flume"
- name: "FLUME_OUTPUT"
value: "/opt/log/log-pilot-out"
- name: "NODE_NAME"
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: sock
mountPath: /var/run/docker.sock
- name: root
mountPath: /host
- name: log-meta
mountPath: /flume/log_meta
- name: logout
mountPath: /opt/log/log-pilot-out
- name: flume-log
mountPath: /opt/log/flume
securityContext:
capabilities:
add:
- SYS_ADMIN
terminationGracePeriodSeconds: 30
volumes:
- name: sock
hostPath:
path: /var/run/docker.sock
- name: root
hostPath:
path: /
- name: logout
hostPath:
path: /opt/log/log-pilot-out
- name: log-meta
hostPath:
path: /opt/log/log-pilot-out/log_meta
- name: flume-log
hostPath:
path: /opt/log/flume
Loading