wget http://apache.tt.co.kr/flume/1.2.0/apache-flume-1.2.0-bin.tar.gz
tar -xvf apache-flume-1.2.0.tar.gz
export FLUME_HOME=/apps/apache-flume-1.2.0
PATH=$PATH:$HOME/bin:${FLUME_HOME}/bin
export PATH
conf/example.conf
# agent 설정
collector_agent.sources = source1
collector_agent.sinks = sink1
collector_agent.channels = channel1
# sources 설정
collector_agent.sources.source1.type = netcat
collector_agent.sources.source1.bind = 10.101.25.173
collector_agent.sources.source1.port = 8994
# sinks 설정
collector_agent.sinks.sink1.type = logger
# channel 설정
collector_agent.channels.channel1.type = memory
collector_agent.channels.channel1.capacity = 1000
collector_agent.channels.channel1.transactionCapactiy = 100
# channel에 sources와 sinks 바인딩
collector_agent.sources.source1.channels = channel1
collector_agent.sinks.sink1.channel = channel1
bin/flume-ng agent --conf-file conf/example.conf --name collector_agent -Dflume.root.logger=INFO,console
# telnet 접속
telnet 10.101.25.173 8994
# Connected 후 메세지를 전송한다.
Connected to 10.101.25.173 (10.101.25.173).
Escape character is '^]'.
Hello World!!
OK
Yes..OK!!
OK
12/09/19 10:19:57 INFO source.NetcatSource: Source starting
12/09/19 10:19:57 INFO source.NetcatSource: Created serverSocket:sun.nio.ch.ServerSocketChannelImpl[/10.101.25.173:8994]
12/09/19 10:20:12 INFO sink.LoggerSink: Event: { headers:{} body: 48 65 6C 6C 6F 20 57 6F 72 6C 64 21 21 0D Hello World!!. }
conf/flume-conf.properties
# agent 설정
collector_agent.sources = avro-source
collector_agent.sinks = fileSink
collector_agent.channels = channel1
# channel 설정
collector_agent.channels.channel1.type = memory
collector_agent.channels.channel1.capacity = 1000
collector_agent.channels.channel1.transactionCapactiy = 100
# sources 설정
collector_agent.sources.avro-source.type = avro
collector_agent.sources.avro-source.bind = 10.101.25.173
collector_agent.sources.avro-source.port = 8994
# sinks 설정
collector_agent.sinks.fileSink.type = FILE_ROLL
collector_agent.sinks.fileSink.sink.directory = /apps/apache-flume-1.2.0/data
# rollInterval단위는 seconds
collector_agent.sinks.fileSink.sink.rollInterval = 3600
# channel에 sources와 sinks 바인딩
collector_agent.sources.avro-source.channels = channel1
collector_agent.sinks.fileSink.channel = channel1
bin/flume-ng agent -n collector_agent --conf ./conf/ -f conf/flume-conf.properties &
conf/flume-transfer-conf.properties
# agent 설정
agent_transfer.channels = channel1
agent_transfer.sources = reader
agent_transfer.sinks = avro-forward-sink
# channel 설정
agent_transfer.channels.channel1.type = memory
agent_transfer.channels.channel1.capacity = 1000000
agent_transfer.channels.channel1.transactionCapacity = 10000
# sources 설정
agent_transfer.sources.reader.type = exec
#agent_transfer.sources.reader.command = tail -F /logs/apache/access.log.`/bin/date +%Y%m%d`
agent_transfer.sources.reader.command = tail -F /logs/apache/access.log.20120919
agent_transfer.sources.reader.logStdErr = true
agent_transfer.sources.reader.restart = true
# sinks 설정
agent_transfer.sinks.avro-forward-sink.type = avro
agent_transfer.sinks.avro-forward-sink.hostname =10.101.25.173
agent_transfer.sinks.avro-forward-sink.port = 8994
agent_transfer.sinks.avro-forward-sink.batch-size = 100
agent_transfer.sinks.avro-forward-sink.runner.type = polling
agent_transfer.sinks.avro-forward-sink.runner.polling.interval = 10
# channel에 sources와 sinks 바인딩
agent_transfer.sources.reader.channels = channel1
agent_transfer.sinks.avro-forward-sink.channel = channel1
bin/flume-ng agent -n agent_transfer --conf ./conf/ -f conf/flume-transfer-conf.properties &