头歌Flume 第2关 采集目录下所有新文件到Hdfs

# 配置source,channel，sink名称

a1.sources = source1

a1.sinks = sink1

a1.channels = channel1




# 配置source

a1.sources.source1.type = spooldir

a1.sources.source1.spoolDir = /opt/flume/data

##定义文件上传完后的后缀,默认是.COMPLETED

a1.sources.source1.fileSuffix=.FINISHED

##默认是2048,如果文件行数据量超过2048字节(1k),会被截断，导致数据丢失

a1.sources.source1.deserializer.maxLineLength=5120




# 配置 channel

a1.channels.channel1.type = memory

## event条数

a1.channels.channel1.capacity = 500000

##flume事务控制所需要的缓存容量600条event

a1.channels.channel1.transactionCapacity = 600



# 配置 sink

a1.sinks.sink1.type = hdfs

a1.sinks.sink1.hdfs.path =hdfs://localhost:9000/flume

#上传文件的前缀

a1.sinks.sink1.hdfs.filePrefix = flume

#上传文件的后缀

a1.sinks.sink1.hdfs.fileSuffix = .log

#积攒多少个Event才flush到HDFS一次

a1.sinks.sink1.hdfs.batchSize= 100

a1.sinks.sink1.hdfs.fileType = DataStream

a1.sinks.sink1.hdfs.writeFormat =Text




#配置source和sink绑定到channel

a1.sources.source1.channels = channel1

a1.sinks.sink1.channel = channel1