????
????????org.springframework.boot
????????spring-boot-starter-web
????????
????????
????????????
????????????????org.springframework.boot
????????????????spring-boot-starter-logging
????????????
????????
?????
?
?
?????org.springframework.boot
?????spring-boot-starter-log4j2
??
???
?????com.lmax
?????disruptor
?????3.3.4
????
?

"1.0"?encoding="UTF-8"?>
"INFO"?schema="Log4J-V2.0.xsd"?monitorInterval="600"?>
????
????????"LOG_HOME">logs
????????"FILE_NAME">collector
????????"patternLayout">[%d{yyyy-MM-dd'T'HH:mm:ss.SSSZZ}]?[%level{length=5}]?[%thread-%tid]?[%logger]?[%X{hostName}]?[%X{ip}]?[%X{applicationName}]?[%F,%L,%C,%M]?[%m]?##?'%ex'%n
????
????
????????"CONSOLE"?target="SYSTEM_OUT">
????????????"${patternLayout}"/>
??????????
????????"appAppender"?fileName="${LOG_HOME}/app-${FILE_NAME}.log"?filePattern="${LOG_HOME}/app-${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"?>
??????????"${patternLayout}"?/>
??????????
??????????????"1"/>
??????????????"500MB"/>
??????????
??????????"20"/>?????????
????????
????????"errorAppender"?fileName="${LOG_HOME}/error-${FILE_NAME}.log"?filePattern="${LOG_HOME}/error-${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"?>
??????????"${patternLayout}"?/>
??????????
??????????????"warn"?onMatch="ACCEPT"?onMismatch="DENY"/>
????????????????????????
??????????
??????????????"1"/>
??????????????"500MB"/>
??????????
??????????"20"/>?????????
????????????????????
????
????
????????
????????"com.bfxy.*"?level="info"?includeLocation="true">
??????????"appAppender"/>
????????
????????"com.bfxy.*"?level="info"?includeLocation="true">
??????????"errorAppender"/>
???????????????
????????"info">
????????????"CONSOLE"/>
????????????"appAppender"/>
????????????"errorAppender"/>
?????????????????
????

@Slf4j
@RestController
public?class?IndexController?{

?@RequestMapping(value?=?"/index")
?public?String?index()?{
??InputMDC.putMDC();
??
??log.info("我是一條info日志");
??
??log.warn("我是一條warn日志");

??log.error("我是一條error日志");
??
??return?"idx";
?}


?@RequestMapping(value?=?"/err")
?public?String?err()?{
??InputMDC.putMDC();
??try?{
???int?a?=?1/0;
??}?catch?(Exception?e)?{
???log.error("算術(shù)異常",?e);
??}
??return?"err";
?}
?
}

@Component
public?class?InputMDC?implements?EnvironmentAware?{

?private?static?Environment?environment;
?
?@Override
?public?void?setEnvironment(Environment?environment)?{
??InputMDC.environment?=?environment;
?}
?
?public?static?void?putMDC()?{
??MDC.put("hostName",?NetUtil.getLocalHostName());
??MDC.put("ip",?NetUtil.getLocalIp());
??MDC.put("applicationName",?environment.getProperty("spring.application.name"));
?}

}

##?解壓命令：
tar?-zxvf?kafka_2.12-2.1.0.tgz?-C?/usr/local/
##?改名命令：
mv?kafka_2.12-2.1.0/?kafka_2.12
##?進(jìn)入解壓后的目錄，修改server.properties文件：
vim?/usr/local/kafka_2.12/config/server.properties
##?修改配置：
broker.id=0
port=9092
host.name=192.168.11.51
advertised.host.name=192.168.11.51
log.dirs=/usr/local/kafka_2.12/kafka-logs
num.partitions=2
zookeeper.connect=192.168.11.111:2181,192.168.11.112:2181,192.168.11.113:2181

##?建立日志文件夾：
mkdir?/usr/local/kafka_2.12/kafka-logs

##啟動(dòng)kafka：
/usr/local/kafka_2.12/bin/kafka-server-start.sh?/usr/local/kafka_2.12/config/server.properties?&

##?創(chuàng)建topic
kafka-topics.sh?--zookeeper?192.168.11.111:2181?--create?--topic?app-log-collector?--partitions?1??--replication-factor?1
kafka-topics.sh?--zookeeper?192.168.11.111:2181?--create?--topic?error-log-collector?--partitions?1??--replication-factor?1?

kafka-topics.sh?--zookeeper?192.168.11.111:2181?--topic?app-log-test?--describe

cd?/usr/local/software
tar?-zxvf?filebeat-6.6.0-linux-x86_64.tar.gz?-C?/usr/local/
cd?/usr/local
mv?filebeat-6.6.0-linux-x86_64/?filebeat-6.6.0

vim?/usr/local/filebeat-5.6.2/filebeat.yml

######################?Filebeat?Configuration?Example?#########################
filebeat.prospectors:

-?input_type:?log

??paths:
????##?app-服務(wù)名稱.log,?為什么寫死，防止發(fā)生輪轉(zhuǎn)抓取歷史數(shù)據(jù)
????-?/usr/local/logs/app-collector.log
??#定義寫入?ES?時(shí)的?_type?值
??document_type:?"app-log"
??multiline:
????#pattern:?'^\s*(\d{4}|\d{2})\-(\d{2}|[a-zA-Z]{3})\-(\d{2}|\d{4})'???#?指定匹配的表達(dá)式（匹配以?2017-11-15?08:04:23:889?時(shí)間格式開頭的字符串）
????pattern:?'^\['??????????????????????????????#?指定匹配的表達(dá)式（匹配以?"{?開頭的字符串）
????negate:?true????????????????????????????????#?是否匹配到
????match:?after????????????????????????????????#?合并到上一行的末尾
????max_lines:?2000?????????????????????????????#?最大的行數(shù)
????timeout:?2s?????????????????????????????????#?如果在規(guī)定時(shí)間沒(méi)有新的日志事件就不等待后面的日志
??fields:
????logbiz:?collector
????logtopic:?app-log-collector???##?按服務(wù)劃分用作kafka?topic
????evn:?dev

-?input_type:?log

??paths:
????-?/usr/local/logs/error-collector.log
??document_type:?"error-log"
??multiline:
????#pattern:?'^\s*(\d{4}|\d{2})\-(\d{2}|[a-zA-Z]{3})\-(\d{2}|\d{4})'???#?指定匹配的表達(dá)式（匹配以?2017-11-15?08:04:23:889?時(shí)間格式開頭的字符串）
????pattern:?'^\['??????????????????????????????#?指定匹配的表達(dá)式（匹配以?"{?開頭的字符串）
????negate:?true????????????????????????????????#?是否匹配到
????match:?after????????????????????????????????#?合并到上一行的末尾
????max_lines:?2000?????????????????????????????#?最大的行數(shù)
????timeout:?2s?????????????????????????????????#?如果在規(guī)定時(shí)間沒(méi)有新的日志事件就不等待后面的日志
??fields:
????logbiz:?collector
????logtopic:?error-log-collector???##?按服務(wù)劃分用作kafka?topic
????evn:?dev
????
output.kafka:
??enabled:?true
??hosts:?["192.168.11.51:9092"]
??topic:?'%{[fields.logtopic]}'
??partition.hash:
????reachable_only:?true
??compression:?gzip
??max_message_bytes:?1000000
??required_acks:?1
logging.to_files:?true

cd?/usr/local/filebeat-6.6.0
./filebeat?-c?filebeat.yml?-configtest
##?Config?OK

/usr/local/filebeat-6.6.0/filebeat?&

ps?-ef?|?grep?filebeat

mkdir?scrpit

cd?scrpit
vim?logstash-script.conf

##?multiline?插件也可以用于其他類似的堆棧式信息，比如?linux?的內(nèi)核日志。
input?{
??kafka?{
????##?app-log-服務(wù)名稱
????topics_pattern?=>?"app-log-.*"
????bootstrap_servers?=>?"192.168.11.51:9092"
?codec?=>?json
?consumer_threads?=>?1?##?增加consumer的并行消費(fèi)線程數(shù)
?decorate_events?=>?true
????#auto_offset_rest?=>?"latest"
?group_id?=>?"app-log-group"
???}
???
???kafka?{
????##?error-log-服務(wù)名稱
????topics_pattern?=>?"error-log-.*"
????bootstrap_servers?=>?"192.168.11.51:9092"
?codec?=>?json
?consumer_threads?=>?1
?decorate_events?=>?true
????#auto_offset_rest?=>?"latest"
?group_id?=>?"error-log-group"
???}
???
}

filter?{
??
??##?時(shí)區(qū)轉(zhuǎn)換
??ruby?{
?code?=>?"event.set('index_time',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"
??}

??if?"app-log"?in?[fields][logtopic]{
????grok?{
????????##?表達(dá)式,這里對(duì)應(yīng)的是Springboot輸出的日志格式
????????match?=>?["message",?"\[%{NOTSPACE:currentDateTime}\]?\[%{NOTSPACE:level}\]?\[%{NOTSPACE:thread-id}\]?\[%{NOTSPACE:class}\]?\[%{DATA:hostName}\]?\[%{DATA:ip}\]?\[%{DATA:applicationName}\]?\[%{DATA:location}\]?\[%{DATA:messageInfo}\]?##?(\'\'|%{QUOTEDSTRING:throwable})"]
????}
??}

??if?"error-log"?in?[fields][logtopic]{
????grok?{
????????##?表達(dá)式
????????match?=>?["message",?"\[%{NOTSPACE:currentDateTime}\]?\[%{NOTSPACE:level}\]?\[%{NOTSPACE:thread-id}\]?\[%{NOTSPACE:class}\]?\[%{DATA:hostName}\]?\[%{DATA:ip}\]?\[%{DATA:applicationName}\]?\[%{DATA:location}\]?\[%{DATA:messageInfo}\]?##?(\'\'|%{QUOTEDSTRING:throwable})"]
????}
??}
??
}

##?測(cè)試輸出到控制臺(tái)：
output?{
??stdout?{?codec?=>?rubydebug?}
}


##?elasticsearch：
output?{

??if?"app-log"?in?[fields][logtopic]{
?##?es插件
?elasticsearch?{
???????#?es服務(wù)地址
????????hosts?=>?["192.168.11.35:9200"]
????????#?用戶名密碼??????
????????user?=>?"elastic"
????????password?=>?"123456"
????????##?索引名，+?號(hào)開頭的，就會(huì)自動(dòng)認(rèn)為后面是時(shí)間格式：
????????##?javalog-app-service-2019.01.23?
????????index?=>?"app-log-%{[fields][logbiz]}-%{index_time}"
????????#?是否嗅探集群ip：一般設(shè)置true；http://192.168.11.35:9200/_nodes/http?pretty
????????#?通過(guò)嗅探機(jī)制進(jìn)行es集群負(fù)載均衡發(fā)日志消息
????????sniffing?=>?true
????????#?logstash默認(rèn)自帶一個(gè)mapping模板，進(jìn)行模板覆蓋
????????template_overwrite?=>?true
????}?
??}
??
??if?"error-log"?in?[fields][logtopic]{
?elasticsearch?{
????????hosts?=>?["192.168.11.35:9200"]????
????????user?=>?"elastic"
????????password?=>?"123456"
????????index?=>?"error-log-%{[fields][logbiz]}-%{index_time}"
????????sniffing?=>?true
????????template_overwrite?=>?true
????}?
??}
??

}

/usr/local/logstash-6.6.0/bin/logstash?-f?/usr/local/logstash-6.6.0/script/logstash-script.conf?&