엘라스틱 서치(2) - metricbeat

 

메트릭비트 확인

http://node-1.kitri.com:5601/app/management/data/index_management/indices

메트릭비트 -> 리눅스 상태정보(log)를 확인할 수

 

[root@ns1 ~]# filebeat modules enable apache

 

[root@ns1 ~]# vi /etc/metricbeat/metricbeat.yml

 

[root@ns1 ~]# vi  /etc/hosts
[root@ns1 ~]# mkdir   /etc/metricbeat/certs
[root@ns1 ~]# systemctl enable metricbeat.service
Created symlink from /etc/systemd/system/multi-user.target.wants/metricbeat.service to /usr/lib/systemd/system/m        etricbeat.service.

 

#vi  /etc/hosts

 vi /etc/metricbeat/metricbeat.yml

 

 

 

파일비트 설치 - 설정

 

- 파일비트 올라오는 확인

 

[root@ns1 ~]# vi /etc/filebeat/modules.d/apache.yml
[root@ns1 ~]# systemctl restart filebeat.service
[root@ns1 ~]# ./apache_log_generator.py

[root@ns1 ~]# vi apache_log_generator.py
    return date

def gen_line():
    pct = random.randint(0,100)
    if pct < 80:
        line = "GET / HTTP/1.1"
    elif pct < 90:
        line = "POST /wp-login.php HTTP/1.1"
    elif pct <95:
        line = "PUT /post-new.php HTTP/1.1"
    else:
        line = "DELETE /post-new.php HTTP/1.1"
    return line

def gen_code():
    pct = random.randint(0,100)
    if pct < 92:      code = 200
    elif pct < 96:    code = 403
    elif pct <98:     code = 404
    else:             code = 500
    return code

def gen_agent():
    pct = random.randint(0,100)
    if pct   < 70:    agent = "Mozilla/5.0 (Windows) Edge"
    elif pct < 96:    agent = "Mozilla/5.0 (Android) Chrome"
    elif pct < 98:    agent = "Mozilla/5.0 (Apple) Sapari"
    else:             agent = "Mozilla/5.0 (Linux) Firefox"
    return agent


while True:
#for n in range(0,5):
    ipv4 = gen_ipv4()
    date = gen_datetime()
    line = gen_line()
    code = gen_code()
    agent = gen_agent()
    log = f'{ipv4} - - [{date}] "{line}" {code} 7000 "-" "{agent}"'
    os.system(f"echo '{log}' >> /var/log/httpd/access_log")
    time.sleep(random.random()*3)

 chmod a+x apache_log_generator.py

 

 

 

[root@ns1 logstash]# cd conf.d
[root@ns1 conf.d]# ls
[root@ns1 conf.d]# cp ../logstash-sample.conf     apache.conf
[root@ns1 conf.d]# cat apache.conf

 

받는 설정, 나가는 설정

 

[root@ns1 conf.d]# which logstash
/usr/bin/which: no logstash in (/usr/lib64/qt-3.3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin)

[root@ns1 conf.d]# find  / -name logstash -and -perm -100  | grep "/bin"
/usr/share/logstash/bin/logstash
[root@ns1 conf.d]# PATH=$PATH:/usr/share/logstash/bin

 

 

 

정규편식을 빠르게 찾기 위한 필터

 

ns1 에서 로그 발생시키기 --> 

[root@ns1 ~]# ./apache_log_generator.py

[root@ns1 conf.d]# logstash -f  /etc/logstash/conf.d/apache.conf

 

 

https://grokdebug.herokuapp.com/

 

정규패턴 확인

# cat  /usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-patterns-core-4.3.4/patterns/ecs-v1/grok-patterns

[root@ns1 conf.d]# cat  /usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-patterns-core-4.3.4/patterns/ecs-v1/grok-patterns

 

단축어로 저장

[root@ns1 conf.d]# GROK=/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-patterns-core-4.3.4/patterns/ecs-v1/grok-patterns
[root@ns1 conf.d]# head $GROK

 

 

패턴 입력

%{IPV4:client_ip}\s%{USERNAME:user}\s%{USERNAME:auth}\s\[%{HTTPDATE:date}\]\s\"%{METHOD:method}\s%{URIPATH:uri}\sHTTP/%{NUMBER:version}\"\s%{INT:status_code}\s%{INT:bytes}\s\"%{REFERER:referer}\"\s\"%{DATA:agent}\"

 

flter - grok 패턴 입력

 

 

패턴 확인 --> 1번째 나온 거 들어가서 uer정의 추가

[root@ns1 conf.d]# find  /usr/share/logstash/ -name "grok-patterns"
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-patterns-core-4.3.4/patterns/ecs-v1/grok-patterns
/usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-patterns-core-4.3.4/patterns/legacy/grok-patterns
[root@ns1 conf.d]#
[root@ns1 conf.d]# vi  //usr/share/logstash/vendor/bundle/jruby/2.6.0/gems/logstash-patterns-core-4.3.4/patterns/ecs-v1/grok-patterns

METHOD [A-Z]+
REFERER ([-]|%{URI})

 

 

[root@ns1 conf.d]# logstash -f  /etc/logstash/conf.d/apache.conf

mutate : 위도, 경도 입력

geoip : ip 위치값 입력

[root@ns1 conf.d]# vi apache.conf

 logstash -f  /etc/logstash/conf.d/apache.conf

 

---->>  위도, 경도 결과값 확인

 

 

-> 추가 OUT 설정

 

*** 권한 설정

[root@ns1 conf.d]# chgrp logstash  /var/log/httpd
[root@ns1 conf.d]# chmod g+rx  /var/log/httpd

[root@ns1 conf.d]# ls -lh  /var/log/httpd
합계 560K
-rw-r--r--. 1 root root 496K 11월 17 09:49 access_log
-rw-r--r--. 1 root root  46K 11월 16 16:53 error_log

 

 

kibana 서버에서 Apache 로그에 Geo_Point 적용

ㄱ) kibana 서버 접속후에 왼쪽 메뉴에서 [Dev Tools] 클릭

ㄴ) index 목록에서 logstash-httpd-2022.04.03 확인
GET _cat/indices
ㄷ) logstash-httpd-2022.04.03 index 제거
DELETE logstash-httpd-2022.04.03
ㄹ) logstash-httpd-2022.04.03 생성 후, geo_point 타입 적용

 

GET /_cat/indices

DELETE  /logstash-httpd-2022.11.17

GET /_cat/indices

PUT  /logstash-httpd-2022.11.17

GET /_cat/indices

PUT logstash-httpd-2022.11.17/_mapping
{
  "properties" : 
  {
   "geoip_client_ip" : 
   {
     "properties" : 
     {
       "geo" :
       {
         "properties" :
         {
           "location" :
           {
             "type" : "geo_point" 
             
           }
          } 
         
       }
       
     } 
     
   }
}
}