OpenResty 配置(nginx.conf)

  1. http {
  2. include mime.types;
  3. default_type application/octet-stream;
  4. #log_format my_log escape=default '[$time_iso8601] [$remote_addr] [$status] [$request_method] [$server_protocol] [$uri] [$query_string] [$http_cookie] [$req_header]';
  5. log_format nginx_access_log escape=json '{'
  6. '"time":"$time_iso8601",'
  7. '"clientIp":"$remote_addr",'
  8. '"serverHost":"$http_host",'
  9. '"protocol":"$server_protocol",'
  10. '"status":"$status",'
  11. '"method":"$request_method",'
  12. '"path":"$uri",'
  13. '"params":"$query_string",'
  14. '"log":"$json_log",'
  15. '"totalTime":"$request_time",'
  16. '"reqSize":"$request_length",'
  17. '"resSize":"$bytes_sent"'
  18. '}';
  19. # 自定义配置
  20. server {
  21. listen 28080;
  22. server_name baidu;
  23. access_log logs/statistical_log.log nginx_access_log;
  24. set $json_log "{}";
  25. rewrite_by_lua '
  26. local cjson = require "cjson"
  27. string.split = function(s, p)
  28. if type(s) == "table" then
  29. return s
  30. end
  31. local rt= {}
  32. if(s ~= nil and string.len(s) >=1 ) then
  33. string.gsub(s, "[^"..p.."]+", function(w) table.insert(rt, w) end )
  34. end
  35. return rt
  36. end
  37. local cookies = ngx.req.get_headers()["Cookie"]
  38. cookies = string.split(cookies, "; ")
  39. if #cookies>=1 then
  40. local map = {}
  41. for i= 0, #cookies do
  42. if type(cookies[i]) == "string" then
  43. local cookie = cookies[i]
  44. local key = string.match(cookies[i], "(.-)=")
  45. local value = string.sub(cookies[i], string.len(key) + 2)
  46. map[key] = value
  47. end
  48. end
  49. cookies = map
  50. else
  51. cookies = {}
  52. end
  53. local log_map = {
  54. cookies = cookies,
  55. headers = ngx.req.get_headers()
  56. };
  57. ngx.var.json_log = cjson.encode(log_map)
  58. ';
  59. location / {
  60. proxy_pass https://www.baidu.com/;
  61. }
  62. location /lua {
  63. default_type 'application/json;charset=utf-8';
  64. content_by_lua '
  65. local cjson = require "cjson"
  66. local t = {}
  67. t["aaa"] = "hello"
  68. t["bbb"] = "world 啦啦啦"
  69. t["headers"] = ngx.req.get_headers()
  70. -- t["cookies"] = ngx.header.set_cookie
  71. t["cookies2"] = "123"
  72. ngx.say(cjson.encode(t))
  73. ';
  74. }
  75. }
  76. }

输出日志样例

  1. {"time":"2019-02-22T13:59:29+08:00","clientIp":"127.0.0.1","serverHost":"127.0.0.1:28080","protocol":"HTTP/1.1","status":"200","method":"GET","path":"/s","params":"ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&tn=baidu&wd=nginx&oq=nginx&rsv_pq=d79ff367000dca6a&rsv_t=448eNrSW7wcxA5qE35VE5bDpvMY6jwu2wniwTPSG3JbId1Fq210C17V168E&rqlang=cn&rsv_enter=0","log":"{\"cookies\":{\"H_PS_645EC\":\"02ddLV%2FjIPNJ8CkcLUU30nwj4m0lUa6phUrCaCxJWGtnXYwZfmTLUwU5sI4\",\"BDSVRTM\":\"17\",\"BD_CK_SAM\":\"1\",\"BD_UPN\":\"12314753\"},\"headers\":{\"host\":\"127.0.0.1:28080\",\"connection\":\"keep-alive\",\"upgrade-insecure-requests\":\"1\",\"cache-control\":\"max-age=0\",\"cookie\":\"BD_UPN=12314753; BD_CK_SAM=1; BDSVRTM=17; H_PS_645EC=02ddLV%2FjIPNJ8CkcLUU30nwj4m0lUa6phUrCaCxJWGtnXYwZfmTLUwU5sI4\",\"accept-encoding\":\"gzip, deflate, br\",\"user-agent\":\"Mozilla\\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/72.0.3626.109 Safari\\/537.36\",\"accept-language\":\"zh-CN,zh;q=0.9,en;q=0.8\",\"accept\":\"text\\/html,application\\/xhtml+xml,application\\/xml;q=0.9,image\\/webp,image\\/apng,*\\/*;q=0.8\"}}","totalTime":"0.821","reqSize":"719","resSize":"108674"}
  2. {"time":"2019-02-22T13:59:31+08:00","clientIp":"127.0.0.1","serverHost":"127.0.0.1:28080","protocol":"HTTP/1.1","status":"200","method":"GET","path":"/favicon.ico","params":"","log":"{\"cookies\":{\"H_PS_645EC\":\"ae28uqQhd8D7lCV2rAHP0DDybC4KB0cpjw%2BHUhQuO00KXc3gKUOAhCEtlAQ\",\"BD_CK_SAM\":\"1\",\"BD_UPN\":\"12314753\"},\"headers\":{\"host\":\"127.0.0.1:28080\",\"connection\":\"keep-alive\",\"cache-control\":\"no-cache\",\"referer\":\"http:\\/\\/127.0.0.1:28080\\/s?ie=utf-8&f=8&rsv_bp=1&rsv_idx=1&tn=baidu&wd=nginx&oq=nginx&rsv_pq=d79ff367000dca6a&rsv_t=448eNrSW7wcxA5qE35VE5bDpvMY6jwu2wniwTPSG3JbId1Fq210C17V168E&rqlang=cn&rsv_enter=0\",\"pragma\":\"no-cache\",\"user-agent\":\"Mozilla\\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/72.0.3626.109 Safari\\/537.36\",\"cookie\":\"BD_UPN=12314753; BD_CK_SAM=1; H_PS_645EC=ae28uqQhd8D7lCV2rAHP0DDybC4KB0cpjw%2BHUhQuO00KXc3gKUOAhCEtlAQ\",\"accept-language\":\"zh-CN,zh;q=0.9,en;q=0.8\",\"accept-encoding\":\"gzip, deflate, br\",\"accept\":\"image\\/webp,image\\/apng,image\\/*,*\\/*;q=0.8\"}}","totalTime":"1.391","reqSize":"693","resSize":"1488"}

FileBeat 配置(filebeat.yml)

  1. filebeat.inputs:
  2. - type: log
  3. enabled: true
  4. paths:
  5. - D:\ServiceSoftware\OpenResty\openresty-1.13.6.2-win64\logs\statistical_log.log
  6. filebeat.config.modules:
  7. path: ${path.config}/modules.d/*.yml
  8. reload.enabled: false
  9. output.logstash:
  10. hosts: ["127.0.0.1:5044"]
  11. # output.elasticsearch:
  12. # enabled: true
  13. # hosts: ["10.7.1.74:9200"]
  14. # index: "nginx-log-%{+yyyy.MM.dd}"
  15. #
  16. # setup.template:
  17. # enabled: true
  18. # name: "nginx-log"
  19. # pattern: "nginx-log-*"
  20. #output.kafka:
  21. # # initial brokers for reading cluster metadata
  22. # hosts: ["127.0.0.1:9092"]
  23. # # message topic selection + partitioning
  24. # topic: 'log'
  25. # partition.round_robin:
  26. # reachable_only: false
  27. #
  28. # required_acks: 1
  29. # compression: gzip
  30. # max_message_bytes: 1000000

Logstash配置(logstash.conf)

  1. input {
  2. beats {
  3. host => "127.0.0.1"
  4. port => 5044
  5. codec => "json"
  6. }
  7. stdin {
  8. codec => "json"
  9. }
  10. }
  11. filter {
  12. json {
  13. source => "log"
  14. remove_field => ["log"]
  15. #target => "jsoncontent"
  16. }
  17. }
  18. output {
  19. elasticsearch {
  20. hosts => ["http://10.7.1.74:9200"]
  21. index => "nginx-log-%{+YYYY.MM}"
  22. #index => "nginx-log-%{+YYYY.MM.dd}"
  23. #document_type => "%{[@metadata][type]}"
  24. #user => "elastic"
  25. #password => "changeme"
  26. template => "../config/nginx-log-index.json"
  27. template_name => "nginx-log"
  28. }
  29. stdout {}
  30. }

elasticsearch index template配置

  1. {
  2. "order": 0,
  3. "index_patterns": [
  4. "nginx-log-*"
  5. ],
  6. "settings": {
  7. "index": {
  8. "max_result_window": "10000",
  9. "number_of_shards": "5",
  10. "number_of_replicas": "1",
  11. "refresh_interval": "5s"
  12. }
  13. },
  14. "mappings": {
  15. "doc": {
  16. "numeric_detection": false,
  17. "dynamic_date_formats": ["yyyy-MM-dd'T'HH:mm:ssZZ", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"],
  18. "dynamic": true,
  19. "dynamic_templates": [
  20. {
  21. "string_fields": {
  22. "match_mapping_type": "string",
  23. "mapping": {
  24. "type": "text",
  25. "index": true,
  26. "analyzer": "standard"
  27. }
  28. }
  29. },
  30. {
  31. "object_fields": {
  32. "match_mapping_type": "object",
  33. "mapping": {
  34. "type": "object",
  35. "index": true
  36. }
  37. }
  38. }
  39. ],
  40. "properties": {
  41. "@timestamp": {
  42. "type": "date",
  43. "format": "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
  44. },
  45. "@version": {
  46. "type": "long",
  47. "index": true
  48. },
  49. "status": {
  50. "type": "long",
  51. "index": true
  52. },
  53. "totalTime": {
  54. "type": "double",
  55. "index": true
  56. },
  57. "reqSize": {
  58. "type": "long",
  59. "index": true
  60. },
  61. "resSize": {
  62. "type": "long",
  63. "index": true
  64. }
  65. }
  66. }
  67. }
  68. }

elasticsearch查询日志格式

  1. {
  2. "_index": "nginx-log-2019.02",
  3. "_type": "doc",
  4. "_id": "Y_DME2kBbppiCK7OUD2R",
  5. "_score": 1,
  6. "_source": {
  7. "headers": {
  8. "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36",
  9. "accept-encoding": "gzip, deflate, br",
  10. "host": "127.0.0.1:28080",
  11. "accept": "*/*",
  12. "bbb": "67890",
  13. "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
  14. "cookie": "BD_UPN=12314753; BD_CK_SAM=1",
  15. "connection": "keep-alive",
  16. "aaa": "12345"
  17. },
  18. "status": "200",
  19. "@version": "1",
  20. "protocol": "HTTP/1.1",
  21. "prospector": {
  22. "type": "log"
  23. },
  24. "offset": 220360,
  25. "serverHost": "127.0.0.1:28080",
  26. "@timestamp": "2019-02-22T05:59:20.272Z",
  27. "cookies": {
  28. "BD_UPN": "12314753",
  29. "BD_CK_SAM": "1"
  30. },
  31. "path": "/lua",
  32. "input": {
  33. "type": "log"
  34. },
  35. "resSize": "615",
  36. "reqSize": "363",
  37. "method": "GET",
  38. "clientIp": "127.0.0.1",
  39. "totalTime": "0.000",
  40. "tags": [
  41. "beats_input_codec_json_applied"
  42. ],
  43. "params": "ie=utf-8&f=8",
  44. "time": "2019-02-22T13:59:17+08:00",
  45. "host": {
  46. "name": "JT-DS998-lzw"
  47. },
  48. "beat": {
  49. "name": "JT-DS998-lzw",
  50. "hostname": "JT-DS998-lzw",
  51. "version": "6.3.2"
  52. },
  53. "source": """D:\ServiceSoftware\OpenResty\openresty-1.13.6.2-win64\logs\statistical_log.log"""
  54. }
  55. }

服务启动命令

  1. # OpenResty
  2. start nginx
  3. nginx -s reload
  4. nginx -s stop
  5. # FileBeat
  6. filebeat.exe -e -c filebeat.yml
  7. # Logstash
  8. logstash.bat -f ../config/logstash.conf
文档更新时间: 2019-04-26 10:32   作者:lizw