Logstash监控指标

环境：Logstash 5.6.1

通过jvm监控

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
[root@ELK-test ~]# curl -s localhost:9600/_node/stats/jvm?pretty=true
{
  "host" : "ELK-test",
  "version" : "5.6.1",
  "http_address" : "127.0.0.1:9600",
  "id" : "6eaac40c-2e2a-4e04-8fce-10e938c9dc0d",
  "name" : "ELK-test",
  "jvm" : {
    "threads" : {
      "count" : 67,
      "peak_count" : 68
    },
    "mem" : {
      "heap_used_percent" : 17,
      "heap_committed_in_bytes" : 340779008,
      "heap_max_in_bytes" : 1056309248,
      "heap_used_in_bytes" : 181136632,
      "non_heap_used_in_bytes" : 112857424,
      "non_heap_committed_in_bytes" : 120733696,
      "pools" : {
        "survivor" : {
          "peak_used_in_bytes" : 17432576,
          "used_in_bytes" : 5437952,
          "peak_max_in_bytes" : 17432576,
          "max_in_bytes" : 17432576,
          "committed_in_bytes" : 17432576
        },
        "old" : {
          "peak_used_in_bytes" : 130825496,
          "used_in_bytes" : 130825496,
          "peak_max_in_bytes" : 899284992,
          "max_in_bytes" : 899284992,
          "committed_in_bytes" : 183754752
        },
        "young" : {
          "peak_used_in_bytes" : 139591680,
          "used_in_bytes" : 44873184,
          "peak_max_in_bytes" : 139591680,
          "max_in_bytes" : 139591680,
          "committed_in_bytes" : 139591680
        }
      }
    },
    "gc" : {
      "collectors" : {
        "old" : {
          "collection_time_in_millis" : 375,
          "collection_count" : 5
        },
        "young" : {
          "collection_time_in_millis" : 44399,
          "collection_count" : 3181
        }
      }
    },
    "uptime_in_millis" : 186066906
  }
}[root@ELK-test ~]#

通过process监控

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
[root@ELK-test ~]# curl -s localhost:9600/_node/stats/process?pretty=true
{
  "host" : "ELK-test",
  "version" : "5.6.1",
  "http_address" : "127.0.0.1:9600",
  "id" : "6eaac40c-2e2a-4e04-8fce-10e938c9dc0d",
  "name" : "ELK-test",
  "process" : {
    "open_file_descriptors" : 202,
    "peak_open_file_descriptors" : 202,
    "max_file_descriptors" : 4096,
    "mem" : {
      "total_virtual_in_bytes" : 3939852288
    },
    "cpu" : {
      "total_in_millis" : 3338270,
      "percent" : 0,
      "load_average" : {
        "1m" : 0.54,
        "5m" : 0.32,
        "15m" : 0.31
      }
    }
  }
}[root@ELK-test ~]#

通过pipeline监控

这里可以看到每个插件的日志处理情况（数量，耗时等），如果使用了grok插件，还显示正则匹配失败的数量、每个字段匹配的正则表达式个数等很有用的排障和性能调优信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
[root@ELK-test ~]# curl -s localhost:9600/_node/stats/pipeline?pretty=true
{
  "host" : "ELK-test",
  "version" : "5.6.1",
  "http_address" : "127.0.0.1:9600",
  "id" : "6eaac40c-2e2a-4e04-8fce-10e938c9dc0d",
  "name" : "ELK-test",
  "pipeline" : {
    "events" : {
      "duration_in_millis" : 3671575,
      "in" : 1621595,
      "out" : 1621595,
      "filtered" : 1621595,
      "queue_push_duration_in_millis" : 295900
    },
    "plugins" : {
      "inputs" : [ {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-2",
        "events" : {
          "out" : 1621595,
          "queue_push_duration_in_millis" : 295900
        },
        "current_connections" : 1,
        "name" : "beats",
        "peak_connections" : 1
      } ],
      "filters" : [ ],
      "outputs" : [ {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-16",
        "events" : {
          "duration_in_millis" : 307125,
          "in" : 411982,
          "out" : 411982
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-37",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-28",
        "events" : {
          "duration_in_millis" : 71536,
          "in" : 11323,
          "out" : 11323
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-30",
        "events" : {
          "duration_in_millis" : 8323,
          "in" : 1200,
          "out" : 1200
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-33",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-40",
        "events" : {
          "duration_in_millis" : 3373,
          "in" : 154,
          "out" : 154
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-17",
        "events" : {
          "duration_in_millis" : 24148,
          "in" : 3444,
          "out" : 3444
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-31",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-3",
        "events" : {
          "duration_in_millis" : 12911,
          "in" : 1007,
          "out" : 1007
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-32",
        "events" : {
          "duration_in_millis" : 3564,
          "in" : 285,
          "out" : 285
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-25",
        "events" : {
          "duration_in_millis" : 717356,
          "in" : 407063,
          "out" : 407063
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-23",
        "events" : {
          "duration_in_millis" : 29538,
          "in" : 2054,
          "out" : 2054
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-11",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-12",
        "events" : {
          "duration_in_millis" : 1805,
          "in" : 104,
          "out" : 104
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-10",
        "events" : {
          "duration_in_millis" : 4454,
          "in" : 838,
          "out" : 838
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-13",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-20",
        "events" : {
          "duration_in_millis" : 35142,
          "in" : 4872,
          "out" : 4872
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-22",
        "events" : {
          "duration_in_millis" : 1334,
          "in" : 107,
          "out" : 107
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-21",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-29",
        "events" : {
          "duration_in_millis" : 4538,
          "in" : 429,
          "out" : 429
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-24",
        "events" : {
          "duration_in_millis" : 73579,
          "in" : 10990,
          "out" : 10990
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-39",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-15",
        "events" : {
          "duration_in_millis" : 34938,
          "in" : 3095,
          "out" : 3095
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-9",
        "events" : {
          "duration_in_millis" : 0,
          "in" : 0,
          "out" : 0
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-26",
        "events" : {
          "duration_in_millis" : 793240,
          "in" : 424502,
          "out" : 424502
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-18",
        "events" : {
          "duration_in_millis" : 32408,
          "in" : 5051,
          "out" : 5051
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-36",
        "events" : {
          "duration_in_millis" : 6709,
          "in" : 2142,
          "out" : 2142
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-7",
        "events" : {
          "duration_in_millis" : 27306,
          "in" : 1791,
          "out" : 1791
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-35",
        "events" : {
          "duration_in_millis" : 2960,
          "in" : 24,
          "out" : 24
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-5",
        "events" : {
          "duration_in_millis" : 867,
          "in" : 1,
          "out" : 1
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-4",
        "events" : {
          "duration_in_millis" : 19048,
          "in" : 3088,
          "out" : 3088
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-8",
        "events" : {
          "duration_in_millis" : 340928,
          "in" : 306042,
          "out" : 306042
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-19",
        "events" : {
          "duration_in_millis" : 26274,
          "in" : 3311,
          "out" : 3311
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-27",
        "events" : {
          "duration_in_millis" : 512,
          "in" : 3,
          "out" : 3
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-14",
        "events" : {
          "duration_in_millis" : 1477,
          "in" : 329,
          "out" : 329
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-38",
        "events" : {
          "duration_in_millis" : 6962,
          "in" : 3773,
          "out" : 3773
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-6",
        "events" : {
          "duration_in_millis" : 2092,
          "in" : 989,
          "out" : 989
        },
        "name" : "elasticsearch"
      }, {
        "id" : "87303f752c45ebe8feaa4dafabb419ceb4188803-34",
        "events" : {
          "duration_in_millis" : 8653,
          "in" : 1152,
          "out" : 1152
        },
        "name" : "elasticsearch"
      } ]
    },
    "reloads" : {
      "last_error" : null,
      "successes" : 0,
      "last_success_timestamp" : null,
      "last_failure_timestamp" : null,
      "failures" : 0
    },
    "queue" : {
      "type" : "memory"
    },
    "id" : "main"
  }
}
[root@ELK-test ~]#

本文出自”Jack Wang Blog”：http://www.yfshare.vip/2017/12/04/Logstash监控/