hi, all
I am using prometheus, but there are some error ouccured in prometheus log.
– error log
time=“2017-12-12T08:19:16Z” level=warning msg=“Scrape health sample discarded” error=“sample timestamp out of order” sample=up{instance=“malldb25”, job=“mysql”} => 1 @[1234] source=“scrape.go:587”
time=“2017-12-12T08:19:16Z” level=warning msg=“Scrape duration sample discarded” error=“sample timestamp out of order” sample=scrape_duration_seconds{instance=“malldb25”, job=“mysql”} => 4.728769441 @[1234] source=“scrape.go:590”
time=“2017-12-12T08:19:16Z” level=warning msg=“Scrape sample count sample discarded” error=“sample timestamp out of order” sample=scrape_duration_seconds{instance=“malldb25”, job=“mysql”} => 4.728769441 @[1234] source=“scrape.go:593”
time=“2017-12-12T08:19:16Z” level=warning msg=“Scrape sample count post-relabeling sample discarded” error=“sample timestamp out of order” sample=scrape_duration_seconds{instance=“malldb25”, job=“mysql”} => 4.728769441 @[1234] source=“scrape.go:596”
too many logs are storing now.
please let me know why these errors are ouccured.
– docker run command
docker run -d
-p 80:80
-e METRICS_MEMORY=48318382080 -e METRICS_RETENTION=2160h -e QUERIES_RETENTION=2160h -e METRICS_RESOLUTION=5s
–volumes-from pmm-data
–name pmm-server
–restart always
percona/pmm-server:latest
– prometheus.yml
Managed by pmm-managed. DO NOT EDIT.
global:
scrape_interval: 5s
scrape_timeout: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: prometheus
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /prometheus/metrics
scheme: http
static_configs: - targets:
- localhost:9090
labels:
instance: prometheus - job_name: linux
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
static_configs: - targets:
- localhost:9100
labels:
instance: pmm-server
consul_sd_configs: - server: localhost:8500
datacenter: dc1
tag_separator: ‘,’
scheme: http
services: - linux:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs: - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,alias([-\w:.]+),.*
target_label: instance
replacement: $1
action: replace - source_labels: [__meta_consul_tags]
separator: ;
regex: .,scheme_https,.
target_label: scheme
replacement: https
action: replace - job_name: proxysql
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
consul_sd_configs: - server: localhost:8500
datacenter: dc1
tag_separator: ‘,’
scheme: http
services: - proxysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs: - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,alias([-\w:.]+),.*
target_label: instance
replacement: $1
action: replace - source_labels: [__meta_consul_tags]
separator: ;
regex: .,scheme_https,.
target_label: scheme
replacement: https
action: replace - job_name: mongodb
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
consul_sd_configs: - server: localhost:8500
datacenter: dc1
tag_separator: ‘,’
scheme: http
services: - mongodb:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs: - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,alias([-\w:.]+),.*
target_label: instance
replacement: $1
action: replace - source_labels: [__meta_consul_tags]
separator: ;
regex: .,scheme_https,.
target_label: scheme
replacement: https
action: replace - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,cluster([-\w:.]+),.*
target_label: cluster
replacement: $1
action: replace - job_name: mysql-hr
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics-hr
scheme: http
consul_sd_configs: - server: localhost:8500
datacenter: dc1
tag_separator: ‘,’
scheme: http
services: - mysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs: - separator: ;
regex: (.*)
target_label: job
replacement: mysql
action: replace - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,alias([-\w:.]+),.*
target_label: instance
replacement: $1
action: replace - source_labels: [__meta_consul_tags]
separator: ;
regex: .,scheme_https,.
target_label: scheme
replacement: https
action: replace - job_name: mysql-mr
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics-mr
scheme: http
consul_sd_configs: - server: localhost:8500
datacenter: dc1
tag_separator: ‘,’
scheme: http
services: - mysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs: - separator: ;
regex: (.*)
target_label: job
replacement: mysql
action: replace - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,alias([-\w:.]+),.*
target_label: instance
replacement: $1
action: replace - source_labels: [__meta_consul_tags]
separator: ;
regex: .,scheme_https,.
target_label: scheme
replacement: https
action: replace - job_name: mysql-lr
scrape_interval: 1m
scrape_timeout: 5s
metrics_path: /metrics-lr
scheme: http
consul_sd_configs: - server: localhost:8500
datacenter: dc1
tag_separator: ‘,’
scheme: http
services: - mysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs: - separator: ;
regex: (.*)
target_label: job
replacement: mysql
action: replace - source_labels: [_meta_consul_tags]
separator: ;
regex: .*,alias([-\w:.]+),.*
target_label: instance
replacement: $1
action: replace - source_labels: [__meta_consul_tags]
separator: ;
regex: .,scheme_https,.
target_label: scheme
replacement: https
action: replace - job_name: rds-mysql-hr
scrape_interval: 1s
scrape_timeout: 1s
metrics_path: /metrics-hr
relabel_configs: - target_label: job
replacement: mysql - job_name: rds-mysql-mr
scrape_interval: 5s
scrape_timeout: 1s
metrics_path: /metrics-mr
relabel_configs: - target_label: job
replacement: mysql - job_name: rds-mysql-lr
scrape_interval: 1m
scrape_timeout: 5s
metrics_path: /metrics-lr
relabel_configs: - target_label: job
replacement: mysql
– top result
top - 18:05:17 up 22 days, 1:04, 1 user, load average: 0.68, 1.16, 1.25
Tasks: 444 total, 1 running, 443 sleeping, 0 stopped, 0 zombie
%Cpu(s): 6.9 us, 0.2 sy, 0.0 ni, 92.9 id, 0.1 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 65692272 total, 33868000 free, 2573836 used, 29250436 buff/cache
KiB Swap: 8191996 total, 8164396 free, 27600 used. 62573256 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
29289 1000 20 0 5564340 1.605g 12892 S 220.2 2.6 94:18.48 prometheus
29520 1000 20 0 4151424 43404 6504 S 2.3 0.1 0:30.39 percona-qan-api
29299 root 20 0 2427432 30316 5048 S 1.3 0.0 0:20.82 node_exporter
5813 root 20 0 148464 2452 1464 R 0.7 0.0 0:00.03 top
29283 polkitd 20 0 10.344g 115144 6864 S 0.7 0.2 0:17.54 mysqld
thanks