Not the answer you need?
Register and ask your own question!

some errors are ouccured in prometheus log

bckimbckim ContributorCurrent User Role Beginner
hi, all

I am using prometheus, but there are some error ouccured in prometheus log.


-- error log

time="2017-12-12T08:19:16Z" level=warning msg="Scrape health sample discarded" error="sample timestamp out of order" sample=up{instance="malldb25", job="mysql"} => 1 @[1234] source="scrape.go:587"
time="2017-12-12T08:19:16Z" level=warning msg="Scrape duration sample discarded" error="sample timestamp out of order" sample=scrape_duration_seconds{instance="malldb25", job="mysql"} => 4.728769441 @[1234] source="scrape.go:590"
time="2017-12-12T08:19:16Z" level=warning msg="Scrape sample count sample discarded" error="sample timestamp out of order" sample=scrape_duration_seconds{instance="malldb25", job="mysql"} => 4.728769441 @[1234] source="scrape.go:593"
time="2017-12-12T08:19:16Z" level=warning msg="Scrape sample count post-relabeling sample discarded" error="sample timestamp out of order" sample=scrape_duration_seconds{instance="malldb25", job="mysql"} => 4.728769441 @[1234] source="scrape.go:596"


too many logs are storing now.

please let me know why these errors are ouccured.




-- docker run command

docker run -d \
-p 80:80 \
-e METRICS_MEMORY=48318382080 -e METRICS_RETENTION=2160h -e QUERIES_RETENTION=2160h -e METRICS_RESOLUTION=5s \
--volumes-from pmm-data \
--name pmm-server \
--restart always \
percona/pmm-server:latest



-- prometheus.yml

# Managed by pmm-managed. DO NOT EDIT.
---
global:
scrape_interval: 5s
scrape_timeout: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: prometheus
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /prometheus/metrics
scheme: http
static_configs:
- targets:
- localhost:9090
labels:
instance: prometheus
- job_name: linux
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
static_configs:
- targets:
- localhost:9100
labels:
instance: pmm-server
consul_sd_configs:
- server: localhost:8500
datacenter: dc1
tag_separator: ','
scheme: http
services:
- linux:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,alias_([-\w:\.]+),.*
target_label: instance
replacement: $1
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,scheme_https,.*
target_label: __scheme__
replacement: https
action: replace
- job_name: proxysql
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
consul_sd_configs:
- server: localhost:8500
datacenter: dc1
tag_separator: ','
scheme: http
services:
- proxysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,alias_([-\w:\.]+),.*
target_label: instance
replacement: $1
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,scheme_https,.*
target_label: __scheme__
replacement: https
action: replace
- job_name: mongodb
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics
scheme: http
consul_sd_configs:
- server: localhost:8500
datacenter: dc1
tag_separator: ','
scheme: http
services:
- mongodb:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs:
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,alias_([-\w:\.]+),.*
target_label: instance
replacement: $1
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,scheme_https,.*
target_label: __scheme__
replacement: https
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,cluster_([-\w:\.]+),.*
target_label: cluster
replacement: $1
action: replace
- job_name: mysql-hr
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics-hr
scheme: http
consul_sd_configs:
- server: localhost:8500
datacenter: dc1
tag_separator: ','
scheme: http
services:
- mysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs:
- separator: ;
regex: (.*)
target_label: job
replacement: mysql
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,alias_([-\w:\.]+),.*
target_label: instance
replacement: $1
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,scheme_https,.*
target_label: __scheme__
replacement: https
action: replace
- job_name: mysql-mr
scrape_interval: 5s
scrape_timeout: 5s
metrics_path: /metrics-mr
scheme: http
consul_sd_configs:
- server: localhost:8500
datacenter: dc1
tag_separator: ','
scheme: http
services:
- mysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs:
- separator: ;
regex: (.*)
target_label: job
replacement: mysql
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,alias_([-\w:\.]+),.*
target_label: instance
replacement: $1
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,scheme_https,.*
target_label: __scheme__
replacement: https
action: replace
- job_name: mysql-lr
scrape_interval: 1m
scrape_timeout: 5s
metrics_path: /metrics-lr
scheme: http
consul_sd_configs:
- server: localhost:8500
datacenter: dc1
tag_separator: ','
scheme: http
services:
- mysql:metrics
basic_auth:
username: pmm
password: pmm
tls_config:
insecure_skip_verify: true
relabel_configs:
- separator: ;
regex: (.*)
target_label: job
replacement: mysql
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,alias_([-\w:\.]+),.*
target_label: instance
replacement: $1
action: replace
- source_labels: [__meta_consul_tags]
separator: ;
regex: .*,scheme_https,.*
target_label: __scheme__
replacement: https
action: replace
- job_name: rds-mysql-hr
scrape_interval: 1s
scrape_timeout: 1s
metrics_path: /metrics-hr
relabel_configs:
- target_label: job
replacement: mysql
- job_name: rds-mysql-mr
scrape_interval: 5s
scrape_timeout: 1s
metrics_path: /metrics-mr
relabel_configs:
- target_label: job
replacement: mysql
- job_name: rds-mysql-lr
scrape_interval: 1m
scrape_timeout: 5s
metrics_path: /metrics-lr
relabel_configs:
- target_label: job
replacement: mysql




-- top result

top - 18:05:17 up 22 days, 1:04, 1 user, load average: 0.68, 1.16, 1.25
Tasks: 444 total, 1 running, 443 sleeping, 0 stopped, 0 zombie
%Cpu(s): 6.9 us, 0.2 sy, 0.0 ni, 92.9 id, 0.1 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 65692272 total, 33868000 free, 2573836 used, 29250436 buff/cache
KiB Swap: 8191996 total, 8164396 free, 27600 used. 62573256 avail Mem

PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
29289 1000 20 0 5564340 1.605g 12892 S 220.2 2.6 94:18.48 prometheus
29520 1000 20 0 4151424 43404 6504 S 2.3 0.1 0:30.39 percona-qan-api
29299 root 20 0 2427432 30316 5048 S 1.3 0.0 0:20.82 node_exporter
5813 root 20 0 148464 2452 1464 R 0.7 0.0 0:00.03 top
29283 polkitd 20 0 10.344g 115144 6864 S 0.7 0.2 0:17.54 mysqld



thanks

Comments

  • PeterPeter Percona CEO Percona Moderator Role
    Unless excessive you should not be concerned. "Prometheus" Dashboard has a graph to show you how many samples are being discarded every second
Sign In or Register to comment.

MySQL, InnoDB, MariaDB and MongoDB are trademarks of their respective owners.
Copyright ©2005 - 2020 Percona LLC. All rights reserved.