I’ve just installed PMM 1.05 and I ran into an issue with the query-analyzer, it apparently tries bind to 127.0.0.1:42001 whereas the other servers bind to [IP_OF_SERVER]:42000/42002
please advise
root@mm01entpp02:/var/log# pmm-admin config --server mm02pmm01.dc.mediamiser.internal --server-user mm --server-password X
OK, PMM server is alive.
PMM Server | mm02pmm01.dc.mediamiser.internal (password-protected)
Client Name | mm01entpp02
Client Address | 10.1.12.11
root@mm01entpp02:/var/log# pmm-admin --version
1.0.5
root@mm01entpp02:/var/log# pmm-admin info
pmm-admin 1.0.5
PMM Server | mm02pmm01.dc.mediamiser.internal (password-protected)
Client Name | mm01entpp02
Client Address | 10.1.12.11
Service manager | linux-upstart
root@mm01entpp02:~# pmm-admin check-network --no-emoji
PMM Network Status
Server | mm02pmm01.dc.mediamiser.internal
Client | 10.1.12.11
* Client --> Server
--------------- -------
SERVER SERVICE STATUS
--------------- -------
Consul API OK
QAN API OK
Prometheus API OK
Connection duration | 5.008357767s
Request duration | 18.156711ms
Full round trip | 5.026514478s
* Client <-- Server
-------------- ------------ ---------------------- -------
SERVICE TYPE NAME REMOTE ENDPOINT STATUS
-------------- ------------ ---------------------- -------
linux:metrics mm01entpp02 10.1.12.11:42000 OK
mysql:metrics mm01entpp02 10.1.12.11:42002 OK
root@mm01entpp02:~#
root@mm01entpp02:~# pmm-admin add linux:metrics
OK, now monitoring this system.
root@mm01entpp02:~# pmm-admin add mysql:metrics --user pmm --password Ahpaing6so7a --disable-tablestats
OK, now monitoring MySQL metrics using DSN pmm:***@unix(/var/run/mysqld/mysqld.sock)
root@mm01entpp02:~# pmm-admin add mysql:queries --user pmm --password X
Error adding MySQL queries: timeout 10s waiting on agent to connect to API.
root@mm01entpp02:~# pmm-admin add mysql:queries --user pmm --password X
We have found system services disconnected from PMM server.
Usually, this happens when data container is wiped before all monitoring services are removed or client is uninstalled.
Orphaned local services: pmm-mysql-queries-42001
To continue, run 'pmm-admin repair' to remove orphaned services.
root@mm01entpp02:~# ls -l /var/log/pmm-
pmm-linux-metrics-42000.log pmm-mysql-metrics-42002.log pmm-mysql-queries-42001.log
root@mm01entpp02:~# ls -l /var/log/pmm-mysql-queries-42001.log
-rw-r--r-- 1 root root 381 Oct 14 16:13 /var/log/pmm-mysql-queries-42001.log
root@mm01entpp02:~# cat /var/log/pmm-mysql-queries-42001.log
# Version: percona-qan-agent 1.0.5
# Basedir: /usr/local/percona/qan-agent
# Listen: 127.0.0.1:42001
# PID: 30269
# API: mm02pmm01.dc.mediamiser.internal/qan-api
# UUID: 738860ad74ff409b7d349beaa93e2c6f
2016/10/14 16:13:51.910550 main.go:165: Starting agent...
2016/10/14 16:13:51.911297 main.go:343: Agent is ready
2016/10/14 16:13:56.948270 main.go:207: API is ready
root@mm01entpp02:~# ps -ef | grep 42001
root 30269 1 0 16:13 ? 00:00:00 /usr/local/percona/qan-agent/bin/percona-qan-agent -listen=127.0.0.1:42001
root 30330 29164 0 16:14 pts/7 00:00:00 grep --color=auto 42001
root@mm01entpp02:~# nslookup mm01entpp02
Server: 10.2.10.200
Address: 10.2.10.200#53
Name: mm01entpp02.mediamiser.internal
Address: 10.1.12.11
root@mm01entpp02:~#
root@mm01entpp02:~#
root@mm01entpp02:~# pmm-admin add mysql:queries --user pmm --password X
Error adding MySQL queries: timeout 10s waiting on agent to connect to API.
root@mm01entpp02:~# pmm-admin add mysql:queries --user pmm --password X
We have found system services disconnected from PMM server.
Usually, this happens when data container is wiped before all monitoring services are removed or client is uninstalled.
Orphaned local services: pmm-mysql-queries-42001
To continue, run 'pmm-admin repair' to remove orphaned services.
root@mm01entpp02:~# ls -l /var/log/pmm-
pmm-linux-metrics-42000.log pmm-mysql-metrics-42002.log pmm-mysql-queries-42001.log
root@mm01entpp02:~# ls -l /var/log/pmm-mysql-queries-42001.log
-rw-r--r-- 1 root root 381 Oct 14 16:13 /var/log/pmm-mysql-queries-42001.log
root@mm01entpp02:~# cat /var/log/pmm-mysql-queries-42001.log
# Version: percona-qan-agent 1.0.5
# Basedir: /usr/local/percona/qan-agent
# Listen: 127.0.0.1:42001
# PID: 30269
# API: mm02pmm01.dc.mediamiser.internal/qan-api
# UUID: 738860ad74ff409b7d349beaa93e2c6f
2016/10/14 16:13:51.910550 main.go:165: Starting agent...
2016/10/14 16:13:51.911297 main.go:343: Agent is ready
2016/10/14 16:13:56.948270 main.go:207: API is ready
root@mm01entpp02:~# ps -ef | grep 42001
root 30269 1 0 16:13 ? 00:00:00 /usr/local/percona/qan-agent/bin/percona-qan-agent -listen=127.0.0.1:42001
root 30330 29164 0 16:14 pts/7 00:00:00 grep --color=auto 42001
root@mm01entpp02:~# nslookup mm01entpp02
Server: 10.2.10.200
Address: 10.2.10.200#53
Name: mm01entpp02.mediamiser.internal
Address: 10.1.12.11
root@mm01entpp02:~#
root@mm01entpp02:~#
singer.wang@mm01entpp02:~$
singer.wang@mm01entpp02:~$ ps -ef | grep 420
root 29893 1 0 16:09 ? 00:00:06 /usr/local/percona/pmm-client/node_exporter -web.listen-address=10.1.12.11:42000 -collectors.enabled=diskstats,filesystem,loadavg,meminfo,netdev,netstat,stat,time,uname,vmstat
root 29977 1 1 16:09 ? 00:00:07 /usr/local/percona/pmm-client/mysqld_exporter -collect.auto_increment.columns=false -collect.binlog_size=true -collect.global_status=true -collect.global_variables=true -collect.info_schema.innodb_metrics=true -collect.info_schema.processlist=true -collect.info_schema.query_response_time=true -collect.info_schema.tables=false -collect.info_schema.tablestats=false -collect.info_schema.userstats=true -collect.perf_schema.eventswaits=true -collect.perf_schema.file_events=true -collect.perf_schema.indexiowaits=false -collect.perf_schema.tableiowaits=false -collect.perf_schema.tablelocks=false -collect.slave_status=true -web.listen-address=10.1.12.11:42002
root 30269 1 0 16:13 ? 00:00:00 /usr/local/percona/qan-agent/bin/percona-qan-agent -listen=127.0.0.1:42001
2005 31027 30922 0 16:20 pts/1 00:00:00 grep --color=auto 420
singer.wang@mm01entpp02:~$ hostname
mm01entpp02
singer.wang@mm01entpp02:~$ ping mm01entpp02
PING mm01entpp02.mediamiser.internal (10.1.12.11) 56(84) bytes of data.
64 bytes from mm01entpp02.mediamiser.internal (10.1.12.11): icmp_req=1 ttl=64 time=0.026 ms
^C
--- mm01entpp02.mediamiser.internal ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.026/0.026/0.026/0.000 ms
singer.wang@mm01entpp02:~$ nslookup mm01entpp02
Server: 10.2.10.200
Address: 10.2.10.200#53
Name: mm01entpp02.mediamiser.internal
Address: 10.1.12.11
singer.wang@mm01entpp02:~$
singer.wang@mm01entpp02:~$ hostname
mm01entpp02
singer.wang@mm01entpp02:~$ ping mm01entpp02
PING mm01entpp02.mediamiser.internal (10.1.12.11) 56(84) bytes of data.
64 bytes from mm01entpp02.mediamiser.internal (10.1.12.11): icmp_req=1 ttl=64 time=0.026 ms
^C
--- mm01entpp02.mediamiser.internal ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.026/0.026/0.026/0.000 ms
singer.wang@mm01entpp02:~$ nslookup mm01entpp02
Server: 10.2.10.200
Address: 10.2.10.200#53
Name: mm01entpp02.mediamiser.internal
Address: 10.1.12.11
singer.wang@mm01entpp02:~$ ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
link/ether 00:25:90:01:53:c2 brd ff:ff:ff:ff:ff:ff
inet 10.1.12.11/24 brd 10.1.12.255 scope global eth0
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN qlen 1000
link/ether 00:25:90:01:53:c3 brd ff:ff:ff:ff:ff:ff
singer.wang@mm01entpp02:~$ ifconfig
eth0 Link encap:Ethernet HWaddr 00:25:90:01:53:c2
inet addr:10.1.12.11 Bcast:10.1.12.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:118033620 errors:0 dropped:204013 overruns:0 frame:0
TX packets:92263751 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:151663870578 (151.6 GB) TX bytes:117332569525 (117.3 GB)
Memory:c0000000-c0020000
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:65536 Metric:1
RX packets:7396683 errors:0 dropped:0 overruns:0 frame:0
TX packets:7396683 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:10960006330 (10.9 GB) TX bytes:10960006330 (10.9 GB)
singer.wang@mm01entpp02:~$