Here is PMM Memory chart showing that used memory keep growing.
03/13 point at timeline is moment of restart. I checked all charts / metrics but cannot explain where does memory go. Maybe you can give me some advice what I can check? Here is my config and some details:
htop
# Top Processes ##############################################
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
2817 mysql 20 0 33.8g 27.0g 11176 S 231.2 57.4 5500:48 mysqld
19711 bitrix 20 0 2619456 174448 146400 S 25.0 0.4 3:18.68 httpd
8520 root 20 0 58684 2396 1480 R 6.2 0.0 0:00.01 top
28359 root 20 0 1504220 235408 7220 S 6.2 0.5 36:50.78 vmagent
1 root 20 0 191664 4572 2604 S 0.0 0.0 11:11.12 systemd
2 root 20 0 0 0 0 S 0.0 0.0 0:00.75 kthreadd
4 root 0 -20 0 0 0 S 0.0 0.0 0:00.00 kworker/0:+
6 root 20 0 0 0 0 S 0.0 0.0 0:21.54 ksoftirqd/0
7 root rt 0 0 0 0 S 0.0 0.0 0:09.96 migration/0
memory
# Memory #####################################################
Total | 47.0G
Free | 865.3M
Used | physical = 34.3G, swap allocated = 2.0G, swap used = 3.5M, virtual = 34.3G
Shared | 756.4M
Buffers | 11.9G
Caches | 11.6G
Dirty | 3888 kB
UsedRSS | 60.8G
Swappiness | 30
DirtyPolicy | 30, 10
DirtyStatus | 0, 0
# Percona Toolkit MySQL Summary Report #######################
System time | 2025-03-14 14:37:17 UTC (local TZ: +03 +0300)
# Instances ##################################################
Port Data Directory Nice OOM Socket
===== ========================== ==== === ======
0 0
# MySQL Executable ###########################################
Path to executable | /usr/sbin/mysqld
Has symbols | No
# Slave Hosts ################################################
No slaves found
# Report On Port 3306 ########################################
User | pmm@127.0.0.1
Time | 2025-03-14 17:37:17 (+03)
Hostname | hostname
Version | 5.7.44-48 Percona Server (GPL), Release 48, Revision 497f936a373
Built On | Linux x86_64
Started | 2025-03-13 09:14 (up 1+08:23:13)
Databases | 7
Datadir | /var/lib/mysql/
Processes | 40 connected, 3 running
Replication | Is not a slave, has 0 slaves connected
Pidfile | /var/run/mysqld/mysqld.pid (exists)
# Processlist ################################################
Command COUNT(*) Working SUM(Time) MAX(Time)
------------------------------ -------- ------- --------- ---------
Query 3 3 1 1
Sleep 35 0 70000 17500
User COUNT(*) Working SUM(Time) MAX(Time)
------------------------------ -------- ------- --------- ---------
bitrix0 10 2 1 1
pmm 5 1 0 0
userdb1 25 0 0 0
Host COUNT(*) Working SUM(Time) MAX(Time)
------------------------------ -------- ------- --------- ---------
127.0.0.1 5 1 0 0
localhost 35 2 1 1
db COUNT(*) Working SUM(Time) MAX(Time)
------------------------------ -------- ------- --------- ---------
NULL 5 1 0 0
userdb1-b24 25 0 0 0
mydb 10 2 1 1
State COUNT(*) Working SUM(Time) MAX(Time)
------------------------------ -------- ------- --------- ---------
35 0 0 0
Sending data 2 2 1 1
starting 1 1 0 0
# Status Counters (Wait 10 Seconds) ##########################
Variable Per day Per second 11 secs
Aborted_clients 300
Aborted_connects 2
Bytes_received 30000000000 350000 300000
Bytes_sent 175000000000 2250000 1000000
Com_admin_commands 25000
Com_begin 12500
Com_change_db 8
Com_commit 12500
Com_delete 1250000 15 30
Com_delete_multi 80000 3
Com_empty_query 50000
Com_insert 1750000 20 20
Com_insert_select 350000 4 5
Com_replace 7000
Com_rollback 10
Com_select 50000000 600 450
Com_set_option 1000000 15 15
Com_show_binlogs 15
Com_show_create_db 25
Com_show_create_table 5
Com_show_databases 20
Com_show_engine_status 9000
Com_show_fields 4000
Com_show_keys 1000
Com_show_master_status 15
Com_show_plugins 1500
Com_show_processlist 15
Com_show_slave_hosts 15
Com_show_slave_status 9000
Com_show_status 17500
Com_show_table_status 4
Com_show_tables 45000
Com_show_variables 4500
Com_show_warnings 2
Com_stmt_execute 25
Com_stmt_close 25
Com_stmt_prepare 25
Com_truncate 2
Com_update 2000000 20 25
Com_update_multi 80000 3
Connections 225000 2 4
Created_tmp_disk_tables 3500000 40 15
Created_tmp_files 12500
Created_tmp_tables 6000000 70 50
Handler_commit 50000000 600 500
Handler_delete 1250000 15 8
Handler_external_lock 225000000 2500 2000
Handler_read_first 7000000 80 60
Handler_read_key 6000000000 70000 17500
Handler_read_last 60000
Handler_read_next 70000000000 800000 1250000
Handler_read_prev 225000000 2500 5
Handler_read_rnd 250000000 3000 700
Handler_read_rnd_next 2500000000 30000 1250
Handler_rollback 300
Handler_update 5000000 60 35
Handler_write 250000000 3000 300
Innodb_background_log_sync 90000 1
Innodb_buffer_pool_bytes_data 12500000000 150000 7000
Innodb_buffer_pool_bytes_dirty 450000000 5000 60000
Innodb_buffer_pool_pages_flushed 1250000 15
Innodb_buffer_pool_pages_made_not_young 1500000 20
Innodb_buffer_pool_pages_made_young 700000 7
Innodb_buffer_pool_pages_old 300000 3
Innodb_buffer_pool_read_ahead 60000
Innodb_buffer_pool_read_requests 100000000000 1250000 1500000
Innodb_buffer_pool_reads 700000 7
Innodb_buffer_pool_write_requests 250000000 3000 1250
Innodb_checkpoint_age 175000000 2000 45000
Innodb_checkpoint_max_age 1250000000 15000
Innodb_data_fsyncs 800000 8 1
Innodb_data_read 12500000000 150000 6000
Innodb_data_reads 800000 8
Innodb_data_writes 5000000 60 60
Innodb_data_written 30000000000 350000 100000
Innodb_dblwr_pages_written 700000 7
Innodb_dblwr_writes 300000 3
Innodb_ibuf_free_list 5000
Innodb_ibuf_segment_size 5000
Innodb_log_write_requests 6000000 70 125
Innodb_log_writes 3500000 40 60
Innodb_lsn_current 1250000000000 15000000 45000
Innodb_lsn_flushed 1250000000000 15000000 40000
Innodb_lsn_last_checkpoint 1250000000000 15000000
Innodb_master_thread_active_loops 70000 1
Innodb_master_thread_idle_loops 12500
Innodb_max_trx_id 2500000000 30000 125
Innodb_mem_adaptive_hash 1000000000 12500
Innodb_mem_dictionary 90000000 1000
Innodb_oldest_view_low_limit_trx_id 2500000000 30000 125
Innodb_os_log_fsyncs 80000 1
Innodb_os_log_written 6000000000 70000 100000
Innodb_pages_created 22500
Innodb_pages_read 800000 8
Innodb_pages0_read 1500
Innodb_pages_written 1250000 15
Innodb_purge_trx_id 2500000000 30000 125
Innodb_row_lock_time 1750
Innodb_row_lock_waits 50
Innodb_rows_deleted 1250000 15 8
Innodb_rows_inserted 250000000 3000 450
Innodb_rows_read 80000000000 900000 1250000
Innodb_rows_updated 1750000 20 30
Innodb_num_open_files 1500
Innodb_available_undo_logs 90
Innodb_secondary_index_triggered_cluster_reads 15000000000 175000 70000
Innodb_secondary_index_triggered_cluster_reads_avoided 125000 1
Innodb_buffered_aio_submitted 60000
Key_read_requests 30
Key_reads 3
Open_table_definitions 1250
Opened_files 17500
Opened_table_definitions 4500
Opened_tables 3500000 40 45
Performance_schema_digest_lost 600000 7 1
Queries 60000000 600 600
Questions 60000000 600 600
Select_full_join 200000 2 1
Select_full_range_join 450000 5
Select_range 5000000 60 35
Select_range_check 4500
Select_scan 3500000 40 50
Sort_merge_passes 70000
Sort_range 3000000 40 40
Sort_rows 225000000 2500 700
Sort_scan 6000000 70 45
Ssl_accepts 350 1
Ssl_finished_accepts 350 1
Table_locks_immediate 125000 1 1
Table_open_cache_hits 100000000 1250 900
Table_open_cache_misses 3500000 40 45
Table_open_cache_overflows 3500000 40 45
Threads_created 70
Uptime 90000 1 1
# Table cache ################################################
Size | 2395
Usage | 100%
# Key Percona Server features ################################
Table & Index Stats | Disabled
Multiple I/O Threads | Enabled
Corruption Resilient | Enabled
Durable Replication | Not Supported
Import InnoDB Tables | Not Supported
Fast Server Restarts | Not Supported
Enhanced Logging | Disabled
Replica Perf Logging | Disabled
Response Time Hist. | Enabled
Smooth Flushing | Not Supported
HandlerSocket NoSQL | Not Supported
Fast Hash UDFs | Unknown
# Percona XtraDB Cluster #####################################
# Plugins ####################################################
InnoDB compression | ACTIVE
# Query cache ################################################
query_cache_type | OFF
Size | 0.0
Usage | 0%
HitToInsertRatio | 0%
# Schema #####################################################
Specify --databases or --all-databases to dump and summarize schemas
# Noteworthy Technologies ####################################
SSL | Yes
Explicit LOCK TABLES | No
Delayed Insert | No
XA Transactions | No
NDB Cluster | No
Prepared Statements | Yes
Prepared statement count | 0
# InnoDB #####################################################
Version | 5.7.44-48
Buffer Pool Size | 24.0G
Buffer Pool Fill | 70%
Buffer Pool Dirty | 2%
File Per Table | ON
Page Size | 16k
Log File Size | 2 * 1.0G = 2.0G
Log Buffer Size | 128M
Flush Method | O_DIRECT
Flush Log At Commit | 2
XA Support | ON
Checksums | ON
Doublewrite | ON
R/W I/O Threads | 8 4
I/O Capacity | 200
Thread Concurrency | 0
Concurrency Tickets | 5000
Commit Concurrency | 0
Txn Isolation Level | READ-COMMITTED
Adaptive Flushing | ON
Adaptive Checkpoint |
Checkpoint Age | 212M
InnoDB Queue | 0 queries inside InnoDB, 0 queries in queue
Oldest Transaction | 0 Seconds
History List Len | 58
Read Views | 2
Undo Log Entries | 0 transactions, 0 total undo, 0 max undo
Pending I/O Reads | 0 buf pool reads, 0 normal AIO, 0 ibuf AIO, 0 preads
Pending I/O Writes | 0 buf pool (0 LRU, 0 flush list, 0 page); 0 AIO, 0 sync, 0 log IO (0 log, 0 chkp); 0 pwrites
Pending I/O Flushes | 0 buf pool, 0 log
Transaction States | 34xnot started
# MyISAM #####################################################
Key Cache | 16.0M
Pct Used | 20%
Unflushed | 0%
# Security ###################################################
Users | 8 users, 0 anon, 0 w/o pw, 0 old pw
Old Passwords | 0
# Encryption #################################################
No keyring plugins found
# Binary Logging #############################################
# Noteworthy Variables #######################################
Auto-Inc Incr/Offset | 1/1
default_storage_engine | InnoDB
flush_time | 0
init_connect | SET NAMES utf8 COLLATE utf8_unicode_ci
init_file |
sql_mode |
join_buffer_size | 2M
sort_buffer_size | 2M
read_buffer_size | 128k
read_rnd_buffer_size | 256k
bulk_insert_buffer | 0.00
max_heap_table_size | 1G
tmp_table_size | 1G
max_allowed_packet | 16M
thread_stack | 128k
log |
log_error | /var/log/mysql/error.log
log_warnings | 2
log_slow_queries |
log_queries_not_using_indexes | OFF
log_slave_updates | OFF
# Configuration File #########################################
Config File | /etc/my.cnf
[client]
port = 3306
socket = /var/lib/mysqld/mysqld.sock
default-character-set = utf8
[mysqld_safe]
nice = 0
socket = /var/lib/mysqld/mysqld.sock
[mysqld]
user = mysql
port = 3306
basedir = /usr
datadir = /var/lib/mysql
socket = /var/lib/mysqld/mysqld.sock
skip-external-locking
default-storage-engine = innodb
pid-file = /var/run/mysqld/mysqld.pid
transaction-isolation = READ-COMMITTED
max_allowed_packet = 16M
myisam-recover-options = BACKUP
explicit_defaults_for_timestamp = 1
expire_logs_days = 10
max_binlog_size = 100M
sql_mode = ""
query_cache_size = 32M
table_open_cache = 4096
thread_cache_size = 32
key_buffer_size = 16M
thread_stack = 128K
join_buffer_size = 2M
sort_buffer_size = 2M
tmpdir = /tmp
max_heap_table_size = 32M
tmp_table_size = 32M
innodb_file_per_table
innodb_buffer_pool_size = 32M
innodb_flush_log_at_trx_commit = 2
innodb_log_file_size = 64M
innodb_flush_method = O_DIRECT
innodb_strict_mode = OFF
character-set-server = utf8
collation-server = utf8_unicode_ci
init-connect = "SET NAMES utf8 COLLATE utf8_unicode_ci"
skip-name-resolve
[mysqldump]
quick
quote-names
max_allowed_packet = 16M
default-character-set = utf8
[mysql]
[isamchk]
key_buffer = 16M
# /etc/mysql/conf.d/logging.cnf
[mysqld_safe]
log-error = /var/log/mysql/error.log
[mysqld]
log-error = /var/log/mysql/error.log
# /etc/mysql/conf.d/bvat.cnf
[mysqld]
query_cache_type = 1
query_cache_size = 128M
query_cache_limit = 16M
innodb_buffer_pool_size = 18432M
max_connections = 205
table_open_cache = 18432
thread_cache_size = 512
max_heap_table_size = 128M
tmp_table_size = 128M
key_buffer_size = 256M
join_buffer_size = 32M
sort_buffer_size = 32M
bulk_insert_buffer_size = 2M
myisam_sort_buffer_size = 32M
# /etc/mysql/conf.d/z_bx_custom.cnf
[mysqld]
query_cache_limit = 64M
max_connections = 200
max_heap_table_size = 1024M
tmp_table_size = 1Gb
query_response_time_stats = on
innodb_read_io_threads = 8
query_cache_type = off
query_cache_size = 0
join_buffer_size = 2M
sort_buffer_size = 2M
key_buffer_size = 16M
table_open_cache = 8192
innodb_buffer_pool_instances = 8
innodb_buffer_pool_size = 24g
innodb_log_file_size = 1g
innodb_log_buffer_size = 128M
max_digest_length = 20480
performance_schema_max_digest_length = 20480
performance_schema_max_sql_text_length = 20480
open_files_limit = 10000
# Memory management library ##################################
jemalloc is not enabled in mysql config for process with id 2817
