Hey there,
We had a crush of traffic for a customer of ours during Black Friday and spun up a cluster to help deal with it, however, there is a slow and steady memory leak that requires us to reboot the cluster about once a day to clear the RAM as it’s not releasing on terminating the process. Is there something we’re not doing in the configuration properly? It applies to both the master + the slave. Everything else works perfectly, just the memory leak is a pain. The DB we’re running on the system is a 17.4GB database.
Any help would be appreciated!
Here’s our my.cnf on the master:
[mysqld]
datadir = /data/mysql
thread_cache_size = 50
tmp_table_size = 32M
max_heap_table_size = 32M
max_allowed_packet=24M
wsrep_provider=/usr/lib64/libgalera_smm.so
wsrep_cluster_address=gcomm://
wsrep_slave_threads=64
wsrep_sst_method=rsync
wsrep_cluster_name=percona_cluster
binlog_format=ROW
innodb_autoinc_lock_mode=2
innodb_locks_unsafe_for_binlog=1
innodb_buffer_pool_size = 20000M
character-set-server = utf8
skip-external-locking
log_warnings
skip_name_resolv
innodb_additional_mem_pool_size = 20M
#innodb_log_file_size = 400M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 2
innodb_lock_wait_timeout = 50
innodb_io_capacity = 1500
innodb_read_io_threads = 8
innodb_write_io_threads = 8
innodb_buffer_pool_restore_at_startup = 500
innodb_locks_unsafe_for_binlog=1
table_cache = 512
thread_cache_size = 1000
query_cache_type = 0
back_log = 128
thread_concurrency = 50
tmpdir = /tmp
max_connections = 5000
max_allowed_packet = 24M
max_join_size = 4294967295
net_buffer_length = 2K
thread_stack = 128K
tmp_table_size = 64M
max_heap_table_size = 64M
log-error = /data/log/mysql/error.log
##new stuff on reboot
innodb_use_sys_malloc =0
#innodb_flush_method = O_DIRECT
#innodb_log_files_in_group = 2
#innodb_file_per_table = 1
log_queries_not_using_indexes = 1
slow_query_log = 1
slow_query_log_file = /data/mysql/mysql-slow.log
table_definition_cache = 4096
table_open_cache = 4096
#new stuff for reboot to solve ram gobbling issue
log-bin=mysql-bin
log_slave_updates=1
max_binlog_size=10G
#end new stuff
[mysqldump]
quick
max_allowed_packet = 24M
[mysql]
no-auto-rehash
Remove the next comment character if you are not familiar with SQL
#safe-updates
Here’s the My.cnf on the slave:
[mysqld]
datadir = /data/mysql
thread_cache_size = 50
tmp_table_size = 32M
max_heap_table_size = 32M
max_allowed_packet=24M
wsrep_provider=/usr/lib64/libgalera_smm.so
wsrep_cluster_address=gcomm://(removed to protect the innocent)
wsrep_slave_threads=64
wsrep_sst_method=rsync
wsrep_cluster_name=percona_cluster
binlog_format=ROW
default_storage_engine=InnoDB
innodb_autoinc_lock_mode=2
innodb_locks_unsafe_for_binlog=1
innodb_buffer_pool_size = 20000M
character-set-server = utf8
skip-external-locking
log_warnings
skip_name_resolv
innodb_additional_mem_pool_size = 20M
#innodb_log_file_size = 400M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 2
innodb_lock_wait_timeout = 50
innodb_file_per_table
innodb_io_capacity = 1500
innodb_read_io_threads = 8
innodb_write_io_threads = 8
innodb_buffer_pool_restore_at_startup = 500
innodb_locks_unsafe_for_binlog=1
table_cache = 512
thread_cache_size = 1000
query_cache_type = 0
back_log = 128
thread_concurrency = 50
tmpdir = /tmp
max_connections = 5000
max_allowed_packet = 24M
max_join_size = 4294967295
net_buffer_length = 2K
thread_stack = 128K
tmp_table_size = 64M
max_heap_table_size = 64M
log-error = /data/log/mysql/error.log
innodb_use_sys_malloc =0
log_queries_not_using_indexes = 1
slow_query_log = 1
slow_query_log_file = /data/mysql/mysql-slow.log
table_definition_cache = 4096
table_open_cache = 4096
#new stuff for reboot to solve ram gobbling issue
log-bin=mysql-bin
#log_slave_updates=1
max_binlog_size=10G
#end new stuff
[mysqldump]
quick
max_allowed_packet = 24M
[mysql]
no-auto-rehash
Remove the next comment character if you are not familiar with SQL
#safe-updates