Not the answer you need?
Register and ask your own question!

Database hangs

chrisjack83chrisjack83 EntrantCurrent User Role Beginner
With no particularly unusual workload, the database hangs up at random times. I have installed the debuginfo packages, here are the pt-pmp output, can someone please tell me what is going on here?


pt-pmp output:

73 __lll_lock_wait(libpthread.so.0),_L_lock_975(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),inline_mysql_mutex_lock(mysql_thread.h:688),fill_status(mysql_thread.h:688),do_fill_table(sql_show.cc:7997),get_schema_tables_result(sql_show.cc:7997),JOIN::prepare_result(sql_select.cc:823),JOIN::exec(sql_executor.cc:116),mysql_execute_select(sql_select.cc:1100),mysql_select(sql_select.cc:1100),handle_select(sql_select.cc:110),execute_sqlcom_select(sql_parse.cc:5687),mysql_execute_command(sql_parse.cc:2966),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

     69 __lll_lock_wait(libpthread.so.0),_L_lock_975(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),inline_mysql_mutex_lock(mysql_thread.h:688),THD::release_resources(mysql_thread.h:688),one_thread_per_connection_end(mysqld.cc:3033),do_handle_one_connection(sql_connect.cc:1555),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

      6 __io_getevents_0_4(libaio.so.1),os_aio_linux_collect(os0file.cc:5173),os_aio_linux_handle(os0file.cc:5173),fil_aio_wait(fil0fil.cc:5837),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)

      3 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_buffer_flush_to_disk(sync0sync.ic:314),innobase_flush_logs(ha_innodb.cc:3867),flush_handlerton(handler.cc:2452),plugin_foreach_with_mask(sql_plugin.cc:2158),ha_flush_logs(handler.cc:2462),reload_acl_and_cache(sql_reload.cc:139),mysql_execute_command(sql_parse.cc:4724),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

      2 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),ibuf_mtr_commit(ibuf0ibuf.ic:58),ibuf_merge_or_delete_for_page(ibuf0ibuf.ic:58),buf_page_io_complete(buf0buf.cc:4495),fil_aio_wait(fil0fil.cc:5873),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)

      2 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_reserve_and_write_fast(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),row_ins_sec_index_entry_low(row0ins.cc:2852),row_ins_sec_index_entry(row0ins.cc:3003),row_ins_index_entry(row0ins.cc:3042),row_ins_index_entry_step(row0ins.cc:3042),row_ins(row0ins.cc:3042),row_ins_step(row0ins.cc:3042),row_insert_for_mysql(row0mysql.cc:1364),ha_innobase::write_row(ha_innodb.cc:7365),handler::ha_write_row(handler.cc:7708),write_record(sql_insert.cc:1947),read_sep_field(sql_load.cc:1139),mysql_load(sql_load.cc:1139),mysql_execute_command(sql_parse.cc:4166),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

      1 sigwait(libpthread.so.0),signal_hand(mysqld.cc:3473),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

      1 sigwaitinfo(libc.so.6),timer_notify_thread(posix_timers.c:78),start_thread(libpthread.so.0),clone(libc.so.6)

      1 select(libc.so.6),os_thread_sleep(os0thread.cc:285),page_cleaner_sleep_if_needed(buf0flu.cc:2581),buf_flush_lru_manager_thread(buf0flu.cc:2581),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),rw_lock_x_lock_func(sync0rw.cc:886),pfs_rw_lock_x_lock_func(sync0rw.ic:890),buf_page_get_gen(sync0rw.ic:890),btr_block_get_func(btr0btr.ic:60),btr_cur_latch_leaves(btr0btr.ic:60),btr_cur_search_to_nth_level(btr0cur.cc:795),btr_pcur_open_low(btr0pcur.ic:440),btr_pcur_open_on_user_rec_func(btr0pcur.ic:440),ibuf_merge_or_delete_for_page(ibuf0ibuf.cc:4774),buf_page_io_complete(buf0buf.cc:4495),fil_aio_wait(fil0fil.cc:5873),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),trx_undo_assign_undo(trx0undo.cc:1813),trx_undo_report_row_operation(trx0rec.cc:1279),btr_cur_del_mark_set_clust_rec(btr0cur.cc:3164),row_upd_del_mark_clust_rec(row0upd.cc:2289),row_upd_clust_step(row0upd.cc:2289),row_upd(row0upd.cc:2553),row_upd_step(row0upd.cc:2553),que_thr_step(que0que.cc:1071),que_run_threads_low(que0que.cc:1071),que_run_threads(que0que.cc:1071),que_eval_sql(que0que.cc:1275),dict_stats_exec_sql(dict0stats.cc:317),dict_stats_save(dict0stats.cc:2424),dict_stats_update(dict0stats.cc:3133),dict_stats_process_entry_from_recalc_pool(dict0stats_bg.cc:313),dict_stats_thread(dict0stats_bg.cc:313),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_write_up_to(sync0sync.ic:314),log_write_up_to(log0log.cc:1733),buf_flush_write_block_low(buf0flu.cc:926),buf_flush_page(buf0flu.cc:926),buf_flush_try_neighbors(buf0flu.cc:1307),buf_flush_page_and_try_neighbors(buf0flu.cc:1398),buf_do_flush_list_batch(buf0flu.cc:1398),buf_flush_batch(buf0flu.cc:1804),buf_flush_list(buf0flu.cc:1804),page_cleaner_do_flush_batch(buf0flu.cc:2358),page_cleaner_flush_pages_if_needed(buf0flu.cc:2358),buf_flush_page_cleaner_thread(buf0flu.cc:2358),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_reserve_and_write_fast(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),trx_undo_report_row_operation(trx0rec.cc:1353),btr_cur_ins_lock_and_undo(btr0cur.cc:1279),btr_cur_optimistic_insert(btr0cur.cc:1279),row_ins_clust_index_entry_low(row0ins.cc:2514),row_ins_clust_index_entry(row0ins.cc:2942),row_ins_index_entry(row0ins.cc:3040),row_ins_index_entry_step(row0ins.cc:3040),row_ins(row0ins.cc:3040),row_ins_step(row0ins.cc:3040),row_insert_for_mysql(row0mysql.cc:1364),ha_innobase::write_row(ha_innodb.cc:7365),handler::ha_write_row(handler.cc:7708),write_record(sql_insert.cc:1947),mysql_insert(sql_insert.cc:1085),mysql_execute_command(sql_parse.cc:3906),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_reserve_and_write_fast(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),ibuf_mtr_commit(ibuf0ibuf.ic:58),ibuf_merge_or_delete_for_page(ibuf0ibuf.ic:58),buf_page_io_complete(buf0buf.cc:4495),fil_aio_wait(fil0fil.cc:5873),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_print(sync0sync.ic:314),srv_printf_innodb_monitor(srv0srv.cc:1388),innodb_show_status(ha_innodb.cc:13379),innobase_show_status(ha_innodb.cc:13379),ha_show_status(handler.cc:7336),mysql_execute_command(sql_parse.cc:3282),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)

      1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_get_lsn(sync0sync.ic:314),srv_error_monitor_thread(sync0sync.ic:314),start_thread(libpthread.so.0),clone(libc.so.6)


Comments

  • chrisjack83chrisjack83 Entrant Current User Role Beginner
    The remainder of the pt-pmp output (reached max characters on the initial post) as well as the gdb backtrace output
    1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_buffer_sync_in_background(sync0sync.ic:314),srv_sync_log_buffer_in_background(srv0srv.cc:2580),srv_master_do_active_tasks(srv0srv.cc:2580),srv_master_thread(srv0srv.cc:2580),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:269),pfs_mutex_enter_func(sync0sync.ic:269),srv_export_innodb_status(sync0sync.ic:269),innodb_export_status(ha_innodb.cc:13330),show_innodb_vars(ha_innodb.cc:13330),show_status_array(sql_show.cc:2642),fill_status(sql_show.cc:7330),do_fill_table(sql_show.cc:7997),get_schema_tables_result(sql_show.cc:7997),JOIN::prepare_result(sql_select.cc:823),JOIN::exec(sql_executor.cc:116),mysql_execute_select(sql_select.cc:1100),mysql_select(sql_select.cc:1100),handle_select(sql_select.cc:110),execute_sqlcom_select(sql_parse.cc:5687),mysql_execute_command(sql_parse.cc:2966),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:269),pfs_mutex_enter_func(sync0sync.ic:269),buf_pool_get_oldest_modification(sync0sync.ic:269),log_close(log0log.cc:585),mtr_log_reserve_and_write(mtr0mtr.cc:300),mtr_commit(mtr0mtr.cc:300),row_ins_sec_index_entry_low(row0ins.cc:2852),row_ins_sec_index_entry(row0ins.cc:3003),row_ins_index_entry(row0ins.cc:3042),row_ins_index_entry_step(row0ins.cc:3042),row_ins(row0ins.cc:3042),row_ins_step(row0ins.cc:3042),row_insert_for_mysql(row0mysql.cc:1364),ha_innobase::write_row(ha_innodb.cc:7365),handler::ha_write_row(handler.cc:7708),write_record(sql_insert.cc:1947),read_sep_field(sql_load.cc:1139),mysql_load(sql_load.cc:1139),mysql_execute_command(sql_parse.cc:4166),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),srv_purge_coordinator_suspend(srv0srv.cc:3257),srv_purge_coordinator_thread(srv0srv.cc:3257),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),buf_dump_thread(buf0dump.cc:594),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_timedwait,os_cond_wait_timed(os0sync.cc:177),os_event_wait_time_low(os0sync.cc:177),srv_monitor_thread(srv0srv.cc:1914),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_timedwait,os_cond_wait_timed(os0sync.cc:177),os_event_wait_time_low(os0sync.cc:177),lock_wait_timeout_thread(lock0wait.cc:503),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 pthread_cond_timedwait,os_cond_wait_timed(os0sync.cc:177),os_event_wait_time_low(os0sync.cc:177),ib_wqueue_timedwait(ut0wqueue.cc:154),fts_optimize_thread(fts0opt.cc:3000),start_thread(libpthread.so.0),clone(libc.so.6)
    
          1 __lll_lock_wait(libpthread.so.0),_L_lock_975(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),inline_mysql_mutex_lock(mysql_thread.h:688),THD::release_resources(mysql_thread.h:688),THD::~THD(sql_class.cc:1806),THD::~THD(sql_class.cc:1860),create_new_thread(mysqld.cc:6531),handle_connections_sockets(mysqld.cc:6531),mysqld_main(mysqld.cc:6057),__libc_start_main(libc.so.6),_start
    


    gdb bt

    #0  0x00007f2c7b7c51bd in __lll_lock_wait () from /lib64/libpthread.so.0
    
    #1  0x00007f2c7b7c0d38 in _L_lock_975 () from /lib64/libpthread.so.0
    
    #2  0x00007f2c7b7c0ce1 in pthread_mutex_lock () from /lib64/libpthread.so.0
    
    #3  0x00000000006ae270 in inline_mysql_mutex_lock (src_file=0xbb7ab8 "/builddir/build/BUILD/percona-server-5.6.27-75.0/sql/sql_class.cc", src_line=1765, that=<optimized out>)
    
        at /builddir/build/BUILD/percona-server-5.6.27-75.0/include/mysql/psi/mysql_thread.h:688
    
    #4  THD::release_resources (this=this&#64;entry=0x17428b40) at /builddir/build/BUILD/percona-server-5.6.27-75.0/sql/sql_class.cc:1765
    
    #5  0x00000000006ae3fe in THD::~THD (this=0x17428b40, __in_chrg=<optimized out>) at /builddir/build/BUILD/percona-server-5.6.27-75.0/sql/sql_class.cc:1806
    
    #6  0x00000000006aebb1 in THD::~THD (this=0x17428b40, __in_chrg=<optimized out>) at /builddir/build/BUILD/percona-server-5.6.27-75.0/sql/sql_class.cc:1860
    
    #7  0x000000000058b3eb in create_new_thread (thd=0x17428b40) at /builddir/build/BUILD/percona-server-5.6.27-75.0/sql/mysqld.cc:6531
    
    #8  handle_connections_sockets () at /builddir/build/BUILD/percona-server-5.6.27-75.0/sql/mysqld.cc:6847
    
    #9  0x000000000059491f in mysqld_main (argc=86, argv=0x2a4ce80) at /builddir/build/BUILD/percona-server-5.6.27-75.0/sql/mysqld.cc:6057
    
    #10 0x00007f2c79b4db35 in __libc_start_main () from /lib64/libc.so.6
    
    #11 0x00000000005858dd in _start ()
    
Sign In or Register to comment.

MySQL, InnoDB, MariaDB and MongoDB are trademarks of their respective owners.
Copyright ©2005 - 2020 Percona LLC. All rights reserved.