With no particularly unusual workload, the database hangs up at random times. I have installed the debuginfo packages, here are the pt-pmp output, can someone please tell me what is going on here?
pt-pmp output:
73 __lll_lock_wait(libpthread.so.0),_L_lock_975(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),inline_mysql_mutex_lock(mysql_thread.h:688),fill_status(mysql_thread.h:688),do_fill_table(sql_show.cc:7997),get_schema_tables_result(sql_show.cc:7997),JOIN::prepare_result(sql_select.cc:823),JOIN::exec(sql_executor.cc:116),mysql_execute_select(sql_select.cc:1100),mysql_select(sql_select.cc:1100),handle_select(sql_select.cc:110),execute_sqlcom_select(sql_parse.cc:5687),mysql_execute_command(sql_parse.cc:2966),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
69 __lll_lock_wait(libpthread.so.0),_L_lock_975(libpthread.so.0),pthread_mutex_lock(libpthread.so.0),inline_mysql_mutex_lock(mysql_thread.h:688),THD::release_resources(mysql_thread.h:688),one_thread_per_connection_end(mysqld.cc:3033),do_handle_one_connection(sql_connect.cc:1555),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
6 __io_getevents_0_4(libaio.so.1),os_aio_linux_collect(os0file.cc:5173),os_aio_linux_handle(os0file.cc:5173),fil_aio_wait(fil0fil.cc:5837),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)
3 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_buffer_flush_to_disk(sync0sync.ic:314),innobase_flush_logs(ha_innodb.cc:3867),flush_handlerton(handler.cc:2452),plugin_foreach_with_mask(sql_plugin.cc:2158),ha_flush_logs(handler.cc:2462),reload_acl_and_cache(sql_reload.cc:139),mysql_execute_command(sql_parse.cc:4724),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
2 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),ibuf_mtr_commit(ibuf0ibuf.ic:58),ibuf_merge_or_delete_for_page(ibuf0ibuf.ic:58),buf_page_io_complete(buf0buf.cc:4495),fil_aio_wait(fil0fil.cc:5873),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)
2 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_reserve_and_write_fast(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),row_ins_sec_index_entry_low(row0ins.cc:2852),row_ins_sec_index_entry(row0ins.cc:3003),row_ins_index_entry(row0ins.cc:3042),row_ins_index_entry_step(row0ins.cc:3042),row_ins(row0ins.cc:3042),row_ins_step(row0ins.cc:3042),row_insert_for_mysql(row0mysql.cc:1364),ha_innobase::write_row(ha_innodb.cc:7365),handler::ha_write_row(handler.cc:7708),write_record(sql_insert.cc:1947),read_sep_field(sql_load.cc:1139),mysql_load(sql_load.cc:1139),mysql_execute_command(sql_parse.cc:4166),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
1 sigwait(libpthread.so.0),signal_hand(mysqld.cc:3473),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
1 sigwaitinfo(libc.so.6),timer_notify_thread(posix_timers.c:78),start_thread(libpthread.so.0),clone(libc.so.6)
1 select(libc.so.6),os_thread_sleep(os0thread.cc:285),page_cleaner_sleep_if_needed(buf0flu.cc:2581),buf_flush_lru_manager_thread(buf0flu.cc:2581),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),rw_lock_x_lock_func(sync0rw.cc:886),pfs_rw_lock_x_lock_func(sync0rw.ic:890),buf_page_get_gen(sync0rw.ic:890),btr_block_get_func(btr0btr.ic:60),btr_cur_latch_leaves(btr0btr.ic:60),btr_cur_search_to_nth_level(btr0cur.cc:795),btr_pcur_open_low(btr0pcur.ic:440),btr_pcur_open_on_user_rec_func(btr0pcur.ic:440),ibuf_merge_or_delete_for_page(ibuf0ibuf.cc:4774),buf_page_io_complete(buf0buf.cc:4495),fil_aio_wait(fil0fil.cc:5873),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),trx_undo_assign_undo(trx0undo.cc:1813),trx_undo_report_row_operation(trx0rec.cc:1279),btr_cur_del_mark_set_clust_rec(btr0cur.cc:3164),row_upd_del_mark_clust_rec(row0upd.cc:2289),row_upd_clust_step(row0upd.cc:2289),row_upd(row0upd.cc:2553),row_upd_step(row0upd.cc:2553),que_thr_step(que0que.cc:1071),que_run_threads_low(que0que.cc:1071),que_run_threads(que0que.cc:1071),que_eval_sql(que0que.cc:1275),dict_stats_exec_sql(dict0stats.cc:317),dict_stats_save(dict0stats.cc:2424),dict_stats_update(dict0stats.cc:3133),dict_stats_process_entry_from_recalc_pool(dict0stats_bg.cc:313),dict_stats_thread(dict0stats_bg.cc:313),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_write_up_to(sync0sync.ic:314),log_write_up_to(log0log.cc:1733),buf_flush_write_block_low(buf0flu.cc:926),buf_flush_page(buf0flu.cc:926),buf_flush_try_neighbors(buf0flu.cc:1307),buf_flush_page_and_try_neighbors(buf0flu.cc:1398),buf_do_flush_list_batch(buf0flu.cc:1398),buf_flush_batch(buf0flu.cc:1804),buf_flush_list(buf0flu.cc:1804),page_cleaner_do_flush_batch(buf0flu.cc:2358),page_cleaner_flush_pages_if_needed(buf0flu.cc:2358),buf_flush_page_cleaner_thread(buf0flu.cc:2358),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_reserve_and_write_fast(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),trx_undo_report_row_operation(trx0rec.cc:1353),btr_cur_ins_lock_and_undo(btr0cur.cc:1279),btr_cur_optimistic_insert(btr0cur.cc:1279),row_ins_clust_index_entry_low(row0ins.cc:2514),row_ins_clust_index_entry(row0ins.cc:2942),row_ins_index_entry(row0ins.cc:3040),row_ins_index_entry_step(row0ins.cc:3040),row_ins(row0ins.cc:3040),row_ins_step(row0ins.cc:3040),row_insert_for_mysql(row0mysql.cc:1364),ha_innobase::write_row(ha_innodb.cc:7365),handler::ha_write_row(handler.cc:7708),write_record(sql_insert.cc:1947),mysql_insert(sql_insert.cc:1085),mysql_execute_command(sql_parse.cc:3906),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_reserve_and_write_fast(sync0sync.ic:314),mtr_log_reserve_and_write(sync0sync.ic:314),mtr_commit(sync0sync.ic:314),ibuf_mtr_commit(ibuf0ibuf.ic:58),ibuf_merge_or_delete_for_page(ibuf0ibuf.ic:58),buf_page_io_complete(buf0buf.cc:4495),fil_aio_wait(fil0fil.cc:5873),io_handler_thread(srv0start.cc:512),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_print(sync0sync.ic:314),srv_printf_innodb_monitor(srv0srv.cc:1388),innodb_show_status(ha_innodb.cc:13379),innobase_show_status(ha_innodb.cc:13379),ha_show_status(handler.cc:7336),mysql_execute_command(sql_parse.cc:3282),mysql_parse(sql_parse.cc:6972),dispatch_command(sql_parse.cc:1442),do_command(sql_parse.cc:1054),do_handle_one_connection(sql_connect.cc:1541),handle_one_connection(sql_connect.cc:1444),pfs_spawn_thread(pfs.cc:1860),start_thread(libpthread.so.0),clone(libc.so.6)
1 pthread_cond_wait,os_cond_wait(os0sync.cc:214),os_event_wait_low(os0sync.cc:214),sync_array_wait_event(sync0arr.cc:433),mutex_spin_wait(sync0sync.cc:683),mutex_enter_func(sync0sync.ic:314),pfs_mutex_enter_func(sync0sync.ic:314),log_get_lsn(sync0sync.ic:314),srv_error_monitor_thread(sync0sync.ic:314),start_thread(libpthread.so.0),clone(libc.so.6)