midenok
commented
1 year ago Database schema filename built
#0 build_schema_filename (database=0x7fed44004360 "mydumper_0", suffix=0x56267084c1bf "schema-create") at ../src/extra/mydumper/src/mydumper_common.c:232
#1 0x0000562670865d6e in create_database_related_job (database=0x7fed44004250, conf=0x7ffd2726ee90, type=JOB_CREATE_DATABASE, suffix=0x56267084c1bf "schema-create") at ../src/extra/mydumper/src/mydumper_jobs.c:968
#2 0x0000562670865dc9 in create_job_to_dump_schema (database=0x7fed44004250, conf=0x7ffd2726ee90) at ../src/extra/mydumper/src/mydumper_jobs.c:975
#3 0x000056267086ad61 in thd_JOB_DUMP_ALL_DATABASES (td=0x562670c92a48, job=0x562670c62ed0) at /home/midenok/src/mariadb/10.6/src/extra/mydumper/src/mydumper_working_thread.c:294View schema written
#0 m_fopen (filename=0x7fc3cf2fd9e8, type=0x55619246636c "w") at ../src/extra/mydumper/src/mydumper_working_thread.c:157
#1 0x000055619247af9b in write_view_definition_into_file (conn=0x7fc3c0000b90, dbt=0x7fc3b4003e90, filename=0x7fc3b400e860 "/home/midenok/src/mariadb/10.6/build/mysql-test/var/tmp/mydumper/test.v1-schema.sql", filename2=0x7fc3b400e8f0 "/home/midenok/src/mariadb/10.6/build/mysql-test/var/tmp/mydumper/test.v1-schema-view.sql", checksum_filename=0) at ../src/extra/mydumper/src/mydumper_jobs.c:393
#2 0x000055619247e91e in do_JOB_VIEW (td=0x5561940ca130, job=0x7fc3b4004560) at ../src/extra/mydumper/src/mydumper_jobs.c:892
#3 0x00005561924867b4 in process_job (td=0x5561940ca130, job=0x7fc3b4004560) at ../src/extra/mydumper/src/mydumper_working_thread.c:767
#4 0x00005561924869a6 in process_queue (queue=0x5561940bdfa0, td=0x5561940ca130, p=0x5561924866e0 <process_job>, f=0x0) at ../src/extra/mydumper/src/mydumper_working_thread.c:819
#5 0x0000556192487ae9 in working_thread (td=0x5561940ca130) at ../src/extra/mydumper/src/mydumper_working_thread.c:1236
#6 0x00007fc3d7106331 in () at /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0Data loaded with INSERT
#9 0x00007ffff78422af in mysql_real_query () from /usr/lib/x86_64-linux-gnu/libmysqlclient.so.21
#10 0x00005555555746f3 in restore_data_in_gstring_by_statement (td=0x5555555c8e80, data=0x7fffc40173c0, is_schema=0, query_counter=0x7fffd1ffaaf4) at ../src/extra/mydumper/src/myloader_restore.c:36
#11 0x00005555555755e1 in split_and_restore_data_in_gstring_by_statement (td=0x5555555c8e80, data=0x7fffe400c000, is_schema=0, query_counter=0x7fffd1ffaaf4, offset_line=33940) at ../src/extra/mydumper/src/myloader_restore.c:129
#12 0x0000555555575c90 in restore_data_from_file (td=0x5555555c8e80, database=0x5555555e3100 "vtrack", table=0x5555555f51c0 "vg_activation", filename=0x7fffc402bef0 "vtrack.vg_activation.00000.sql", is_schema=0) at ../src/extra/mydumper/src/myloader_restore.c:213
#13 0x0000555555577fa8 in process_restore_job (td=0x5555555c8e80, rj=0x7fffc402c820) at ../src/extra/mydumper/src/myloader_restore_job.c:258
#14 0x0000555555578d19 in process_job (td=0x5555555c8e80, job=0x7fffb4009080) at ../src/extra/mydumper/src/myloader_control_job.c:78
#15 0x000055555557c3c1 in process_loader_thread (td=0x5555555c8e80) at ../src/extra/mydumper/src/myloader_worker_loader.c:126
#16 0x000055555557bf2a in loader_thread (td=0x5555555c8e80) at ../src/extra/mydumper/src/myloader_worker_loader.c:172
#17 0x00007ffff7144331 in ?? () from /usr/lib/x86_64-linux-gnu/libglib-2.0.so.0
#18 0x00007ffff6897ada in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:444
#19 0x00007ffff692847c in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:78Mydumper splitting data into multiple files
Myloader multi-threading works on file-level. Multiple data-files must exist and they may be scheduled to different threads.
sub_part determines file number.
gboolean update_files_on_table_job(struct table_job *tj){
if (tj->sql_file == 0){
if (tj->chunk_step_item->chunk_type == INTEGER && tj->chunk_step_item->chunk_step->integer_step.is_step_fixed_length ){
if (tj->chunk_step_item->chunk_step->integer_step.is_unsigned)
tj->sub_part = tj->chunk_step_item->chunk_step->integer_step.type.unsign.min / tj->chunk_step_item->chunk_step->integer_step.step + 1;
else
tj->sub_part = tj->chunk_step_item->chunk_step->integer_step.type.sign.min / tj->chunk_step_item->chunk_step->integer_step.step + 1;
}
if (load_data){
initialize_load_data_fn(tj);
tj->sql_filename = build_data_filename(tj->dbt->database->filename, tj->dbt->table_filename, tj->nchunk, tj->sub_part);
tj->sql_file = m_open(&(tj->sql_filename),"w");
return TRUE;
}else{
initialize_sql_fn(tj);
}
}
return FALSE;
}initialize_chunk_step_item() must be called to set chunk_type to non-zero.
void set_chunk_strategy_for_dbt(MYSQL *conn, struct db_table *dbt){
g_mutex_lock(dbt->chunks_mutex);
struct chunk_step_item * csi = NULL;
guint64 rows = get_rows_from_explain(conn, dbt, NULL ,NULL);
if (rows > dbt->min_chunk_step_size){
GList *partitions=NULL;
if (split_partitions || dbt->partition_regex){
csi = g_new0(struct chunk_step_item, 1);
csi->chunk_step=NULL;
csi->chunk_functions.process=&process_none_chunk;
partitions = get_partitions_for_table(conn, dbt);
csi->chunk_type=PARTITION;
csi->chunk_functions.process = &process_partition_chunk;
csi->chunk_functions.get_next = &get_next_partition_chunk;
csi->chunk_step=new_real_partition_step(partitions,0,0);
}else{
if (dbt->starting_chunk_step_size > 0) {
csi = initialize_chunk_step_item(conn, dbt, 0, NULL, rows);
}else{
csi = new_none_chunk_step();
}
}
}else{
csi = new_none_chunk_step();
}
// dbt->initial_chunk_step=csi;
dbt->chunks=g_list_prepend(dbt->chunks,csi);
g_async_queue_push(dbt->chunks_queue, csi);
dbt->status=READY;
g_mutex_unlock(dbt->chunks_mutex);
}Num Type Disp Enb Address What
1 breakpoint keep n 0x0000558670d478c9 in write_row_into_file_in_sql_mode at ../src/extra/mydumper/src/mydumper_write.c:548
btc
2 breakpoint keep n 0x0000558670d2ec0c in update_files_on_table_job at ../src/extra/mydumper/src/mydumper_jobs.c:1058
btc
3 hw watchpoint keep n -location tj->sub_part
4 breakpoint keep y 0x0000558670d4dadf in new_integer_step_item at ../src/extra/mydumper/src/mydumper_integer_chunks.c:96
btc
5 breakpoint keep y 0x0000558670d39c73 in initialize_chunk_step_item at ../src/extra/mydumper/src/mydumper_chunks.c:100
btc
6 breakpoint keep n 0x0000558670d3a760 in set_chunk_strategy_for_dbt at ../src/extra/mydumper/src/mydumper_chunks.c:276
breakpoint already hit 3 times
btc
7 breakpoint keep y 0x0000558670d3a7a5 in set_chunk_strategy_for_dbt at ../src/extra/mydumper/src/mydumper_chunks.c:281
breakpoint already hit 1 time
btc
https://github.com/mydumper/mydumper/issues
Create job to dump table schema
Dumping data
Dumping sequence