-
Notifications
You must be signed in to change notification settings - Fork 8.3k
02844_max_backup_bandwidth_s3 is flaky #85084
Copy link
Copy link
Closed
Labels
flaky testflaky test found by CIflaky test found by CI
Description
Describe the bug
2025-08-05 04:15:16 Reason: result differs with reference:
2025-08-05 04:15:16 --- /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.reference 2025-08-05 03:56:44.892803405 +1200
2025-08-05 04:15:16 +++ /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stdout 2025-08-05 04:15:16.161876261 +1200
2025-08-05 04:15:16 @@ -1,2 +1,2 @@
2025-08-05 04:15:16 native_copy 0
2025-08-05 04:15:16 -no_native_copy 1
2025-08-05 04:15:16 +no_native_copy 0
2025-08-05 04:15:16
2025-08-05 04:15:16
2025-08-05 04:15:16 /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/test_mhsg6g5p/02844_max_backup_bandwidth_s3.sh.debuglog:
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:12] client_opts=(--allow_repeated_settings --send_logs_level 'error')
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:14] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error -m -q '
2025-08-05 04:15:16 drop table if exists data;
2025-08-05 04:15:16 create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, disk='\''s3_disk'\'';
2025-08-05 04:15:16 -- reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds
2025-08-05 04:15:16 insert into data select * from numbers(1e6);
2025-08-05 04:15:16 '
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:21] random_str 10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] local n=10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] shift
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] tr -cd '[:lower:]'
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] head -c10
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:21] query_id=vltlhifbio
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:22] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error --query_id vltlhifbio -q 'backup table data to S3(s3_conn, '\''backups/test_mhsg6g5p/data/backup2'\'') SETTINGS allow_s3_native_copy=1' --max_backup_bandwidth=1M
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:23] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error -m -q '
2025-08-05 04:15:16 SYSTEM FLUSH LOGS query_log;
2025-08-05 04:15:16 SELECT
2025-08-05 04:15:16 '\''native_copy'\'',
2025-08-05 04:15:16 query_duration_ms >= 7e3
2025-08-05 04:15:16 FROM system.query_log
2025-08-05 04:15:16 WHERE current_database = '\''test_mhsg6g5p'\'' AND query_id = '\''vltlhifbio'\'' AND type != '\''QueryStart'\''
2025-08-05 04:15:16 '
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:32] random_str 10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] local n=10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] shift
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] tr -cd '[:lower:]'
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] head -c10
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:32] query_id=jgmxlxxiwy
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:33] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error --query_id jgmxlxxiwy -q 'backup table data to S3(s3_conn, '\''backups/test_mhsg6g5p/data/backup3'\'') SETTINGS allow_s3_native_copy=0' --max_backup_bandwidth=1M
2025-08-05 04:15:16 + [2025-08-05 04:15:15] [:34] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error -m -q '
2025-08-05 04:15:16 SYSTEM FLUSH LOGS query_log;
2025-08-05 04:15:16 SELECT
2025-08-05 04:15:16 '\''no_native_copy'\'',
2025-08-05 04:15:16 query_duration_ms >= 7e3
2025-08-05 04:15:16 FROM system.query_log
2025-08-05 04:15:16 WHERE current_database = '\''test_mhsg6g5p'\'' AND query_id = '\''jgmxlxxiwy'\'' AND type != '\''QueryStart'\''
2025-08-05 04:15:16 '
2025-08-05 04:15:16
2025-08-05 04:15:16
2025-08-05 04:15:16 MergeTree settings used in test: --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1
2025-08-05 04:15:16
2025-08-05 04:15:16 Database: test_mhsg6g5p
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
flaky testflaky test found by CIflaky test found by CI