Skip to content

02844_max_backup_bandwidth_s3 is flaky #85084

@divanik

Description

@divanik

Describe the bug

https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=eff379cdcaf50d408952c527a312ada1aa61d280&name_0=MasterCI&name_1=Stateless%20tests%20%28amd_debug%2C%20distributed%20plan%2C%20s3%20storage%2C%20parallel%29&name_1=Stateless%20tests%20%28amd_debug%2C%20distributed%20plan%2C%20s3%20storage%2C%20parallel%29

2025-08-05 04:15:16 Reason: result differs with reference:  
2025-08-05 04:15:16 --- /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.reference	2025-08-05 03:56:44.892803405 +1200
2025-08-05 04:15:16 +++ /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stdout	2025-08-05 04:15:16.161876261 +1200
2025-08-05 04:15:16 @@ -1,2 +1,2 @@
2025-08-05 04:15:16  native_copy	0
2025-08-05 04:15:16 -no_native_copy	1
2025-08-05 04:15:16 +no_native_copy	0
2025-08-05 04:15:16 
2025-08-05 04:15:16 
2025-08-05 04:15:16 /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/test_mhsg6g5p/02844_max_backup_bandwidth_s3.sh.debuglog:
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:12] client_opts=(--allow_repeated_settings --send_logs_level 'error')
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:14] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error -m -q '
2025-08-05 04:15:16     drop table if exists data;
2025-08-05 04:15:16     create table data (key UInt64 CODEC(NONE)) engine=MergeTree() order by tuple() settings min_bytes_for_wide_part=1e9, disk='\''s3_disk'\'';
2025-08-05 04:15:16     -- reading 1e6*8 bytes with 1M bandwith it should take (8-1)/1=7 seconds
2025-08-05 04:15:16     insert into data select * from numbers(1e6);
2025-08-05 04:15:16 '
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:21] random_str 10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] local n=10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] shift
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] tr -cd '[:lower:]'
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] head -c10
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:21] query_id=vltlhifbio
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:22] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error --query_id vltlhifbio -q 'backup table data to S3(s3_conn, '\''backups/test_mhsg6g5p/data/backup2'\'') SETTINGS allow_s3_native_copy=1' --max_backup_bandwidth=1M
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:23] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error -m -q '
2025-08-05 04:15:16     SYSTEM FLUSH LOGS query_log;
2025-08-05 04:15:16     SELECT
2025-08-05 04:15:16         '\''native_copy'\'',
2025-08-05 04:15:16         query_duration_ms >= 7e3
2025-08-05 04:15:16     FROM system.query_log
2025-08-05 04:15:16     WHERE current_database = '\''test_mhsg6g5p'\'' AND query_id = '\''vltlhifbio'\'' AND type != '\''QueryStart'\''
2025-08-05 04:15:16 '
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:32] random_str 10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] local n=10
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:187] shift
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] tr -cd '[:lower:]'
2025-08-05 04:15:16 ++ [2025-08-05 04:15:08] [:188] head -c10
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:32] query_id=jgmxlxxiwy
2025-08-05 04:15:16 + [2025-08-05 04:15:08] [:33] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error --query_id jgmxlxxiwy -q 'backup table data to S3(s3_conn, '\''backups/test_mhsg6g5p/data/backup3'\'') SETTINGS allow_s3_native_copy=0' --max_backup_bandwidth=1M
2025-08-05 04:15:16 + [2025-08-05 04:15:15] [:34] /home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/ci/tmp/clickhouse-client --send_logs_level=warning --database=test_mhsg6g5p --log_comment 02844_max_backup_bandwidth_s3.sh --allow_merge_tree_settings --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1 --allow_repeated_settings --client_logs_file=/home/ubuntu/actions-runner/_work/ClickHouse/ClickHouse/tests/queries/0_stateless/02844_max_backup_bandwidth_s3.stderr-fatal --allow_repeated_settings --send_logs_level error -m -q '
2025-08-05 04:15:16     SYSTEM FLUSH LOGS query_log;
2025-08-05 04:15:16     SELECT
2025-08-05 04:15:16         '\''no_native_copy'\'',
2025-08-05 04:15:16         query_duration_ms >= 7e3
2025-08-05 04:15:16     FROM system.query_log
2025-08-05 04:15:16     WHERE current_database = '\''test_mhsg6g5p'\'' AND query_id = '\''jgmxlxxiwy'\'' AND type != '\''QueryStart'\''
2025-08-05 04:15:16 '
2025-08-05 04:15:16 
2025-08-05 04:15:16 
2025-08-05 04:15:16 MergeTree settings used in test: --ratio_of_defaults_for_sparse_serialization 0.6348864611320207 --prefer_fetch_merged_part_size_threshold 1 --vertical_merge_algorithm_min_rows_to_activate 738154 --vertical_merge_algorithm_min_columns_to_activate 100 --allow_vertical_merges_from_compact_to_wide_parts 1 --min_merge_bytes_to_use_direct_io 10737418240 --index_granularity_bytes 6139104 --merge_max_block_size 13388 --index_granularity 61353 --min_bytes_for_wide_part 0 --marks_compress_block_size 57283 --primary_key_compress_block_size 75764 --replace_long_file_name_to_hash 0 --max_file_name_length 16 --min_bytes_for_full_part_storage 399540033 --compact_parts_max_bytes_to_buffer 110500836 --compact_parts_max_granules_to_buffer 121 --compact_parts_merge_max_bytes_to_prefetch_part 8826193 --cache_populated_by_fetch 0 --concurrent_part_removal_threshold 0 --old_parts_lifetime 10 --prewarm_mark_cache 0 --use_const_adaptive_granularity 1 --enable_index_granularity_compression 0 --enable_block_number_column 1 --enable_block_offset_column 0 --use_primary_key_cache 1 --prewarm_primary_key_cache 1
2025-08-05 04:15:16 
2025-08-05 04:15:16 Database: test_mhsg6g5p

Metadata

Metadata

Assignees

No one assigned

    Labels

    flaky testflaky test found by CI

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions