pytest --count 100 -x test_backup_restore_on_cluster/test_concurrency.py -k test_concurrent_backups_on_same_node


test_backup_restore_on_cluster/test_concurrency.py::test_concurrent_backups_on_same_node[35-100] PASSED                                                             [ 16%]
test_backup_restore_on_cluster/test_concurrency.py::test_concurrent_backups_on_same_node[36-100] FAILED                                                             [ 17%]

================================================================================ FAILURES =================================================================================
______________________________________________________________ test_concurrent_backups_on_same_node[36-100] _______________________________________________________________

    def test_concurrent_backups_on_same_node():
        create_and_fill_table()

        backup_names = [new_backup_name() for _ in range(num_concurrent_backups)]

        ids = []
        for backup_name in backup_names:
            id = node0.query(
                f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name} ASYNC"
            ).split("\t")[0]
            ids.append(id)

        ids_list = "[" + ", ".join([f"'{id}'" for id in ids]) + "]"

>       assert_eq_with_retry(
            node0,
            f"SELECT status FROM system.backups WHERE status == 'CREATING_BACKUP' AND id IN {ids_list}",
            "",
        )

test_backup_restore_on_cluster/test_concurrency.py:122:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

instance = <helpers.cluster.ClickHouseInstance object at 0x7f17c07d03a0>
query = "SELECT status FROM system.backups WHERE status == 'CREATING_BACKUP' AND id IN ['7a95bd38-4241-4134-acbe-714ef4517c4f', 'c963cfe2-a95c-4885-a1a3-eafd79539e36', '53213f9a-58eb-4bf5-bde1-50919b5e7ad5', '601ec446-cfe9-43ba-adec-b3f99a14c8cf']"
expectation = '', retry_count = 20, sleep_time = 0.5, stdin = None, timeout = None, settings = None, user = None, ignore_error = False
get_result = <function <lambda> at 0x7f17c12e09d0>

    def assert_eq_with_retry(
        instance,
        query,
        expectation,
        retry_count=20,
        sleep_time=0.5,
        stdin=None,
        timeout=None,
        settings=None,
        user=None,
        ignore_error=False,
        get_result=lambda x: x,
    ):
        expectation_tsv = TSV(expectation)
        for i in range(retry_count):
            try:
                if (
                    TSV(
                        get_result(
                            instance.query(
                                query,
                                user=user,
                                stdin=stdin,
                                timeout=timeout,
                                settings=settings,
                                ignore_error=ignore_error,
                            )
                        )
                    )
                    == expectation_tsv
                ):
                    break
                time.sleep(sleep_time)
            except Exception as ex:
                logging.exception(f"assert_eq_with_retry retry {i+1} exception {ex}")
                time.sleep(sleep_time)
        else:
            val = TSV(
                get_result(
                    instance.query(
                        query,
                        user=user,
                        stdin=stdin,
                        timeout=timeout,
                        settings=settings,
                        ignore_error=ignore_error,
                    )
                )
            )
            if expectation_tsv != val:
>               raise AssertionError(
                    "'{}' != '{}'\n{}".format(
                        expectation_tsv,
                        val,
                        "\n".join(expectation_tsv.diff(val, n1="expectation", n2="query")),
                    )
                )
E               AssertionError: '' != 'CREATING_BACKUP
E               CREATING_BACKUP
E               CREATING_BACKUP
E               CREATING_BACKUP'
E               @@ -0,0 +1,4 @@
E               +CREATING_BACKUP
E               +CREATING_BACKUP
E               +CREATING_BACKUP
E               +CREATING_BACKUP

helpers/test_tools.py:114: AssertionError