Skip to content

Commit 3debdc2

Browse files
excitoonexcitoon
committed
Added integration tests for ALTER MOVE PARTITION and fixed minor things.
1 parent 9446fd2 commit 3debdc2

2 files changed

Lines changed: 108 additions & 31 deletions

File tree

‎dbms/src/Storages/MergeTree/MergeTreeData.cpp‎

Lines changed: 22 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2723,19 +2723,18 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String &
27232723
if (!disk)
27242724
throw Exception("Disk " + name + " does not exists on policy " + storage_policy->getName(), ErrorCodes::UNKNOWN_DISK);
27252725

2726-
parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) {
2727-
return part_ptr->disk->getName() == disk->getName();
2728-
}), parts.end());
2726+
parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr)
2727+
{
2728+
return part_ptr->disk->getName() == disk->getName();
2729+
}), parts.end());
27292730

2730-
if (!parts.empty())
2731+
if (parts.empty())
27312732
{
2732-
if (!movePartsToSpace(parts, std::static_pointer_cast<const DiskSpace::Space>(disk)))
2733-
throw Exception("Cannot move parts because moves are manually disabled.", ErrorCodes::ABORTED);
2734-
}
2735-
else
2736-
{
2737-
LOG_DEBUG(log, "No parts of partition " << partition_id << " to move to disk " << disk->getName());
2733+
throw Exception("All parts of partition " + partition_id + " are already on disk '" + disk->getName() + "'", ErrorCodes::UNKNOWN_DISK);
27382734
}
2735+
2736+
if (!movePartsToSpace(parts, std::static_pointer_cast<const DiskSpace::Space>(disk)))
2737+
throw Exception("Cannot move parts because moves are manually disabled.", ErrorCodes::ABORTED);
27392738
}
27402739

27412740

@@ -2763,32 +2762,25 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String
27632762
if (!volume)
27642763
throw Exception("Volume " + name + " does not exists on policy " + storage_policy->getName(), ErrorCodes::UNKNOWN_DISK);
27652764

2766-
for (const auto & part : parts)
2767-
for (const auto & disk : volume->disks)
2768-
if (part->disk->getName() == disk->getName())
2769-
throw Exception("Part " + part->name + " already on volume '" + name + "'", ErrorCodes::UNKNOWN_DISK);
2770-
2771-
parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) {
2772-
for (const auto & disk : volume->disks)
2765+
parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr)
27732766
{
2774-
if (part_ptr->disk->getName() == disk->getName())
2767+
for (const auto & disk : volume->disks)
27752768
{
2776-
return true;
2769+
if (part_ptr->disk->getName() == disk->getName())
2770+
{
2771+
return true;
2772+
}
27772773
}
2778-
}
2779-
return false;
2780-
}), parts.end());
2781-
2774+
return false;
2775+
}), parts.end());
27822776

2783-
if (!parts.empty())
2777+
if (parts.empty())
27842778
{
2785-
if (!movePartsToSpace(parts, std::static_pointer_cast<const DiskSpace::Space>(volume)))
2786-
throw Exception("Cannot move parts because moves are manually disabled.", ErrorCodes::ABORTED);
2787-
}
2788-
else
2789-
{
2790-
LOG_DEBUG(log, "No parts of partition " << partition_id << " to move to volume " << volume->getName());
2779+
throw Exception("All parts of partition " + partition_id + " are already on volume '" + volume->getName() + "'", ErrorCodes::UNKNOWN_DISK);
27912780
}
2781+
2782+
if (!movePartsToSpace(parts, std::static_pointer_cast<const DiskSpace::Space>(volume)))
2783+
throw Exception("Cannot move parts because moves are manually disabled.", ErrorCodes::ABORTED);
27922784
}
27932785

27942786

‎dbms/tests/integration/test_multiple_disks/test.py‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@ def test_alter_move(start_cluster, name, engine):
462462
node1.query("INSERT INTO {} VALUES(toDate('2019-04-10'), 42)".format(name))
463463
node1.query("INSERT INTO {} VALUES(toDate('2019-04-11'), 43)".format(name))
464464
used_disks = get_used_disks_for_table(node1, name)
465-
assert all(d.startswith("jbod") for d in used_disks), "All writes shoud go to jbods"
465+
assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods"
466466

467467
first_part = node1.query("SELECT name FROM system.parts WHERE table = '{}' and active = 1 ORDER BY modification_time LIMIT 1".format(name)).strip()
468468

@@ -498,6 +498,91 @@ def test_alter_move(start_cluster, name, engine):
498498
finally:
499499
node1.query("DROP TABLE IF EXISTS {name}".format(name=name))
500500

501+
502+
@pytest.mark.parametrize("volume_or_disk", [
503+
"DISK",
504+
"VOLUME"
505+
])
506+
def test_alter_move_half_of_partition(start_cluster, volume_or_disk):
507+
name = "alter_move_half_of_partition"
508+
engine = "MergeTree()"
509+
try:
510+
node1.query("""
511+
CREATE TABLE {name} (
512+
EventDate Date,
513+
number UInt64
514+
) ENGINE = {engine}
515+
ORDER BY tuple()
516+
PARTITION BY toYYYYMM(EventDate)
517+
SETTINGS storage_policy='jbods_with_external'
518+
""".format(name=name, engine=engine))
519+
520+
node1.query("SYSTEM STOP MERGES {}".format(name))
521+
522+
node1.query("INSERT INTO {} VALUES(toDate('2019-03-15'), 65)".format(name))
523+
node1.query("INSERT INTO {} VALUES(toDate('2019-03-16'), 42)".format(name))
524+
used_disks = get_used_disks_for_table(node1, name)
525+
assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods"
526+
527+
time.sleep(1)
528+
parts = node1.query("SELECT name FROM system.parts WHERE table = '{}' and active = 1".format(name)).splitlines()
529+
assert len(parts) == 2
530+
531+
node1.query("ALTER TABLE {} MOVE PART '{}' TO VOLUME 'external'".format(name, parts[0]))
532+
disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and name = '{}' and active = 1".format(name, parts[0])).splitlines()
533+
assert disks == ["external"]
534+
535+
time.sleep(1)
536+
node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, volume_or_disk=volume_or_disk))
537+
disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format(name)).splitlines()
538+
assert disks == ["external"]*2
539+
540+
assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n"
541+
542+
finally:
543+
node1.query("DROP TABLE IF EXISTS {name}".format(name=name))
544+
545+
546+
@pytest.mark.parametrize("volume_or_disk", [
547+
"DISK",
548+
"VOLUME"
549+
])
550+
def test_alter_double_move_partition(start_cluster, volume_or_disk):
551+
name = "alter_double_move_partition"
552+
engine = "MergeTree()"
553+
try:
554+
node1.query("""
555+
CREATE TABLE {name} (
556+
EventDate Date,
557+
number UInt64
558+
) ENGINE = {engine}
559+
ORDER BY tuple()
560+
PARTITION BY toYYYYMM(EventDate)
561+
SETTINGS storage_policy='jbods_with_external'
562+
""".format(name=name, engine=engine))
563+
564+
node1.query("SYSTEM STOP MERGES {}".format(name))
565+
566+
node1.query("INSERT INTO {} VALUES(toDate('2019-03-15'), 65)".format(name))
567+
node1.query("INSERT INTO {} VALUES(toDate('2019-03-16'), 42)".format(name))
568+
used_disks = get_used_disks_for_table(node1, name)
569+
assert all(d.startswith("jbod") for d in used_disks), "All writes should go to jbods"
570+
571+
time.sleep(1)
572+
node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, volume_or_disk=volume_or_disk))
573+
disks = node1.query("SELECT disk_name FROM system.parts WHERE table = '{}' and partition = '201903' and active = 1".format(name)).splitlines()
574+
assert disks == ["external"]*2
575+
576+
assert node1.query("SELECT COUNT() FROM {}".format(name)) == "2\n"
577+
578+
time.sleep(1)
579+
with pytest.raises(QueryRuntimeException):
580+
node1.query("ALTER TABLE {} MOVE PARTITION 201903 TO {volume_or_disk} 'external'".format(name, volume_or_disk=volume_or_disk))
581+
582+
finally:
583+
node1.query("DROP TABLE IF EXISTS {name}".format(name=name))
584+
585+
501586
def produce_alter_move(node, name):
502587
move_type = random.choice(["PART", "PARTITION"])
503588
if move_type == "PART":

0 commit comments

Comments
 (0)