Skip to content

Commit 9dbf0e6

Browse files
andreidanandreidan
authored
ILM: migrate action configures the _tier_preference setting (elastic#62829)
The `migrate` action will now configure the `index.routing.allocation.include._tier_preference` setting to the corresponding tiers. For the HOT phase it will configure `data_hot`, for the WARM phase it will configure `data_warm,data_hot` and for the COLD phase `data_cold,data_warm,data_cold`.
1 parent e62d9f8 commit 9dbf0e6

6 files changed

Lines changed: 115 additions & 50 deletions

File tree

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierAllocationDecider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ private enum OpType {
238238
* exist. If no nodes for any of the tiers are available, returns an empty
239239
* {@code Optional<String>}.
240240
*/
241-
static Optional<String> preferredAvailableTier(String prioritizedTiers, DiscoveryNodes nodes) {
241+
public static Optional<String> preferredAvailableTier(String prioritizedTiers, DiscoveryNodes nodes) {
242242
String[] tiers = Strings.tokenizeToStringArray(prioritizedTiers, ",");
243243
return Arrays.stream(tiers).filter(tier -> tierNodesPresent(tier, nodes)).findFirst();
244244
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStep.java‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010
import org.elasticsearch.action.support.ActiveShardCount;
1111
import org.elasticsearch.cluster.ClusterState;
1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
13-
import org.elasticsearch.cluster.node.DiscoveryNode;
14-
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
1513
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
1614
import org.elasticsearch.common.Strings;
1715
import org.elasticsearch.common.settings.ClusterSettings;
@@ -24,9 +22,9 @@
2422
import java.util.HashSet;
2523
import java.util.List;
2624
import java.util.Locale;
25+
import java.util.Optional;
2726
import java.util.Set;
2827

29-
import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_ROLE;
3028
import static org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING;
3129
import static org.elasticsearch.xpack.core.ilm.AllocationRoutedStep.getPendingAllocations;
3230
import static org.elasticsearch.xpack.core.ilm.step.info.AllocationInfo.waitingForActiveShardsAllocationInfo;
@@ -73,44 +71,49 @@ public Result isConditionMet(Index index, ClusterState clusterState) {
7371
logger.debug("[{}] lifecycle action for index [{}] executed but index no longer exists", getKey().getAction(), index.getName());
7472
return new Result(false, null);
7573
}
76-
String destinationTier = INDEX_ROUTING_PREFER_SETTING.get(idxMeta.getSettings());
74+
String preferredTierConfiguration = INDEX_ROUTING_PREFER_SETTING.get(idxMeta.getSettings());
75+
Optional<String> availableDestinationTier = DataTierAllocationDecider.preferredAvailableTier(preferredTierConfiguration,
76+
clusterState.getNodes());
77+
7778
if (ActiveShardCount.ALL.enoughShardsActive(clusterState, index.getName()) == false) {
78-
if (Strings.isEmpty(destinationTier)) {
79+
if (Strings.isEmpty(preferredTierConfiguration)) {
7980
logger.debug("[{}] lifecycle action for index [{}] cannot make progress because not all shards are active",
8081
getKey().getAction(), index.getName());
8182
} else {
82-
logger.debug("[{}] migration of index [{}] to the [{}] tier cannot progress, as not all shards are active",
83-
getKey().getAction(), index.getName(), destinationTier);
83+
if (availableDestinationTier.isPresent()) {
84+
logger.debug("[{}] migration of index [{}] to the [{}] tier preference cannot progress, as not all shards are active",
85+
getKey().getAction(), index.getName(), preferredTierConfiguration);
86+
} else {
87+
logger.debug("[{}] migration of index [{}] to the next tier cannot progress as there is no available tier for the " +
88+
"configured preferred tiers [{}] and not all shards are active", getKey().getAction(), index.getName(),
89+
preferredTierConfiguration);
90+
}
8491
}
8592
return new Result(false, waitingForActiveShardsAllocationInfo(idxMeta.getNumberOfReplicas()));
8693
}
8794

88-
if (Strings.isEmpty(destinationTier)) {
89-
logger.debug("index [{}] has no data tier routing setting configured and all its shards are active. considering the [{}] " +
90-
"step condition met and continuing to the next step", index.getName(), getKey().getName());
95+
if (Strings.isEmpty(preferredTierConfiguration)) {
96+
logger.debug("index [{}] has no data tier routing preference setting configured and all its shards are active. considering " +
97+
"the [{}] step condition met and continuing to the next step", index.getName(), getKey().getName());
9198
// the user removed the tier routing setting and all the shards are active so we'll cary on
9299
return new Result(true, null);
93100
}
94101

95102
int allocationPendingAllShards = getPendingAllocations(index, ALLOCATION_DECIDERS, clusterState);
96103

97104
if (allocationPendingAllShards > 0) {
98-
boolean targetTierNodeFound = false;
99-
for (DiscoveryNode node : clusterState.nodes()) {
100-
for (DiscoveryNodeRole role : node.getRoles()) {
101-
if (role.roleName().equals(DATA_ROLE.roleName()) || role.roleName().equals(destinationTier)) {
102-
targetTierNodeFound = true;
103-
break;
104-
}
105-
}
106-
}
107-
String statusMessage = String.format(Locale.ROOT, "%s lifecycle action [%s] waiting for [%s] shards to be moved to the [%s] " +
108-
"tier" + (targetTierNodeFound ? "" : " but there are currently no [%s] nodes in the cluster"),
109-
index, getKey().getAction(), allocationPendingAllShards, destinationTier, destinationTier);
105+
String statusMessage = availableDestinationTier.map(
106+
s -> String.format(Locale.ROOT, "[%s] lifecycle action [%s] waiting for [%s] shards to be moved to the [%s] tier (tier " +
107+
"migration preference configuration is [%s])", index.getName(), getKey().getAction(), allocationPendingAllShards, s,
108+
preferredTierConfiguration)
109+
).orElseGet(
110+
() -> String.format(Locale.ROOT, "index [%s] has a preference for tiers [%s], but no nodes for any of those tiers are " +
111+
"available in the cluster", index.getName(), preferredTierConfiguration));
110112
logger.debug(statusMessage);
111113
return new Result(false, new AllocationInfo(idxMeta.getNumberOfReplicas(), allocationPendingAllShards, true, statusMessage));
112114
} else {
113-
logger.debug("[{}] migration of index [{}] to tier [{}] complete", getKey().getAction(), index, destinationTier);
115+
logger.debug("[{}] migration of index [{}] to tier [{}] (preference [{}]) complete",
116+
getKey().getAction(), index, availableDestinationTier, preferredTierConfiguration);
114117
return new Result(true, null);
115118
}
116119
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MigrateAction.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.Arrays;
2323
import java.util.List;
2424
import java.util.Objects;
25+
import java.util.stream.Collectors;
2526

2627
/**
2728
* A {@link LifecycleAction} which enables or disables the automatic migration of data between
@@ -31,6 +32,9 @@ public class MigrateAction implements LifecycleAction {
3132
public static final String NAME = "migrate";
3233
public static final ParseField ENABLED_FIELD = new ParseField("enabled");
3334

35+
// Represents an ordered list of data tiers from cold to hot (or slow to fast)
36+
private static final List<String> COLD_TO_HOT_TIERS = List.of(DataTier.DATA_COLD, DataTier.DATA_WARM, DataTier.DATA_HOT);
37+
3438
private static final ConstructingObjectParser<MigrateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
3539
a -> new MigrateAction(a[0] == null ? true : (boolean) a[0]));
3640

@@ -92,7 +96,7 @@ public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
9296
Settings.Builder migrationSettings = Settings.builder();
9397
String dataTierName = "data_" + phase;
9498
assert DataTier.validTierName(dataTierName) : "invalid data tier name:" + dataTierName;
95-
migrationSettings.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, dataTierName);
99+
migrationSettings.put(DataTierAllocationDecider.INDEX_ROUTING_PREFER, getPreferredTiersConfiguration(dataTierName));
96100
UpdateSettingsStep updateMigrationSettingStep = new UpdateSettingsStep(migrationKey, migrationRoutedKey, client,
97101
migrationSettings.build());
98102
DataTierMigrationRoutedStep migrationRoutedStep = new DataTierMigrationRoutedStep(migrationRoutedKey, nextStepKey);
@@ -102,6 +106,19 @@ public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
102106
}
103107
}
104108

109+
/**
110+
* Based on the provided target tier it will return a comma separated list of preferred tiers.
111+
* ie. if `data_cold` is the target tier, it will return `data_cold,data_warm,data_hot`
112+
* This is usually used in conjunction with {@link DataTierAllocationDecider#INDEX_ROUTING_PREFER_SETTING}
113+
*/
114+
static String getPreferredTiersConfiguration(String targetTier) {
115+
int indexOfTargetTier = COLD_TO_HOT_TIERS.indexOf(targetTier);
116+
if (indexOfTargetTier == -1) {
117+
throw new IllegalArgumentException("invalid data tier [" + targetTier + "]");
118+
}
119+
return COLD_TO_HOT_TIERS.stream().skip(indexOfTargetTier).collect(Collectors.joining(","));
120+
}
121+
105122
@Override
106123
public int hashCode() {
107124
return Objects.hash(enabled);

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/DataTierMigrationRoutedStepTests.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ public void testExecuteWithPendingShards() {
112112
DataTierMigrationRoutedStep step = createRandomInstance();
113113
Result expectedResult = new Result(false, new AllocationInfo(0, 1, true,
114114
"[" + index.getName() + "] lifecycle action [" + step.getKey().getAction() + "] waiting for " +
115-
"[1] shards to be moved to the [data_warm] tier")
115+
"[1] shards to be moved to the [data_warm] tier (tier migration preference configuration is [data_warm])")
116116
);
117117

118118
Result actualResult = step.isConditionMet(index, clusterState);
@@ -137,9 +137,8 @@ public void testExecuteWithPendingShardsAndTargetRoleNotPresentInCluster() {
137137
.build();
138138
DataTierMigrationRoutedStep step = createRandomInstance();
139139
Result expectedResult = new Result(false, new AllocationInfo(0, 1, true,
140-
"[" + index.getName() + "] lifecycle action [" + step.getKey().getAction() + "] waiting for " +
141-
"[1] shards to be moved to the [data_warm] tier but there are currently no [data_warm] nodes in the cluster")
142-
);
140+
"index [" + index.getName() + "] has a preference for tiers [data_warm], but no nodes for any of those tiers are available " +
141+
"in the cluster"));
143142

144143
Result actualResult = step.isConditionMet(index, clusterState);
145144
assertThat(actualResult.isComplete(), is(false));

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MigrateActionTests.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,21 @@
77

88
import org.elasticsearch.common.io.stream.Writeable.Reader;
99
import org.elasticsearch.common.xcontent.XContentParser;
10+
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
1011
import org.elasticsearch.xpack.core.ilm.Step.StepKey;
1112

1213
import java.io.IOException;
1314
import java.util.List;
1415

16+
import static org.elasticsearch.xpack.core.DataTier.DATA_COLD;
17+
import static org.elasticsearch.xpack.core.DataTier.DATA_HOT;
18+
import static org.elasticsearch.xpack.core.DataTier.DATA_WARM;
19+
import static org.elasticsearch.xpack.core.ilm.MigrateAction.getPreferredTiersConfiguration;
20+
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.COLD_PHASE;
1521
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.DELETE_PHASE;
22+
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.HOT_PHASE;
23+
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.WARM_PHASE;
24+
import static org.hamcrest.CoreMatchers.is;
1625

1726
public class MigrateActionTests extends AbstractActionTestCase<MigrateAction> {
1827

@@ -56,4 +65,36 @@ public void testToSteps() {
5665
assertEquals(0, steps.size());
5766
}
5867
}
68+
69+
public void testGetPreferredTiersConfiguration() {
70+
assertThat(getPreferredTiersConfiguration(DATA_HOT), is(DATA_HOT));
71+
assertThat(getPreferredTiersConfiguration(DATA_WARM), is(DATA_WARM + "," + DATA_HOT));
72+
assertThat(getPreferredTiersConfiguration(DATA_COLD), is(DATA_COLD + "," + DATA_WARM + "," + DATA_HOT));
73+
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> getPreferredTiersConfiguration("no_tier"));
74+
assertThat(exception.getMessage(), is("invalid data tier [no_tier]"));
75+
}
76+
77+
public void testMigrateActionsConfiguresTierPreference() {
78+
StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
79+
randomAlphaOfLengthBetween(1, 10));
80+
MigrateAction action = new MigrateAction();
81+
{
82+
List<Step> steps = action.toSteps(null, HOT_PHASE, nextStepKey);
83+
UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
84+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(firstStep.getSettings()),
85+
is(DATA_HOT));
86+
}
87+
{
88+
List<Step> steps = action.toSteps(null, WARM_PHASE, nextStepKey);
89+
UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
90+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(firstStep.getSettings()),
91+
is(DATA_WARM + "," + DATA_HOT));
92+
}
93+
{
94+
List<Step> steps = action.toSteps(null, COLD_PHASE, nextStepKey);
95+
UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
96+
assertThat(DataTierAllocationDecider.INDEX_ROUTING_PREFER_SETTING.get(firstStep.getSettings()),
97+
is(DATA_COLD + "," + DATA_WARM + "," + DATA_HOT));
98+
}
99+
}
59100
}

‎x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java‎

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,19 @@ public static Settings coldNode(final Settings settings) {
9292

9393
public void testIndexDataTierMigration() throws Exception {
9494
internalCluster().startMasterOnlyNodes(1, Settings.EMPTY);
95-
logger.info("starting hot data node");
95+
logger.info("starting 2 hot data nodes");
9696
internalCluster().startNode(hotNode(Settings.EMPTY));
97+
internalCluster().startNode(hotNode(Settings.EMPTY));
98+
99+
// it's important we start one node of each tear as otherwise all phases will be allocated on the 2 available hot nodes (as our
100+
// tier preference configuration will not detect any available warm/cold tier node and will fallback to the available hot tier)
101+
// we want ILM to stop in the check-migration step in the warm and cold phase so we can unblock it manually by starting another
102+
// node in the corresponding tier (so that the index replica is allocated)
103+
logger.info("starting a warm data node");
104+
internalCluster().startNode(warmNode(Settings.EMPTY));
105+
106+
logger.info("starting a cold data node");
107+
internalCluster().startNode(coldNode(Settings.EMPTY));
97108

98109
Phase hotPhase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap());
99110
Phase warmPhase = new Phase("warm", TimeValue.ZERO, Collections.emptyMap());
@@ -104,7 +115,7 @@ public void testIndexDataTierMigration() throws Exception {
104115
assertAcked(putLifecycleResponse);
105116

106117
Settings settings = Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1)
107-
.put(SETTING_NUMBER_OF_REPLICAS, 0).put(LifecycleSettings.LIFECYCLE_NAME, policy).build();
118+
.put(SETTING_NUMBER_OF_REPLICAS, 1).put(LifecycleSettings.LIFECYCLE_NAME, policy).build();
108119
CreateIndexResponse res = client().admin().indices().prepareCreate(managedIndex).setSettings(settings).get();
109120
assertTrue(res.isAcknowledged());
110121

@@ -118,7 +129,7 @@ public void testIndexDataTierMigration() throws Exception {
118129
assertThat(indexLifecycleExplainResponse.getStep(), is(DataTierMigrationRoutedStep.NAME));
119130
});
120131

121-
logger.info("starting warm data node");
132+
logger.info("starting a warm data node");
122133
internalCluster().startNode(warmNode(Settings.EMPTY));
123134
assertBusy(() -> {
124135
ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
@@ -130,7 +141,7 @@ public void testIndexDataTierMigration() throws Exception {
130141
assertThat(indexLifecycleExplainResponse.getStep(), is(DataTierMigrationRoutedStep.NAME));
131142
});
132143

133-
logger.info("starting cold data node");
144+
logger.info("starting a cold data node");
134145
internalCluster().startNode(coldNode(Settings.EMPTY));
135146

136147
// wait for lifecycle to complete in the cold phase after the index has been migrated to the cold node
@@ -147,9 +158,15 @@ public void testIndexDataTierMigration() throws Exception {
147158

148159
public void testUserOptsOutOfTierMigration() throws Exception {
149160
internalCluster().startMasterOnlyNodes(1, Settings.EMPTY);
150-
logger.info("starting hot data node");
161+
logger.info("starting a hot data node");
151162
internalCluster().startNode(hotNode(Settings.EMPTY));
152163

164+
logger.info("starting a warm data node");
165+
internalCluster().startNode(warmNode(Settings.EMPTY));
166+
167+
logger.info("starting a cold data node");
168+
internalCluster().startNode(coldNode(Settings.EMPTY));
169+
153170
Phase hotPhase = new Phase("hot", TimeValue.ZERO, Collections.emptyMap());
154171
Phase warmPhase = new Phase("warm", TimeValue.ZERO, Collections.emptyMap());
155172
Phase coldPhase = new Phase("cold", TimeValue.ZERO, Collections.emptyMap());
@@ -171,26 +188,14 @@ public void testUserOptsOutOfTierMigration() throws Exception {
171188
IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
172189
assertThat(indexLifecycleExplainResponse.getPhase(), is("warm"));
173190
assertThat(indexLifecycleExplainResponse.getStep(), is(DataTierMigrationRoutedStep.NAME));
174-
});
191+
assertReplicaIsUnassigned();
192+
}, 30, TimeUnit.SECONDS);
175193

176194
Settings removeTierRoutingSetting = Settings.builder().putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER).build();
177195
UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(managedIndex).settings(removeTierRoutingSetting);
178196
assertAcked(client().admin().indices().updateSettings(updateSettingsRequest).actionGet());
179197

180-
assertBusy(() -> {
181-
ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex);
182-
ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE,
183-
explainRequest).get();
184-
185-
IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
186-
assertThat(indexLifecycleExplainResponse.getPhase(), is("warm"));
187-
assertThat(indexLifecycleExplainResponse.getStep(), is(DataTierMigrationRoutedStep.NAME));
188-
assertReplicaIsUnassigned();
189-
}, 30, TimeUnit.SECONDS);
190-
191-
internalCluster().startNode(coldNode(Settings.EMPTY));
192-
193-
// the index should successfully allocate
198+
// the index should successfully allocate on any nodes
194199
ensureGreen(managedIndex);
195200

196201
// the index is successfully allocated but the migrate action from the cold phase re-configured the tier migration setting to the
@@ -206,7 +211,7 @@ public void testUserOptsOutOfTierMigration() throws Exception {
206211
IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex);
207212
assertThat(indexLifecycleExplainResponse.getPhase(), is("cold"));
208213
assertThat(indexLifecycleExplainResponse.getStep(), is(DataTierMigrationRoutedStep.NAME));
209-
});
214+
}, 30, TimeUnit.SECONDS);
210215

211216
// remove the tier routing setting again
212217
assertAcked(client().admin().indices().updateSettings(updateSettingsRequest).actionGet());

0 commit comments

Comments
 (0)