Skip to content

Commit 76e515a

Browse files
rjernstrjernst
authored
Removed multiple paths from MetadataStateFormat (#72821)
relates #71205
1 parent f3ce700 commit 76e515a

3 files changed

Lines changed: 134 additions & 218 deletions

File tree

‎server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java‎

Lines changed: 63 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import org.apache.lucene.store.IndexInput;
2020
import org.apache.lucene.store.IndexOutput;
2121
import org.apache.lucene.store.SimpleFSDirectory;
22-
import org.elasticsearch.ExceptionsHelper;
22+
import org.elasticsearch.ElasticsearchException;
2323
import org.elasticsearch.common.collect.Tuple;
2424
import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
2525
import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
@@ -39,12 +39,10 @@
3939
import java.nio.file.NoSuchFileException;
4040
import java.nio.file.Path;
4141
import java.util.ArrayList;
42-
import java.util.Arrays;
4342
import java.util.Collections;
4443
import java.util.List;
4544
import java.util.regex.Matcher;
4645
import java.util.regex.Pattern;
47-
import java.util.stream.Collectors;
4846

4947
/**
5048
* MetadataStateFormat is a base class to write checksummed
@@ -168,10 +166,10 @@ private static void performStateDirectoriesFsync(List<Tuple<Path, Directory>> st
168166
/**
169167
* Writes the given state to the given directories and performs cleanup of old state files if the write succeeds or
170168
* newly created state file if write fails.
171-
* See also {@link #write(Object, Path...)} and {@link #cleanupOldFiles(long, Path[])}.
169+
* See also {@link #write(Object, Path)} and {@link #cleanupOldFiles(long, Path)}.
172170
*/
173-
public final long writeAndCleanup(final T state, final Path... locations) throws WriteStateException {
174-
return write(state, true, locations);
171+
public final long writeAndCleanup(final T state, final Path location) throws WriteStateException {
172+
return write(state, true, location);
175173
}
176174

177175
/**
@@ -186,29 +184,26 @@ public final long writeAndCleanup(final T state, final Path... locations) throws
186184
* If this method fails with an exception, it performs cleanup of newly created state file.
187185
* But if this method succeeds, it does not perform cleanup of old state files.
188186
* If this write succeeds, but some further write fails, you may want to rollback the transaction and keep old file around.
189-
* After transaction is finished use {@link #cleanupOldFiles(long, Path[])} for the clean-up.
190-
* If this write is not a part of bigger transaction, consider using {@link #writeAndCleanup(Object, Path...)} method instead.
187+
* After transaction is finished use {@link #cleanupOldFiles(long, Path)} for the clean-up.
188+
* If this write is not a part of bigger transaction, consider using {@link #writeAndCleanup(Object, Path)} method instead.
191189
*
192190
* @param state the state object to write
193-
* @param locations the locations where the state should be written to.
191+
* @param location the data dir the state should be written into
194192
* @throws WriteStateException if some exception during writing state occurs. See also {@link WriteStateException#isDirty()}.
195193
* @return generation of newly written state.
196194
*/
197-
public final long write(final T state, final Path... locations) throws WriteStateException {
198-
return write(state, false, locations);
195+
public final long write(final T state, final Path location) throws WriteStateException {
196+
return write(state, false, location);
199197
}
200198

201-
private long write(final T state, boolean cleanup, final Path... locations) throws WriteStateException {
202-
if (locations == null) {
199+
private long write(final T state, boolean cleanup, final Path location) throws WriteStateException {
200+
if (location == null) {
203201
throw new IllegalArgumentException("Locations must not be null");
204202
}
205-
if (locations.length <= 0) {
206-
throw new IllegalArgumentException("One or more locations required");
207-
}
208203

209204
final long oldGenerationId, newGenerationId;
210205
try {
211-
oldGenerationId = findMaxGenerationId(prefix, locations);
206+
oldGenerationId = findMaxGenerationId(prefix, location);
212207
newGenerationId = oldGenerationId + 1;
213208
} catch (Exception e) {
214209
throw new WriteStateException(false, "exception during looking up new generation id", e);
@@ -220,13 +215,11 @@ private long write(final T state, boolean cleanup, final Path... locations) thro
220215
List<Tuple<Path, Directory>> directories = new ArrayList<>();
221216

222217
try {
223-
for (Path location : locations) {
224-
Path stateLocation = location.resolve(STATE_DIR_NAME);
225-
try {
226-
directories.add(new Tuple<>(location, newDirectory(stateLocation)));
227-
} catch (IOException e) {
228-
throw new WriteStateException(false, "failed to open state directory " + stateLocation, e);
229-
}
218+
Path stateLocation = location.resolve(STATE_DIR_NAME);
219+
try {
220+
directories.add(new Tuple<>(location, newDirectory(stateLocation)));
221+
} catch (IOException e) {
222+
throw new WriteStateException(false, "failed to open state directory " + stateLocation, e);
230223
}
231224

232225
writeStateToFirstLocation(state, directories.get(0).v1(), directories.get(0).v2(), tmpFileName);
@@ -235,7 +228,7 @@ private long write(final T state, boolean cleanup, final Path... locations) thro
235228
performStateDirectoriesFsync(directories);
236229
} catch (WriteStateException e) {
237230
if (cleanup) {
238-
cleanupOldFiles(oldGenerationId, locations);
231+
cleanupOldFiles(oldGenerationId, location);
239232
}
240233
throw e;
241234
} finally {
@@ -246,7 +239,7 @@ private long write(final T state, boolean cleanup, final Path... locations) thro
246239
}
247240

248241
if (cleanup) {
249-
cleanupOldFiles(newGenerationId, locations);
242+
cleanupOldFiles(newGenerationId, location);
250243
}
251244

252245
return newGenerationId;
@@ -307,68 +300,61 @@ protected Directory newDirectory(Path dir) throws IOException {
307300
* Clean ups all state files not matching passed generation.
308301
*
309302
* @param currentGeneration state generation to keep.
310-
* @param locations state paths.
303+
* @param location data dir.
311304
*/
312-
public void cleanupOldFiles(final long currentGeneration, Path... locations) {
305+
public void cleanupOldFiles(final long currentGeneration, Path location) {
313306
final String fileNameToKeep = getStateFileName(currentGeneration);
314-
for (Path location : locations) {
315-
logger.trace("cleanupOldFiles: cleaning up {}", location);
316-
Path stateLocation = location.resolve(STATE_DIR_NAME);
317-
try (Directory stateDir = newDirectory(stateLocation)) {
318-
for (String file : stateDir.listAll()) {
319-
if (file.startsWith(prefix) && file.equals(fileNameToKeep) == false) {
320-
deleteFileIgnoreExceptions(stateLocation, stateDir, file);
321-
}
307+
logger.trace("cleanupOldFiles: cleaning up {}", location);
308+
Path stateLocation = location.resolve(STATE_DIR_NAME);
309+
try (Directory stateDir = newDirectory(stateLocation)) {
310+
for (String file : stateDir.listAll()) {
311+
if (file.startsWith(prefix) && file.equals(fileNameToKeep) == false) {
312+
deleteFileIgnoreExceptions(stateLocation, stateDir, file);
322313
}
323-
} catch (Exception e) {
324-
logger.trace("clean up failed for state location {}", stateLocation);
325314
}
315+
} catch (Exception e) {
316+
logger.trace("clean up failed for state location {}", stateLocation);
326317
}
327318
}
328319

329320
/**
330321
* Finds state file with maximum id.
331322
*
332323
* @param prefix - filename prefix
333-
* @param locations - paths to directories with state folder
324+
* @param dataLocation - path to directory with state folder
334325
* @return maximum id of state file or -1 if no such files are found
335326
* @throws IOException if IOException occurs
336327
*/
337-
private long findMaxGenerationId(final String prefix, Path... locations) throws IOException {
328+
private long findMaxGenerationId(final String prefix, Path dataLocation) throws IOException {
338329
long maxId = -1;
339-
for (Path dataLocation : locations) {
340-
final Path resolve = dataLocation.resolve(STATE_DIR_NAME);
341-
if (Files.exists(resolve)) {
342-
try (DirectoryStream<Path> stream = Files.newDirectoryStream(resolve, prefix + "*")) {
343-
for (Path stateFile : stream) {
344-
final Matcher matcher = stateFilePattern.matcher(stateFile.getFileName().toString());
345-
if (matcher.matches()) {
346-
final long id = Long.parseLong(matcher.group(1));
347-
maxId = Math.max(maxId, id);
348-
}
330+
final Path resolve = dataLocation.resolve(STATE_DIR_NAME);
331+
if (Files.exists(resolve)) {
332+
try (DirectoryStream<Path> stream = Files.newDirectoryStream(resolve, prefix + "*")) {
333+
for (Path stateFile : stream) {
334+
final Matcher matcher = stateFilePattern.matcher(stateFile.getFileName().toString());
335+
if (matcher.matches()) {
336+
final long id = Long.parseLong(matcher.group(1));
337+
maxId = Math.max(maxId, id);
349338
}
350339
}
351340
}
352341
}
353342
return maxId;
354343
}
355344

356-
private List<Path> findStateFilesByGeneration(final long generation, Path... locations) {
357-
List<Path> files = new ArrayList<>();
345+
private Path findStateFilesByGeneration(final long generation, Path dataLocation) {
358346
if (generation == -1) {
359-
return files;
347+
return null;
360348
}
361349

362350
final String fileName = getStateFileName(generation);
363-
for (Path dataLocation : locations) {
364-
final Path stateFilePath = dataLocation.resolve(STATE_DIR_NAME).resolve(fileName);
365-
if (Files.exists(stateFilePath)) {
366-
logger.trace("found state file: {}", stateFilePath);
367-
files.add(stateFilePath);
368-
}
351+
final Path stateFilePath = dataLocation.resolve(STATE_DIR_NAME).resolve(fileName);
352+
if (Files.exists(stateFilePath)) {
353+
logger.trace("found state file: {}", stateFilePath);
354+
return stateFilePath;
369355
}
370356

371-
return files;
357+
return null;
372358
}
373359

374360
public String getStateFileName(long generation) {
@@ -381,52 +367,40 @@ public String getStateFileName(long generation) {
381367
*
382368
* @param logger a logger instance.
383369
* @param generation the generation to be loaded.
384-
* @param dataLocations the data-locations to try.
370+
* @param dataLocation the data dir to read from
385371
* @return the state of asked generation or <code>null</code> if no state was found.
386372
*/
387-
public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, long generation, Path... dataLocations) {
388-
List<Path> stateFiles = findStateFilesByGeneration(generation, dataLocations);
373+
public T loadGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, long generation, Path dataLocation) {
374+
Path stateFile = findStateFilesByGeneration(generation, dataLocation);
389375

390-
final List<Throwable> exceptions = new ArrayList<>();
391-
for (Path stateFile : stateFiles) {
376+
if (stateFile != null) {
392377
try {
393378
T state = read(namedXContentRegistry, stateFile);
394379
logger.trace("generation id [{}] read from [{}]", generation, stateFile.getFileName());
395380
return state;
396381
} catch (Exception e) {
397-
exceptions.add(new IOException("failed to read " + stateFile, e));
398-
logger.debug(() -> new ParameterizedMessage(
399-
"{}: failed to read [{}], ignoring...", stateFile, prefix), e);
382+
logger.debug(() -> new ParameterizedMessage("{}: failed to read [{}], ignoring...", stateFile, prefix), e);
383+
throw new ElasticsearchException("failed to read " + stateFile, e);
400384
}
401385
}
402-
// if we reach this something went wrong
403-
ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptions);
404-
if (stateFiles.size() > 0) {
405-
// We have some state files but none of them gave us a usable state
406-
throw new IllegalStateException("Could not find a state file to recover from among " +
407-
stateFiles.stream().map(Object::toString).collect(Collectors.joining(", ")));
408-
}
409386
return null;
410387
}
411388

412389
/**
413390
* Tries to load the latest state from the given data-locations.
414391
*
415392
* @param logger a logger instance.
416-
* @param dataLocations the data-locations to try.
393+
* @param dataLocation the data dir to read from
417394
* @return tuple of the latest state and generation. (null, -1) if no state is found.
418395
*/
419-
public Tuple<T, Long> loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations)
396+
public Tuple<T, Long> loadLatestStateWithGeneration(Logger logger, NamedXContentRegistry namedXContentRegistry, Path dataLocation)
420397
throws IOException {
421-
long generation = findMaxGenerationId(prefix, dataLocations);
422-
T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocations);
398+
long generation = findMaxGenerationId(prefix, dataLocation);
399+
T state = loadGeneration(logger, namedXContentRegistry, generation, dataLocation);
423400

424401
if (generation > -1 && state == null) {
425402
throw new IllegalStateException("unable to find state files with generation id " + generation +
426-
" returned by findMaxGenerationId function, in data folders [" +
427-
Arrays.stream(dataLocations).
428-
map(Object::toString).collect(Collectors.joining(", ")) +
429-
"], concurrent writes?");
403+
" returned by findMaxGenerationId function, in data folder [" + dataLocation + "], concurrent writes?");
430404
}
431405
return Tuple.tuple(state, generation);
432406
}
@@ -435,24 +409,19 @@ public Tuple<T, Long> loadLatestStateWithGeneration(Logger logger, NamedXContent
435409
* Tries to load the latest state from the given data-locations.
436410
*
437411
* @param logger a logger instance.
438-
* @param dataLocations the data-locations to try.
412+
* @param dataLocation the data dir to read from
439413
* @return the latest state or <code>null</code> if no state was found.
440414
*/
441-
public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path... dataLocations) throws
442-
IOException {
443-
return loadLatestStateWithGeneration(logger, namedXContentRegistry, dataLocations).v1();
415+
public T loadLatestState(Logger logger, NamedXContentRegistry namedXContentRegistry, Path dataLocation) throws IOException {
416+
return loadLatestStateWithGeneration(logger, namedXContentRegistry, dataLocation).v1();
444417
}
445418

446419
/**
447420
* Deletes all meta state directories recursively for the given data locations
448-
* @param dataLocations the data location to delete
421+
* @param dataLocation the data dir to delete state from
449422
*/
450-
public static void deleteMetaState(Path... dataLocations) throws IOException {
451-
Path[] stateDirectories = new Path[dataLocations.length];
452-
for (int i = 0; i < dataLocations.length; i++) {
453-
stateDirectories[i] = dataLocations[i].resolve(STATE_DIR_NAME);
454-
}
455-
IOUtils.rm(stateDirectories);
423+
public static void deleteMetaState(Path dataLocation) throws IOException {
424+
IOUtils.rm(dataLocation.resolve(STATE_DIR_NAME));
456425
}
457426

458427
public String getPrefix() {

0 commit comments

Comments
 (0)