The test fails with
org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
at __randomizedtesting.SeedInfo.seed([37BFB39821208C41:7E43DFB03F6667E0]:0)
at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:349)
at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:669)
at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:589)
at org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:531)
at org.apache.lucene.index.IndexWriter.checkpoint(IndexWriter.java:2563)
at org.apache.lucene.index.IndexWriter.publishFlushedSegment(IndexWriter.java:2641)
at org.apache.lucene.index.IndexWriter.lambda$publishFlushedSegments$10(IndexWriter.java:5079)
at org.apache.lucene.index.DocumentsWriterFlushQueue.innerPurge(DocumentsWriterFlushQueue.java:115)
at org.apache.lucene.index.DocumentsWriterFlushQueue.forcePurge(DocumentsWriterFlushQueue.java:136)
at org.apache.lucene.index.DocumentsWriter.purgeFlushTickets(DocumentsWriter.java:199)
at org.apache.lucene.index.IndexWriter.publishFlushedSegments(IndexWriter.java:5059)
at org.apache.lucene.index.IndexWriter.rollbackInternalNoCommit(IndexWriter.java:2292)
at org.apache.lucene.index.IndexWriter.rollbackInternal(IndexWriter.java:2266)
at org.apache.lucene.index.IndexWriter.maybeCloseOnTragicEvent(IndexWriter.java:4907)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1610)
at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1213)
at org.elasticsearch.index.engine.InternalEngine.addDocs(InternalEngine.java:1105)
at org.elasticsearch.index.engine.InternalEngine.indexIntoLucene(InternalEngine.java:1050)
at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:884)
at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:790)
at org.elasticsearch.index.shard.IndexShard.applyIndexOperation(IndexShard.java:763)
at org.elasticsearch.index.shard.IndexShard.applyIndexOperationOnPrimary(IndexShard.java:720)
at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:263)
at org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:159)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:197)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:116)
at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:77)
at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:927)
at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:104)
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.runWithPrimaryShardReference(TransportReplicationAction.java:364)
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.lambda$doRun$0(TransportReplicationAction.java:311)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:62)
at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:269)
at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:236)
at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2513)
at org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryOperationPermit(TransportReplicationAction.java:865)
at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:310)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at org.elasticsearch.action.support.replication.TransportReplicationAction.handlePrimaryRequest(TransportReplicationAction.java:273)
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:703)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:747)
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.lang.Thread.run(Thread.java:834)
Caused by: java.io.FileNotFoundException: a random IOException (_2.fdt)
at org.apache.lucene.store.MockDirectoryWrapper.maybeThrowIOExceptionOnOpen(MockDirectoryWrapper.java:577)
at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:645)
at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:74)
at org.elasticsearch.index.store.ByteSizeCachingDirectory.createOutput(ByteSizeCachingDirectory.java:130)
at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:74)
at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44)
at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:115)
at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183)
at org.apache.lucene.index.StoredFieldsConsumer.initStoredFieldsWriter(StoredFieldsConsumer.java:39)
at org.apache.lucene.index.StoredFieldsConsumer.startDocument(StoredFieldsConsumer.java:46)
at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:366)
at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:402)
at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:251)
at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:494)
at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1594)
... 32 more
Does not reproduce:
./gradlew :server:integTest \
-Dtests.seed=37BFB39821208C41 \
-Dtests.class=org.elasticsearch.search.basic.SearchWithRandomIOExceptionsIT \
-Dtests.method="testRandomDirectoryIOExceptions" \
-Dtests.security.manager=true \
-Dtests.locale=en-MY \
-Dtests.timezone=America/St_Kitts \
-Dcompiler.java=12 \
-Druntime.java=11
There have been a number of failures on the master branch
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+internalClusterTest/4690/console
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+internalClusterTest/4683/console
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+multijob-unix-compatibility/os=centos-7&&immutable/320/console
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+intake/3000/console
The test fails with
Does not reproduce:
There have been a number of failures on the master branch
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+internalClusterTest/4690/console
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+internalClusterTest/4683/console
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+multijob-unix-compatibility/os=centos-7&&immutable/320/console
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+intake/3000/console