Skip to content

Commit 459d8ed

Browse files
original-brownbearoriginal-brownbear
authored
Make BlobStoreRepository Aware of ClusterState (#49639)
This is a preliminary to #49060. It does not introduce any substantial behavior change to how the blob store repository operates. What it does is to add all the infrastructure changes around passing the cluster service to the blob store, associated test changes and a best effort approach to tracking the latest repository generation on all nodes from cluster state updates. This brings a slight improvement to the consistency by which non-master nodes (or master directly after a failover) will be able to determine the latest repository generation. It does not however do any tricky checks for the situation after a repository operation (create, delete or cleanup) that could theoretically be used to get even greater accuracy to keep this change simple. This change does not in any way alter the behavior of the blobstore repository other than adding a better "guess" for the value of the latest repo generation and is mainly intended to isolate the actual logical change to how the repository operates in #49060
1 parent cb3ba97 commit 459d8ed

46 files changed

Lines changed: 329 additions & 153 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@
1919

2020
package org.elasticsearch.plugin.repository.url;
2121

22+
import org.elasticsearch.cluster.service.ClusterService;
2223
import org.elasticsearch.common.settings.Setting;
2324
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2425
import org.elasticsearch.env.Environment;
2526
import org.elasticsearch.plugins.Plugin;
2627
import org.elasticsearch.plugins.RepositoryPlugin;
2728
import org.elasticsearch.repositories.Repository;
2829
import org.elasticsearch.repositories.url.URLRepository;
29-
import org.elasticsearch.threadpool.ThreadPool;
3030

3131
import java.util.Arrays;
3232
import java.util.Collections;
@@ -46,8 +46,8 @@ public List<Setting<?>> getSettings() {
4646

4747
@Override
4848
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry,
49-
ThreadPool threadPool) {
49+
ClusterService clusterService) {
5050
return Collections.singletonMap(URLRepository.TYPE,
51-
metadata -> new URLRepository(metadata, env, namedXContentRegistry, threadPool));
51+
metadata -> new URLRepository(metadata, env, namedXContentRegistry, clusterService));
5252
}
5353
}

‎modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.logging.log4j.LogManager;
2323
import org.apache.logging.log4j.Logger;
2424
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
25+
import org.elasticsearch.cluster.service.ClusterService;
2526
import org.elasticsearch.common.blobstore.BlobContainer;
2627
import org.elasticsearch.common.blobstore.BlobPath;
2728
import org.elasticsearch.common.blobstore.BlobStore;
@@ -33,7 +34,6 @@
3334
import org.elasticsearch.env.Environment;
3435
import org.elasticsearch.repositories.RepositoryException;
3536
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
36-
import org.elasticsearch.threadpool.ThreadPool;
3737

3838
import java.net.MalformedURLException;
3939
import java.net.URISyntaxException;
@@ -81,8 +81,8 @@ public class URLRepository extends BlobStoreRepository {
8181
* Constructs a read-only URL-based repository
8282
*/
8383
public URLRepository(RepositoryMetaData metadata, Environment environment,
84-
NamedXContentRegistry namedXContentRegistry, ThreadPool threadPool) {
85-
super(metadata, namedXContentRegistry, threadPool, BlobPath.cleanPath());
84+
NamedXContentRegistry namedXContentRegistry, ClusterService clusterService) {
85+
super(metadata, namedXContentRegistry, clusterService, BlobPath.cleanPath());
8686

8787
if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(environment.settings()) == false) {
8888
throw new RepositoryException(metadata.name(), "missing url");

‎modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
import org.elasticsearch.env.Environment;
2626
import org.elasticsearch.env.TestEnvironment;
2727
import org.elasticsearch.repositories.RepositoryException;
28+
import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
2829
import org.elasticsearch.test.ESTestCase;
29-
import org.elasticsearch.threadpool.ThreadPool;
3030

3131
import java.io.IOException;
3232
import java.nio.file.Path;
@@ -35,13 +35,12 @@
3535
import static org.hamcrest.CoreMatchers.is;
3636
import static org.hamcrest.CoreMatchers.not;
3737
import static org.hamcrest.CoreMatchers.nullValue;
38-
import static org.mockito.Mockito.mock;
3938

4039
public class URLRepositoryTests extends ESTestCase {
4140

4241
private URLRepository createRepository(Settings baseSettings, RepositoryMetaData repositoryMetaData) {
4342
return new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
44-
new NamedXContentRegistry(Collections.emptyList()), mock(ThreadPool.class)) {
43+
new NamedXContentRegistry(Collections.emptyList()), BlobStoreTestUtil.mockClusterService()) {
4544
@Override
4645
protected void assertSnapshotOrGenericThread() {
4746
// eliminate thread name check as we create repo manually on test/main threads

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.logging.log4j.Logger;
2525
import org.apache.logging.log4j.message.ParameterizedMessage;
2626
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
27+
import org.elasticsearch.cluster.service.ClusterService;
2728
import org.elasticsearch.common.Strings;
2829
import org.elasticsearch.common.blobstore.BlobPath;
2930
import org.elasticsearch.common.blobstore.BlobStore;
@@ -32,7 +33,6 @@
3233
import org.elasticsearch.common.unit.ByteSizeValue;
3334
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3435
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
35-
import org.elasticsearch.threadpool.ThreadPool;
3636

3737
import java.util.Locale;
3838
import java.util.function.Function;
@@ -79,8 +79,8 @@ public AzureRepository(
7979
final RepositoryMetaData metadata,
8080
final NamedXContentRegistry namedXContentRegistry,
8181
final AzureStorageService storageService,
82-
final ThreadPool threadPool) {
83-
super(metadata, namedXContentRegistry, threadPool, buildBasePath(metadata));
82+
final ClusterService clusterService) {
83+
super(metadata, namedXContentRegistry, clusterService, buildBasePath(metadata));
8484
this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
8585
this.storageService = storageService;
8686

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepositoryPlugin.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22+
import org.elasticsearch.cluster.service.ClusterService;
2223
import org.elasticsearch.common.settings.Setting;
2324
import org.elasticsearch.common.settings.Settings;
2425
import org.elasticsearch.common.settings.SettingsException;
@@ -31,7 +32,6 @@
3132
import org.elasticsearch.repositories.Repository;
3233
import org.elasticsearch.threadpool.ExecutorBuilder;
3334
import org.elasticsearch.threadpool.ScalingExecutorBuilder;
34-
import org.elasticsearch.threadpool.ThreadPool;
3535

3636
import java.util.Arrays;
3737
import java.util.Collections;
@@ -60,9 +60,9 @@ AzureStorageService createAzureStoreService(final Settings settings) {
6060

6161
@Override
6262
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry,
63-
ThreadPool threadPool) {
63+
ClusterService clusterService) {
6464
return Collections.singletonMap(AzureRepository.TYPE,
65-
(metadata) -> new AzureRepository(metadata, namedXContentRegistry, azureStoreService, threadPool));
65+
(metadata) -> new AzureRepository(metadata, namedXContentRegistry, azureStoreService, clusterService));
6666
}
6767

6868
@Override

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@
2626
import org.elasticsearch.common.unit.ByteSizeValue;
2727
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2828
import org.elasticsearch.env.Environment;
29+
import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil;
2930
import org.elasticsearch.test.ESTestCase;
30-
import org.elasticsearch.threadpool.ThreadPool;
3131

3232
import static org.hamcrest.Matchers.is;
3333
import static org.hamcrest.Matchers.nullValue;
@@ -42,8 +42,7 @@ private AzureRepository azureRepository(Settings settings) {
4242
.put(settings)
4343
.build();
4444
final AzureRepository azureRepository = new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
45-
NamedXContentRegistry.EMPTY, mock(AzureStorageService.class),
46-
mock(ThreadPool.class));
45+
NamedXContentRegistry.EMPTY, mock(AzureStorageService.class), BlobStoreTestUtil.mockClusterService());
4746
assertThat(azureRepository.getBlobStore(), is(nullValue()));
4847
return azureRepository;
4948
}

‎plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.repositories.gcs;
2121

22+
import org.elasticsearch.cluster.service.ClusterService;
2223
import org.elasticsearch.common.settings.Setting;
2324
import org.elasticsearch.common.settings.Settings;
2425
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -27,7 +28,6 @@
2728
import org.elasticsearch.plugins.ReloadablePlugin;
2829
import org.elasticsearch.plugins.RepositoryPlugin;
2930
import org.elasticsearch.repositories.Repository;
30-
import org.elasticsearch.threadpool.ThreadPool;
3131

3232
import java.util.Arrays;
3333
import java.util.Collections;
@@ -52,9 +52,9 @@ protected GoogleCloudStorageService createStorageService() {
5252

5353
@Override
5454
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry,
55-
ThreadPool threadPool) {
55+
ClusterService clusterService) {
5656
return Collections.singletonMap(GoogleCloudStorageRepository.TYPE,
57-
metadata -> new GoogleCloudStorageRepository(metadata, namedXContentRegistry, this.storageService, threadPool));
57+
metadata -> new GoogleCloudStorageRepository(metadata, namedXContentRegistry, this.storageService, clusterService));
5858
}
5959

6060
@Override

‎plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.logging.log4j.LogManager;
2323
import org.apache.logging.log4j.Logger;
2424
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
25+
import org.elasticsearch.cluster.service.ClusterService;
2526
import org.elasticsearch.common.Strings;
2627
import org.elasticsearch.common.blobstore.BlobPath;
2728
import org.elasticsearch.common.settings.Setting;
@@ -30,7 +31,6 @@
3031
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3132
import org.elasticsearch.repositories.RepositoryException;
3233
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
33-
import org.elasticsearch.threadpool.ThreadPool;
3434

3535
import java.util.function.Function;
3636

@@ -64,8 +64,8 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
6464
final RepositoryMetaData metadata,
6565
final NamedXContentRegistry namedXContentRegistry,
6666
final GoogleCloudStorageService storageService,
67-
final ThreadPool threadPool) {
68-
super(metadata, namedXContentRegistry, threadPool, buildBasePath(metadata));
67+
final ClusterService clusterService) {
68+
super(metadata, namedXContentRegistry, clusterService, buildBasePath(metadata));
6969
this.storageService = storageService;
7070

7171
this.chunkSize = getSetting(CHUNK_SIZE, metadata);

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import fixture.gcs.FakeOAuth2HttpHandler;
2828
import fixture.gcs.GoogleCloudStorageHttpHandler;
2929
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
30+
import org.elasticsearch.cluster.service.ClusterService;
3031
import org.elasticsearch.common.SuppressForbidden;
3132
import org.elasticsearch.common.io.Streams;
3233
import org.elasticsearch.common.settings.MockSecureSettings;
@@ -38,7 +39,6 @@
3839
import org.elasticsearch.plugins.Plugin;
3940
import org.elasticsearch.repositories.Repository;
4041
import org.elasticsearch.repositories.blobstore.ESMockAPIBasedRepositoryIntegTestCase;
41-
import org.elasticsearch.threadpool.ThreadPool;
4242
import org.threeten.bp.Duration;
4343

4444
import java.io.IOException;
@@ -170,9 +170,10 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien
170170
}
171171

172172
@Override
173-
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry registry, ThreadPool threadPool) {
173+
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry registry,
174+
ClusterService clusterService) {
174175
return Collections.singletonMap(GoogleCloudStorageRepository.TYPE,
175-
metadata -> new GoogleCloudStorageRepository(metadata, registry, this.storageService, threadPool) {
176+
metadata -> new GoogleCloudStorageRepository(metadata, registry, this.storageService, clusterService) {
176177
@Override
177178
protected GoogleCloudStorageBlobStore createBlobStore() {
178179
return new GoogleCloudStorageBlobStore("bucket", "test", storageService) {

‎plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,13 @@
3030
import org.apache.hadoop.security.KerberosInfo;
3131
import org.apache.hadoop.security.SecurityUtil;
3232
import org.elasticsearch.SpecialPermission;
33+
import org.elasticsearch.cluster.service.ClusterService;
3334
import org.elasticsearch.common.SuppressForbidden;
3435
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3536
import org.elasticsearch.env.Environment;
3637
import org.elasticsearch.plugins.Plugin;
3738
import org.elasticsearch.plugins.RepositoryPlugin;
3839
import org.elasticsearch.repositories.Repository;
39-
import org.elasticsearch.threadpool.ThreadPool;
4040

4141
public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
4242

@@ -112,7 +112,7 @@ private static Void eagerInit() {
112112

113113
@Override
114114
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry,
115-
ThreadPool threadPool) {
116-
return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env, namedXContentRegistry, threadPool));
115+
ClusterService clusterService) {
116+
return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env, namedXContentRegistry, clusterService));
117117
}
118118
}

0 commit comments

Comments
 (0)