Skip to content

Commit e680a64

Browse files
alpar-talpar-t
authored
cluster formation DSL - Gradle integration - part 2 (#32028)
* Implement Version in java - This allows to move all all .java files from .groovy. - Will prevent eclipse from tangling up in this setup - make it possible to use Version from Java * PR review comments * Cluster formation plugin with reference counting ``` > Task :plugins:ingest-user-agent:listElasticSearchClusters Starting cluster: myTestCluster * myTestCluster: /home/alpar/work/elastic/elasticsearch/plugins/ingest-user-agent/foo Asked to unClaimAndStop myTestCluster, since cluster still has 1 claim it will not be stopped > Task :plugins:ingest-user-agent:testme UP-TO-DATE Stopping myTestCluster, since no of claims is 0 ``` - Meant to auto manage the clusters lifecycle - Add integration test for cluster formation * Fix rebase * Change to `useCluster` method on task
1 parent 068d03f commit e680a64

10 files changed

Lines changed: 674 additions & 0 deletions

File tree

‎buildSrc/build.gradle‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,15 @@ plugins {
2323
id 'groovy'
2424
}
2525

26+
gradlePlugin {
27+
plugins {
28+
simplePlugin {
29+
id = 'elasticsearch.clusterformation'
30+
implementationClass = 'org.elasticsearch.gradle.clusterformation.ClusterformationPlugin'
31+
}
32+
}
33+
}
34+
2635
group = 'org.elasticsearch.gradle'
2736

2837
String minimumGradleVersion = file('src/main/resources/minimumGradleVersion').text.trim()

‎buildSrc/src/main/java/org/elasticsearch/GradleServicesAdapter.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch;
20+
21+
import org.gradle.api.Action;
22+
import org.gradle.api.Project;
23+
import org.gradle.api.file.CopySpec;
24+
import org.gradle.api.file.FileTree;
25+
import org.gradle.api.tasks.WorkResult;
26+
import org.gradle.process.ExecResult;
27+
import org.gradle.process.JavaExecSpec;
28+
29+
import java.io.File;
30+
31+
/**
32+
* Facilitate access to Gradle services without a direct dependency on Project.
33+
*
34+
* In a future release Gradle will offer service injection, this adapter plays that role until that time.
35+
* It exposes the service methods that are part of the public API as the classes implementing them are not.
36+
* Today service injection is <a href="https://github.com/gradle/gradle/issues/2363">not available</a> for
37+
* extensions.
38+
*
39+
* Everything exposed here must be thread safe. That is the very reason why project is not passed in directly.
40+
*/
41+
public class GradleServicesAdapter {
42+
43+
public final Project project;
44+
45+
public GradleServicesAdapter(Project project) {
46+
this.project = project;
47+
}
48+
49+
public static GradleServicesAdapter getInstance(Project project) {
50+
return new GradleServicesAdapter(project);
51+
}
52+
53+
public WorkResult copy(Action<? super CopySpec> action) {
54+
return project.copy(action);
55+
}
56+
57+
public WorkResult sync(Action<? super CopySpec> action) {
58+
return project.sync(action);
59+
}
60+
61+
public ExecResult javaexec(Action<? super JavaExecSpec> action) {
62+
return project.javaexec(action);
63+
}
64+
65+
public FileTree zipTree(File zipPath) {
66+
return project.zipTree(zipPath);
67+
}
68+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle;
20+
21+
public enum Distribution {
22+
23+
INTEG_TEST("integ-test-zip"),
24+
ZIP("zip"),
25+
ZIP_OSS("zip-oss");
26+
27+
private final String name;
28+
29+
Distribution(String name) {
30+
this.name = name;
31+
}
32+
33+
public String getName() {
34+
return name;
35+
}
36+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ClusterFormationTaskExecutionListener.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle.clusterformation;
20+
21+
import org.gradle.api.Task;
22+
import org.gradle.api.execution.TaskActionListener;
23+
import org.gradle.api.execution.TaskExecutionListener;
24+
import org.gradle.api.tasks.TaskState;
25+
26+
public class ClusterFormationTaskExecutionListener implements TaskExecutionListener, TaskActionListener {
27+
@Override
28+
public void afterExecute(Task task, TaskState state) {
29+
// always unclaim the cluster, even if _this_ task is up-to-date, as others might not have been and caused the
30+
// cluster to start.
31+
ClusterFormationTaskExtension.getForTask(task).getClaimedClusters().forEach(ElasticsearchConfiguration::unClaimAndStop);
32+
}
33+
34+
@Override
35+
public void beforeActions(Task task) {
36+
// we only start the cluster before the actions, so we'll not start it if the task is up-to-date
37+
ClusterFormationTaskExtension.getForTask(task).getClaimedClusters().forEach(ElasticsearchConfiguration::start);
38+
}
39+
40+
@Override
41+
public void beforeExecute(Task task) {
42+
}
43+
44+
@Override
45+
public void afterActions(Task task) {
46+
}
47+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ClusterFormationTaskExtension.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle.clusterformation;
20+
21+
import org.gradle.api.Task;
22+
import org.gradle.api.logging.Logger;
23+
import org.gradle.api.logging.Logging;
24+
25+
import java.util.ArrayList;
26+
import java.util.Collections;
27+
import java.util.List;
28+
29+
public class ClusterFormationTaskExtension {
30+
31+
private final Task task;
32+
33+
private final List<ElasticsearchConfiguration> claimedClusters = new ArrayList<>();
34+
35+
private final Logger logger = Logging.getLogger(ClusterFormationTaskExtension.class);
36+
37+
public ClusterFormationTaskExtension(Task task) {
38+
this.task = task;
39+
}
40+
41+
public void call(ElasticsearchConfiguration cluster) {
42+
// not likely to configure the same task from multiple threads as of Gradle 4.7, but it's the right thing to do
43+
synchronized (claimedClusters) {
44+
if (claimedClusters.contains(cluster)) {
45+
logger.warn("{} already claimed cluster {} will not claim it again",
46+
task.getPath(), cluster.getName()
47+
);
48+
return;
49+
}
50+
claimedClusters.add(cluster);
51+
}
52+
logger.info("CF: the {} task will use cluster: {}", task.getName(), cluster.getName());
53+
}
54+
55+
public List<ElasticsearchConfiguration> getClaimedClusters() {
56+
synchronized (claimedClusters) {
57+
return Collections.unmodifiableList(claimedClusters);
58+
}
59+
}
60+
61+
static ClusterFormationTaskExtension getForTask(Task task) {
62+
return task.getExtensions().getByType(ClusterFormationTaskExtension.class);
63+
}
64+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ClusterformationPlugin.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle.clusterformation;
20+
21+
import groovy.lang.Closure;
22+
import org.elasticsearch.GradleServicesAdapter;
23+
import org.gradle.api.NamedDomainObjectContainer;
24+
import org.gradle.api.Plugin;
25+
import org.gradle.api.Project;
26+
import org.gradle.api.Task;
27+
import org.gradle.api.logging.Logger;
28+
import org.gradle.api.logging.Logging;
29+
import org.gradle.api.plugins.ExtraPropertiesExtension;
30+
31+
public class ClusterformationPlugin implements Plugin<Project> {
32+
33+
public static final String LIST_TASK_NAME = "listElasticSearchClusters";
34+
public static final String EXTENSION_NAME = "elasticSearchClusters";
35+
public static final String TASK_EXTENSION_NAME = "useClusterExt";
36+
37+
private final Logger logger = Logging.getLogger(ClusterformationPlugin.class);
38+
39+
@Override
40+
public void apply(Project project) {
41+
NamedDomainObjectContainer<? extends ElasticsearchConfiguration> container = project.container(
42+
ElasticsearchNode.class,
43+
(name) -> new ElasticsearchNode(name, GradleServicesAdapter.getInstance(project))
44+
);
45+
project.getExtensions().add(EXTENSION_NAME, container);
46+
47+
Task listTask = project.getTasks().create(LIST_TASK_NAME);
48+
listTask.setGroup("ES cluster formation");
49+
listTask.setDescription("Lists all ES clusters configured for this project");
50+
listTask.doLast((Task task) ->
51+
container.forEach((ElasticsearchConfiguration cluster) ->
52+
logger.lifecycle(" * {}: {}", cluster.getName(), cluster.getDistribution())
53+
)
54+
);
55+
56+
// register an extension for all current and future tasks, so that any task can declare that it wants to use a
57+
// specific cluster.
58+
project.getTasks().all((Task task) -> {
59+
ClusterFormationTaskExtension taskExtension = task.getExtensions().create(
60+
TASK_EXTENSION_NAME, ClusterFormationTaskExtension.class, task
61+
);
62+
// Gradle doesn't look for extensions that might implement `call` and instead only looks for task methods
63+
// work around by creating a closure in the extra properties extensions which does work
64+
task.getExtensions().findByType(ExtraPropertiesExtension.class)
65+
.set(
66+
"useCluster",
67+
new Closure<Void>(this, this) {
68+
public void doCall(ElasticsearchConfiguration conf) {
69+
taskExtension.call(conf);
70+
}
71+
}
72+
);
73+
});
74+
75+
// Make sure we only claim the clusters for the tasks that will actually execute
76+
project.getGradle().getTaskGraph().whenReady(taskExecutionGraph ->
77+
taskExecutionGraph.getAllTasks().forEach(
78+
task -> ClusterFormationTaskExtension.getForTask(task).getClaimedClusters().forEach(
79+
ElasticsearchConfiguration::claim
80+
)
81+
)
82+
);
83+
84+
// create the listener to start the clusters on-demand and terminate when no longer claimed.
85+
// we need to use a task execution listener, as tasl
86+
project.getGradle().addListener(new ClusterFormationTaskExecutionListener());
87+
}
88+
89+
}

‎buildSrc/src/main/java/org/elasticsearch/gradle/clusterformation/ElasticsearchConfiguration.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.gradle.clusterformation;
20+
21+
import org.elasticsearch.gradle.Distribution;
22+
import org.elasticsearch.gradle.Version;
23+
24+
import java.util.concurrent.Future;
25+
26+
public interface ElasticsearchConfiguration {
27+
String getName();
28+
29+
Version getVersion();
30+
31+
void setVersion(Version version);
32+
33+
default void setVersion(String version) {
34+
setVersion(Version.fromString(version));
35+
}
36+
37+
Distribution getDistribution();
38+
39+
void setDistribution(Distribution distribution);
40+
41+
void claim();
42+
43+
Future<Void> start();
44+
45+
void unClaimAndStop();
46+
}

0 commit comments

Comments
 (0)