Skip to content

Commit 6fa1a25

Browse files
cwperkscwperks
committed
Fix flaky test SlowIntegrationTests.testDelayInSecurityIndexInitialization (#3763)
Signed-off-by: Craig Perkins <cwperx@amazon.com> (cherry picked from commit 7498eb0)
1 parent 3003149 commit 6fa1a25

5 files changed

Lines changed: 126 additions & 20 deletions

File tree

‎DEVELOPER_GUIDE.md‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,11 +225,30 @@ curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:admin' -
225225

226226
Launch IntelliJ IDEA, choose **Project from Existing Sources**, and select directory with Gradle build script (`build.gradle`).
227227

228-
## Running integration tests
228+
## Running tests
229229

230230
Locally these can be run with `./gradlew test` with detailed results being avaliable at `${project-root}/build/reports/tests/test/index.html`, or run through an IDEs JUnit test runner.
231231

232-
Integration tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are avaliable by going to the GitHub action workflow's summary view and downloading the associated jdk version run of the tests, after extracting this file onto your local machine integration tests results are at `./tests/tests/index.html`.
232+
Tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`.
233+
234+
### Running an individual test multiple times
235+
236+
This repo has a `@Repeat` annotation which you can import to annotate a test to run many times repeatedly. To use the annotation, add the following code to your test suite.
237+
238+
```
239+
@Rule
240+
public RepeatRule repeatRule = new RepeatRule();
241+
242+
@Test
243+
@Repeat(10)
244+
public void testMethod() {
245+
...
246+
}
247+
```
248+
249+
## Running tests in the integrationTest package
250+
251+
Tests in the integrationTest package can be run with `./gradlew integrationTest`.
233252
234253
### Bulk test runs
235254
To collect reliability data on test runs there is a manual GitHub action workflow called `Bulk Integration Test`. The workflow is started for a branch on this project or in a fork by going to [GitHub action workflows](https://github.com/opensearch-project/security/actions/workflows/integration-tests.yml) and selecting `Run Workflow`.

‎build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,9 @@ dependencies {
670670
testImplementation "org.springframework:spring-beans:${spring_version}"
671671
testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1'
672672
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1'
673+
testImplementation('org.awaitility:awaitility:4.2.0') {
674+
exclude(group: 'org.hamcrest', module: 'hamcrest')
675+
}
673676
// Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available
674677
if (osdetector.classifier in ["osx-x86_64", "osx-aarch_64", "linux-x86_64", "linux-aarch_64", "windows-x86_64"]) {
675678
testImplementation "io.netty:netty-tcnative-classes:2.0.61.Final"

‎src/test/java/org/opensearch/security/SlowIntegrationTests.java‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,14 @@
2929
import java.io.IOException;
3030

3131
import com.google.common.collect.Lists;
32-
import org.apache.http.HttpStatus;
32+
import org.awaitility.Awaitility;
3333
import org.junit.Assert;
3434
import org.junit.Test;
3535

3636
import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
3737
import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
3838
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
39+
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
3940
import org.opensearch.cluster.health.ClusterHealthStatus;
4041
import org.opensearch.common.settings.Settings;
4142
import org.opensearch.common.unit.TimeValue;
@@ -50,6 +51,9 @@
5051
import org.opensearch.security.test.helper.rest.RestHelper;
5152
import org.opensearch.transport.Netty4Plugin;
5253

54+
import static org.hamcrest.Matchers.equalTo;
55+
import static org.junit.Assert.assertThrows;
56+
5357
public class SlowIntegrationTests extends SingleClusterTest {
5458

5559
@Test
@@ -223,27 +227,34 @@ public void testDelayInSecurityIndexInitialization() throws Exception {
223227
.put(ConfigConstants.SECURITY_ALLOW_DEFAULT_INIT_SECURITYINDEX, true)
224228
.put("cluster.routing.allocation.exclude._ip", "127.0.0.1")
225229
.build();
226-
try {
230+
assertThrows(IOException.class, () -> {
227231
setup(Settings.EMPTY, null, settings, false);
228-
Assert.fail("Expected IOException here due to red cluster state");
229-
} catch (IOException e) {
230-
// Index request has a default timeout of 1 minute, adding buffer between nodes initialization and cluster health check
231-
Thread.sleep(1000 * 80);
232-
// Ideally, we would want to remove this cluster setting, but default settings cannot be removed. So overriding with a reserved
233-
// IP address
234232
clusterHelper.nodeClient()
235233
.admin()
236-
.cluster()
237-
.updateSettings(
238-
new ClusterUpdateSettingsRequest().transientSettings(
239-
Settings.builder().put("cluster.routing.allocation.exclude._ip", "192.0.2.0").build()
240-
)
241-
);
242-
this.clusterInfo = clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), 3);
243-
}
234+
.indices()
235+
.create(new CreateIndexRequest("test-index").timeout(TimeValue.timeValueSeconds(10)))
236+
.actionGet();
237+
clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(5), ClusterConfiguration.DEFAULT.getNodes());
238+
});
239+
// Ideally, we would want to remove this cluster setting, but default settings cannot be removed. So overriding with a reserved
240+
// IP address
241+
clusterHelper.nodeClient()
242+
.admin()
243+
.cluster()
244+
.updateSettings(
245+
new ClusterUpdateSettingsRequest().transientSettings(
246+
Settings.builder().put("cluster.routing.allocation.exclude._ip", "192.0.2.0").build()
247+
)
248+
);
249+
this.clusterInfo = clusterHelper.waitForCluster(ClusterHealthStatus.GREEN, TimeValue.timeValueSeconds(10), 3);
244250
RestHelper rh = nonSslRestHelper();
245-
Thread.sleep(10000);
246-
Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("", encodeBasicHeader("admin", "admin")).getStatusCode());
251+
Awaitility.await()
252+
.alias("Wait until Security is initialized")
253+
.until(
254+
() -> rh.executeGetRequest("/_plugins/_security/health", encodeBasicHeader("admin", "admin"))
255+
.getTextFromJsonBody("/status"),
256+
equalTo("UP")
257+
);
247258
}
248259

249260
}

‎src/test/java/org/opensearch/security/util/Repeat.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*
8+
* Modifications Copyright OpenSearch Contributors. See
9+
* GitHub history for details.
10+
*/
11+
12+
package org.opensearch.security.util;
13+
14+
import java.lang.annotation.Retention;
15+
import java.lang.annotation.RetentionPolicy;
16+
import java.lang.annotation.Target;
17+
18+
import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
19+
import static java.lang.annotation.ElementType.METHOD;
20+
21+
@Retention(RetentionPolicy.RUNTIME)
22+
@Target({ METHOD, ANNOTATION_TYPE })
23+
public @interface Repeat {
24+
int value() default 1;
25+
}

‎src/test/java/org/opensearch/security/util/RepeatRule.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*
8+
* Modifications Copyright OpenSearch Contributors. See
9+
* GitHub history for details.
10+
*/
11+
12+
package org.opensearch.security.util;
13+
14+
import org.junit.rules.TestRule;
15+
import org.junit.runner.Description;
16+
import org.junit.runners.model.Statement;
17+
18+
public class RepeatRule implements TestRule {
19+
20+
private static class RepeatStatement extends Statement {
21+
private final Statement statement;
22+
private final int repeat;
23+
24+
public RepeatStatement(Statement statement, int repeat) {
25+
this.statement = statement;
26+
this.repeat = repeat;
27+
}
28+
29+
@Override
30+
public void evaluate() throws Throwable {
31+
for (int i = 0; i < repeat; i++) {
32+
statement.evaluate();
33+
}
34+
}
35+
36+
}
37+
38+
@Override
39+
public Statement apply(Statement statement, Description description) {
40+
Statement result = statement;
41+
Repeat repeat = description.getAnnotation(Repeat.class);
42+
if (repeat != null) {
43+
int times = repeat.value();
44+
result = new RepeatStatement(statement, times);
45+
}
46+
return result;
47+
}
48+
}

0 commit comments

Comments
 (0)