Skip to content

Commit e4f4817

Browse files
pawel-gudel-eliatrapawel-gudel-eliatra
authored
[Enhancement] Parallel test jobs for CI (#2861)
* Split multiple tests into separate gradle tasks. * Tasks are configured in "splitTestConfig" map in build.gradle file. Map allows to use all patterns from TestFilter like: includeTestsMatching, excludeTestsMatching, includeTest etc. * Tasks are automatically generated from "splitTestConfig" map. * Two new Gradle tasks: listTasksAsJSON and listTasksAsParam to output task names to console. First one outputs them as a JSON and second - in gradlew "-x <TASK>" format to use in CLI. * Patterns included in tasks are automatically excluded from main "test" task but at the same time generated tasks are dependencies for "test". Running "gradlew test" will run whole suite at once. * CI pipeline has been configured to accomodate all changes. * New 'master' task to generate list of jobs to run in parallel. * Updated matrix strategy to include task name to start. Signed-off-by: Pawel Gudel <pawel.gudel@eliatra.com>
1 parent ab1afce commit e4f4817

2 files changed

Lines changed: 173 additions & 27 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,34 @@ env:
66
GRADLE_OPTS: -Dhttp.keepAlive=false
77

88
jobs:
9-
build:
10-
name: build
9+
generate-test-list:
10+
runs-on: ubuntu-latest
11+
outputs:
12+
separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
13+
steps:
14+
- name: Set up JDK for build and test
15+
uses: actions/setup-java@v2
16+
with:
17+
distribution: temurin # Temurin is a distribution of adoptium
18+
java-version: 17
19+
20+
- name: Checkout security
21+
uses: actions/checkout@v2
22+
23+
- name: Generate list of tasks
24+
id: set-matrix
25+
run: |
26+
echo "separateTestsNames=$(./gradlew listTasksAsJSON -q --console=plain | tail -n 1)" >> $GITHUB_OUTPUT
27+
28+
test:
29+
name: test
30+
needs: generate-test-list
1131
strategy:
1232
fail-fast: false
1333
matrix:
34+
gradle_task: ${{ fromJson(needs.generate-test-list.outputs.separateTestsNames) }}
35+
platform: [windows-latest, ubuntu-latest]
1436
jdk: [11, 17]
15-
platform: ["ubuntu-latest", "windows-latest"]
1637
runs-on: ${{ matrix.platform }}
1738

1839
steps:
@@ -29,12 +50,8 @@ jobs:
2950
uses: gradle/gradle-build-action@v2
3051
with:
3152
arguments: |
32-
build test -Dbuild.snapshot=false
33-
-x integrationTest
34-
-x spotlessCheck
35-
-x checkstyleMain
36-
-x checkstyleTest
37-
-x spotbugsMain
53+
${{ matrix.gradle_task }} -Dbuild.snapshot=false
54+
-x test
3855
3956
- name: Coverage
4057
uses: codecov/codecov-action@v1
@@ -59,7 +76,7 @@ jobs:
5976
fail-fast: false
6077
matrix:
6178
jdk: [17]
62-
platform: ["ubuntu-latest", "windows-latest"]
79+
platform: [ubuntu-latest, windows-latest]
6380
runs-on: ${{ matrix.platform }}
6481

6582
steps:
@@ -78,18 +95,13 @@ jobs:
7895
with:
7996
arguments: |
8097
integrationTest -Dbuild.snapshot=false
81-
-x spotlessCheck
82-
-x checkstyleMain
83-
-x checkstyleTest
84-
-x spotbugsMain
8598
8699
backward-compatibility:
87-
88100
strategy:
89101
fail-fast: false
90102
matrix:
91103
jdk: [11, 17]
92-
platform: ["ubuntu-latest", "windows-latest"]
104+
platform: [ubuntu-latest, windows-latest]
93105
runs-on: ${{ matrix.platform }}
94106

95107
steps:

‎build.gradle‎

Lines changed: 145 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
import com.diffplug.gradle.spotless.JavaExtension
1414
import org.opensearch.gradle.test.RestIntegTestTask
15+
import groovy.json.JsonBuilder
1516

1617
buildscript {
1718
ext {
@@ -105,12 +106,85 @@ tasks.whenTaskAdded {task ->
105106
}
106107
}
107108

109+
def splitTestConfig = [
110+
ciSecurityIntegrationTest: [
111+
description: "Runs integration tests from all classes.",
112+
filters: [
113+
includeTestsMatching: [
114+
"org.opensearch.security.*Integ*"
115+
],
116+
excludeTestsMatching: [
117+
"org.opensearch.security.sanity.tests.*"
118+
]
119+
]
120+
],
121+
crossClusterTest: [
122+
description: "Runs cross-cluster tests.",
123+
filters: [
124+
includeTestsMatching: [
125+
"org.opensearch.security.ccstest.*"
126+
]
127+
]
128+
],
129+
dlicDlsflsTest: [
130+
description: "Runs Document- and Field-Level Security tests.",
131+
filters: [
132+
includeTestsMatching: [
133+
"org.opensearch.security.dlic.dlsfls.*"
134+
]
135+
]
136+
],
137+
dlicRestApiTest: [
138+
description: "Runs REST Management API tests.",
139+
filters: [
140+
includeTestsMatching: [
141+
"org.opensearch.security.dlic.rest.*"
142+
]
143+
]
144+
],
145+
indicesTest: [
146+
description: "Runs indices tests from all classes.",
147+
filters: [
148+
includeTestsMatching: [
149+
"org.opensearch.security.*indices*"
150+
],
151+
excludeTestsMatching: [
152+
"org.opensearch.security.sanity.tests.*"
153+
]
154+
]
155+
],
156+
opensslCITest: [
157+
description: "Runs portion of SSL tests related to OpenSSL. Explained in https://github.com/opensearch-project/security/pull/2301",
158+
include: '**/OpenSSL*.class'
159+
],
160+
sslTest: [
161+
description: "Runs most of the SSL tests.",
162+
filters: [
163+
includeTestsMatching: [
164+
"org.opensearch.security.ssl.*"
165+
],
166+
excludeTestsMatching: [
167+
"org.opensearch.security.ssl.OpenSSL*"
168+
]
169+
]
170+
]
171+
] as ConfigObject
172+
173+
List<String> taskNames = splitTestConfig.keySet() as List
174+
175+
task listTasksAsJSON {
176+
// We are using `doLast` to explicitly specify when we
177+
// want this action to be started. Without it the output
178+
// is not shown at all or can be mixed with other outputs.
179+
doLast {
180+
System.out.println(new JsonBuilder(["citest"] + taskNames))
181+
}
182+
}
108183

109184
test {
110185
include '**/*.class'
111186
filter {
112187
excludeTestsMatching "org.opensearch.security.sanity.tests.*"
113-
excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
114188
}
115189
maxParallelForks = 8
116190
jvmArgs += "-Xmx3072m"
@@ -138,14 +212,37 @@ test {
138212
}
139213
}
140214

141-
//add new task that runs OpenSSL tests
142-
task opensslTest(type: Test) {
143-
include '**/OpenSSL*.class'
144-
retry {
215+
task copyExtraTestResources(dependsOn: testClasses) {
216+
217+
copy {
218+
from 'src/test/resources'
219+
into 'build/testrun/test/src/test/resources'
220+
}
221+
222+
taskNames.each { testName ->
223+
copy {
224+
from 'src/test/resources'
225+
into "build/testrun/${testName}/src/test/resources"
226+
}
227+
}
228+
229+
copy {
230+
from 'src/test/resources'
231+
into 'build/testrun/citest/src/test/resources'
232+
}
233+
}
234+
235+
def setCommonTestConfig(Test task) {
236+
task.maxParallelForks = 8
237+
task.jvmArgs += "-Xmx3072m"
238+
if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
239+
task.jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
240+
}
241+
task.retry {
145242
failOnPassedAfterRetry = false
146243
maxRetries = 5
147244
}
148-
jacoco {
245+
task.jacoco {
149246
excludes = [
150247
"com.sun.jndi.dns.*",
151248
"com.sun.security.sasl.gsskerb.*",
@@ -160,21 +257,58 @@ task opensslTest(type: Test) {
160257
"sun.util.resources.provider.*"
161258
]
162259
}
260+
task.dependsOn copyExtraTestResources
261+
task.finalizedBy jacocoTestReport
163262
}
164263

165-
task copyExtraTestResources(dependsOn: testClasses) {
166-
copy {
167-
from 'src/test/resources'
168-
into 'build/testrun/test/src/test/resources'
264+
task citest(type: Test) {
265+
group = "Github Actions tests"
266+
description = "Runs the test suite on classes not covered by rest of the task in this group."
267+
include '**/*.class'
268+
filter {
269+
excludeTestsMatching "org.opensearch.security.sanity.tests.*"
270+
excludeTestsMatching "org.opensearch.security.ssl.OpenSSL*"
271+
splitTestConfig.each { entry ->
272+
entry.value.filters.each{ test ->
273+
if (test.key == "includeTestsMatching") {
274+
test.value.each{
275+
excludeTestsMatching "${it}"
276+
}
277+
} else if (test.key == "includeTest") {
278+
test.value.each{
279+
excludeTest "${it}"
280+
}
281+
}
282+
}
283+
}
284+
}
285+
setCommonTestConfig(it)
286+
}
287+
288+
splitTestConfig.each{ testName, testCfg ->
289+
task "${testName}"(type: Test) {
290+
group = testCfg.group ?: "Github Actions tests"
291+
description = testCfg.description
292+
include testCfg.include ?: '**/*.class'
293+
filter {
294+
testCfg.filters.each{ filter, values ->
295+
values.each{ value ->
296+
"${filter}" "${value}"
297+
}
298+
}
299+
}
300+
setCommonTestConfig(it)
169301
}
170302
}
171-
tasks.test.dependsOn(copyExtraTestResources, opensslTest)
303+
304+
tasks.test.dependsOn(copyExtraTestResources)
172305

173306
jacoco {
174307
reportsDirectory = file("$buildDir/reports/jacoco")
175308
}
176309

177310
jacocoTestReport {
311+
getExecutionData().setFrom(fileTree(buildDir).include("/jacoco/*.exec"))
178312
reports {
179313
xml.required = true
180314
}

0 commit comments

Comments
 (0)