Skip to content

Commit 4faa839

Browse files
dougqhdougqh
authored
Merge branch 'master' into dougqh/conflating-metrics-background-work
2 parents d1df95e + 970f5ee commit 4faa839

75 files changed

Lines changed: 792 additions & 411 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/pull_request_template.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,14 @@
1212
Use `solves` instead, and assign the PR [milestone](https://github.com/DataDog/dd-trace-java/milestones) to the issue
1313
- Update the [CODEOWNERS](https://github.com/DataDog/dd-trace-java/blob/master/.github/CODEOWNERS) file on source file addition, migration, or deletion
1414
- Update [public documentation](https://docs.datadoghq.com/tracing/trace_collection/library_config/java/) with any new configuration flags or behaviors
15+
- Add your completed PR to the merge queue by commenting `/merge`. You can also:
16+
- Customize the commit message associated with the merge with `/merge --commit-message "..."`
17+
- Remove your PR from the merge queue with `/merge -c`
18+
- Skip all merge queue checks with `/merge -f --reason "reason"`; please use this judiciously, as some checks do not run at the PR-level
19+
- Get more information in [this doc](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/3121612126/MergeQueue)
1520

1621
Jira ticket: [PROJ-IDENT]
1722

18-
***Note:*** **Once your PR is ready to merge, add it to the merge queue by commenting `/merge`.** `/merge -c` cancels the queue request. `/merge -f --reason "reason"` skips all merge queue checks; please use this judiciously, as some checks do not run at the PR-level. For more information, see [this doc](https://datadoghq.atlassian.net/wiki/spaces/DEVX/pages/3121612126/MergeQueue).
19-
2023
<!--
2124
# Opening vs Drafting a PR:
2225
When opening a pull request, please open it as a draft to not auto assign reviewers before you feel the pull request is in a reviewable state.

‎.github/workflows/analyze-changes.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030
${{ runner.os }}-gradle-
3131
3232
- name: Initialize CodeQL
33-
uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
33+
uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5
3434
with:
3535
languages: 'java'
3636
build-mode: 'manual'
@@ -43,7 +43,7 @@ jobs:
4343
./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=4
4444
4545
- name: Perform CodeQL Analysis and upload results to GitHub Security tab
46-
uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
46+
uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5
4747

4848
trivy:
4949
name: Analyze changes with Trivy
@@ -102,7 +102,7 @@ jobs:
102102
TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
103103

104104
- name: Upload Trivy scan results to GitHub Security tab
105-
uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4
105+
uses: github/codeql-action/upload-sarif@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5
106106
if: always()
107107
with:
108108
sarif_file: 'trivy-results.sarif'

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ target
88
#########
99
!**/gradle/wrapper/*
1010
/.gradle
11-
*/.gradle
11+
**/.gradle
1212
**/build/
1313
examples/**/build/
1414

‎.gitlab-ci.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,14 @@ default:
229229
# replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties
230230
- sed -i "s|https://repo.maven.apache.org/maven2/|$MAVEN_REPOSITORY_PROXY|g" .mvn/wrapper/maven-wrapper.properties
231231
- mkdir -p .mvn/caches
232+
# Redirect Spotless's Equo/Solstice P2 cache into the project tree so it is captured by the GitLab cache.
233+
# Solstice (https://github.com/equodev/equo-ide) defaults to ~/.m2/repository/dev/equo/p2-data, which is outside $CI_PROJECT_DIR.
234+
- |
235+
mkdir -p .mvn/caches/equo "$HOME/.m2/repository/dev"
236+
if [ ! -L "$HOME/.m2/repository/dev/equo" ]; then
237+
rm -rf "$HOME/.m2/repository/dev/equo"
238+
ln -s "$(pwd)/.mvn/caches/equo" "$HOME/.m2/repository/dev/equo"
239+
fi
232240
- export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
233241
- export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
234242
- *normalize_node_index
@@ -369,6 +377,9 @@ populate_dep_cache:
369377
CACHE_TYPE: "latestdep"
370378
- GRADLE_TARGET: ":smokeTest"
371379
CACHE_TYPE: "smoke"
380+
- GRADLE_TARGET: "spotlessCheck"
381+
CACHE_TYPE: "spotless"
382+
GRADLE_MEMORY_MAX: "6G"
372383

373384
publish-artifacts-to-s3:
374385
image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
@@ -408,6 +419,7 @@ spotless:
408419
needs: []
409420
variables:
410421
GRADLE_MEMORY_MAX: 6G
422+
CACHE_TYPE: "spotless"
411423
script:
412424
- ./gradlew --version
413425
- ./gradlew spotlessCheck $GRADLE_ARGS

‎.gitlab/java-benchmark-configs.yml‎

Lines changed: 22 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,26 @@
1818
interruptible: true
1919
allow_failure: true
2020

21+
# Startup benchmarks run on PRs as well
22+
.parallel_startup_benchmark_rules: &parallel_startup_benchmark_rules
23+
- if: '$POPULATE_CACHE'
24+
when: never
25+
- if: '$CI_COMMIT_TAG =~ /^v?[0-9]+\.[0-9]+\.[0-9]+$/'
26+
when: manual
27+
allow_failure: true
28+
- if: '$CI_COMMIT_BRANCH == "master"'
29+
when: on_success
30+
interruptible: false
31+
- if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
32+
when: on_success
33+
interruptible: true
34+
- if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue\//'
35+
when: on_success
36+
interruptible: true
37+
- when: on_success
38+
interruptible: true
39+
allow_failure: true
40+
2141
# Ensure the tracer artifact publish finishes before the benchmark jobs start.
2242
linux-java-spring-petclinic-parallel:
2343
needs: ["publish-artifacts-to-s3"]
@@ -32,11 +52,11 @@ linux-java-spring-petclinic-load-parallel:
3252

3353
linux-java-insecure-bank-startup-parallel:
3454
needs: ["publish-artifacts-to-s3"]
35-
rules: *parallel_benchmark_rules
55+
rules: *parallel_startup_benchmark_rules
3656

3757
linux-java-spring-petclinic-startup-parallel:
3858
needs: ["publish-artifacts-to-s3"]
39-
rules: *parallel_benchmark_rules
59+
rules: *parallel_startup_benchmark_rules
4060

4161
linux-java-dacapo-parallel-1:
4262
needs: ["publish-artifacts-to-s3"]
@@ -45,12 +65,3 @@ linux-java-dacapo-parallel-1:
4565
linux-java-dacapo-parallel-2:
4666
needs: ["publish-artifacts-to-s3"]
4767
rules: *parallel_benchmark_rules
48-
49-
java-startup-parallel-check-slo-breaches:
50-
rules: *parallel_benchmark_rules
51-
52-
java-load-parallel-check-slo-breaches:
53-
rules: *parallel_benchmark_rules
54-
55-
java-dacapo-parallel-check-slo-breaches:
56-
rules: *parallel_benchmark_rules

‎buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzleMavenRepoUtils.kt‎

Lines changed: 125 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,21 @@ import org.eclipse.aether.connector.basic.BasicRepositoryConnectorFactory
99
import org.eclipse.aether.repository.LocalRepository
1010
import org.eclipse.aether.repository.RemoteRepository
1111
import org.eclipse.aether.resolution.VersionRangeRequest
12+
import org.eclipse.aether.resolution.VersionRangeResolutionException
1213
import org.eclipse.aether.resolution.VersionRangeResult
1314
import org.eclipse.aether.spi.connector.RepositoryConnectorFactory
1415
import org.eclipse.aether.spi.connector.transport.TransporterFactory
1516
import org.eclipse.aether.transport.file.FileTransporterFactory
1617
import org.eclipse.aether.transport.http.HttpTransporterFactory
1718
import org.eclipse.aether.version.Version
1819
import org.gradle.api.GradleException
20+
import org.gradle.api.logging.Logging
1921
import java.nio.file.Files
2022

2123
internal object MuzzleMavenRepoUtils {
24+
private val log = Logging.getLogger(MuzzleMavenRepoUtils::class.java)
25+
private val backoffDelaysSeconds = listOf(5L, 10L, 30L)
26+
2227
/**
2328
* Remote repositories used to query version ranges and fetch dependencies.
2429
*
@@ -122,12 +127,15 @@ internal object MuzzleMavenRepoUtils {
122127
/**
123128
* Resolves the version range for a given MuzzleDirective using the provided RepositorySystem and RepositorySystemSession.
124129
* Equivalent to the Groovy implementation in MuzzlePlugin.
130+
*
131+
* @param enableBackoffRetries if true, waits 5s, 10s, and 30s after the first three immediate retries
125132
*/
126133
fun resolveVersionRange(
127134
muzzleDirective: MuzzleDirective,
128135
system: RepositorySystem,
129136
session: RepositorySystemSession,
130-
defaultRepos: List<RemoteRepository> = defaultMuzzleRepos()
137+
defaultRepos: List<RemoteRepository> = defaultMuzzleRepos(),
138+
enableBackoffRetries: Boolean = true
131139
): VersionRangeResult {
132140
val directiveArtifact: Artifact = DefaultArtifact(
133141
muzzleDirective.group,
@@ -142,16 +150,56 @@ internal object MuzzleMavenRepoUtils {
142150
}
143151

144152
// In rare cases, the version resolution range silently failed with the maven proxy,
145-
// retries 3 times at most then suggest to restart the job later.
146-
var range = system.resolveVersionRange(session, rangeRequest)
147-
for (i in 0..3) {
148-
if (range.lowestVersion != null && range.highestVersion != null) {
153+
// retries 3 times immediately, then backs off before suggesting to restart the job later.
154+
var attemptCount = 0
155+
var range: VersionRangeResult? = null
156+
var failure: VersionRangeResolutionException? = null
157+
fun attemptResolve(): VersionRangeResult? {
158+
attemptCount++
159+
return try {
160+
range = system.resolveVersionRange(session, rangeRequest)
161+
failure = null
162+
range?.takeIf { it.hasBounds() }
163+
} catch (e: VersionRangeResolutionException) {
164+
failure = e
165+
range = e.result ?: range
166+
null
167+
}
168+
}
169+
170+
repeat(4) {
171+
attemptResolve()?.let { range ->
149172
return range
150173
}
151-
range = system.resolveVersionRange(session, rangeRequest)
152174
}
153175

154-
throw IllegalStateException("The version range resolution failed during report, this is not expected. Advised course of action: Restart the job later.")
176+
var waitedSeconds = 0L
177+
if (enableBackoffRetries) {
178+
for (delaySeconds in backoffDelaysSeconds) {
179+
sleepBeforeBackoffRetry(delaySeconds, directiveArtifact)
180+
waitedSeconds += delaySeconds
181+
attemptResolve()?.let { resolvedRange ->
182+
log.warn(
183+
"Muzzle version range resolution for ${artifactCoordinates(directiveArtifact)} " +
184+
"succeeded after waiting ${waitedSeconds}s across $attemptCount attempts"
185+
)
186+
return resolvedRange
187+
}
188+
}
189+
}
190+
191+
throw IllegalStateException(
192+
versionRangeFailureMessage(
193+
directiveArtifact,
194+
rangeRequest.repositories,
195+
range,
196+
failure,
197+
attemptCount,
198+
waitedSeconds,
199+
enableBackoffRetries
200+
),
201+
failure
202+
)
155203
}
156204

157205
/**
@@ -205,6 +253,76 @@ internal object MuzzleMavenRepoUtils {
205253
*/
206254
fun lowest(a: Version, b: Version): Version = if (a < b) a else b
207255

256+
private fun VersionRangeResult.hasBounds(): Boolean =
257+
lowestVersion != null && highestVersion != null
258+
259+
private fun sleepBeforeBackoffRetry(delaySeconds: Long, artifact: Artifact) {
260+
try {
261+
Thread.sleep(delaySeconds * 1000L)
262+
} catch (e: InterruptedException) {
263+
Thread.currentThread().interrupt()
264+
throw IllegalStateException(
265+
"Interrupted while waiting ${delaySeconds}s before retrying version range resolution for " +
266+
artifactCoordinates(artifact),
267+
e
268+
)
269+
}
270+
}
271+
272+
private fun versionRangeFailureMessage(
273+
artifact: Artifact,
274+
repositories: List<RemoteRepository>,
275+
range: VersionRangeResult?,
276+
failure: VersionRangeResolutionException?,
277+
attemptCount: Int,
278+
waitedSeconds: Long,
279+
enableBackoffRetries: Boolean
280+
): String {
281+
val backoffDetails =
282+
if (enableBackoffRetries) {
283+
"enabled; waited ${waitedSeconds}s using delays ${backoffDelaysSeconds.joinToString(", ") { "${it}s" }}"
284+
} else {
285+
"disabled"
286+
}
287+
return buildString {
288+
appendLine("Muzzle version range resolution failed.")
289+
appendLine("Artifact:")
290+
appendLine(" ${artifactCoordinates(artifact)}")
291+
appendLine("Repositories:")
292+
repositories.forEach { appendLine(" - ${it.id}: ${it.url}") }
293+
appendLine("Attempts:")
294+
appendLine(" $attemptCount")
295+
appendLine("Backoff:")
296+
appendLine(" $backoffDetails")
297+
appendLine("Last resolution result:")
298+
if (range == null) {
299+
appendLine(" <none returned>")
300+
} else {
301+
appendLine(" lowestVersion=${range.lowestVersion ?: "<missing>"}")
302+
appendLine(" highestVersion=${range.highestVersion ?: "<missing>"}")
303+
appendLine(" versionCount=${range.versions.size}")
304+
}
305+
if (failure != null) {
306+
appendLine("Last resolution failure:")
307+
appendLine(" ${failure.javaClass.name}: ${failure.message ?: "<no message>"}")
308+
}
309+
appendLine()
310+
appendLine("Maven metadata resolution may have returned an incomplete range, especially through a proxy.")
311+
appendLine("Restart the job later if the repositories above are reachable.")
312+
}.trimEnd()
313+
}
314+
315+
private fun artifactCoordinates(artifact: Artifact): String {
316+
val classifier = artifact.classifier?.takeUnless { it.isEmpty() }
317+
return listOfNotNull(
318+
artifact.groupId,
319+
artifact.artifactId,
320+
classifier,
321+
artifact.extension,
322+
artifact.version
323+
).joinToString(":")
324+
}
325+
208326
/**
209327
* Convert a muzzle directive to a set of artifacts for all filtered versions.
210328
* Throws GradleException if no artifacts are found.

0 commit comments

Comments
 (0)