Skip to content

Commit b7410c6

Browse files
jbachorikjbachorik
committed
ci: migrate CI pipeline from java-profiler-build into java-profiler
- Merge all GitLab CI config from java-profiler-build repository - Register project with ADMS and add octo-sts policies - Move gitlab/ to .gitlab/ for unified CI structure - Switch to java-profiler IAM service account and SSM prefix - Flatten pipeline and fix cross-cutting CI issues - Gate push pipelines on GitHub PR existence; cancel gracefully when no PR - Fix ARM runner tags: arch:arm64 instead of runner:apm-k8s-arm-metal PROF-14208
1 parent 29263cc commit b7410c6

71 files changed

Lines changed: 8126 additions & 113 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.adms/python/gitlab.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# File generated and managed by #dependency-management.
2+
# Changes are subject to overwriting.
3+
# DO NOT EDIT
4+
5+
variables:
6+
PIP_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple"
7+
PIP_EXTRA_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple"
8+
UV_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple"
9+
UV_DEFAULT_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple"

‎.github/chainguard/async-profiler-build.ci.sts.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
# Allow async-profiler-build CI to publish to gh-pages
22
issuer: https://gitlab.ddbuild.io
33

4-
subject_pattern: "project_path:DataDog/apm-reliability/async-profiler-build:ref_type:branch:ref:.*"
4+
subject_pattern: "project_path:DataDog/java-profiler:ref_type:branch:ref:.*"
55

66
claim_pattern:
7-
project_path: "DataDog/apm-reliability/async-profiler-build"
7+
project_path: "DataDog/java-profiler"
88
ref_type: "branch"
99
ref: ".*"
1010

‎.github/chainguard/gh-pages.sts.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Octo-STS Trust Policy for GitHub Pages Publishing
2+
# This policy allows GitLab CI to push integration test reports to gh-pages branch
3+
#
4+
# Trust Policy Location: .github/chainguard/gh-pages.sts.yaml
5+
# Referenced by: scripts/get-github-token-via-octo-sts.sh (OCTO_STS_POLICY=gh-pages)
6+
#
7+
# How it works:
8+
# 1. GitLab CI generates OIDC token with issuer: https://gitlab.ddbuild.io
9+
# 2. Token includes claims: project_path, ref, namespace_path, etc.
10+
# 3. Octo-STS validates token against this policy
11+
# 4. If valid, Octo-STS returns short-lived GitHub token with specified permissions
12+
13+
# GitLab OIDC issuer
14+
issuer: https://gitlab.ddbuild.io
15+
16+
# Match GitLab CI jobs from any branch (needed for PR comments)
17+
# GitLab token includes: project_path=DataDog/java-profiler, ref=<branch>
18+
subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.*
19+
20+
# GitHub API permissions for the returned token
21+
# contents:write - Required to push to gh-pages branch
22+
permissions:
23+
contents: write
24+
25+
# Token lifetime (default: 1 hour)
26+
# Short-lived tokens reduce security risk

‎.github/chainguard/update-images.sts.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Octo-STS Trust Policy for Image Update PRs
2+
#
3+
# Allows the GitLab CI check-image-updates and rebuild-images-pr jobs to push
4+
# branches and create pull requests for CI image reference updates.
5+
#
6+
# Referenced by: scripts/create-image-update-pr.sh (OCTO_STS_POLICY=update-images)
7+
8+
# GitLab OIDC issuer
9+
issuer: https://gitlab.ddbuild.io
10+
11+
# Match GitLab CI jobs from the async-profiler-build project on any branch
12+
subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.*
13+
14+
# GitHub API permissions
15+
permissions:
16+
contents: write
17+
pull_requests: write

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
**/build_*/
33
**/build-*/
44
!build-logic/
5+
!.gitlab/build-deploy/
56
/nbproject/
67
/out/
78
/.idea/

‎.gitlab-ci.yml‎

Lines changed: 152 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,159 @@
1-
# Triggers a build within the Datadog infrastructure in the ddprof-build repository
2-
trigger_internal_build:
1+
image: alpine
2+
3+
variables:
4+
REGISTRY: registry.ddbuild.io
5+
PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest
6+
# Image with dd-octo-sts for GitHub token exchange (check-image-updates, rebuild-images-pr)
7+
DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
8+
FORCE_BUILD:
9+
value: ""
10+
description: "Force build even if no new commits (any non-empty value)"
11+
12+
default:
13+
tags: ["arch:amd64"]
14+
interruptible: true
15+
before_script:
16+
- '[ "${CANCELLED:-}" != "true" ] || { echo "No PR for this branch — skipping job"; exit 0; }'
17+
18+
stages:
19+
- images
20+
- generate-signing-key
21+
- prepare
22+
- build
23+
- stresstest
24+
- deploy
25+
- integration-test
26+
- reliability
27+
- benchmarks
28+
- notify
29+
30+
# Detects newer images in registry and creates GitHub PR with updates
31+
check-image-updates:
32+
stage: images
333
rules:
4-
- if: $CI_COMMIT_BRANCH =~ /release\/.*/
34+
- if: '$CI_PIPELINE_SOURCE == "schedule" && $CHECK_IMAGE_UPDATES == "true"'
35+
when: always
36+
- if: '$CI_PIPELINE_SOURCE == "web"'
37+
when: manual
38+
allow_failure: true
39+
extends: .bootstrap-gh-tools
40+
tags: ["arch:amd64"]
41+
image: ${DD_OCTO_STS_IMAGE}
42+
id_tokens:
43+
DDOCTOSTS_ID_TOKEN:
44+
aud: dd-octo-sts
45+
script:
46+
- set -euo pipefail
47+
- echo "Checking for image updates..."
48+
- .gitlab/scripts/check-image-updates.sh > updates.json
49+
- |
50+
update_count=$(jq 'length' updates.json)
51+
echo "Found ${update_count} update(s)"
52+
if [ "$update_count" -gt 0 ]; then
53+
echo "Updates available:"
54+
jq . updates.json
55+
.gitlab/scripts/create-image-update-pr.sh updates.json
56+
else
57+
echo "All images are up to date"
58+
fi
59+
artifacts:
60+
when: always
61+
paths:
62+
- updates.json
63+
expire_in: 7 days
64+
65+
rebuild-images:
66+
stage: images
67+
rules:
68+
- if: '$CI_COMMIT_TAG'
569
when: never
6-
- when: always
7-
allow_failure: false
70+
- if: '$CI_PIPELINE_SOURCE == "schedule"'
71+
when: never
72+
- when: manual
73+
allow_failure: true
74+
tags: ["arch:amd64"]
75+
variables:
76+
REBUILD_IMAGES: "" # comma/space-separated short names, or empty = all
77+
image: ${DOCKER_IMAGE}
78+
id_tokens:
79+
DDSIGN_ID_TOKEN:
80+
aud: image-integrity
81+
script:
82+
- set -euo pipefail
83+
- .gitlab/scripts/rebuild-images.sh
84+
artifacts:
85+
when: always
86+
paths:
87+
- updates.json
88+
expire_in: 1 day
89+
90+
rebuild-images-pr:
91+
stage: images
92+
rules:
93+
- if: '$CI_COMMIT_TAG'
94+
when: never
95+
- if: '$CI_PIPELINE_SOURCE == "schedule"'
96+
when: never
97+
- when: on_success
98+
needs:
99+
- job: rebuild-images
100+
artifacts: true
101+
extends: .bootstrap-gh-tools
102+
tags: ["arch:amd64"]
103+
image: ${DD_OCTO_STS_IMAGE}
104+
id_tokens:
105+
DDOCTOSTS_ID_TOKEN:
106+
aud: dd-octo-sts
107+
script:
108+
- set -euo pipefail
109+
- .gitlab/scripts/create-image-update-pr.sh updates.json
110+
111+
create_key:
112+
stage: generate-signing-key
113+
when: manual
114+
needs: []
115+
tags: ["arch:amd64"]
8116
variables:
9-
DOWNSTREAM_BRANCH: "main"
10-
UPSTREAM_PROJECT: ${CI_PROJECT_PATH}
11-
UPSTREAM_PROJECT_NAME: ${CI_PROJECT_NAME}
12-
UPSTREAM_BRANCH: ${CI_COMMIT_BRANCH}
13-
UPSTREAM_COMMIT_SHA: ${CI_COMMIT_SHA}
14-
DDPROF_DEFAULT_BRANCH: "main"
15-
DDPROF_COMMIT_BRANCH: ${CI_COMMIT_BRANCH}
16-
DDROF_COMMIT_SHA: ${CI_COMMIT_SHA}
17-
DPROF_SHORT_COMMIT_SHA: ${CI_COMMIT_SHORT_SHA}
18-
DDPROF_COMMIT_TAG: ${CI_COMMIT_TAG}
117+
PROJECT_NAME: "java-profiler"
118+
EXPORT_TO_KEYSERVER: "true"
119+
KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler
120+
image: $REGISTRY/ci/agent-key-management-tools/gpg:1
121+
script:
122+
- /create.sh
123+
artifacts:
124+
expire_in: 13 mos
125+
paths:
126+
- pubkeys
127+
128+
# Shared version detection used by benchmarks and reliability pipelines
129+
get-versions:
130+
extends: .get-versions
131+
needs:
132+
- job: prepare:start
133+
artifacts: false
134+
135+
# Triggered externally from async-profiler-build with JDK build parameters;
136+
# kept as a child pipeline because it is mutually exclusive with the main build
137+
jdk-integration-test:
138+
stage: build
139+
rules:
140+
- if: '$JDK_VERSION == null || $DEBUG_LEVEL == null || $HASH == null || $DOWNSTREAM == null'
141+
when: never
142+
- if: '$CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "web"'
143+
when: always
144+
allow_failure: false
145+
- when: always
19146
trigger:
20-
project: DataDog/apm-reliability/async-profiler-build
147+
include: .gitlab/jdk-integration/.gitlab-ci.yml
21148
strategy: depend
22-
branch: $DOWNSTREAM_BRANCH
23149
forward:
24150
pipeline_variables: true
151+
152+
include:
153+
- local: .gitlab/common.yml
154+
- local: .adms/python/gitlab.yaml
155+
- local: .gitlab/benchmarks/images.yml
156+
- local: .gitlab/build-deploy/images.yml
157+
- local: .gitlab/build-deploy/.gitlab-ci.yml
158+
- local: .gitlab/benchmarks/.gitlab-ci.yml
159+
- local: .gitlab/reliability/.gitlab-ci.yml

‎.gitlab/Dockerfile.datadog-ci‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
ARG BASEIMAGE=registry.ddbuild.io/images/base/gbi-ubuntu_2404:release
2+
FROM ${BASEIMAGE}
3+
4+
USER root
5+
6+
# Create non-root user for security
7+
RUN useradd --create-home --shell /bin/bash --uid 1001 ci-user
8+
9+
# Install Node.js 20 and npm
10+
# Default seems to be 14 which does not work with datadog-ci
11+
RUN set -x \
12+
&& apt-get update && apt-get -y install --no-install-recommends curl xz-utils\
13+
&& curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
14+
&& apt-get install -y nodejs \
15+
pipx=1.4.3-1 \
16+
binutils \
17+
jq \
18+
&& npm install -g @datadog/datadog-ci@3.16.0 \
19+
&& apt-get -y clean \
20+
&& rm -rf /var/lib/apt/lists/*
21+
22+
# Install GitHub CLI
23+
RUN set -x \
24+
&& curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
25+
-o /usr/share/keyrings/githubcli-archive-keyring.gpg \
26+
&& chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
27+
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
28+
> /etc/apt/sources.list.d/github-cli.list \
29+
&& apt-get update \
30+
&& apt-get install -y gh \
31+
&& apt-get -y clean \
32+
&& rm -rf /var/lib/apt/lists/*
33+
34+
# awscli is not available in Ubuntu 2404 for some inexplicable reason so lets install in via other means
35+
RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install awscli
36+
37+
# Install Go 1.22.3
38+
RUN set -x \
39+
&& curl -LO https://golang.org/dl/go1.22.3.linux-amd64.tar.gz \
40+
&& tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz \
41+
&& rm go1.22.3.linux-amd64.tar.gz
42+
43+
# Set up Go environment for root and install Crane
44+
ENV PATH="/usr/local/go/bin:${PATH}"
45+
ENV GOPATH="/root/go"
46+
ENV GOBIN="/usr/local/bin"
47+
48+
# Install Crane version 0.19.1 directly to /usr/local/bin so it's available for all users
49+
RUN set -x \
50+
&& go install github.com/google/go-containerregistry/cmd/crane@v0.19.1
51+
52+
# Switch to non-root user
53+
USER ci-user
54+
WORKDIR /home/ci-user
55+
56+
# Set PATH for the ci-user (crane is now in /usr/local/bin)
57+
ENV PATH="/usr/local/go/bin:/usr/local/bin:${PATH}"
58+
59+
# Verify installation (as non-root user)
60+
RUN node -v && npm -v && go version && crane version && datadog-ci --help && jq --version && gh --version

‎.gitlab/base/Dockerfile‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
ARG BASE_IMAGE=openjdk:11-slim-buster
2+
FROM ${BASE_IMAGE} as base
3+
ARG CI_JOB_TOKEN
4+
WORKDIR /root
5+
6+
RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debian-artifacts/issues/24
7+
RUN (apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential wget bsdmainutils clang libclang-rt-dev jq zip unzip maven) || true
8+
RUN (apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq gradle zip unzip) || true
9+
RUN npm install -g --save-dev @datadog/datadog-ci
10+
RUN rm -rf "/var/lib/apt/lists/*"

‎.gitlab/base/centos7/Dockerfile‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
ARG BASE_IMAGE=openjdk:11-slim-buster
2+
FROM ${BASE_IMAGE} as base
3+
ARG CI_JOB_TOKEN
4+
WORKDIR /root
5+
6+
# 1. Replace dead mirrorlist entries with HTTPS vault URLs
7+
RUN set -eux; \
8+
sed -i -e 's/^mirrorlist/#mirrorlist/' \
9+
-e 's|^#baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \
10+
/etc/yum.repos.d/CentOS-*.repo
11+
12+
# 2. Add a vault mirror that still contains Software Collections
13+
RUN cat > /etc/yum.repos.d/CentOS-SCLo-Vault.repo <<'EOF'
14+
[centos-sclo-rh]
15+
name=CentOS-7 - SCLo rh (Rocky Vault)
16+
baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/rh/
17+
gpgcheck=0
18+
enabled=1
19+
20+
[centos-sclo-sclo]
21+
name=CentOS-7 - SCLo sclo (Rocky Vault)
22+
baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/sclo/
23+
gpgcheck=0
24+
enabled=1
25+
EOF
26+
27+
# 3. Expose devtoolset-11 binaries & libs by default (they are installed a bit later)
28+
ENV PATH="/opt/rh/devtoolset-11/root/usr/bin:${PATH}" \
29+
LD_LIBRARY_PATH="/opt/rh/devtoolset-11/root/usr/lib64:${LD_LIBRARY_PATH}"
30+
31+
RUN yum -y clean all
32+
RUN yum -y update && yum -y install scl-utils devtoolset-11 devtoolset-11-toolchain curl zip unzip git libstdc++-static make which wget cmake binutils
33+
RUN yum -y clean all
34+
RUN (curl -s "https://get.sdkman.io" | bash)
35+
RUN (source ~/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem)
36+
RUN (curl -sL https://rpm.nodesource.com/setup_16.x | bash -)
37+
# installing JQ requires two steps - adding the repo and then installing the tool
38+
RUN yum install -y epel-release
39+
RUN yum install -y jq
40+
# now install nodejs and datadog CI support
41+
RUN yum -y install nodejs
42+
RUN npm install -g --save-dev @datadog/datadog-ci
43+
RUN rm -rf "/var/lib/apt/lists/*"

0 commit comments

Comments
 (0)