Skip to content

Commit 3a0c156

Browse files
DiscountchubbsDiscountchubbs
committed
bc devtekve said. also, this is repetitive af
1 parent c38e644 commit 3a0c156

File tree

2 files changed

+234
-133
lines changed

2 files changed

+234
-133
lines changed

‎.github/workflows/build-all-tinygrad-models.yaml‎

Lines changed: 152 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,28 @@ name: Build All Tinygrad Models and Push to GitLab
33
on:
44
workflow_dispatch:
55
inputs:
6+
runner_type:
7+
description: 'Runner type'
8+
required: false
9+
default: '[self-hosted, james-mac]'
10+
type: choice
11+
options:
12+
- ubuntu-latest
13+
- '[self-hosted, james-mac]'
614
branch:
715
description: 'Branch to run workflow from'
816
required: false
917
default: 'master-new'
1018
type: string
1119

1220
jobs:
13-
build-all:
14-
runs-on: ubuntu-latest
15-
env:
16-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
17-
GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
18-
CI_SUNNYPILOT_DOCS_PRIVATE_KEY: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
21+
setup:
22+
runs-on: ${{ github.event.inputs.runner_type }}
23+
outputs:
24+
models: ${{ steps.get-models.outputs.models }}
25+
RECOMPILED_DIR: ${{ steps.set-recompiled.outputs.RECOMPILED_DIR }}
26+
JSON_FILE: ${{ steps.get-json.outputs.JSON_FILE }}
27+
SRC_JSON_FILE: ${{ steps.get-json.outputs.SRC_JSON_FILE }}
1928
steps:
2029
- name: Set up SSH
2130
uses: webfactory/ssh-agent@v0.9.0
@@ -83,138 +92,148 @@ jobs:
8392
echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
8493
echo "SRC_JSON_FILE: docs/docs/driving_models_v${latest}.json"
8594
86-
- name: Install dependencies
95+
- name: Get tinygrad models from JSON
96+
id: get-models
97+
working-directory: docs/docs
8798
run: |
88-
sudo apt-get update
89-
sudo apt-get install -y jq gh unzip
99+
MODELS=$(jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name, is_20hz}]' "$(basename "${SRC_JSON_FILE}")")
100+
echo "models=${MODELS}" >> $GITHUB_OUTPUT
101+
echo "Parsed models: ${MODELS}"
102+
shell: bash
103+
104+
get-models:
105+
needs: setup
106+
runs-on: ${{ github.event.inputs.runner_type }}
107+
strategy:
108+
matrix:
109+
model: ${{ fromJson(needs.setup.outputs.models) }}
110+
fail-fast: false
111+
env:
112+
OUTPUT_DIR: ${{ github.workspace }}/output/${{ matrix.model.display_name }}-${{ matrix.model.ref }}
113+
steps:
114+
- name: Checkout commaai/openpilot
115+
id: checkout_upstream
116+
continue-on-error: true
117+
uses: actions/checkout@v4
118+
with:
119+
repository: commaai/openpilot
120+
ref: ${{ matrix.model.ref }}
121+
submodules: recursive
122+
path: openpilot
90123

91-
- name: Build all tinygrad models
92-
id: trigger-builds
124+
- name: Fallback to sunnypilot/sunnypilot
125+
if: steps.checkout_upstream.outcome == 'failure'
126+
uses: actions/checkout@v4
127+
with:
128+
repository: sunnypilot/sunnypilot
129+
ref: ${{ matrix.model.ref }}
130+
submodules: recursive
131+
path: openpilot
132+
133+
- name: Get commit date
134+
id: commit-date
93135
run: |
94-
set -e
95-
> triggered_run_ids.txt
96-
BRANCH="${{ github.event.inputs.branch }}"
97-
SRC_JSON_FILE="$SRC_JSON_FILE"
98-
if [ ! -f "$SRC_JSON_FILE" ]; then
99-
echo "ERROR: Source JSON file $SRC_JSON_FILE not found!"
100-
exit 1
101-
fi
102-
jq -c '.bundles[] | select(.runner=="tinygrad")' "$SRC_JSON_FILE" | while read -r bundle; do
103-
ref=$(echo "$bundle" | jq -r '.ref')
104-
display_name=$(echo "$bundle" | jq -r '.display_name' | sed 's/ ([^)]*)//g')
105-
is_20hz=$(echo "$bundle" | jq -r '.is_20hz')
106-
echo "Triggering build for: $display_name ($ref) [20Hz: $is_20hz]"
107-
START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
108-
gh workflow run sunnypilot-build-model.yaml \
109-
--repo sunnypilot/sunnypilot \
110-
--ref "$BRANCH" \
111-
-f upstream_branch="$ref" \
112-
-f custom_name="$display_name" \
113-
-f is_20hz="$is_20hz"
114-
echo "Triggered workflow for $display_name ($ref), waiting for run to appear..."
115-
for i in {1..36}; do
116-
RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="$BRANCH" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
117-
if [ -n "$RUN_ID" ]; then
118-
echo "Found run ID: $RUN_ID for $display_name ($ref)"
119-
break
120-
fi
121-
done
122-
if [ -z "$RUN_ID" ]; then
123-
echo "Could not find the triggered workflow run for $display_name ($ref)."
124-
exit 1
125-
fi
126-
echo "$RUN_ID" >> triggered_run_ids.txt
127-
done
128-
129-
- name: Wait for all model builds to finish
136+
cd openpilot
137+
commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
138+
echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
139+
cat $GITHUB_OUTPUT
140+
141+
- name: Pull lfs
130142
run: |
131-
set -e
132-
SUCCESS_RUNS=()
133-
FAILED_RUNS=()
134-
declare -A RUN_ID_TO_NAME
135-
136-
# Get artifact names for each run
137-
while read -r RUN_ID; do
138-
ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name' || echo "unknown")
139-
RUN_ID_TO_NAME["$RUN_ID"]="$ARTIFACT_NAME"
140-
done < triggered_run_ids.txt
141-
142-
# Poll all runs together, sleep between checks to avoid 90,000 line run logs
143-
RUN_IDS=($(cat triggered_run_ids.txt))
144-
declare -A RUN_STATUS
145-
for RUN_ID in "${RUN_IDS[@]}"; do
146-
RUN_STATUS["$RUN_ID"]="in_progress"
147-
done
148-
149-
while :; do
150-
ALL_DONE=true
151-
for RUN_ID in "${RUN_IDS[@]}"; do
152-
if [[ "${RUN_STATUS[$RUN_ID]}" == "in_progress" ]]; then
153-
# Try to get status, skip on API error
154-
CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion' 2>/dev/null || echo "api_error")
155-
if [[ "$CONCLUSION" == "api_error" ]]; then
156-
echo "Warning: Could not fetch status for run $RUN_ID, will retry."
157-
ALL_DONE=false
158-
continue
159-
fi
160-
if [[ -z "$CONCLUSION" || "$CONCLUSION" == "null" ]]; then
161-
ALL_DONE=false
162-
continue
163-
fi
164-
ARTIFACT_NAME="${RUN_ID_TO_NAME[$RUN_ID]}"
165-
echo "Run $RUN_ID ($ARTIFACT_NAME) concluded with: $CONCLUSION"
166-
RUN_STATUS["$RUN_ID"]="$CONCLUSION"
167-
if [[ "$CONCLUSION" == "success" ]]; then
168-
SUCCESS_RUNS+=("$RUN_ID")
169-
else
170-
FAILED_RUNS+=("$RUN_ID")
171-
fi
172-
fi
173-
done
174-
if $ALL_DONE; then
175-
break
176-
fi
177-
echo "Waiting for unfinished runs... sleeping for 2 minutes"
178-
sleep 120
179-
done
180-
181-
if [[ ${#SUCCESS_RUNS[@]} -eq 0 ]]; then
182-
echo "All model builds failed. Aborting."
183-
exit 1
184-
fi
143+
cd openpilot
144+
git lfs pull
185145
186-
if [[ ${#FAILED_RUNS[@]} -gt 0 ]]; then
187-
echo "WARNING: The following model builds failed:"
188-
for RUN_ID in "${FAILED_RUNS[@]}"; do
189-
echo "- $RUN_ID (${RUN_ID_TO_NAME[$RUN_ID]})"
190-
done
191-
echo "You may want to rerun these models manually."
192-
fi
146+
- name: Copy models
147+
run: |
148+
mkdir -p "${{ env.OUTPUT_DIR }}"
149+
cp openpilot/selfdrive/modeld/models/*.onnx "${{ env.OUTPUT_DIR }}/" || echo "No models found."
193150
194-
echo "${SUCCESS_RUNS[@]}" > success_run_ids.txt
151+
- name: Upload model artifacts
152+
uses: actions/upload-artifact@v4
153+
with:
154+
name: onnx-${{ matrix.model.display_name }}-${{ matrix.model.ref }}-${{ github.run_number }}
155+
path: ${{ env.OUTPUT_DIR }}
195156

196-
- name: Download and extract all model artifacts
197-
run: |
198-
ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
199-
mkdir -p "$ARTIFACT_DIR"
200-
for RUN_ID in $(cat success_run_ids.txt); do
201-
ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
202-
echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
203-
mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
204-
echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
205-
gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
206-
echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
207-
ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
208-
if [ -n "$ZIP_PATH" ]; then
209-
echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
210-
unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
211-
rm -f "$ZIP_PATH"
212-
echo "Unzipped and removed $ZIP_PATH"
213-
else
214-
echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME (This is NOT an error)."
215-
fi
216-
echo "Done processing $ARTIFACT_NAME"
217-
done
157+
build-models:
158+
needs: [setup, get-models]
159+
runs-on: [self-hosted, tici]
160+
strategy:
161+
matrix:
162+
model: ${{ fromJson(needs.setup.outputs.models) }}
163+
fail-fast: false
164+
env:
165+
BUILD_DIR: /data/openpilot
166+
OUTPUT_DIR: ${{ github.workspace }}/output/${{ matrix.model.display_name }}-${{ matrix.model.ref }}
167+
SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
168+
TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
169+
MODELS_DIR: ${{ github.workspace }}/output/${{ matrix.model.display_name }}-${{ matrix.model.ref }}
170+
POWERSAVE_SCRIPT: ${{ github.workspace }}/scripts/manage-powersave.py
171+
MODEL_GENERATOR: ${{ github.workspace }}/release/ci/model_generator.py
172+
GET_MODEL_METADATA: ${{ github.workspace }}/selfdrive/modeld/get_model_metadata.py
173+
steps:
174+
- uses: actions/checkout@v4
175+
with:
176+
ref: ${{ github.event.inputs.branch }}
177+
submodules: recursive
178+
- run: git lfs pull
179+
- uses: actions/cache@v4
180+
with:
181+
path: ${{ env.SCONS_CACHE_DIR }}
182+
key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
183+
restore-keys: |
184+
scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
185+
scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
186+
scons-${{ runner.os }}-${{ runner.arch }}-master-new-model
187+
scons-${{ runner.os }}-${{ runner.arch }}-master-model
188+
scons-${{ runner.os }}-${{ runner.arch }}-master-new
189+
scons-${{ runner.os }}-${{ runner.arch }}-master
190+
scons-${{ runner.os }}-${{ runner.arch }}
191+
- run: |
192+
source /etc/profile
193+
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
194+
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
195+
printenv >> $GITHUB_ENV
196+
- uses: actions/download-artifact@v4
197+
with:
198+
name: onnx-${{ matrix.model.display_name }}-${{ matrix.model.ref }}-${{ github.run_number }}
199+
path: ${{ env.MODELS_DIR }}
200+
- run: |
201+
python3 release/ci/build_model.py \
202+
--build-dir "${BUILD_DIR}" \
203+
--output-dir "${OUTPUT_DIR}" \
204+
--scons-cache-dir "${SCONS_CACHE_DIR}" \
205+
--tinygrad-path "${TINYGRAD_PATH}" \
206+
--models-dir "${MODELS_DIR}" \
207+
--custom-name "${{ matrix.model.display_name }}" \
208+
--upstream-branch "${{ matrix.model.ref }}" \
209+
--is-20hz "${{ matrix.model.is_20hz }}" \
210+
--powersave-script "${POWERSAVE_SCRIPT}" \
211+
--model-generator "${MODEL_GENERATOR}" \
212+
--get-model-metadata "${GET_MODEL_METADATA}"
213+
- run: |
214+
sudo rsync -avm \
215+
--include='*.dlc' \
216+
--include='*.thneed' \
217+
--include='*.pkl' \
218+
--include='*.onnx' \
219+
--exclude='*' \
220+
--delete-excluded \
221+
--chown=comma:comma \
222+
"${MODELS_DIR}/" "${OUTPUT_DIR}/"
223+
- uses: actions/upload-artifact@v4
224+
with:
225+
name: model-${{ matrix.model.display_name }}-${{ matrix.model.ref }}-${{ github.run_number }}
226+
path: ${{ env.OUTPUT_DIR }}
227+
228+
postprocess:
229+
needs: [setup, build-models]
230+
runs-on: ${{ github.event.inputs.runner_type }}
231+
if: needs.build-models.result == 'success'
232+
steps:
233+
- name: Download all model artifacts
234+
uses: actions/download-artifact@v4
235+
with:
236+
path: gitlab_docs/models/${{ needs.setup.outputs.RECOMPILED_DIR }}
218237

219238
- name: Push recompiled dir to GitLab
220239
env:
@@ -229,7 +248,7 @@ jobs:
229248
git commit -m "Add $(basename $RECOMPILED_DIR) from build-all-tinygrad-models"
230249
git push origin main
231250
232-
- name: Run json_parser.py to update JSON
251+
- name: Update JSON with new models
233252
run: |
234253
python3 docs/json_parser.py \
235254
--json-path "$JSON_FILE" \
@@ -242,5 +261,5 @@ jobs:
242261
git config --global user.email "action@github.com"
243262
git checkout gh-pages
244263
git add docs/"$(basename $JSON_FILE)"
245-
git commit -m "Update $(basename $JSON_FILE) after recompiling models" || echo "No changes to commit"
264+
git commit -m "Update $(basename $JSON_FILE) after recompiling models" || true
246265
git push origin gh-pages

0 commit comments

Comments
 (0)