|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +# This is where the local pytorch install in the docker image is located |
| 4 | +pt_checkout="/var/lib/jenkins/workspace" |
| 5 | + |
| 6 | +source "$pt_checkout/.ci/pytorch/common_utils.sh" |
| 7 | + |
| 8 | +echo "python_doc_push_script.sh: Invoked with $*" |
| 9 | + |
| 10 | +set -ex |
| 11 | + |
| 12 | +# for statements like ${1:-${DOCS_INSTALL_PATH:-docs/}} |
| 13 | +# the order of operations goes: |
| 14 | +# 1. Check if there's an argument $1 |
| 15 | +# 2. If no argument check for environment var DOCS_INSTALL_PATH |
| 16 | +# 3. If no environment var fall back to default 'docs/' |
| 17 | + |
| 18 | +# NOTE: It might seem weird to gather the second argument before gathering the first argument |
| 19 | +# but since DOCS_INSTALL_PATH can be derived from DOCS_VERSION it's probably better to |
| 20 | +# try and gather it first, just so we don't potentially break people who rely on this script |
| 21 | +# Argument 2: What version of the docs we are building. |
| 22 | +version="${2:-${DOCS_VERSION:-main}}" |
| 23 | +if [ -z "$version" ]; then |
| 24 | +echo "error: python_doc_push_script.sh: version (arg2) not specified" |
| 25 | + exit 1 |
| 26 | +fi |
| 27 | + |
| 28 | +# Argument 1: Where to copy the built documentation to |
| 29 | +# (pytorch.github.io/$install_path) |
| 30 | +install_path="${1:-${DOCS_INSTALL_PATH:-docs/${DOCS_VERSION}}}" |
| 31 | +if [ -z "$install_path" ]; then |
| 32 | +echo "error: python_doc_push_script.sh: install_path (arg1) not specified" |
| 33 | + exit 1 |
| 34 | +fi |
| 35 | + |
| 36 | +is_main_doc=false |
| 37 | +if [ "$version" == "main" ]; then |
| 38 | + is_main_doc=true |
| 39 | +fi |
| 40 | + |
| 41 | +# Argument 3: The branch to push to. Usually is "site" |
| 42 | +branch="${3:-${DOCS_BRANCH:-site}}" |
| 43 | +if [ -z "$branch" ]; then |
| 44 | +echo "error: python_doc_push_script.sh: branch (arg3) not specified" |
| 45 | + exit 1 |
| 46 | +fi |
| 47 | + |
| 48 | +echo "install_path: $install_path version: $version" |
| 49 | + |
| 50 | + |
| 51 | +build_docs () { |
| 52 | + set +e |
| 53 | + set -o pipefail |
| 54 | + make "$1" 2>&1 | tee /tmp/docs_build.txt |
| 55 | + code=$? |
| 56 | + if [ $code -ne 0 ]; then |
| 57 | + set +x |
| 58 | + echo ========================= |
| 59 | + grep "WARNING:" /tmp/docs_build.txt |
| 60 | + echo ========================= |
| 61 | + echo Docs build failed. If the failure is not clear, scan back in the log |
| 62 | + echo for any WARNINGS or for the line "build finished with problems" |
| 63 | + echo "(tried to echo the WARNINGS above the ==== line)" |
| 64 | + echo ========================= |
| 65 | + fi |
| 66 | + set -ex |
| 67 | + return $code |
| 68 | +} |
| 69 | + |
| 70 | + |
| 71 | +git clone https://github.com/pytorch/pytorch.github.io -b "$branch" --depth 1 |
| 72 | +pushd pytorch.github.io |
| 73 | + |
| 74 | +export LC_ALL=C |
| 75 | +export PATH=/opt/conda/bin:$PATH |
| 76 | +if [ -n "$ANACONDA_PYTHON_VERSION" ]; then |
| 77 | + export PATH=/opt/conda/envs/py_$ANACONDA_PYTHON_VERSION/bin:$PATH |
| 78 | +fi |
| 79 | + |
| 80 | +rm -rf pytorch || true |
| 81 | + |
| 82 | +# Get all the documentation sources, put them in one place |
| 83 | +pushd "$pt_checkout" |
| 84 | +pushd docs |
| 85 | + |
| 86 | +# Build the docs |
| 87 | +if [ "$is_main_doc" = true ]; then |
| 88 | + if ! build_docs html; then |
| 89 | + exit $? |
| 90 | + fi |
| 91 | + make coverage |
| 92 | + # Now we have the coverage report, we need to make sure it is empty. |
| 93 | + # Count the number of lines in the file and turn that number into a variable |
| 94 | + # $lines. The `cut -f1 ...` is to only parse the number, not the filename |
| 95 | + # Skip the report header by subtracting 2: the header will be output even if |
| 96 | + # there are no undocumented items. |
| 97 | + # |
| 98 | + # Also: see docs/source/conf.py for "coverage_ignore*" items, which should |
| 99 | + # be documented then removed from there. |
| 100 | + lines=$(wc -l build/coverage/python.txt 2>/dev/null |cut -f1 -d' ') |
| 101 | + undocumented=$((lines - 2)) |
| 102 | + if [ $undocumented -lt 0 ]; then |
| 103 | + echo coverage output not found |
| 104 | + exit 1 |
| 105 | + elif [ $undocumented -gt 0 ]; then |
| 106 | + echo undocumented objects found: |
| 107 | + cat build/coverage/python.txt |
| 108 | + echo "Make sure you've updated relevant .rsts in docs/source!" |
| 109 | + exit 1 |
| 110 | + fi |
| 111 | +else |
| 112 | + # skip coverage, format for stable or tags |
| 113 | + if ! build_docs html-stable; then |
| 114 | + exit $? |
| 115 | + fi |
| 116 | +fi |
| 117 | + |
| 118 | +# Move them into the docs repo |
| 119 | +popd |
| 120 | +popd |
| 121 | +git rm -rf "$install_path" || true |
| 122 | +mv "$pt_checkout/docs/build/html" "$install_path" |
| 123 | + |
| 124 | +# Prevent Google from indexing $install_path/_modules. This folder contains |
| 125 | +# generated source files. |
| 126 | +# NB: the following only works on gnu sed. The sed shipped with mac os is different. |
| 127 | +# One can `brew install gnu-sed` on a mac and then use "gsed" instead of "sed". |
| 128 | +find "$install_path/_modules" -name "*.html" -print0 | xargs -0 sed -i '/<head>/a \ \ <meta name="robots" content="noindex">' |
| 129 | + |
| 130 | +git add "$install_path" || true |
| 131 | +git status |
| 132 | +git config user.email "soumith+bot@pytorch.org" |
| 133 | +git config user.name "pytorchbot" |
| 134 | +# If there aren't changes, don't make a commit; push is no-op |
| 135 | +git commit -m "Generate Python docs from pytorch/pytorch@${GITHUB_SHA}" || true |
| 136 | +git status |
| 137 | + |
| 138 | +if [[ "${WITH_PUSH:-}" == true ]]; then |
| 139 | + # push to a temp branch first to trigger CLA check and satisfy branch protections |
| 140 | + git push -u origin HEAD:pytorchbot/temp-branch-py -f |
| 141 | + git push -u origin HEAD^:pytorchbot/base -f |
| 142 | + sleep 30 |
| 143 | + git push -u origin "${branch}" |
| 144 | +fi |
| 145 | + |
| 146 | +popd |
0 commit comments