Skip to content

Commit 542880f

Browse files
author
Ailing Zhang
committed
Merge branch 'master' of github.com:pytorch/pytorch into fix_dropout_test
2 parents 0614af5 + 9c82b57 commit 542880f

114 files changed

Lines changed: 3801 additions & 2429 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.circleci/cimodel/data/simple/android_definitions.py‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import cimodel.data.simple.util.branch_filters
1+
import cimodel.data.simple.util.branch_filters as branch_filters
22
from cimodel.data.simple.util.docker_constants import DOCKER_IMAGE_NDK
33

44

@@ -37,7 +37,7 @@ def gen_tree(self):
3737
}
3838

3939
if self.is_master_only:
40-
props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
40+
props_dict["filters"] = branch_filters.gen_filter_dict(branch_filters.NON_PR_BRANCH_LIST)
4141

4242
return [{self.template_name: props_dict}]
4343

@@ -47,12 +47,14 @@ def __init__(self,
4747
job_name,
4848
template_name,
4949
dependencies,
50-
is_master_only=True):
50+
is_master_only=True,
51+
is_pr_only=False):
5152

5253
self.job_name = job_name
5354
self.template_name = template_name
5455
self.dependencies = dependencies
5556
self.is_master_only = is_master_only
57+
self.is_pr_only = is_pr_only
5658

5759
def gen_tree(self):
5860

@@ -62,7 +64,9 @@ def gen_tree(self):
6264
}
6365

6466
if self.is_master_only:
65-
props_dict["filters"] = cimodel.data.simple.util.branch_filters.gen_filter_dict()
67+
props_dict["filters"] = branch_filters.gen_filter_dict(branch_filters.NON_PR_BRANCH_LIST)
68+
elif self.is_pr_only:
69+
props_dict["filters"] = branch_filters.gen_filter_dict(branch_filters.PR_BRANCH_LIST)
6670

6771
return [{self.template_name: props_dict}]
6872

@@ -77,7 +81,14 @@ def gen_tree(self):
7781
"pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32",
7882
"pytorch_android_gradle_build-x86_32",
7983
["pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build"],
80-
is_master_only=False),
84+
is_master_only=False,
85+
is_pr_only=True),
86+
AndroidGradleJob(
87+
"pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single",
88+
"pytorch_android_gradle_custom_build_single",
89+
[],
90+
is_master_only=False,
91+
is_pr_only=True),
8192
AndroidGradleJob(
8293
"pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build",
8394
"pytorch_android_gradle_build",

‎.circleci/cimodel/data/simple/util/branch_filters.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,11 @@
44
r"/release\/.*/",
55
]
66

7+
PR_BRANCH_LIST = [
8+
r"/gh\/.*\/head/",
9+
r"/pull\/.*/",
10+
]
11+
712
RC_PATTERN = r"/v[0-9]+(\.[0-9]+)*-rc[0-9]+/"
813

914
def gen_filter_dict(

‎.circleci/config.yml‎

Lines changed: 111 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,49 @@ commands:
169169
echo "This is not a pull request, skipping..."
170170
fi
171171
172-
172+
upload_binary_size_for_android_build:
173+
description: "Upload binary size data for Android build"
174+
parameters:
175+
build_type:
176+
type: string
177+
default: ""
178+
artifacts:
179+
type: string
180+
default: ""
181+
steps:
182+
- run:
183+
name: "Binary Size - Set Python Version"
184+
no_output_timeout: "1m"
185+
command: pyenv global 3.7.0
186+
- run:
187+
name: "Binary Size - Install Dependencies"
188+
no_output_timeout: "5m"
189+
command: |
190+
retry () {
191+
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
192+
}
193+
retry pip3 install requests
194+
- run:
195+
name: "Binary Size - Untar Artifacts"
196+
no_output_timeout: "5m"
197+
command: |
198+
# The artifact file is created inside docker container, which contains the result binaries.
199+
# Now unpackage it into the project folder. The subsequent script will scan project folder
200+
# to locate result binaries and report their sizes.
201+
# If artifact file is not provided it assumes that the project folder has been mounted in
202+
# the docker during build and already contains the result binaries, so this step can be skipped.
203+
export ARTIFACTS="<< parameters.artifacts >>"
204+
if [ -n "${ARTIFACTS}" ]; then
205+
tar xf "${ARTIFACTS}" -C ~/project
206+
fi
207+
- run:
208+
name: "Binary Size - Upload << parameters.build_type >>"
209+
no_output_timeout: "5m"
210+
command: |
211+
cd ~/project
212+
export ANDROID_BUILD_TYPE="<< parameters.build_type >>"
213+
export COMMIT_TIME=$(git log --max-count=1 --format=%ct || echo 0)
214+
python3 .circleci/scripts/upload_binary_size_to_scuba.py android
173215
174216
##############################################################################
175217
# Binary build (nightlies nightly build) defaults
@@ -538,6 +580,7 @@ jobs:
538580
# See https://circleci.com/docs/2.0/env-vars/#using-parameters-and-bash-environment
539581
echo "export PARALLEL_FLAGS=\"${PARALLEL_FLAGS}\"" >> $BASH_ENV
540582
echo "export id=$id" >> $BASH_ENV
583+
echo "export COMMIT_DOCKER_IMAGE=${COMMIT_DOCKER_IMAGE}" >> $BASH_ENV
541584
- run:
542585
name: Check for no AVX instruction by default
543586
no_output_timeout: "20m"
@@ -554,8 +597,8 @@ jobs:
554597
}
555598
556599
if is_vanilla_build; then
557-
echo "apt-get update && apt-get install -y qemu-user" | docker exec -u root -i "$id" bash
558-
echo "cd workspace/build; qemu-x86_64 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU" | docker exec -u jenkins -i "$id" bash
600+
echo "apt-get update && apt-get install -y qemu-user gdb" | docker exec -u root -i "$id" bash
601+
echo "cd workspace/build; qemu-x86_64 -g 2345 -cpu Broadwell -E ATEN_CPU_CAPABILITY=default ./bin/basic --gtest_filter=BasicTest.BasicTestCPU & gdb ./bin/basic -ex 'set pagination off' -ex 'target remote :2345' -ex 'continue' -ex 'bt' -ex='set confirm off' -ex 'quit \$_isvoid(\$_exitcode)'" | docker exec -u jenkins -i "$id" bash
559602
else
560603
echo "Skipping for ${BUILD_ENVIRONMENT}"
561604
fi
@@ -580,6 +623,12 @@ jobs:
580623
echo "cd workspace; python test/print_test_stats.py test" | docker exec -u jenkins -i "$id" bash
581624
echo "Retrieving test reports"
582625
docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
626+
echo "Stopping current container"
627+
docker container stop $id
628+
echo "Pruning stopped containers"
629+
docker container prune -f
630+
echo "Deleting the image"
631+
docker image rm ${COMMIT_DOCKER_IMAGE}
583632
when: always
584633
- store_test_results:
585634
path: test-reports
@@ -1529,26 +1578,9 @@ jobs:
15291578
output_image=$docker_image_libtorch_android_x86_32-gradle
15301579
docker commit "$id_x86_32" ${output_image}
15311580
time docker push ${output_image}
1532-
- run:
1533-
name: save binary size
1534-
no_output_timeout: "5m"
1535-
command: |
1536-
docker_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32-gradle
1537-
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image})
1538-
echo "docker-id: $id"
1539-
cat \<< EOL | docker exec -u jenkins -i "$id" bash
1540-
# ============================== Begin Docker ==============================
1541-
cd workspace
1542-
source ./env
1543-
export ANDROID_BUILD_TYPE="prebuild"
1544-
export COMMIT_TIME=\$(git log --max-count=1 --format=%ct || echo 0)
1545-
export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM}"
1546-
export CIRCLE_SHA1="${CIRCLE_SHA1}"
1547-
export CIRCLE_BRANCH="${CIRCLE_BRANCH}"
1548-
export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
1549-
python .circleci/scripts/upload_binary_size_to_scuba.py android
1550-
# ============================== End Docker ==============================
1551-
EOL
1581+
- upload_binary_size_for_android_build:
1582+
build_type: prebuilt
1583+
artifacts: /home/circleci/workspace/build_android_artifacts/artifacts.tgz
15521584
- store_artifacts:
15531585
path: ~/workspace/build_android_artifacts/artifacts.tgz
15541586
destination: artifacts.tgz
@@ -1599,14 +1631,6 @@ jobs:
15991631
image: ubuntu-1604:201903-01
16001632
steps:
16011633
- checkout
1602-
- run:
1603-
name: filter out not PR runs
1604-
no_output_timeout: "5m"
1605-
command: |
1606-
echo "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST:-}"
1607-
if [ -z "${CIRCLE_PULL_REQUEST:-}" ]; then
1608-
circleci step halt
1609-
fi
16101634
- setup_linux_system_environment
16111635
- checkout
16121636
- setup_ci_environment
@@ -1631,30 +1655,53 @@ jobs:
16311655
output_image=${docker_image_libtorch_android_x86_32}-gradle
16321656
docker commit "$id" ${output_image}
16331657
time docker push ${output_image}
1634-
- run:
1635-
name: save binary size
1636-
no_output_timeout: "5m"
1637-
command: |
1638-
docker_image=${DOCKER_IMAGE}-${CIRCLE_SHA1}-android-x86_32-gradle
1639-
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${docker_image})
1640-
echo "docker-id: $id"
1641-
cat \<< EOL | docker exec -u jenkins -i "$id" bash
1642-
# ============================== Begin Docker ==============================
1643-
cd workspace
1644-
source ./env
1645-
export ANDROID_BUILD_TYPE="prebuild-single"
1646-
export COMMIT_TIME=\$(git log --max-count=1 --format=%ct || echo 0)
1647-
export CIRCLE_BUILD_NUM="${CIRCLE_BUILD_NUM}"
1648-
export CIRCLE_SHA1="${CIRCLE_SHA1}"
1649-
export CIRCLE_BRANCH="${CIRCLE_BRANCH}"
1650-
export SCRIBE_GRAPHQL_ACCESS_TOKEN="${SCRIBE_GRAPHQL_ACCESS_TOKEN}"
1651-
python .circleci/scripts/upload_binary_size_to_scuba.py android
1652-
# ============================== End Docker ==============================
1653-
EOL
1658+
- upload_binary_size_for_android_build:
1659+
build_type: prebuilt-single
1660+
artifacts: /home/circleci/workspace/build_android_x86_32_artifacts/artifacts.tgz
16541661
- store_artifacts:
16551662
path: ~/workspace/build_android_x86_32_artifacts/artifacts.tgz
16561663
destination: artifacts.tgz
16571664

1665+
pytorch_android_gradle_custom_build_single:
1666+
environment:
1667+
BUILD_ENVIRONMENT: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single
1668+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:209062ef-ab58-422a-b295-36c4eed6e906"
1669+
PYTHON_VERSION: "3.6"
1670+
resource_class: large
1671+
machine:
1672+
image: ubuntu-1604:201903-01
1673+
steps:
1674+
- checkout
1675+
- setup_linux_system_environment
1676+
- checkout
1677+
- setup_ci_environment
1678+
- run:
1679+
name: pytorch android gradle custom build single architecture (for PR)
1680+
no_output_timeout: "1h"
1681+
command: |
1682+
set -e
1683+
# Unlike other gradle jobs, it's not worth building libtorch in a separate CI job and share via docker, because:
1684+
# 1) Not shareable: it's custom selective build, which is different from default libtorch mobile build;
1685+
# 2) Not parallelizable by architecture: it only builds libtorch for one architecture;
1686+
1687+
echo "DOCKER_IMAGE: ${DOCKER_IMAGE}"
1688+
time docker pull ${DOCKER_IMAGE} >/dev/null
1689+
export id=$(docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
1690+
1691+
git submodule sync && git submodule update -q --init --recursive
1692+
1693+
VOLUME_MOUNTS="-v /home/circleci/project/:/var/lib/jenkins/workspace"
1694+
export id=$(docker run ${VOLUME_MOUNTS} --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -t -d -w /var/lib/jenkins ${DOCKER_IMAGE})
1695+
1696+
export COMMAND='((echo "source ./workspace/env" && echo "export BUILD_ENVIRONMENT=${BUILD_ENVIRONMENT}" && echo "export GRADLE_OFFLINE=1" && echo "sudo chown -R jenkins workspace && cd workspace && ./.circleci/scripts/build_android_gradle.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
1697+
echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
1698+
1699+
# Skip docker push as this job is purely for size analysis purpose.
1700+
# Result binaries are already in `/home/circleci/project/` as it's mounted instead of copied.
1701+
1702+
- upload_binary_size_for_android_build:
1703+
build_type: custom-build-single
1704+
16581705
pytorch_ios_build:
16591706
<<: *pytorch_ios_params
16601707
macos:
@@ -1903,6 +1950,7 @@ jobs:
19031950
update_s3_htmls: &update_s3_htmls
19041951
machine:
19051952
image: ubuntu-1604:201903-01
1953+
resource_class: medium
19061954
steps:
19071955
- checkout
19081956
- setup_linux_system_environment
@@ -7546,9 +7594,21 @@ workflows:
75467594
docker_image: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-py3-clang5-android-ndk-r19c:fff7795428560442086f7b2bb6004b65245dc11a"
75477595
name: pytorch_linux_xenial_py3_clang5_android_ndk_r19c_vulkan_x86_32_build
75487596
- pytorch_android_gradle_build-x86_32:
7597+
filters:
7598+
branches:
7599+
only:
7600+
- /gh\/.*\/head/
7601+
- /pull\/.*/
75497602
name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-build-x86_32
75507603
requires:
75517604
- pytorch_linux_xenial_py3_clang5_android_ndk_r19c_x86_32_build
7605+
- pytorch_android_gradle_custom_build_single:
7606+
filters:
7607+
branches:
7608+
only:
7609+
- /gh\/.*\/head/
7610+
- /pull\/.*/
7611+
name: pytorch-linux-xenial-py3-clang5-android-ndk-r19c-gradle-custom-build-single
75527612
- pytorch_android_gradle_build:
75537613
filters:
75547614
branches:

‎.circleci/scripts/build_android_gradle.sh‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
#!/usr/bin/env bash
22
set -eux -o pipefail
33

4+
env
5+
echo "BUILD_ENVIRONMENT:$BUILD_ENVIRONMENT"
6+
47
export ANDROID_NDK_HOME=/opt/ndk
8+
export ANDROID_NDK=/opt/ndk
59
export ANDROID_HOME=/opt/android/sdk
610

711
# Must be in sync with GRADLE_VERSION in docker image for android
@@ -10,6 +14,31 @@ export GRADLE_VERSION=4.10.3
1014
export GRADLE_HOME=/opt/gradle/gradle-$GRADLE_VERSION
1115
export GRADLE_PATH=$GRADLE_HOME/bin/gradle
1216

17+
# touch gradle cache files to prevent expiration
18+
while IFS= read -r -d '' file
19+
do
20+
touch "$file" || true
21+
done < <(find /var/lib/jenkins/.gradle -type f -print0)
22+
23+
export GRADLE_LOCAL_PROPERTIES=~/workspace/android/local.properties
24+
rm -f $GRADLE_LOCAL_PROPERTIES
25+
echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES
26+
echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES
27+
echo "cmake.dir=/usr/local" >> $GRADLE_LOCAL_PROPERTIES
28+
29+
retry () {
30+
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
31+
}
32+
33+
# Run custom build script
34+
if [[ "${BUILD_ENVIRONMENT}" == *-gradle-custom-build* ]]; then
35+
# Install torch & torchvision - used to download & dump used ops from test model.
36+
retry pip install torch torchvision --progress-bar off
37+
38+
exec "$(dirname "${BASH_SOURCE[0]}")/../../android/build_test_app_custom.sh" armeabi-v7a
39+
fi
40+
41+
# Run default build
1342
BUILD_ANDROID_INCLUDE_DIR_x86=~/workspace/build_android/install/include
1443
BUILD_ANDROID_LIB_DIR_x86=~/workspace/build_android/install/lib
1544

@@ -44,9 +73,6 @@ ln -s ${BUILD_ANDROID_INCLUDE_DIR_arm_v8a} ${JNI_INCLUDE_DIR}/arm64-v8a
4473
ln -s ${BUILD_ANDROID_LIB_DIR_arm_v8a} ${JNI_LIBS_DIR}/arm64-v8a
4574
fi
4675

47-
env
48-
echo "BUILD_ENVIRONMENT:$BUILD_ENVIRONMENT"
49-
5076
GRADLE_PARAMS="-p android assembleRelease --debug --stacktrace"
5177
if [[ "${BUILD_ENVIRONMENT}" == *-gradle-build-only-x86_32* ]]; then
5278
GRADLE_PARAMS+=" -PABI_FILTERS=x86"
@@ -56,20 +82,6 @@ if [ -n "{GRADLE_OFFLINE:-}" ]; then
5682
GRADLE_PARAMS+=" --offline"
5783
fi
5884

59-
# touch gradle cache files to prevent expiration
60-
while IFS= read -r -d '' file
61-
do
62-
touch "$file" || true
63-
done < <(find /var/lib/jenkins/.gradle -type f -print0)
64-
65-
env
66-
67-
export GRADLE_LOCAL_PROPERTIES=~/workspace/android/local.properties
68-
rm -f $GRADLE_LOCAL_PROPERTIES
69-
echo "sdk.dir=/opt/android/sdk" >> $GRADLE_LOCAL_PROPERTIES
70-
echo "ndk.dir=/opt/ndk" >> $GRADLE_LOCAL_PROPERTIES
71-
echo "cmake.dir=/usr/local" >> $GRADLE_LOCAL_PROPERTIES
72-
7385
$GRADLE_PATH $GRADLE_PARAMS
7486

7587
find . -type f -name "*.a" -exec ls -lh {} \;

‎.circleci/scripts/python_doc_push_script.sh‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ sudo apt-get -y install expect-dev
77
# This is where the local pytorch install in the docker image is located
88
pt_checkout="/var/lib/jenkins/workspace"
99

10+
source "$pt_checkout/.jenkins/pytorch/common_utils.sh"
11+
1012
echo "python_doc_push_script.sh: Invoked with $*"
1113

1214
set -ex
@@ -59,12 +61,7 @@ pip install -q https://s3.amazonaws.com/ossci-linux/wheels/tensorboard-1.14.0a0-
5961

6062
# Get all the documentation sources, put them in one place
6163
pushd "$pt_checkout"
62-
git clone https://github.com/pytorch/vision
63-
pushd vision
64-
git checkout c2e8a00885e68ae1200eb6440f540e181d9125de
65-
conda install -q pillow
66-
time python setup.py install
67-
popd
64+
checkout_install_torchvision
6865
pushd docs
6966
rm -rf source/torchvision
7067
cp -a ../vision/docs/source source/torchvision

0 commit comments

Comments
 (0)