Skip to content

Commit 68ccd29

Browse files
huydhnhuydhn
committed
Merge branch 'master' into remove-cuda11.6
2 parents 460f661 + 752e43c commit 68ccd29

164 files changed

Lines changed: 2329 additions & 1069 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ci/pytorch/build-tsan.sh‎

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎.ci/pytorch/build.sh‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,6 @@ if [[ "$BUILD_ENVIRONMENT" == *-clang7-asan* ]]; then
1515
exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"
1616
fi
1717

18-
if [[ "$BUILD_ENVIRONMENT" == *-clang7-tsan* ]]; then
19-
exec "$(dirname "${BASH_SOURCE[0]}")/build-tsan.sh" "$@"
20-
fi
21-
2218
if [[ "$BUILD_ENVIRONMENT" == *-mobile-*build* ]]; then
2319
exec "$(dirname "${BASH_SOURCE[0]}")/build-mobile.sh" "$@"
2420
fi

‎.ci/pytorch/macos-build.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ build_lite_interpreter() {
7272
"${CPP_BUILD}/caffe2/build/bin/test_lite_interpreter_runtime"
7373
}
7474

75+
print_cmake_info
76+
7577
if [[ ${BUILD_ENVIRONMENT} = *arm64* ]]; then
7678
if [[ $(uname -m) == "arm64" ]]; then
7779
compile_arm64

‎.ci/pytorch/macos-common.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,22 @@ sysctl -a | grep machdep.cpu
1212
export MACOSX_DEPLOYMENT_TARGET=10.9
1313
export CXX=clang++
1414
export CC=clang
15+
16+
print_cmake_info() {
17+
CMAKE_EXEC=$(which cmake)
18+
echo "$CMAKE_EXEC"
19+
20+
CONDA_INSTALLATION_DIR=$(dirname "$CMAKE_EXEC")
21+
# Print all libraries under cmake rpath for debugging
22+
ls -la "$CONDA_INSTALLATION_DIR/../lib"
23+
24+
export CMAKE_EXEC
25+
# Explicitly add conda env lib folder to cmake rpath to address the flaky issue
26+
# where cmake dependencies couldn't be found. This seems to point to how conda
27+
# links $CMAKE_EXEC to its package cache when cloning a new environment
28+
install_name_tool -add_rpath @executable_path/../lib "${CMAKE_EXEC}" || true
29+
# Adding the rpath will invalidate cmake signature, so signing it again here
30+
# to trust the executable. EXC_BAD_ACCESS (SIGKILL (Code Signature Invalid))
31+
# with an exit code 137 otherwise
32+
codesign -f -s - "${CMAKE_EXEC}" || true
33+
}

‎.ci/pytorch/macos-test.sh‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -82,25 +82,6 @@ test_libtorch() {
8282
fi
8383
}
8484

85-
print_cmake_info() {
86-
CMAKE_EXEC=$(which cmake)
87-
echo "$CMAKE_EXEC"
88-
89-
CONDA_INSTALLATION_DIR=$(dirname "$CMAKE_EXEC")
90-
# Print all libraries under cmake rpath for debugging
91-
ls -la "$CONDA_INSTALLATION_DIR/../lib"
92-
93-
export CMAKE_EXEC
94-
# Explicitly add conda env lib folder to cmake rpath to address the flaky issue
95-
# where cmake dependencies couldn't be found. This seems to point to how conda
96-
# links $CMAKE_EXEC to its package cache when cloning a new environment
97-
install_name_tool -add_rpath @executable_path/../lib "${CMAKE_EXEC}" || true
98-
# Adding the rpath will invalidate cmake signature, so signing it again here
99-
# to trust the executable. EXC_BAD_ACCESS (SIGKILL (Code Signature Invalid))
100-
# with an exit code 137 otherwise
101-
codesign -f -s - "${CMAKE_EXEC}" || true
102-
}
103-
10485
test_custom_backend() {
10586
print_cmake_info
10687

‎.ci/pytorch/test.sh‎

Lines changed: 50 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -177,10 +177,6 @@ if [[ "$BUILD_ENVIRONMENT" == *asan* ]]; then
177177
(cd test && ! get_exit_code python -c "import torch; torch._C._crash_if_aten_asan(3)")
178178
fi
179179

180-
if [[ "$BUILD_ENVIRONMENT" == *-tsan* ]]; then
181-
export PYTORCH_TEST_WITH_TSAN=1
182-
fi
183-
184180
if [[ $TEST_CONFIG == 'nogpu_NO_AVX2' ]]; then
185181
export ATEN_CPU_CAPABILITY=default
186182
elif [[ $TEST_CONFIG == 'nogpu_AVX512' ]]; then
@@ -273,7 +269,7 @@ elif [[ "${TEST_CONFIG}" == *inductor* && "${TEST_CONFIG}" != *perf* ]]; then
273269
fi
274270

275271
if [[ "${TEST_CONFIG}" == *dynamic* ]]; then
276-
DYNAMO_BENCHMARK_FLAGS+=(--dynamic-shapes)
272+
DYNAMO_BENCHMARK_FLAGS+=(--dynamic-shapes --dynamic-batch-only)
277273
fi
278274

279275
if [[ "${TEST_CONFIG}" == *cpu_accuracy* ]]; then
@@ -507,10 +503,8 @@ test_libtorch() {
507503
TEST_REPORTS_DIR=test/test-reports/cpp-unittest/test_libtorch
508504
mkdir -p $TEST_REPORTS_DIR
509505

510-
if [[ "$BUILD_ENVIRONMENT" != *-tsan* ]]; then
511-
# Run JIT cpp tests
512-
python test/cpp/jit/tests_setup.py setup
513-
fi
506+
# Run JIT cpp tests
507+
python test/cpp/jit/tests_setup.py setup
514508

515509
if [[ "$BUILD_ENVIRONMENT" == *cuda* ]]; then
516510
"$TORCH_BIN_DIR"/test_jit --gtest_output=xml:$TEST_REPORTS_DIR/test_jit.xml
@@ -526,9 +520,7 @@ test_libtorch() {
526520
"$TORCH_BIN_DIR"/test_lazy --gtest_output=xml:$TEST_REPORTS_DIR/test_lazy.xml
527521
fi
528522

529-
if [[ "$BUILD_ENVIRONMENT" != *-tsan* ]]; then
530-
python test/cpp/jit/tests_setup.py shutdown
531-
fi
523+
python test/cpp/jit/tests_setup.py shutdown
532524

533525
# Wait for background download to finish
534526
wait
@@ -790,7 +782,51 @@ test_bazel() {
790782

791783
tools/bazel test --config=cpu-only --test_timeout=480 --test_output=all --test_tag_filters=-gpu-required --test_filter=-*CUDA :all_tests
792784
else
793-
tools/bazel test //c10/test:core_tests //c10/test:typeid_test //c10/test:util_base_tests
785+
tools/bazel test \
786+
//:any_test \
787+
//:autograd_test \
788+
//:dataloader_test \
789+
//:dispatch_test \
790+
//:enum_test \
791+
//:expanding_array_test \
792+
//:fft_test \
793+
//:functional_test \
794+
//:grad_mode_test \
795+
//:inference_mode_test \
796+
//:init_test \
797+
//:jit_test \
798+
//:memory_test \
799+
//:meta_tensor_test \
800+
//:misc_test \
801+
//:moduledict_test \
802+
//:modulelist_test \
803+
//:modules_test \
804+
//:namespace_test \
805+
//:nested_test \
806+
//:nn_utils_test \
807+
//:operations_test \
808+
//:ordered_dict_test \
809+
//:parallel_benchmark_test \
810+
//:parameterdict_test \
811+
//:parameterlist_test \
812+
//:sequential_test \
813+
//:serialize_test \
814+
//:special_test \
815+
//:static_test \
816+
//:support_test \
817+
//:tensor_flatten_test \
818+
//:tensor_indexing_test \
819+
//:tensor_options_cuda_test \
820+
//:tensor_options_test \
821+
//:tensor_test \
822+
//:torch_dist_autograd_test \
823+
//:torch_include_test \
824+
//:transformer_test \
825+
//c10/cuda/test:test \
826+
//c10/test:core_tests \
827+
//c10/test:typeid_test \
828+
//c10/test:util/ssize_test \
829+
//c10/test:util_base_tests
794830
fi
795831
}
796832

@@ -846,7 +882,7 @@ test_executorch() {
846882
assert_git_not_dirty
847883
}
848884

849-
if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* || "${BUILD_ENVIRONMENT}" == *-bazel-* || "${BUILD_ENVIRONMENT}" == *-tsan* ]]; then
885+
if ! [[ "${BUILD_ENVIRONMENT}" == *libtorch* || "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
850886
(cd test && python -c "import torch; print(torch.__config__.show())")
851887
(cd test && python -c "import torch; print(torch.__config__.parallel_info())")
852888
fi
@@ -935,10 +971,6 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-bazel-* ]]; then
935971
test_bazel
936972
elif [[ "${BUILD_ENVIRONMENT}" == *-mobile-lightweight-dispatch* ]]; then
937973
test_libtorch
938-
elif [[ "${BUILD_ENVIRONMENT}" == *-tsan* ]]; then
939-
# TODO: TSAN check is currently failing with 415 data race warnings. This will
940-
# be addressed later, the first PR can be merged first to setup the CI jobs
941-
test_libtorch || true
942974
elif [[ "${TEST_CONFIG}" = docs_test ]]; then
943975
test_docs_test
944976
else

‎.github/workflows/mac-mps.yml‎

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -33,23 +33,6 @@ jobs:
3333
MACOS_SCCACHE_S3_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
3434
MACOS_SCCACHE_S3_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
3535

36-
macos-12-py3-arm64-test:
37-
name: macos-12-py3-arm64
38-
uses: ./.github/workflows/_mac-test.yml
39-
needs: macos-12-py3-arm64-build
40-
with:
41-
build-environment: macos-12-py3-arm64
42-
test-matrix: |
43-
{ include: [
44-
{ config: "default", shard: 1, num_shards: 3, runner: "macos-m1-12" },
45-
{ config: "default", shard: 2, num_shards: 3, runner: "macos-m1-12" },
46-
{ config: "default", shard: 3, num_shards: 3, runner: "macos-m1-12" },
47-
]}
48-
arch: arm64
49-
secrets:
50-
AWS_OSSCI_METRICS_V2_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_V2_ACCESS_KEY_ID }}
51-
AWS_OSSCI_METRICS_V2_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_V2_SECRET_ACCESS_KEY }}
52-
5336
macos-12-py3-arm64-mps-test:
5437
name: macos-12-py3-arm64-mps
5538
uses: ./.github/workflows/_mac-test-mps.yml

‎.github/workflows/periodic.yml‎

Lines changed: 28 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -93,57 +93,32 @@ jobs:
9393
AWS_OSSCI_METRICS_V2_ACCESS_KEY_ID: ${{ secrets.AWS_OSSCI_METRICS_V2_ACCESS_KEY_ID }}
9494
AWS_OSSCI_METRICS_V2_SECRET_ACCESS_KEY: ${{ secrets.AWS_OSSCI_METRICS_V2_SECRET_ACCESS_KEY }}
9595

96-
linux-bionic-cuda11_7-py3_9-gcc7-build:
97-
name: linux-bionic-cuda11.7-py3.9-gcc7
96+
linux-bionic-cuda11_8-py3_9-gcc7-build:
97+
name: linux-bionic-cuda11.8-py3.9-gcc7
9898
uses: ./.github/workflows/_linux-build.yml
9999
with:
100-
build-environment: linux-bionic-cuda11.7-py3.9-gcc7
101-
docker-image-name: pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7
100+
build-environment: linux-bionic-cuda11.8-py3.9-gcc7
101+
docker-image-name: pytorch-linux-bionic-cuda11.8-cudnn8-py3-gcc7
102102
test-matrix: |
103103
{ include: [
104104
{ config: "multigpu", shard: 1, num_shards: 1, runner: "linux.16xlarge.nvidia.gpu" },
105105
]}
106106
build-with-debug: false
107107

108-
linux-bionic-cuda11_7-py3_9-gcc7-test:
109-
name: linux-bionic-cuda11.7-py3.9-gcc7
110-
uses: ./.github/workflows/_linux-test.yml
111-
needs: linux-bionic-cuda11_7-py3_9-gcc7-build
112-
with:
113-
build-environment: linux-bionic-cuda11.7-py3.9-gcc7
114-
docker-image: ${{ needs.linux-bionic-cuda11_7-py3_9-gcc7-build.outputs.docker-image }}
115-
test-matrix: ${{ needs.linux-bionic-cuda11_7-py3_9-gcc7-build.outputs.test-matrix }}
116-
117-
linux-bionic-cuda11_7-py3_10-gcc7-debug-build:
118-
name: linux-bionic-cuda11.7-py3.10-gcc7-debug
119-
uses: ./.github/workflows/_linux-build.yml
120-
with:
121-
build-environment: linux-bionic-cuda11.7-py3.10-gcc7-debug
122-
docker-image-name: pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7
123-
build-with-debug: true
124-
test-matrix: |
125-
{ include: [
126-
{ config: "default", shard: 1, num_shards: 5, runner: "linux.4xlarge.nvidia.gpu" },
127-
{ config: "default", shard: 2, num_shards: 5, runner: "linux.4xlarge.nvidia.gpu" },
128-
{ config: "default", shard: 3, num_shards: 5, runner: "linux.4xlarge.nvidia.gpu" },
129-
{ config: "default", shard: 4, num_shards: 5, runner: "linux.4xlarge.nvidia.gpu" },
130-
{ config: "default", shard: 5, num_shards: 5, runner: "linux.4xlarge.nvidia.gpu" },
131-
]}
132-
133-
linux-bionic-cuda11_7-py3_10-gcc7-debug-test:
134-
name: linux-bionic-cuda11.7-py3.10-gcc7-debug
108+
linux-bionic-cuda11_8-py3_9-gcc7-test:
109+
name: linux-bionic-cuda11.8-py3.9-gcc7
135110
uses: ./.github/workflows/_linux-test.yml
136-
needs: linux-bionic-cuda11_7-py3_10-gcc7-debug-build
111+
needs: linux-bionic-cuda11_8-py3_9-gcc7-build
137112
with:
138-
build-environment: linux-bionic-cuda11.7-py3.10-gcc7-debug
139-
docker-image: ${{ needs.linux-bionic-cuda11_7-py3_10-gcc7-debug-build.outputs.docker-image }}
140-
test-matrix: ${{ needs.linux-bionic-cuda11_7-py3_10-gcc7-debug-build.outputs.test-matrix }}
113+
build-environment: linux-bionic-cuda11.8-py3.9-gcc7
114+
docker-image: ${{ needs.linux-bionic-cuda11_8-py3_9-gcc7-build.outputs.docker-image }}
115+
test-matrix: ${{ needs.linux-bionic-cuda11_8-py3_9-gcc7-build.outputs.test-matrix }}
141116

142-
linux-bionic-cuda11_8-py3_8-gcc7-debug-build:
143-
name: linux-bionic-cuda11.8-py3.8-gcc7-debug
117+
linux-bionic-cuda11_8-py3_10-gcc7-debug-build:
118+
name: linux-bionic-cuda11.8-py3.10-gcc7-debug
144119
uses: ./.github/workflows/_linux-build.yml
145120
with:
146-
build-environment: linux-bionic-cuda11.8-py3.8-gcc7-debug
121+
build-environment: linux-bionic-cuda11.8-py3.10-gcc7-debug
147122
docker-image-name: pytorch-linux-bionic-cuda11.8-cudnn8-py3-gcc7
148123
build-with-debug: true
149124
test-matrix: |
@@ -155,26 +130,14 @@ jobs:
155130
{ config: "default", shard: 5, num_shards: 5, runner: "linux.4xlarge.nvidia.gpu" },
156131
]}
157132
158-
linux-bionic-cuda11_8-py3_8-gcc7-debug-test:
159-
name: linux-bionic-cuda11.8-py3.8-gcc7-debug
133+
linux-bionic-cuda11_8-py3_10-gcc7-debug-test:
134+
name: linux-bionic-cuda11.8-py3.10-gcc7-debug
160135
uses: ./.github/workflows/_linux-test.yml
161-
needs: linux-bionic-cuda11_8-py3_8-gcc7-debug-build
162-
with:
163-
build-environment: linux-bionic-cuda11.8-py3.8-gcc7-debug
164-
docker-image: ${{ needs.linux-bionic-cuda11_8-py3_8-gcc7-debug-build.outputs.docker-image }}
165-
test-matrix: ${{ needs.linux-bionic-cuda11_8-py3_8-gcc7-debug-build.outputs.test-matrix }}
166-
167-
libtorch-linux-bionic-cuda11_8-gcc7-build:
168-
name: libtorch-linux-bionic-cuda11.8-gcc7
169-
uses: ./.github/workflows/_linux-build.yml
136+
needs: linux-bionic-cuda11_8-py3_10-gcc7-debug-build
170137
with:
171-
build-environment: libtorch-linux-bionic-cuda11.8-gcc7
172-
docker-image-name: pytorch-linux-bionic-cuda11.8-cudnn8-py3-gcc7
173-
build-generates-artifacts: false
174-
test-matrix: |
175-
{ include: [
176-
{ config: "default", shard: 1, num_shards: 1 },
177-
]}
138+
build-environment: linux-bionic-cuda11.8-py3.10-gcc7-debug
139+
docker-image: ${{ needs.linux-bionic-cuda11_8-py3_10-gcc7-debug-build.outputs.docker-image }}
140+
test-matrix: ${{ needs.linux-bionic-cuda11_8-py3_10-gcc7-debug-build.outputs.test-matrix }}
178141

179142
win-vs2019-cuda11_8-py3-build:
180143
name: win-vs2019-cuda11.8-py3
@@ -200,18 +163,6 @@ jobs:
200163
cuda-version: "11.8"
201164
test-matrix: ${{ needs.win-vs2019-cuda11_8-py3-build.outputs.test-matrix }}
202165

203-
libtorch-linux-bionic-cuda11_7-gcc7-build:
204-
name: libtorch-linux-bionic-cuda11.7-gcc7
205-
uses: ./.github/workflows/_linux-build.yml
206-
with:
207-
build-environment: libtorch-linux-bionic-cuda11.7-gcc7
208-
docker-image-name: pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7
209-
build-generates-artifacts: false
210-
test-matrix: |
211-
{ include: [
212-
{ config: "default", shard: 1, num_shards: 1 },
213-
]}
214-
215166
ios-12-5-1-x86-64-coreml:
216167
name: ios-12-5-1-x86-64-coreml
217168
uses: ./.github/workflows/_ios-build-test.yml
@@ -329,3 +280,12 @@ jobs:
329280
secrets:
330281
MACOS_SCCACHE_S3_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_S3_ACCESS_KEY_ID }}
331282
MACOS_SCCACHE_S3_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_S3_SECRET_ACCESS_KEY }}
283+
284+
android-emulator-build-test:
285+
name: android-emulator-build-test
286+
uses: ./.github/workflows/_run_android_tests.yml
287+
with:
288+
test-matrix: |
289+
{ include: [
290+
{ config: "default", shard: 1, num_shards: 1, runner: "ubuntu-latest" },
291+
]}

0 commit comments

Comments
 (0)