[2022-03-02T04:46:50.986Z] [TensorRT] ERROR: 2: [ltWrapper.cpp::setupHeuristic::327] Error Code 2: Internal Error (Assertion cublasStatus == CUBLAS_STATUS_SUCCESS failed.)

[2022-03-02T04:46:50.986Z] ------------------------------ Captured log call -------------------------------

[2022-03-02T04:46:50.986Z] INFO     mmdeploy:init_plugins.py:32 Successfully loaded tensorrt plugins from /opt/mmdeploy/build/lib/libmmdeploy_tensorrt_ops.so

[2022-03-02T04:46:50.986Z] ______________________________ test_create_input _______________________________

[2022-03-02T04:46:50.986Z] 

[2022-03-02T04:46:50.986Z]     def test_create_input():

[2022-03-02T04:46:50.986Z]         model_cfg = load_config(model_cfg_path)[0]

[2022-03-02T04:46:50.986Z]         deploy_cfg = mmcv.Config(

[2022-03-02T04:46:50.986Z]             dict(

[2022-03-02T04:46:50.986Z]                 backend_config=dict(type=Backend.ONNXRUNTIME.value),

[2022-03-02T04:46:50.986Z]                 codebase_config=dict(

[2022-03-02T04:46:50.986Z]                     type=Codebase.MMPOSE.value, task=Task.POSE_DETECTION.value),

[2022-03-02T04:46:50.986Z]                 onnx_config=dict(

[2022-03-02T04:46:50.986Z]                     type='onnx',

[2022-03-02T04:46:50.986Z]                     export_params=True,

[2022-03-02T04:46:50.986Z]                     keep_initializers_as_inputs=False,

[2022-03-02T04:46:50.986Z]                     opset_version=11,

[2022-03-02T04:46:50.986Z]                     save_file='end2end.onnx',

[2022-03-02T04:46:50.986Z]                     input_names=['input'],

[2022-03-02T04:46:50.986Z]                     output_names=['output'],

[2022-03-02T04:46:50.986Z]                     input_shape=None)))

[2022-03-02T04:46:50.986Z]         task_processor = build_task_processor(model_cfg, deploy_cfg, 'cpu')

[2022-03-02T04:46:50.986Z] >       inputs = task_processor.create_input(img, input_shape=img_shape)

[2022-03-02T04:46:50.986Z] 

[2022-03-02T04:46:50.986Z] tests/test_codebase/test_mmpose/test_pose_detection.py:65: 

[2022-03-02T04:46:50.986Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2022-03-02T04:46:50.986Z] mmdeploy/codebase/mmpose/deploy/pose_detection.py:84: in create_input

[2022-03-02T04:46:50.986Z]     from mmpose.apis.inference import LoadImage, _box2cs

[2022-03-02T04:46:50.986Z] ../conda/lib/python3.7/site-packages/mmpose/apis/__init__.py:10: in <module>

[2022-03-02T04:46:50.987Z]     from .train import init_random_seed, train_model

[2022-03-02T04:46:50.987Z] ../conda/lib/python3.7/site-packages/mmpose/apis/train.py:12: in <module>

[2022-03-02T04:46:50.987Z]     from mmpose.core.distributed_wrapper import DistributedDataParallelWrapper

[2022-03-02T04:46:50.987Z] ../conda/lib/python3.7/site-packages/mmpose/core/distributed_wrapper.py:10: in <module>

[2022-03-02T04:46:50.987Z]     class DistributedDataParallelWrapper(nn.Module):

[2022-03-02T04:46:50.987Z] ../mmcv/mmcv/utils/registry.py:312: in _register

[2022-03-02T04:46:50.987Z]     module_class=cls, module_name=name, force=force)

[2022-03-02T04:46:50.987Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z] self = Registry(name=module wrapper, items={'DataParallel': <class 'torch.nn.parallel.data_parallel.DataParallel'>, 'Distribu...arallel'>, 'DistributedDataParallelWrapper': <class 'mmedit.core.distributed_wrapper.DistributedDataParallelWrapper'>})

[2022-03-02T04:46:50.987Z] module_class = <class 'mmpose.core.distributed_wrapper.DistributedDataParallelWrapper'>

[2022-03-02T04:46:50.987Z] module_name = ['DistributedDataParallelWrapper'], force = False

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z]     def _register_module(self, module_class, module_name=None, force=False):

[2022-03-02T04:46:50.987Z]         if not inspect.isclass(module_class):

[2022-03-02T04:46:50.987Z]             raise TypeError('module must be a class, '

[2022-03-02T04:46:50.987Z]                             f'but got {type(module_class)}')

[2022-03-02T04:46:50.987Z]     

[2022-03-02T04:46:50.987Z]         if module_name is None:

[2022-03-02T04:46:50.987Z]             module_name = module_class.__name__

[2022-03-02T04:46:50.987Z]         if isinstance(module_name, str):

[2022-03-02T04:46:50.987Z]             module_name = [module_name]

[2022-03-02T04:46:50.987Z]         for name in module_name:

[2022-03-02T04:46:50.987Z]             if not force and name in self._module_dict:

[2022-03-02T04:46:50.987Z] >               raise KeyError(f'{name} is already registered '

[2022-03-02T04:46:50.987Z]                                f'in {self.name}')

[2022-03-02T04:46:50.987Z] E               KeyError: 'DistributedDataParallelWrapper is already registered in module wrapper'

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z] ../mmcv/mmcv/utils/registry.py:246: KeyError

[2022-03-02T04:46:50.987Z] ___________________________ test_init_pytorch_model ____________________________

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z]     def test_init_pytorch_model():

[2022-03-02T04:46:50.987Z]         from mmpose.models.detectors.base import BasePose

[2022-03-02T04:46:50.987Z] >       model = task_processor.init_pytorch_model(None)

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z] tests/test_codebase/test_mmpose/test_pose_detection.py:71: 

[2022-03-02T04:46:50.987Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2022-03-02T04:46:50.987Z] mmdeploy/codebase/mmpose/deploy/pose_detection.py:63: in init_pytorch_model

[2022-03-02T04:46:50.987Z]     from mmpose.apis import init_pose_model

[2022-03-02T04:46:50.987Z] ../conda/lib/python3.7/site-packages/mmpose/apis/__init__.py:10: in <module>

[2022-03-02T04:46:50.987Z]     from .train import init_random_seed, train_model

[2022-03-02T04:46:50.987Z] ../conda/lib/python3.7/site-packages/mmpose/apis/train.py:12: in <module>

[2022-03-02T04:46:50.987Z]     from mmpose.core.distributed_wrapper import DistributedDataParallelWrapper

[2022-03-02T04:46:50.987Z] ../conda/lib/python3.7/site-packages/mmpose/core/distributed_wrapper.py:10: in <module>

[2022-03-02T04:46:50.987Z]     class DistributedDataParallelWrapper(nn.Module):

[2022-03-02T04:46:50.987Z] ../mmcv/mmcv/utils/registry.py:312: in _register

[2022-03-02T04:46:50.987Z]     module_class=cls, module_name=name, force=force)

[2022-03-02T04:46:50.987Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z] self = Registry(name=module wrapper, items={'DataParallel': <class 'torch.nn.parallel.data_parallel.DataParallel'>, 'Distribu...arallel'>, 'DistributedDataParallelWrapper': <class 'mmedit.core.distributed_wrapper.DistributedDataParallelWrapper'>})

[2022-03-02T04:46:50.987Z] module_class = <class 'mmpose.core.distributed_wrapper.DistributedDataParallelWrapper'>

[2022-03-02T04:46:50.987Z] module_name = ['DistributedDataParallelWrapper'], force = False

[2022-03-02T04:46:50.987Z] 

[2022-03-02T04:46:50.987Z]     def _register_module(self, module_class, module_name=None, force=False):

[2022-03-02T04:46:50.987Z]         if not inspect.isclass(module_class):

[2022-03-02T04:46:50.987Z]             raise TypeError('module must be a class, '

[2022-03-02T04:46:50.988Z]                             f'but got {type(module_class)}')

[2022-03-02T04:46:50.988Z]     

[2022-03-02T04:46:50.988Z]         if module_name is None:

[2022-03-02T04:46:50.988Z]             module_name = module_class.__name__

[2022-03-02T04:46:50.988Z]         if isinstance(module_name, str):

[2022-03-02T04:46:50.988Z]             module_name = [module_name]

[2022-03-02T04:46:50.988Z]         for name in module_name:

[2022-03-02T04:46:50.988Z]             if not force and name in self._module_dict:

[2022-03-02T04:46:50.988Z] >               raise KeyError(f'{name} is already registered '

[2022-03-02T04:46:50.988Z]                                f'in {self.name}')

[2022-03-02T04:46:50.988Z] E               KeyError: 'DistributedDataParallelWrapper is already registered in module wrapper'

[2022-03-02T04:46:50.988Z] 

[2022-03-02T04:46:50.988Z] ../mmcv/mmcv/utils/registry.py:246: KeyError

[2022-03-02T04:46:50.988Z] ______________________ test_single_gpu_test_and_evaluate _______________________

[2022-03-02T04:46:50.988Z] 

[2022-03-02T04:46:50.988Z]     def test_single_gpu_test_and_evaluate():

[2022-03-02T04:46:50.988Z]         from mmcv.parallel import MMDataParallel

[2022-03-02T04:46:50.988Z]         dataset = task_processor.build_dataset(

[2022-03-02T04:46:50.988Z]             dataset_cfg=model_cfg, dataset_type='test')

[2022-03-02T04:46:50.988Z]         dataloader = task_processor.build_dataloader(dataset, 1, 1)

[2022-03-02T04:46:50.988Z]     

[2022-03-02T04:46:50.988Z]         # Prepare dummy model

[2022-03-02T04:46:50.988Z]         model = DummyModel(outputs=[torch.rand([1, 1000])])

[2022-03-02T04:46:50.988Z]         model = MMDataParallel(model, device_ids=[0])

[2022-03-02T04:46:50.988Z]         assert model is not None

[2022-03-02T04:46:50.988Z]         # Run test

[2022-03-02T04:46:50.988Z] >       outputs = task_processor.single_gpu_test(model, dataloader)

[2022-03-02T04:46:50.988Z] 

[2022-03-02T04:46:50.988Z] tests/test_codebase/test_mmpose/test_pose_detection.py:146: 

[2022-03-02T04:46:50.988Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2022-03-02T04:46:50.988Z] mmdeploy/codebase/base/task.py:138: in single_gpu_test

[2022-03-02T04:46:50.988Z]     out_dir, **kwargs)

[2022-03-02T04:46:50.988Z] mmdeploy/codebase/mmpose/deploy/mmpose.py:131: in single_gpu_test

[2022-03-02T04:46:50.988Z]     from mmpose.apis import single_gpu_test

[2022-03-02T04:46:50.988Z] ../conda/lib/python3.7/site-packages/mmpose/apis/__init__.py:10: in <module>

[2022-03-02T04:46:50.988Z]     from .train import init_random_seed, train_model

[2022-03-02T04:46:50.988Z] ../conda/lib/python3.7/site-packages/mmpose/apis/train.py:12: in <module>

[2022-03-02T04:46:50.988Z]     from mmpose.core.distributed_wrapper import DistributedDataParallelWrapper

[2022-03-02T04:46:50.988Z] ../conda/lib/python3.7/site-packages/mmpose/core/distributed_wrapper.py:10: in <module>

[2022-03-02T04:46:50.988Z]     class DistributedDataParallelWrapper(nn.Module):

[2022-03-02T04:46:50.988Z] ../mmcv/mmcv/utils/registry.py:312: in _register

[2022-03-02T04:46:50.988Z]     module_class=cls, module_name=name, force=force)

[2022-03-02T04:46:50.988Z] _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

[2022-03-02T04:46:50.988Z] 

[2022-03-02T04:46:50.988Z] self = Registry(name=module wrapper, items={'DataParallel': <class 'torch.nn.parallel.data_parallel.DataParallel'>, 'Distribu...arallel'>, 'DistributedDataParallelWrapper': <class 'mmedit.core.distributed_wrapper.DistributedDataParallelWrapper'>})

[2022-03-02T04:46:50.988Z] module_class = <class 'mmpose.core.distributed_wrapper.DistributedDataParallelWrapper'>

[2022-03-02T04:46:50.988Z] module_name = ['DistributedDataParallelWrapper'], force = False

[2022-03-02T04:46:50.988Z] 

[2022-03-02T04:46:50.989Z]     def _register_module(self, module_class, module_name=None, force=False):

[2022-03-02T04:46:50.989Z]         if not inspect.isclass(module_class):

[2022-03-02T04:46:50.989Z]             raise TypeError('module must be a class, '

[2022-03-02T04:46:50.989Z]                             f'but got {type(module_class)}')

[2022-03-02T04:46:50.989Z]     

[2022-03-02T04:46:50.989Z]         if module_name is None:

[2022-03-02T04:46:50.989Z]             module_name = module_class.__name__

[2022-03-02T04:46:50.989Z]         if isinstance(module_name, str):

[2022-03-02T04:46:50.989Z]             module_name = [module_name]

[2022-03-02T04:46:50.989Z]         for name in module_name:

[2022-03-02T04:46:50.989Z]             if not force and name in self._module_dict:

[2022-03-02T04:46:50.989Z] >               raise KeyError(f'{name} is already registered '

[2022-03-02T04:46:50.989Z]                                f'in {self.name}')

[2022-03-02T04:46:50.989Z] E               KeyError: 'DistributedDataParallelWrapper is already registered in module wrapper'

[2022-03-02T04:46:50.989Z] 

[2022-03-02T04:46:50.989Z] ../mmcv/mmcv/utils/registry.py:246: KeyError

[2022-03-02T04:46:50.989Z] ----------------------------- Captured stdout call -----------------------------

[2022-03-02T04:46:50.989Z] loading annotations into memory...

[2022-03-02T04:46:50.989Z] Done (t=0.00s)

[2022-03-02T04:46:50.989Z] creating index...

[2022-03-02T04:46:50.989Z] index created!

[2022-03-02T04:46:50.989Z] => num_images: 1

[2022-03-02T04:46:50.989Z] => load 0 samples

[2022-03-02T04:46:50.954Z] 2022-03-02 12:36:49,003 - mmdeploy - INFO - **********Environmental information**********
[2022-03-02T04:46:50.954Z] 2022-03-02 12:36:51,131 - mmdeploy - INFO - sys.platform: linux
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - Python: 3.7.10 (default, Feb 26 2021, 18:47:35) [GCC 7.3.0]
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - CUDA available: True
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - GPU 0: Tesla T4
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - CUDA_HOME: /usr/local/cuda
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - NVCC: Cuda compilation tools, release 10.2, V10.2.89
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - GCC: gcc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - PyTorch: 1.9.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,132 - mmdeploy - INFO - PyTorch compiling details: PyTorch built with:
[2022-03-02T04:46:50.955Z]   - GCC 7.3
[2022-03-02T04:46:50.955Z]   - C++ Version: 201402
[2022-03-02T04:46:50.955Z]   - Intel(R) oneAPI Math Kernel Library Version 2021.2-Product Build 20210312 for Intel(R) 64 architecture applications
[2022-03-02T04:46:50.955Z]   - Intel(R) MKL-DNN v2.1.2 (Git Hash 98be7e8afa711dc9b66c8ff3504129cb82013cdb)
[2022-03-02T04:46:50.955Z]   - OpenMP 201511 (a.k.a. OpenMP 4.5)
[2022-03-02T04:46:50.955Z]   - NNPACK is enabled
[2022-03-02T04:46:50.955Z]   - CPU capability usage: AVX
[2022-03-02T04:46:50.955Z]   - CUDA Runtime 10.2
[2022-03-02T04:46:50.955Z]   - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_37,code=compute_37
[2022-03-02T04:46:50.955Z]   - CuDNN 7.6.5
[2022-03-02T04:46:50.955Z]   - Magma 2.5.2
[2022-03-02T04:46:50.955Z]   - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=10.2, CUDNN_VERSION=7.6.5, CXX_COMPILER=/opt/rh/devtoolset-7/root/usr/bin/c++, CXX_FLAGS= -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -fopenmp -DNDEBUG -DUSE_KINETO -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wno-narrowing -Wall -Wextra -Werror=return-type -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-sign-compare -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wno-unused-result -Wno-unused-local-typedefs -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_VERSION=1.9.0, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, USE_NNPACK=ON, USE_OPENMP=ON, 
[2022-03-02T04:46:50.955Z] 
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - TorchVision: 0.10.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - OpenCV: 4.5.4
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - MMCV: 1.4.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - MMCV Compiler: GCC 7.5
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - MMCV CUDA Compiler: 10.2
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - MMDeployment: 0.3.0+34879e6
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - 
[2022-03-02T04:46:50.955Z] 
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,133 - mmdeploy - INFO - **********Backend information**********
[2022-03-02T04:46:50.955Z] �[00m[2022-03-02 12:36:51.522] [mmdeploy] [info] Register 'DirectoryModel'
[2022-03-02T04:46:50.955Z] �[00m2022-03-02 12:36:51,555 - mmdeploy - INFO - onnxruntime: 1.10.0 ops_is_avaliable : True
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,557 - mmdeploy - INFO - tensorrt: 8.0.3.4 ops_is_avaliable : True
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,559 - mmdeploy - INFO - ncnn: None ops_is_avaliable : False
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,560 - mmdeploy - INFO - pplnn_is_avaliable: False
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,562 - mmdeploy - INFO - openvino_is_avaliable: True
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,562 - mmdeploy - INFO - 
[2022-03-02T04:46:50.955Z] 
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,562 - mmdeploy - INFO - **********Codebase information**********
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,563 - mmdeploy - INFO - mmcls: 0.19.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,564 - mmdeploy - INFO - mmdet: 2.20.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,565 - mmdeploy - INFO - mmedit: 0.12.0
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,566 - mmdeploy - INFO - mmocr: 0.4.1
[2022-03-02T04:46:50.955Z] 2022-03-02 12:36:51,566 - mmdeploy - INFO - mmseg: 0.21.1