localhost = <tests.common.devices.Localhost object at 0x7f9e38bb7cd0>
duthost = <tests.common.devices.SonicHost object at 0x7f9e3a4d0750>
request = <SubRequest 'sanity_check' for <Function test_bgp_facts>>
fanouthosts = <omitted>
testbed = <omitted>

    @pytest.fixture(scope="module", autouse=True)
    def sanity_check(localhost, duthost, request, fanouthosts, testbed):
        logger.info("Start pre-test sanity check")
    
        skip_sanity = False
        allow_recover = False
        recover_method = "adaptive"
        check_items = set(copy.deepcopy(constants.DEFAULT_CHECK_ITEMS))  # Default check items
        post_check = False
    
        customized_sanity_check = None
        for m in request.node.iter_markers():
            logger.info("Found marker: m.name=%s, m.args=%s, m.kwargs=%s" % (m.name, m.args, m.kwargs))
            if m.name == "sanity_check":
                customized_sanity_check = m
                break
    
        if customized_sanity_check:
            logger.info("Process marker %s in script. m.args=%s, m.kwargs=%s" % (m.name, str(m.args), str(m.kwargs)))
            skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False)
            allow_recover = customized_sanity_check.kwargs.get("allow_recover", False)
            recover_method = customized_sanity_check.kwargs.get("recover_method", "adaptive")
            if allow_recover and recover_method not in constants.RECOVER_METHODS:
                pytest.warning("Unsupported recover method")
                logger.info("Fall back to use default recover method 'config_reload'")
                recover_method = "config_reload"
    
            check_items = _update_check_items(check_items,
                                              customized_sanity_check.kwargs.get("check_items", []),
                                              constants.SUPPORTED_CHECK_ITEMS)
            post_check = customized_sanity_check.kwargs.get("post_check", False)
    
        if request.config.option.skip_sanity:
            skip_sanity = True
        if request.config.option.allow_recover:
            allow_recover = True
        items = request.config.getoption("--check_items")
        if items:
            items_array=str(items).split(',')
            check_items = _update_check_items(check_items, items_array, constants.SUPPORTED_CHECK_ITEMS)
    
        # ignore BGP check for particular topology type
        if testbed['topo']['type'] == 'ptf' and 'bgp' in check_items:
            check_items.remove('bgp')
    
        logger.info("Sanity check settings: skip_sanity=%s, check_items=%s, allow_recover=%s, recover_method=%s, post_check=%s" % \
            (skip_sanity, check_items, allow_recover, recover_method, post_check))
    
        if skip_sanity:
            logger.info("Skip sanity check according to command line argument or configuration of test script.")
            yield
            return
    
        if not check_items:
            logger.info("No sanity check item is specified, no pre-test sanity check")
            yield
            logger.info("No sanity check item is specified, no post-test sanity check")
            return
    
        print_logs(duthost, constants.PRINT_LOGS)
        check_results = do_checks(duthost, check_items)
        logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check results: !!!!!!!!!!!!!!!!\n%s" % \
                    json.dumps(check_results, indent=4))
        if any([result["failed"] for result in check_results]):
            if not allow_recover:
                pytest.fail("Pre-test sanity check failed, allow_recover=False {}".format(check_results))
                return
    
            logger.info("Pre-test sanity check failed, try to recover, recover_method=%s" % recover_method)
            recover(duthost, localhost, fanouthosts, check_results, recover_method)
            logger.info("Run sanity check again after recovery")
            new_check_results = do_checks(duthost, check_items)
            logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \
                        json.dumps(new_check_results, indent=4))
            if any([result["failed"] for result in new_check_results]):
>               pytest.fail("Pre-test sanity check failed again after recovered by '%s'" % recover_method)
E               Failed: Pre-test sanity check failed again after recovered by 'adaptive'

allow_recover = True
check_items = set(['bgp', 'dbmemory', 'interfaces', 'processes', 'services'])
check_results = [{'check_item': 'services', 'failed': False, 'services_status': {'bgp': True, 'database': True, 'lldp': True, 'pmon': ...tatus': {'bgp': True, 'database': True, 'lldp': True, 'pmon': True, ...}}, {'check_item': 'dbmemory', 'failed': False}]
customized_sanity_check = None
duthost    = <tests.common.devices.SonicHost object at 0x7f9e3a4d0750>
fanouthosts = <omitted>
items      = False
localhost  = <tests.common.devices.Localhost object at 0x7f9e38bb7cd0>
m          = Mark(name='device_type', args=('vs',), kwargs={})
new_check_results = [{'check_item': 'services', 'failed': False, 'services_status': {'bgp': True, 'database': True, 'lldp': True, 'pmon': ...tatus': {'bgp': True, 'database': True, 'lldp': True, 'pmon': True, ...}}, {'check_item': 'dbmemory', 'failed': False}]
post_check = False
recover_method = 'adaptive'
request    = <SubRequest 'sanity_check' for <Function test_bgp_facts>>
result     = {'check_item': 'dbmemory', 'failed': False}
skip_sanity = False
testbed    = <omitted>

common/plugins/sanity_check/__init__.py:118: Failed

<same as previous>
...
if any([result["failed"] for result in new_check_results]):
                failed_items = json.dumps([result for result in new_check_results if result["failed"]], indent=4)
                logger.error("Failed check items:\n{}".format(failed_items))
>               pytest.fail("Pre-test sanity check failed again after recovered by '{}' with failed items:\n{}".format(recover_method, failed_items))
E               Failed: Pre-test sanity check failed again after recovered by 'adaptive' with failed items:
E               [
E                   {
E                       "check_item": "bgp",
E                       "failed": true,
E                       "down_neighbors": [
E                           "10.0.0.59",
E                           "10.0.0.61",
E                           "10.0.0.63",
E                           "fc00::7e",
E                           "fc00::7a",
E                           "10.0.0.57",
E                           "fc00::76",
E                           "fc00::72"
E                       ]
E                   },
E                   {
E                       "check_item": "interfaces",
E                       "failed": true,
E                       "down_ports": [
E                           "PortChannel0001",
E                           "PortChannel0003",
E                           "PortChannel0002",
E                           "PortChannel0004"
E                       ]
E                   }
E               ]
...
<same as previous>