benchmarks/dynamo/torchbench.py -d cuda --inductor --training --float32 --accuracy --no-skip --only resnet50_quantized_qat

cuda train resnet50_quantized_qat             WARNING:common:fp64 golden ref were not generated for resnet50_quantized_qat
[2022-10-28 17:52:12,635] torch._dynamo.convert_frame: [ERROR] WON'T CONVERT forward <eval_with_key>.5 line 4 
due to: 
Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/torch/_ops.py", line 257, in __call__
    return self._op(*args, **kwargs or {})
TypeError: meta__fused_moving_avg_obs_fq_helper() missing 2 required positional arguments: 'per_row_fake_quant' and 'symmetric_quant'

from user code:
   File "<eval_with_key>.5", line 5, in forward
    activation_post_process_0 = self.activation_post_process_0(x);  x = None

Set torch._dynamo.config.verbose=True for more information
==========
[2022-10-28 17:52:12,635] torch._dynamo.eval_frame: [ERROR] Error while processing frame
Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 380, in _compile
    out_code = transform_code_object(code, transform)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object
    transformations(instructions, code_options)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 368, in transform
    tracer.run()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 1448, in run
    super().run()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 349, in run
    and self.step()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 322, in step
    getattr(self, inst.opname)(inst)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 174, in wrapper
    return inner_fn(self, inst)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 264, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/nn_module.py", line 201, in call_function
    return variables.TensorVariable.create(
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 201, in create
    example_value = _get_fake_value(proxy.node, tx)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 131, in _get_fake_value
    return wrap_fake_exception(
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/utils.py", line 709, in wrap_fake_exception
    return fn()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 132, in <lambda>
    lambda: _run_node(tx.output, node, args, kwargs, nnmodule)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 56, in _run_node
    return nnmodule(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/nn/modules/module.py", line 1423, in _call_impl
    return forward_call(*input, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/ao/quantization/fake_quantize.py", line 342, in forward
    return torch.fused_moving_avg_obs_fake_quant(
  File "/scratch/ybliang/work/repos/pytorch/torch/_subclasses/fake_tensor.py", line 829, in __torch_dispatch__
    r = func(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_ops.py", line 257, in __call__
    return self._op(*args, **kwargs or {})
TypeError: meta__fused_moving_avg_obs_fq_helper() missing 2 required positional arguments: 'per_row_fake_quant' and 'symmetric_quant'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 249, in catch_errors
    return callback(frame, cache_size)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 452, in _convert_frame
    result = inner_convert(frame, cache_size)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn
    return fn(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/utils.py", line 87, in time_wrapper
    r = func(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 325, in _convert_frame_assert
    return _compile(
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 442, in _compile
    raise InternalTorchDynamoError() from e
torch._dynamo.exc.InternalTorchDynamoError
TorchDynamo optimized model failed to run because of following error
ERROR:common:
Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 380, in _compile
    out_code = transform_code_object(code, transform)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/bytecode_transformation.py", line 341, in transform_code_object
    transformations(instructions, code_options)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 368, in transform
    tracer.run()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 1448, in run
    super().run()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 349, in run
    and self.step()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 322, in step
    getattr(self, inst.opname)(inst)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 174, in wrapper
    return inner_fn(self, inst)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 754, in CALL_FUNCTION
    self.call_function(fn, args, {})
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/symbolic_convert.py", line 264, in call_function
    self.push(fn.call_function(self, args, kwargs))
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/nn_module.py", line 201, in call_function
    return variables.TensorVariable.create(
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 201, in create
    example_value = _get_fake_value(proxy.node, tx)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 131, in _get_fake_value
    return wrap_fake_exception(
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/utils.py", line 709, in wrap_fake_exception
    return fn()
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 132, in <lambda>
    lambda: _run_node(tx.output, node, args, kwargs, nnmodule)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/variables/tensor.py", line 56, in _run_node
    return nnmodule(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/nn/modules/module.py", line 1423, in _call_impl
    return forward_call(*input, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/ao/quantization/fake_quantize.py", line 342, in forward
    return torch.fused_moving_avg_obs_fake_quant(
  File "/scratch/ybliang/work/repos/pytorch/torch/_subclasses/fake_tensor.py", line 829, in __torch_dispatch__
    r = func(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_ops.py", line 257, in __call__
    return self._op(*args, **kwargs or {})
TypeError: meta__fused_moving_avg_obs_fq_helper() missing 2 required positional arguments: 'per_row_fake_quant' and 'symmetric_quant'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/scratch/ybliang/work/repos/pytorch/benchmarks/dynamo/common.py", line 1042, in check_accuracy
    new_result = optimized_model_iter_fn(model, example_inputs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 166, in _fn
    return fn(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/benchmarks/dynamo/common.py", line 946, in run_n_iterations
    self.model_iter_fn(mod, inputs, collect_outputs=False)
  File "/scratch/ybliang/work/repos/pytorch/benchmarks/dynamo/torchbench.py", line 332, in forward_and_backward_pass
    cloned_inputs = clone_inputs(inputs)
  File "/scratch/ybliang/work/repos/pytorch/benchmarks/dynamo/torchbench.py", line 335, in <graph break in forward_and_backward_pass>
    pred = mod(*cloned_inputs)
  File "/scratch/ybliang/work/repos/pytorch/torch/fx/graph_module.py", line 660, in call_wrapped
    return self._wrapped_call(self, *args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/fx/graph_module.py", line 279, in __call__
    raise e
  File "/scratch/ybliang/work/repos/pytorch/torch/fx/graph_module.py", line 269, in __call__
    return super(self.cls, obj).__call__(*args, **kwargs)  # type: ignore[misc]
  File "/scratch/ybliang/work/repos/pytorch/torch/nn/modules/module.py", line 1423, in _call_impl
    return forward_call(*input, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/eval_frame.py", line 249, in catch_errors
    return callback(frame, cache_size)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 452, in _convert_frame
    result = inner_convert(frame, cache_size)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 118, in _fn
    return fn(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/utils.py", line 87, in time_wrapper
    r = func(*args, **kwargs)
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 325, in _convert_frame_assert
    return _compile(
  File "/scratch/ybliang/work/repos/pytorch/torch/_dynamo/convert_frame.py", line 442, in _compile
    raise InternalTorchDynamoError() from e
torch._dynamo.exc.InternalTorchDynamoError
FAIL