import torch
from transformers import (GPT2Config, GPT2Model, GPT2Tokenizer)

# use_cache is True by default in GPT2Model. Here we wrap a class to disable past state output.
class GPT2DoubleHeadsModelNoPastState(GPT2DoubleHeadsModel):
    def __init__(self, config):
        super().__init__(config)

    def forward(self, input_ids, token_type_ids):
        return super().forward(input_ids, past=None, attention_mask=None, token_type_ids=token_type_ids, use_cache=False)

model_name="gpt2"
config = GPT2Config.from_pretrained(model_name)
tokenizer = GPT2Tokenizer.from_pretrained(model_name)
model = GPT2ModelNoPastState.from_pretrained(model_name)

example_inputs = tokenizer.encode_plus("This is a sample input", return_tensors="pt")
del example_inputs["attention_mask"]
example_outputs = model(**example_inputs)

input_names = ['input_ids',  'token_type_ids']
output_names=['output_1', 'output_2']
dynamic_axes={'input_ids': {0: 'batch_size', 1: 'num_choices', 2: 'seq_len'}, 
     'token_type_ids': {0: 'batch_size', 1: 'num_choices', 2: 'seq_len'},
     'output_1': {0: 'batch_size', 1: 'num_choices': 2: 'seq_len', 3: 'vocab_size'}, 
     'output_2': {0: 'batch_size', 1: 'num_choices'}
}
output_path='gpt2.onnx'
torch.onnx.export(model=model,
                  args=(example_inputs[input_names[0]].unsqueeze(0), example_inputs[input_names[1]].unsqueeze(0)),
                  f=output_path,
                  input_names=input_names,
                  output_names=output_names,
                  example_outputs=example_outputs,
                  dynamic_axes=dynamic_axes,
                  do_constant_folding=True,
                  opset_version=11,
                  use_external_data_format=False)

.../torch/onnx/symbolic_helper.py", line 87...
RuntimeError: Unexpected node type: onnx::Sub