-        encoder = list(self.added_tokens_encoder.keys()) + [str(token) f
-        # if some of the special tokens are strings, we check if we don'
-        tokens_to_add += [
-            token for token in self.all_special_tokens_extended if token
-        ]
-        if len(tokens_to_add) > 0:
-            # super hack: if a token.special is set, tokenizer ignores i
-            # Accumulate added tokens into batches of special/non-specia
-            # individual tokens would repeatedly rebuild a trie, which c
-            is_last_special = None
-            tokens = []
-            special_tokens = self.all_special_tokens
-            for token in tokens_to_add:
-                is_special = (
-                    (token.special or str(token) in special_tokens)
-                    if isinstance(token, AddedToken)
-                    else str(token) in special_tokens
-                )
-                if is_last_special is None or is_last_special == is_spec
-                    tokens.append(token)
-                else:
-                    self._add_tokens(tokens, special_tokens=is_last_spec
-                    tokens = [token]
-                is_last_special = is_special
-            if tokens:
-                self._add_tokens(tokens, special_tokens=is_last_special)
+ self._add_tokens(tokens_to_add)