import numpy as np
import re
from sklearn.datasets import fetch_20newsgroups
from collections import defaultdict
from time import perf_counter
from statistics import mean, stdev
from sklearn.feature_extraction._hashing_fast import transform as _hashing_transform


def tokens(doc):
    return (tok.lower() for tok in re.findall(r"\w+", doc))


def token_freqs(doc):
    freq = defaultdict(int)
    for tok in tokens(doc):
        freq[tok] += 1
    return freq


n_features = 2**20
n_repeats = 100

raw_data, _ = fetch_20newsgroups(subset="all", return_X_y=True)
raw_X = [token_freqs(d).items() for d in raw_data]

durations = []
for i in range(n_repeats):
    t0 = perf_counter()
    _hashing_transform(
        raw_X, n_features, np.float64, alternate_sign=True, seed=n_repeats
    )
    duration = perf_counter() - t0
    durations.append(duration)

mean_dur = mean(durations)
std_dur = stdev(durations)
print(f"{mean_dur:.4f} +/- {std_dur:.4f}")