Hynek Kydlíček hynky1999
hynky1999
/ count_hf_parquet_doc_sizes.py
Created
July 10, 2025 14:36
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import pyarrow.parquet as pq | |
| from huggingface_hub import HfFileSystem | |
| from datasets import get_dataset_config_names, get_dataset_infos | |
| import os | |
| import json | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| from datetime import datetime | |
| DATA_BASE_PATH = "HuggingFaceFW/fineweb-edu" |
hynky1999
/ gist:8d98b98ae5863919195ee273bef755c9
Created
February 29, 2024 14:28
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| f |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.