dill>=0.3.0
fsspec>=2023.12.2
huggingface-hub>=0.17.0
humanize
loguru>=0.7.0
multiprocess
numpy>=1.25.0
tqdm

[all]
datatrove[quality]
datatrove[testing]

[cli]
rich

[dev]
datatrove[all]

[io]
faust-cchardet
pyarrow
python-magic
warcio
datasets>=2.18.0

[processing]
fasttext-wheel
nltk
inscriptis
tldextract
trafilatura>=1.8.0
tokenizers
ftfy
fasteners
xxhash

[quality]
ruff>=0.1.5

[s3]
s3fs>=2023.12.2

[testing]
datatrove[cli]
datatrove[io]
datatrove[processing]
datatrove[s3]
pytest
pytest-timeout
pytest-xdist
moto[s3,server]
