dill>=0.3.0
fsspec>=2023.12.2
huggingface-hub>=0.17.0
humanize
loguru>=0.7.0
multiprocess
numpy<2.0.0,>=1.25.0
tqdm

[all]
datatrove[quality]
datatrove[testing]

[cli]
rich

[decont]
lighteval>=0.3.0

[dev]
datatrove[all]

[io]
faust-cchardet
pyarrow
python-magic
warcio
datasets>=2.18.0
orjson
zstandard

[multilingual]
spacy
stanza
pyvi
pythainlp
jieba
indic-nlp-library
kiwipiepy

[processing]
fasttext-wheel
nltk
inscriptis
tldextract
trafilatura<1.12.0,>=1.8.0
tokenizers
ftfy
fasteners
xxhash
kenlm
pyahocorasick

[quality]
ruff>=0.1.5

[s3]
s3fs>=2023.12.2

[testing]
datatrove[cli]
datatrove[io]
datatrove[processing]
datatrove[multilingual]
datatrove[s3]
datatrove[decont]
pytest
pytest-timeout
pytest-xdist
moto[s3,server]
