LICENSE
README.md
setup.py
src/mwtokenizer/__init__.py
src/mwtokenizer/tokenizer.py
src/mwtokenizer/utils.py
src/mwtokenizer.egg-info/PKG-INFO
src/mwtokenizer.egg-info/SOURCES.txt
src/mwtokenizer.egg-info/dependency_links.txt
src/mwtokenizer.egg-info/not-zip-safe
src/mwtokenizer.egg-info/requires.txt
src/mwtokenizer.egg-info/top_level.txt
src/mwtokenizer/assets/__init__.py
src/mwtokenizer/assets/dict_abbr_filtered.json
src/mwtokenizer/assets/spcmodels/__init__.py
src/mwtokenizer/assets/spcmodels/nws_bpe_gan.model
src/mwtokenizer/assets/spcmodels/nws_bpe_gan.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_ja.model
src/mwtokenizer/assets/spcmodels/nws_bpe_ja.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_km.model
src/mwtokenizer/assets/spcmodels/nws_bpe_km.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_lo.model
src/mwtokenizer/assets/spcmodels/nws_bpe_lo.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_my.model
src/mwtokenizer/assets/spcmodels/nws_bpe_my.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_th.model
src/mwtokenizer/assets/spcmodels/nws_bpe_th.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_wuu.model
src/mwtokenizer/assets/spcmodels/nws_bpe_wuu.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_zh.model
src/mwtokenizer/assets/spcmodels/nws_bpe_zh.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_zh_classical.model
src/mwtokenizer/assets/spcmodels/nws_bpe_zh_classical.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_zh_min_nan.model
src/mwtokenizer/assets/spcmodels/nws_bpe_zh_min_nan.vocab
src/mwtokenizer/assets/spcmodels/nws_bpe_zh_yue.model
src/mwtokenizer/assets/spcmodels/nws_bpe_zh_yue.vocab
src/mwtokenizer/assets/spcmodels/spc_bpe_tib.model
src/mwtokenizer/assets/spcmodels/spc_bpe_tib.vocab
src/mwtokenizer/config/__init__.py
src/mwtokenizer/config/symbols.py
tests/test_sentence_segmentation.py
tests/test_word_tokenization.py