mytecz_omnitoken-0.1.5.dist-info/licenses/LICENSE,sha256=hM271Qfw9gbzBQ04OoLFNtFXbNGxmcukVZuruxX4oIM,1082
omnitoken/__init__.py,sha256=l3SaT0Px4DPNp1kcAfVu_Kh_lYeRBvrStUFuQ8T0E2U,3667
omnitoken/cli.py,sha256=ncCTCGW1jOPmpH_CYORG3pB3mzqRm0bRgH3iEBIcPo4,9817
omnitoken/tokenizer_base.py,sha256=bgo7a5v980VF-7xgEstIkTBHgo8wj29l1gChBkd2wm8,15034
omnitoken/tokenizer_bpe.py,sha256=tAH8GJUHUpSRn4wudu0QxNx2ZgDV4djMuAVOSXiw5A4,12179
omnitoken/tokenizer_hybrid.py,sha256=ubzd0xL6CyL6a_fhKeN0TfJdJky1eycO7DY0gMe_sog,21107
omnitoken/tokenizer_sentencepiece.py,sha256=TTWnZ0qmDxqYkcUpc2RBkVTMO5_e7CjTOtIRVYB8dGo,15210
omnitoken/tokenizer_wordpiece.py,sha256=1EazY_q1vM3uUJkMh3H4YHQgjZYSS97oCETt-7ZYaMI,15615
omnitoken/trainer.py,sha256=3fA5WEn2T9q-KleiLxGmw378RArA01uEjgl8HMUP-7o,15355
omnitoken/utils.py,sha256=O4bX3f6gpHjvJwPdB-UVPr1GNPZXo0-n2tkYUupUSaY,15962
omnitoken/data/test_samples/roundtrip_expected.json,sha256=p89JyZCYhwPOq3dE9rYG0uxbrNxJptw6Lrq5BEU5TsI,356
omnitoken/data/test_samples/small_texts.txt,sha256=7fV33s9l1X3SUKX0QfljEM78vCDNDMXHSWtxzXghbWM,112
omnitoken/data/test_samples/unicode_cases.txt,sha256=1AOeSpjBdYnYXyb4TjZbzBs0kYY-ayp2I06xyug4GoQ,138
omnitoken/data/training_corpus/code_snippets.txt,sha256=2FWmKP36ztJFaubOa5jIVfzkBGGarkgFCRd4DLDhGAw,427
omnitoken/data/training_corpus/emojis.txt,sha256=xKb0SHdudI4nBXcZ8w8byb7cxvVHIKE64By_kQn-oqU,248
omnitoken/data/training_corpus/english.txt,sha256=V6L2CHiy7fV5FaSbyIjuuTYzliEaEpJq1SDl8tiFB2U,272
omnitoken/data/training_corpus/json_samples.json,sha256=3AzOpvq8wH55m53T4AgnqprfblNaiFtTyL2qhPPzee8,454
omnitoken/data/training_corpus/multilingual.txt,sha256=ZCBRRAP70nTR4Agm6J-M4RiU3iqhzstzpruc-g-8UsM,303
mytecz_omnitoken-0.1.5.dist-info/METADATA,sha256=0CivTC_Z63Moc0PydG5CAPGcmjdR0xtG8G4cCO_MEVE,15985
mytecz_omnitoken-0.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
mytecz_omnitoken-0.1.5.dist-info/entry_points.txt,sha256=t6FhOPzVlQdTbwC0LHlAwhei0DqjMlFs_0EJzGJywFY,49
mytecz_omnitoken-0.1.5.dist-info/top_level.txt,sha256=VmoX1_k4RPJfhLzfotjgG2KQLGUodCx0VKieuNWt-hY,10
mytecz_omnitoken-0.1.5.dist-info/RECORD,,
