datatrove/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/data.py,sha256=V4RctBl3AJoml2cvr-SwqgEjiDl_IItu4n8HOHDlSAo,1145
datatrove/io.py,sha256=c_o7rFcINKJONHJ3UXin-D84uQkmYQqJLxerBhcLwpg,16136
datatrove/assets/banned_subwords.txt,sha256=R3BEXNjSsWSV796IvE78OlIL3A8xN9FNhc09ibGUojI,1673
datatrove/assets/banned_words.txt,sha256=eq5RCczE2DVxZxaPo6kTMfovr9jrjMnDjMkUfCYQ-rA,1732
datatrove/assets/soft_banned_words.txt,sha256=5QQwMKzMozsONoppOZdXa-6kxYJ52mQb_0C04CKBsGM,674
datatrove/assets/tokenizer_assignment.csv,sha256=vykJL2Wpc59Na37jDPGOF4MTQtyAzZ_y5c58Jd4SJjQ,153012
datatrove/assets/url_filterblacklistsv0_3_0.tar.gz,sha256=s37v4fMQPfpGshs_QHuLI87ClY6iIChwDmzETYD1Xp4,18172875
datatrove/executor/__init__.py,sha256=Y5UZbjEfyIFHO1bU70U701ILfD9TM4jJUYoQjCdprEQ,119
datatrove/executor/base.py,sha256=aUJyhji86QqiXw0IyXw1iX9TWVZVFXny8IIqd5soqdM,6684
datatrove/executor/local.py,sha256=S88LLVNuvBt4GDvy-99pc-p66qH3JbtFONktSI-O3ZY,6545
datatrove/executor/ray.py,sha256=6FivBgfH6NVyFnRDypVnVlmhbkO7UcUtVmR7_h9A4Sk,11007
datatrove/executor/slurm.py,sha256=hSrFsdlPY7UM_88x-2PbnFxqTBj62_DAtFQfUT4hoog,16156
datatrove/pipeline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/pipeline/base.py,sha256=TeUGOtySvcsJmj_jbHqUhIq_RCGZXWtqPtGP9Tu4iqc,4175
datatrove/pipeline/decont/__init__.py,sha256=_prPzXmokM5Zc98r-tTgAUS3z-5Se2guMt-MBzu_MFE,81
datatrove/pipeline/decont/n_grams.py,sha256=J7xiNOf8SZ-TLNzBg6G57avx6RNqT33yrVtEV9OHz_U,10136
datatrove/pipeline/dedup/__init__.py,sha256=_PF3nVW0EYa1x4o6S79XsSqHBrYj0EL8QESDuNq1Gow,493
datatrove/pipeline/dedup/bloom_filter.py,sha256=mvt7ebLuLjIuYilnAa3fqHhLHBs9-QPwKoEvjtPapsg,7969
datatrove/pipeline/dedup/exact_substrings.py,sha256=K5gmh4ekRX-ZZwfg8Mq6g_z1RwqHLw6_iynt_BVLhzY,13976
datatrove/pipeline/dedup/minhash.py,sha256=gG7rcSW8bB4cibF3kLXc4A0TT2hVgWzwvRSQTgVKg2c,30094
datatrove/pipeline/dedup/sentence_dedup.py,sha256=skQ3Sn3Kw7MZX4kAQUd4_74mtzW1Ikd1vs4JTydsops,21385
datatrove/pipeline/dedup/url_dedup.py,sha256=CToVwutt5oyRcMiGDnd4vxycjPLt2phNGd1FX3ZlZMY,16756
datatrove/pipeline/extractors/__init__.py,sha256=sHzyoqkwven6X4XdKoWzvmHXq8BUbS8Kl-t1-kvrs8k,80
datatrove/pipeline/extractors/base.py,sha256=oV71zr0smsIysyfEFQZmFrTRXBDFkDZKMRGg5o3wxtQ,6172
datatrove/pipeline/extractors/modular.py,sha256=OzDkwS3vDQeTeNYcRImezh025qcelhcN8cYqE7u-z90,2049
datatrove/pipeline/extractors/trafilatura.py,sha256=zemM3rp_0PzA2gXuIaXMDN7xoX8qA_45P90ZXOzu7E8,1752
datatrove/pipeline/filters/__init__.py,sha256=YmKhWjPyC7XNL9qpghktatbZy6IIJISoEHvJNmfyW8M,554
datatrove/pipeline/filters/base_filter.py,sha256=jjAkLlpSb9FXln97EyIPOFg9EzA46NxFPon2aGYJ7Jc,3337
datatrove/pipeline/filters/c4_filters.py,sha256=JapWlDIKEc0LVDKouipRFBt1Qe0VdE1GqT0MDl6CyG4,11604
datatrove/pipeline/filters/fasttext_filter.py,sha256=vhXPtw3-gckY3S2JQP-2v4vV0c2gKnamAFdbrCGy6O4,5194
datatrove/pipeline/filters/fineweb_quality_filter.py,sha256=ufmwDfNhGu9TEKaU5jg9WsjT_J_naUX4omCLGs2n6CU,2314
datatrove/pipeline/filters/gopher_quality_filter.py,sha256=sp2Ui3fEoLfOB3C8y0piALVYoTOkJZqjwdnieEpeKrI,5195
datatrove/pipeline/filters/gopher_repetition_filter.py,sha256=7G3Ki2tbzJF1KTS9o13AJPvRPBBKMeexGl3vO3t-8Iw,4902
datatrove/pipeline/filters/lambda_filter.py,sha256=TcEIMbpHB0pRecvoUDQ-2MnFwfpL8udGPgqZsbiIGs0,827
datatrove/pipeline/filters/language_filter.py,sha256=7Ap0G1EuwVRzI3Vd_s1Icod1TUhKDL7qUHd9vYFoNkk,2642
datatrove/pipeline/filters/regex_filter.py,sha256=0heUgDmmP98qXuxocMNCZy63WLsl6F_DD2bAMgtOp6A,737
datatrove/pipeline/filters/sampler_filter.py,sha256=QhDn-6_L_kVALYIDMWSNmSR0A2NkS3rrS3iu1sIf2OI,741
datatrove/pipeline/filters/unigram_log_probs.py,sha256=TDh0l353sbUGmoUk5suIsNKW4RjJuSuprcO1cK4udDM,2778
datatrove/pipeline/filters/url_filter.py,sha256=-2ERJyn68h-Ggr9QPKLGHSF3pcItr5oxrRxyltkjUn0,5418
datatrove/pipeline/formatters/__init__.py,sha256=9AexxNkQhphgOuMftuPpfLVxQCnxeg8mC4OjvloRd3g,117
datatrove/pipeline/formatters/base.py,sha256=F3-AojKl3ZA2nTr_IoXIqfW_mTRcBFRfwWw2odjlgdc,665
datatrove/pipeline/formatters/ftfy.py,sha256=kV6Y3y7uO4PzD4ebYQWWkHAo9UELO8wmefiPuzc2-p4,2594
datatrove/pipeline/formatters/pii.py,sha256=V73FNrwuV-8B9802NDt5f_HjJml2cOXL3hyBAkptUmU,3535
datatrove/pipeline/formatters/symbol_lines_remover.py,sha256=B6pVCkpjB5iKCaYwmCuSuXe1zCJcwTivq__73vEpQ2Q,1406
datatrove/pipeline/readers/__init__.py,sha256=bTCTLGdjs7YduMD5sY6q7_WM1iy43q5FL5h7mt64j5E,199
datatrove/pipeline/readers/base.py,sha256=EKy588LPb4bVRPZDmS46OcMk1Rdtkpqhj7xLWK7UYZs,9934
datatrove/pipeline/readers/csv.py,sha256=qXM_Jg9tGkAAJoXFbEekUm26qWchYeMlb73JIl6s7bc,3141
datatrove/pipeline/readers/huggingface.py,sha256=w6u1r_RkxwXNRYRQJHn8lZ9tNRwcKdavZaDkuES5__k,5724
datatrove/pipeline/readers/ipc.py,sha256=ntBQB9T4Ae62qS0IySLiJflBpQy9ZS1Wv9PCQXyLFHU,3877
datatrove/pipeline/readers/jsonl.py,sha256=pvBItv8u7HXzasGMnXABbH-KqQjqqfu8pGM8dPFSZuo,3609
datatrove/pipeline/readers/parquet.py,sha256=7BPb7uRJ2UrYe4caG6SANHL52saRutqC94hNJXuaQOc,3615
datatrove/pipeline/readers/warc.py,sha256=MLj_VwlIGZXIE_7gJfIEFe8R8DEmIiST9ybf90RVsNw,5343
datatrove/pipeline/stats/__init__.py,sha256=mmd0EjLUXvl52Qg33ZX1i9Wm9_kOthsbg5wpQh8O_mA,750
datatrove/pipeline/stats/base.py,sha256=XPNdXNy5osgsU9W-hcNG51NnhCYQugE3rOD8qpMO4qY,5609
datatrove/pipeline/stats/config.py,sha256=79mqjq_RWWIXVnDbkkIadqPZNCQ6yrwhicofEcO7FPc,771
datatrove/pipeline/stats/contamination_stats.py,sha256=-i1G3IS6eRTVCk50idcqBI9JMcma0ULYNnJZQgPNmCA,1797
datatrove/pipeline/stats/doc_stats.py,sha256=FJThwh9HqSc_VLdcoEUuS1c0R-K3z8XLfJDWqOOFS98,2097
datatrove/pipeline/stats/lang_stats.py,sha256=w9Vn8ArXjwlIkAOuplWB8RTS1Stm1bV0REanFTOXmzM,1296
datatrove/pipeline/stats/line_stats.py,sha256=H8vm6zRxE6ZsJ429K0WxiT16lUEles14h5bY2VAI_0A,3674
datatrove/pipeline/stats/merger.py,sha256=ltpKwjbBUejAA189Lcflr-BXw_ypQySYGDOW5EZevog,3478
datatrove/pipeline/stats/paragraph_stats.py,sha256=aLTjPt_9BTp9WS_WgOW62ksl6qvl8JAwDHs11Gt2Jt8,2931
datatrove/pipeline/stats/perplexity_stats.py,sha256=684cn1r8BBGFiMoRJY9sevJ33bk9GD40dR3ay2VJKhc,1338
datatrove/pipeline/stats/sentence_stats.py,sha256=bmepjeMh3nVF4HRYsCAuISdUxKbv4XsEyQ0Y0Bgle20,2521
datatrove/pipeline/stats/token_stats.py,sha256=dJZBr0mK6abnj5rWX1dG9mP7MqYnax7kGk9-o9QYzr4,1362
datatrove/pipeline/stats/word_stats.py,sha256=8N0yli9Yc6QrtB0knTWWGASVXahpP-CB0YcAuA9XD54,3369
datatrove/pipeline/tokens/__init__.py,sha256=RuODjtej5Zl4wW1n0ziZHsLXwDsk5VLYIoZKUkMr2KE,256
datatrove/pipeline/tokens/context_shuffler.py,sha256=MOT2zQMR1jp4XPpid_NCRPqVCG31gPq03l7t5YhZ4dU,3331
datatrove/pipeline/tokens/counter.py,sha256=vfjKxIL96Dicw2lABlPnTqPogV2GtQ4XuRVcpYh6uwA,3192
datatrove/pipeline/tokens/megatron_tokenizer.py,sha256=UMpu7WPtDSVUX2cIVmNvZ_7Er3hmEIIPR4xaaQsc9fM,9574
datatrove/pipeline/tokens/merger.py,sha256=nAhMAWSie49uJ4yIB6Sa8kVUbNSYFuy0vKB2alfaGhU,11129
datatrove/pipeline/tokens/tokenizer.py,sha256=fJJSEAOX3bInfD7lMmk1Pa_dTPlsz5rhiTvKuZDUfEk,23057
datatrove/pipeline/writers/__init__.py,sha256=4UPe-fekukvJ2cYpFACUlYd2Y0TTL-AS0Np6PcxJLq0,132
datatrove/pipeline/writers/disk_base.py,sha256=nDYh4K1h0jImu5bZ8xEuIZ-cZeq4ELv9TTzbazc124M,7256
datatrove/pipeline/writers/huggingface.py,sha256=1-DOY10vGMHotzktNm8HnnXruTrtyB-mp_pARL_Hmow,6031
datatrove/pipeline/writers/jsonl.py,sha256=rIVNIOda0DRonf6c3Mkr0ovYdDFPKr0bJSTrJRL0faU,1668
datatrove/pipeline/writers/parquet.py,sha256=D38fujwE3Gc-4oQAnx-2tFVHEJeIfxUrE2CdlfyBg90,3251
datatrove/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/tools/check_dataset.py,sha256=yyS0cbbVEu8w656PzL02zjI_tnrBSnXSpsiCrcY5nec,3853
datatrove/tools/failed_logs.py,sha256=bi0I_xmPZ1XLdpKlRuC9kJlRQxE0RnjcydnMHSov024,2636
datatrove/tools/inspect_data.py,sha256=mf4T0spOJGehci39PWDMVpakuqIda6Pxd9_u_L9LUEA,5741
datatrove/tools/jobs_status.py,sha256=x3F8pmWc2YA-um_gsWLZTGAMDef1aWndqe9uPPcvnOc,3451
datatrove/tools/launch_pickled_pipeline.py,sha256=HKHZ7cSEOmPA6KTldiFO20twAxY-5czGopi5PonMYX0,512
datatrove/tools/merge_stats.py,sha256=2n0VJRiV_0UTTpCtjK9HbecEjOaSS4bP5Uenv1Y3cR8,1175
datatrove/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
datatrove/utils/_import_utils.py,sha256=uwmPwjYJ6xM1enKNU8PlyYwfYV9e-06rg1orUO0qwHs,3448
datatrove/utils/batching.py,sha256=g6BinvLQZrhNQr6dYbITJPfAiUt8F6aE5UOGCgRrJew,447
datatrove/utils/binaryio.py,sha256=Qasnh3KsdKy5PUco2gVc3Gio5BGfuiU39KZ73HdON1E,3049
datatrove/utils/dataset.py,sha256=6Ty0Lg4xLMbS77-LBkZgMOyWPA3j4kGuaibSN3ontBc,14962
datatrove/utils/hashing.py,sha256=tMofxOQsCMj2mbkKCceMzJWra5nr-ApL3SNhuc2INS4,1522
datatrove/utils/japanese_tokenizer.py,sha256=fqBOwsriOEqsO8vIQi56fo4ozh0h6BRqp0v2gJnKqcw,12056
datatrove/utils/lid.py,sha256=aAlalVvC49G456gDsu6jaLX0pFy0PpQ-7cFuXFignP8,3073
datatrove/utils/logging.py,sha256=NoVn8IrdtHOOWf00cuvXVpj1GdREiSJFR7grrxAOt7I,2090
datatrove/utils/perplexity.py,sha256=o_OY6RTwacK_wORWhEmnvBTwIibIr__gtCQJ_CLGtxI,4656
datatrove/utils/stats.py,sha256=P_uAhM-3f6FG-wEBi9rsaO32xX_BU463n4Tn6E6VJ_U,14395
datatrove/utils/text.py,sha256=Ft3vs_Hpf-4WIL0mY-4xt3tJhl_NVru0k_MtOKMYdsk,7146
datatrove/utils/tokenization.py,sha256=id-1lsWiuNo-fwtJZwYP0NzXQ5dcIc5pyD2B8muT0sM,3309
datatrove/utils/typeshelper.py,sha256=q9WxDzJR6RlwKfljs3M0GlI8gNJ3VCTw1gfjK5eBJF8,117156
datatrove/utils/word_tokenizers.py,sha256=JxSllQh2D1kQ0zHKftdDMrYeiUwcPYliH7XQ5ULudbY,17664
datatrove/utils/hashes/sha1.py,sha256=wffxM0B1aED2J_IScq8qzhk8WyrdGpZx8qfg4A-Z7yo,689
datatrove/utils/hashes/xxhash.py,sha256=kSE2atuhcReX8oOmFPCkr1OxBVK-VYrdRvsU1k36Z0k,148
datatrove-0.5.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
datatrove-0.5.0.dist-info/METADATA,sha256=CBbnYKmcqEOXIPeLNeSgwoMpx2-Cw1TUHk4I_s-TAkQ,30864
datatrove-0.5.0.dist-info/WHEEL,sha256=ooBFpIzZCPdw3uqIQsOo4qqbA4ZRPxHnOH7peeONza0,91
datatrove-0.5.0.dist-info/entry_points.txt,sha256=LMMMud15s4s6jucjsr57crbMppkvg6xRtN6unsgp2MM,330
datatrove-0.5.0.dist-info/top_level.txt,sha256=EA6CAg36D1YzT-oXcFMx0ImOvfCRDHZasZHwxcjFXSQ,10
datatrove-0.5.0.dist-info/RECORD,,
