Tokenizers

Tokenizers is used at run time in 382 crates (of which 91 optionally). It is a direct run-time dependency in 219 crates. It's used only as a dev dependency in 7 crates (of which 4 directly).

Number of dependers	Tokenizers version	Downloads/month
141	0.22.2	259K
125	0.21.4	232K
37	0.20.4	61K
28	0.19.1	17K
33	0.15.2	55K
10	0.14.1	12K
10	0.13.4	5.4K
4	0.11.3	140
1	0.10.1	0

Depender (with downloads and own rev deps)			Tokenizers version
46K	47	fastembed		^0.22.0
39K	2	toktrie_hf_tokenizers		^0.21.2
89K	30	text-splitter	optional	^0.22
12K	21	lancedb	optional	^0.19.1
2.8K		xgrammar		^0.21.2
2.7K		musicgpt		^0.19.1
2.3K		gpt_sovits_rs		^0.20.0
1.5K	5	model2vec-rs		^0.21
4.2K	22	rust-bert	optional	^0.20
1.0K	1	parakeet-rs		^0.22.2
900		glowrs		^0.19.1
800	2	claude-tokenizer		^0.20.0
650		ohms-adaptq		^0.21
650		llms-from-scratch-rs		^0.21.1
600	2	candle-coreml		^0.21.4
600	1	outlines-core		^0.22.2
550		coze		^0.15.2
550		embed_anything		^0.22.1
460	2	oar-ocr-core		^0.22
440		rai-models		^0.15.0
440		bundle_repo		^0.21
430		r2md		^0.21.0
430	1	rbert		^0.21.0
420		vectordb-cli		^0.21.1
370		sbv2_core		^0.20.0
320		wimbd		^0.15.1
320		rphi		^0.19.1
310	2	gibberish-or-not		^0.14.1
300	1	rwhisper		^0.21.0
290	5	candle-examples		^0.21.0
1.3K	5	bitnet-core	optional	^0.15
280		mpnet-rs		^0.15.1
270	1	kalosm-ocr		^0.21.0
270	1	rwuerstchen		^0.21.0
260	1	kalosm-llama		^0.21.0
250		moshi-backend		^0.15.2
230		bm42		^0.19
230		gline-rs		^0.21.0
220		pylate-rs		^0.21.1
220		candle_embed		^0.19.1
170	4	ck-chunk		^0.22
170	1	stable-diffusion		^0.15.0
150	5	candle-datasets		^0.21.0
900	2	reasonkit-mem	optional	^0.21
110		depbank		^0.21.1
100		gte-rs		^0.21.0
		smartgpt		^0.13.3
	4	sapiens		^0.14.1
	1	mlx-lm-utils		^0.22
		rusty-dawg		^0.20.0
750	1	usls	optional	^0.22.1
		mlx-lm		^0.22.0
750		octocode	optional	^0.21.4
		ragtime		^0.19
		ug-llama		^0.20.3
		sbert		^0.11
		bge		^0.15.2
		oxidized-transformers		^0.15
		callm		^0.19
		metal-candle		^0.20
		aha		^0.22.1
		reco-forge		^0.15.0
	4	alith-models		^0.21.1
		tkn-cli		^0.21.1
		edge-transformers		^0.13
		promptbox		^0.15.0
		minillm		^0.22.1
		piper-tts-rust		^0.21.1
		fashion-clip-rs		^0.15
	7	awful_aj		^0.22.1
		modeling		^0.11.0
		model2vec		^0.21
		dynamo-llm		^0.21.4
	2	lmonade-runtime		^0.20
	1	wonnx-preprocessing		^0.13.3
		enfer_core		^0.15.0
	1	vtt-rs		^0.21
		dataflow_nlp		^0.11
		aiochatgpt		^0.13.2
		moz_genai		^0.15.0
	1	ruvector-onnx-embeddings		^0.20
	1	clip_txt		^0.13.3
		memvid-rs		^0.20
		gliclass-rs		^0.21.0
		lsp-ai		^0.14.1
		cracken		~0.11.0
	5	abaddon		^0.22
		aleph-alpha-api		^0.15.0
		aleph-alpha-client		^0.22
	1	alith-inference		^0.21.1
		anonymize_rs		^0.13.3
		askit-llm-agents		^0.22.2
		aurora-semantic		^0.20
		autoagents-onnx		^0.22.0
		bbpe		^0.22.1
		bitmamba		^0.20
		bridge-common		^0.19.1
		candle-chat-types		^0.13.4
		candle-pipelines		^0.21.1
		candle-semantic-router		^0.21.0
		capsa		^0.22
	1	caro		^0.22
		catgrad-llm		^0.21.2
		cetk		^0.22.1
		claudev		^0.20
		cmdai		^0.15
		codegeex4-candle		^0.19.1
	3	codeprysm-search		^0.20
	2	diffuse_rs_common		^0.21.0
	1	diffuse_rs_core		^0.21.0
	2	diffusion_rs_common		^0.21.0
	1	diffusion_rs_core		^0.21.0
	1	dumpfs		^0.21.0
		edgebert		^0.15
		encoderfile		^0.22.1
	1	encoderfile-core		^0.22.1
		git-semantic		^0.22
		glimpse		^0.21.0
	1	gllm		^0.22.1
		gpt-sovits		^0.22.0
		hanzo-datasets		^0.21.0
		heapswap		^0.15.1
		hy-mt		^0.22.2
		inference_core		^0.15.0
		inferno-ai		^0.14
		inspector-gguf		^0.22
	3	kodegen_candle_agent		^0.22
	1	kodegen_simd		^0.22
	1	koharu-models		^0.22
		lance-examples		^0.15.2
	2	llm-shield-models		^0.20
	2	llm-tokenizer		^0.22.0
		llmnop		^0.22.1
		lumen-rag		^0.22.0
		m2m-protocol		^0.21
		milli-core		^0.15.2
		nanogpt		^0.22.1
	2	next-plaid-onnx		^0.21.1
		niblits		^0.22.1
		nihility-listener		^0.22
		npc-neural-affect-matrix		^0.22.1
		oar-ocr-vl		^0.22
	2	oxirs-chat		^0.22
	1	pocket-tts		^0.21.0
		prefrontal		^0.15
		product-os-models		^0.21.1
		rag-module		^0.22
	1	reflex-cache		^0.22
		rmistral		^0.13.4
		ruvector-onnx-embeddings-wasm		^0.20
		searchgrep		^0.15
		shodh-memory		^0.21
		simple-whisper		^0.19.1
		smgrep		^0.22
		spark-bert		^0.21.1
		tauformer		^0.22.1
	2	tessera-embeddings		^0.22.1
		tl-lang		^0.19.1
	1	tl_runtime		^0.19.1
		todozi		^0.20
		token-counter		^0.19.1
	1	tokstream-core		^0.22.2
		topolog-embed		^0.22.1
	2	torsh-text		^0.22
	2	trustformers-tokenizers		^0.21
		turboprop		^0.21
		unsloth-rs-core		^0.19
		valentinus		>=0.13.4
		vecboost		^0.15
		vectorlite		^0.19
		veritas-nexus		^0.15
		vibedev		^0.20
		xm-cli		^0.21
		yolors		^0.15.0
300	2	feagi-agent	optional	^0.19
260		code-splitter	optional	^0.19.1
260	4	scirs2-text	optional	^0.22.2
230		bitnet-inference	optional	^0.15
210	2	memvid-core	optional	^0.22
180		norma	optional	^0.20.0
180	1	pdf_oxide	optional	^0.15
100		kotadb	optional	^0.20
		ctranslate2	optional	^0.22.1
	1	octolib	optional	^0.21.4
	2	ct2rs	optional	^0.22
		chonkier	optional	^0.21.1
		aleph-alpha-tokenizer	optional	^0.10.1
		anno	optional	^0.22
		aumate	optional	^0.21
		autoagents-burn	optional	^0.22.0
	2	beet_ml	optional	^0.22
	1	burn-lm-llama	optional	^0.19
		ceylon-next	optional	^0.21
		chonkie	optional	^0.21.1
		copyforward	optional	^0.15
		helios-engine	optional	^0.20
		inference-lab	optional	^0.22
	1	kproc-llm	optional	^0.22
	3	lmonade-models	optional	^0.20
		manx-cli	optional	^0.15
		memvid-cli	optional	^0.20
		mlmf	optional	^0.22
		narsil-mcp	optional	^0.20
		natural-tts	optional	^0.22.2
		neomemx	optional	^0.21
		openlb	optional	^0.22.0
	2	optirs-learned	optional	^0.22.2
	5	oxirs-vec	optional	^0.22
		pii	optional	^0.15.2
	1	reasonkit-core	optional	^0.21
		rig-experimental	optional	^0.21.1
		rusty-gradients	optional	^0.19
	1	ruvllm	optional	^0.20
		seekdb-rs	optional	^0.15
		sentinellm_ai	optional	^0.20
		spnl	optional	^0.22.0
	2	symbi-runtime	optional	^0.15
	1	vecstore	optional	^0.22
		xgrammar-rs	optional	^0.22.2
250		mtc-token-healing	dev	^0.22.0
	3	general-sam	dev	^0.22.0
		nano-vectordb-rs	dev	^0.21.0
		normy	dev	^0.22.2