chore(deps): bump tokenizers from 0.21.4 to 0.22.2

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.21.4 to 0.22.2.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.21.4...v0.22.2)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-version: 0.22.2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
This commit is contained in:
dependabot[bot]
2026-05-29 20:48:08 +00:00
committed by GitHub
parent 13f7be2ed5
commit 8ee48100ac
2 changed files with 4 additions and 37 deletions
Generated
+3 -36
View File
@@ -1727,7 +1727,7 @@ dependencies = [
"rayon",
"safetensors 0.7.0",
"thiserror 2.0.18",
"tokenizers 0.22.2",
"tokenizers",
"yoke 0.8.2",
"zip 7.2.0",
]
@@ -4544,7 +4544,7 @@ dependencies = [
"test-case",
"thiserror 1.0.69",
"tiktoken-rs",
"tokenizers 0.21.4",
"tokenizers",
"tokio",
"tokio-cron-scheduler",
"tokio-stream",
@@ -7683,7 +7683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.13.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -10760,39 +10760,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "tokenizers"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
dependencies = [
"ahash",
"aho-corasick",
"compact_str 0.9.0",
"dary_heap",
"derive_builder",
"esaxx-rs",
"getrandom 0.3.4",
"itertools 0.14.0",
"log",
"macro_rules_attribute",
"monostate",
"onig",
"paste",
"rand 0.9.4",
"rayon",
"rayon-cond",
"regex",
"regex-syntax",
"serde",
"serde_json",
"spm_precompiled",
"thiserror 2.0.18",
"unicode-normalization-alignments",
"unicode-segmentation",
"unicode_categories",
]
[[package]]
name = "tokenizers"
version = "0.22.2"
+1 -1
View File
@@ -164,7 +164,7 @@ candle-core = { workspace = true, optional = true }
candle-nn = { workspace = true, optional = true }
candle-transformers = { version = "0.10", default-features = false, optional = true }
byteorder = { version = "1.5", default-features = false, features = ["std"], optional = true }
tokenizers = { version = "0.21", default-features = false, features = ["onig"], optional = true }
tokenizers = { version = "0.22", default-features = false, features = ["onig"], optional = true }
symphonia = { version = "0.5", default-features = false, features = ["aac", "adpcm", "alac", "isomp4", "mkv", "mp3", "pcm", "vorbis", "wav"], optional = true }
rubato = { version = "0.16", default-features = false, optional = true }
zip = { workspace = true }