BDEPEND=test? ( hf-hub? ( dev-python/huggingface_hub[python_targets_python3_12(-)?,python_targets_python3_13(-)?,python_targets_python3_14(-)?] ) >=dev-python/pytest-7.4.4[python_targets_python3_12(-)?,python_targets_python3_13(-)?,python_targets_python3_14(-)?] ) || ( >=dev-lang/rust-bin-1.74.1:* >=dev-lang/rust-1.74.1:* ) python_targets_python3_12? ( dev-lang/python:3.12 ) python_targets_python3_13? ( dev-lang/python:3.13 ) python_targets_python3_14? ( dev-lang/python:3.14 ) >=dev-python/gpep517-16[python_targets_python3_12(-)?,python_targets_python3_13(-)?,python_targets_python3_14(-)?] >=dev-util/maturin-1.8.2[python_targets_python3_12(-)?,python_targets_python3_13(-)?,python_targets_python3_14(-)?]
DEFINED_PHASES=compile configure install prepare setup test unpack
DEPEND=hf-hub? ( dev-python/huggingface_hub[python_targets_python3_12(-)?,python_targets_python3_13(-)?,python_targets_python3_14(-)?] )
DESCRIPTION=Fast, efficient, Rust-backed tokenizers for Python (HuggingFace)
EAPI=8
HOMEPAGE=https://github.com/huggingface/tokenizers https://pypi.org/project/tokenizers/
IUSE=onig hf-hub indicatif fancy-regex test debug python_targets_python3_12 python_targets_python3_13 python_targets_python3_14
KEYWORDS=~amd64 ~arm64 ~x86
LICENSE=Apache-2.0
RDEPEND=hf-hub? ( dev-python/huggingface_hub[python_targets_python3_12(-)?,python_targets_python3_13(-)?,python_targets_python3_14(-)?] ) python_targets_python3_12? ( dev-lang/python:3.12 ) python_targets_python3_13? ( dev-lang/python:3.13 ) python_targets_python3_14? ( dev-lang/python:3.14 )
REQUIRED_USE=|| ( python_targets_python3_12 python_targets_python3_13 python_targets_python3_14 )
RESTRICT=!test? ( test )
SLOT=0
SRC_URI=https://files.pythonhosted.org/packages/source/t/tokenizers/tokenizers-0.22.1.tar.gz https://crates.io/api/v1/crates/addr2line/0.24.2/download -> addr2line-0.24.2.crate https://crates.io/api/v1/crates/adler2/2.0.1/download -> adler2-2.0.1.crate https://crates.io/api/v1/crates/ahash/0.8.12/download -> ahash-0.8.12.crate https://crates.io/api/v1/crates/aho-corasick/1.1.3/download -> aho-corasick-1.1.3.crate https://crates.io/api/v1/crates/anstream/0.6.20/download -> anstream-0.6.20.crate https://crates.io/api/v1/crates/anstyle/1.0.11/download -> anstyle-1.0.11.crate https://crates.io/api/v1/crates/anstyle-parse/0.2.7/download -> anstyle-parse-0.2.7.crate https://crates.io/api/v1/crates/anstyle-query/1.1.4/download -> anstyle-query-1.1.4.crate https://crates.io/api/v1/crates/anstyle-wincon/3.0.10/download -> anstyle-wincon-3.0.10.crate https://crates.io/api/v1/crates/autocfg/1.5.0/download -> autocfg-1.5.0.crate https://crates.io/api/v1/crates/backtrace/0.3.75/download -> backtrace-0.3.75.crate https://crates.io/api/v1/crates/base64/0.13.1/download -> base64-0.13.1.crate https://crates.io/api/v1/crates/bitflags/2.9.4/download -> bitflags-2.9.4.crate https://crates.io/api/v1/crates/bumpalo/3.19.0/download -> bumpalo-3.19.0.crate https://crates.io/api/v1/crates/castaway/0.2.4/download -> castaway-0.2.4.crate https://crates.io/api/v1/crates/cc/1.2.38/download -> cc-1.2.38.crate https://crates.io/api/v1/crates/cfg-if/1.0.3/download -> cfg-if-1.0.3.crate https://crates.io/api/v1/crates/colorchoice/1.0.4/download -> colorchoice-1.0.4.crate https://crates.io/api/v1/crates/compact_str/0.9.0/download -> compact_str-0.9.0.crate https://crates.io/api/v1/crates/console/0.15.11/download -> console-0.15.11.crate https://crates.io/api/v1/crates/crossbeam-deque/0.8.6/download -> crossbeam-deque-0.8.6.crate https://crates.io/api/v1/crates/crossbeam-epoch/0.9.18/download -> crossbeam-epoch-0.9.18.crate https://crates.io/api/v1/crates/crossbeam-utils/0.8.21/download -> crossbeam-utils-0.8.21.crate https://crates.io/api/v1/crates/darling/0.20.11/download -> darling-0.20.11.crate https://crates.io/api/v1/crates/darling_core/0.20.11/download -> darling_core-0.20.11.crate https://crates.io/api/v1/crates/darling_macro/0.20.11/download -> darling_macro-0.20.11.crate https://crates.io/api/v1/crates/dary_heap/0.3.8/download -> dary_heap-0.3.8.crate https://crates.io/api/v1/crates/derive_builder/0.20.2/download -> derive_builder-0.20.2.crate https://crates.io/api/v1/crates/derive_builder_core/0.20.2/download -> derive_builder_core-0.20.2.crate https://crates.io/api/v1/crates/derive_builder_macro/0.20.2/download -> derive_builder_macro-0.20.2.crate https://crates.io/api/v1/crates/either/1.15.0/download -> either-1.15.0.crate https://crates.io/api/v1/crates/encode_unicode/1.0.0/download -> encode_unicode-1.0.0.crate https://crates.io/api/v1/crates/env_filter/0.1.3/download -> env_filter-0.1.3.crate https://crates.io/api/v1/crates/env_logger/0.11.8/download -> env_logger-0.11.8.crate https://crates.io/api/v1/crates/errno/0.3.14/download -> errno-0.3.14.crate https://crates.io/api/v1/crates/esaxx-rs/0.1.10/download -> esaxx-rs-0.1.10.crate https://crates.io/api/v1/crates/fastrand/2.3.0/download -> fastrand-2.3.0.crate https://crates.io/api/v1/crates/find-msvc-tools/0.1.2/download -> find-msvc-tools-0.1.2.crate https://crates.io/api/v1/crates/fnv/1.0.7/download -> fnv-1.0.7.crate https://crates.io/api/v1/crates/futures/0.3.31/download -> futures-0.3.31.crate https://crates.io/api/v1/crates/futures-channel/0.3.31/download -> futures-channel-0.3.31.crate https://crates.io/api/v1/crates/futures-core/0.3.31/download -> futures-core-0.3.31.crate https://crates.io/api/v1/crates/futures-executor/0.3.31/download -> futures-executor-0.3.31.crate https://crates.io/api/v1/crates/futures-io/0.3.31/download -> futures-io-0.3.31.crate https://crates.io/api/v1/crates/futures-macro/0.3.31/download -> futures-macro-0.3.31.crate https://crates.io/api/v1/crates/futures-sink/0.3.31/download -> futures-sink-0.3.31.crate https://crates.io/api/v1/crates/futures-task/0.3.31/download -> futures-task-0.3.31.crate https://crates.io/api/v1/crates/futures-util/0.3.31/download -> futures-util-0.3.31.crate https://crates.io/api/v1/crates/getrandom/0.3.3/download -> getrandom-0.3.3.crate https://crates.io/api/v1/crates/gimli/0.31.1/download -> gimli-0.31.1.crate https://crates.io/api/v1/crates/heck/0.5.0/download -> heck-0.5.0.crate https://crates.io/api/v1/crates/ident_case/1.0.1/download -> ident_case-1.0.1.crate https://crates.io/api/v1/crates/indicatif/0.17.11/download -> indicatif-0.17.11.crate https://crates.io/api/v1/crates/indoc/2.0.6/download -> indoc-2.0.6.crate https://crates.io/api/v1/crates/io-uring/0.7.10/download -> io-uring-0.7.10.crate https://crates.io/api/v1/crates/is_terminal_polyfill/1.70.1/download -> is_terminal_polyfill-1.70.1.crate https://crates.io/api/v1/crates/itertools/0.14.0/download -> itertools-0.14.0.crate https://crates.io/api/v1/crates/itoa/1.0.15/download -> itoa-1.0.15.crate https://crates.io/api/v1/crates/jiff/0.2.15/download -> jiff-0.2.15.crate https://crates.io/api/v1/crates/jiff-static/0.2.15/download -> jiff-static-0.2.15.crate https://crates.io/api/v1/crates/js-sys/0.3.80/download -> js-sys-0.3.80.crate https://crates.io/api/v1/crates/libc/0.2.175/download -> libc-0.2.175.crate https://crates.io/api/v1/crates/linux-raw-sys/0.11.0/download -> linux-raw-sys-0.11.0.crate https://crates.io/api/v1/crates/log/0.4.28/download -> log-0.4.28.crate https://crates.io/api/v1/crates/macro_rules_attribute/0.2.2/download -> macro_rules_attribute-0.2.2.crate https://crates.io/api/v1/crates/macro_rules_attribute-proc_macro/0.2.2/download -> macro_rules_attribute-proc_macro-0.2.2.crate https://crates.io/api/v1/crates/matrixmultiply/0.3.10/download -> matrixmultiply-0.3.10.crate https://crates.io/api/v1/crates/memchr/2.7.5/download -> memchr-2.7.5.crate https://crates.io/api/v1/crates/memoffset/0.9.1/download -> memoffset-0.9.1.crate https://crates.io/api/v1/crates/minimal-lexical/0.2.1/download -> minimal-lexical-0.2.1.crate https://crates.io/api/v1/crates/miniz_oxide/0.8.9/download -> miniz_oxide-0.8.9.crate https://crates.io/api/v1/crates/mio/1.0.4/download -> mio-1.0.4.crate https://crates.io/api/v1/crates/monostate/0.1.16/download -> monostate-0.1.16.crate https://crates.io/api/v1/crates/monostate-impl/0.1.16/download -> monostate-impl-0.1.16.crate https://crates.io/api/v1/crates/ndarray/0.16.1/download -> ndarray-0.16.1.crate https://crates.io/api/v1/crates/nom/7.1.3/download -> nom-7.1.3.crate https://crates.io/api/v1/crates/number_prefix/0.4.0/download -> number_prefix-0.4.0.crate https://crates.io/api/v1/crates/num-complex/0.4.6/download -> num-complex-0.4.6.crate https://crates.io/api/v1/crates/num-integer/0.1.46/download -> num-integer-0.1.46.crate https://crates.io/api/v1/crates/numpy/0.25.0/download -> numpy-0.25.0.crate https://crates.io/api/v1/crates/num-traits/0.2.19/download -> num-traits-0.2.19.crate https://crates.io/api/v1/crates/object/0.36.7/download -> object-0.36.7.crate https://crates.io/api/v1/crates/once_cell/1.21.3/download -> once_cell-1.21.3.crate https://crates.io/api/v1/crates/once_cell_polyfill/1.70.1/download -> once_cell_polyfill-1.70.1.crate https://crates.io/api/v1/crates/onig/6.5.1/download -> onig-6.5.1.crate https://crates.io/api/v1/crates/onig_sys/69.9.1/download -> onig_sys-69.9.1.crate https://crates.io/api/v1/crates/paste/1.0.15/download -> paste-1.0.15.crate https://crates.io/api/v1/crates/pin-project-lite/0.2.16/download -> pin-project-lite-0.2.16.crate https://crates.io/api/v1/crates/pin-utils/0.1.0/download -> pin-utils-0.1.0.crate https://crates.io/api/v1/crates/pkg-config/0.3.32/download -> pkg-config-0.3.32.crate https://crates.io/api/v1/crates/portable-atomic/1.11.1/download -> portable-atomic-1.11.1.crate https://crates.io/api/v1/crates/portable-atomic-util/0.2.4/download -> portable-atomic-util-0.2.4.crate https://crates.io/api/v1/crates/ppv-lite86/0.2.21/download -> ppv-lite86-0.2.21.crate https://crates.io/api/v1/crates/proc-macro2/1.0.101/download -> proc-macro2-1.0.101.crate https://crates.io/api/v1/crates/pyo3/0.25.1/download -> pyo3-0.25.1.crate https://crates.io/api/v1/crates/pyo3-async-runtimes/0.25.0/download -> pyo3-async-runtimes-0.25.0.crate https://crates.io/api/v1/crates/pyo3-build-config/0.25.1/download -> pyo3-build-config-0.25.1.crate https://crates.io/api/v1/crates/pyo3-ffi/0.25.1/download -> pyo3-ffi-0.25.1.crate https://crates.io/api/v1/crates/pyo3-macros/0.25.1/download -> pyo3-macros-0.25.1.crate https://crates.io/api/v1/crates/pyo3-macros-backend/0.25.1/download -> pyo3-macros-backend-0.25.1.crate https://crates.io/api/v1/crates/quote/1.0.40/download -> quote-1.0.40.crate https://crates.io/api/v1/crates/rand/0.9.2/download -> rand-0.9.2.crate https://crates.io/api/v1/crates/rand_chacha/0.9.0/download -> rand_chacha-0.9.0.crate https://crates.io/api/v1/crates/rand_core/0.9.3/download -> rand_core-0.9.3.crate https://crates.io/api/v1/crates/rawpointer/0.2.1/download -> rawpointer-0.2.1.crate https://crates.io/api/v1/crates/rayon/1.11.0/download -> rayon-1.11.0.crate https://crates.io/api/v1/crates/rayon-cond/0.4.0/download -> rayon-cond-0.4.0.crate https://crates.io/api/v1/crates/rayon-core/1.13.0/download -> rayon-core-1.13.0.crate https://crates.io/api/v1/crates/r-efi/5.3.0/download -> r-efi-5.3.0.crate https://crates.io/api/v1/crates/regex/1.11.2/download -> regex-1.11.2.crate https://crates.io/api/v1/crates/regex-automata/0.4.10/download -> regex-automata-0.4.10.crate https://crates.io/api/v1/crates/regex-syntax/0.8.6/download -> regex-syntax-0.8.6.crate https://crates.io/api/v1/crates/rustc-demangle/0.1.26/download -> rustc-demangle-0.1.26.crate https://crates.io/api/v1/crates/rustc-hash/2.1.1/download -> rustc-hash-2.1.1.crate https://crates.io/api/v1/crates/rustix/1.1.2/download -> rustix-1.1.2.crate https://crates.io/api/v1/crates/rustversion/1.0.22/download -> rustversion-1.0.22.crate https://crates.io/api/v1/crates/ryu/1.0.20/download -> ryu-1.0.20.crate https://crates.io/api/v1/crates/serde/1.0.225/download -> serde-1.0.225.crate https://crates.io/api/v1/crates/serde_core/1.0.225/download -> serde_core-1.0.225.crate https://crates.io/api/v1/crates/serde_derive/1.0.225/download -> serde_derive-1.0.225.crate https://crates.io/api/v1/crates/serde_json/1.0.145/download -> serde_json-1.0.145.crate https://crates.io/api/v1/crates/shlex/1.3.0/download -> shlex-1.3.0.crate https://crates.io/api/v1/crates/signal-hook-registry/1.4.6/download -> signal-hook-registry-1.4.6.crate https://crates.io/api/v1/crates/slab/0.4.11/download -> slab-0.4.11.crate https://crates.io/api/v1/crates/smallvec/1.15.1/download -> smallvec-1.15.1.crate https://crates.io/api/v1/crates/spm_precompiled/0.1.4/download -> spm_precompiled-0.1.4.crate https://crates.io/api/v1/crates/static_assertions/1.1.0/download -> static_assertions-1.1.0.crate https://crates.io/api/v1/crates/strsim/0.11.1/download -> strsim-0.11.1.crate https://crates.io/api/v1/crates/syn/2.0.106/download -> syn-2.0.106.crate https://crates.io/api/v1/crates/target-lexicon/0.13.3/download -> target-lexicon-0.13.3.crate https://crates.io/api/v1/crates/tempfile/3.22.0/download -> tempfile-3.22.0.crate https://crates.io/api/v1/crates/thiserror/2.0.16/download -> thiserror-2.0.16.crate https://crates.io/api/v1/crates/thiserror-impl/2.0.16/download -> thiserror-impl-2.0.16.crate https://crates.io/api/v1/crates/tokenizers/0.22.1/download -> tokenizers-0.22.1.crate https://crates.io/api/v1/crates/tokio/1.47.1/download -> tokio-1.47.1.crate https://crates.io/api/v1/crates/tokio-macros/2.5.0/download -> tokio-macros-2.5.0.crate https://crates.io/api/v1/crates/unicode_categories/0.1.1/download -> unicode_categories-0.1.1.crate https://crates.io/api/v1/crates/unicode-ident/1.0.19/download -> unicode-ident-1.0.19.crate https://crates.io/api/v1/crates/unicode-normalization-alignments/0.1.12/download -> unicode-normalization-alignments-0.1.12.crate https://crates.io/api/v1/crates/unicode-segmentation/1.12.0/download -> unicode-segmentation-1.12.0.crate https://crates.io/api/v1/crates/unicode-width/0.2.1/download -> unicode-width-0.2.1.crate https://crates.io/api/v1/crates/unindent/0.2.4/download -> unindent-0.2.4.crate https://crates.io/api/v1/crates/utf8parse/0.2.2/download -> utf8parse-0.2.2.crate https://crates.io/api/v1/crates/version_check/0.9.5/download -> version_check-0.9.5.crate https://crates.io/api/v1/crates/wasi/0.11.1+wasi-snapshot-preview1/download -> wasi-0.11.1+wasi-snapshot-preview1.crate https://crates.io/api/v1/crates/wasi/0.14.7+wasi-0.2.4/download -> wasi-0.14.7+wasi-0.2.4.crate https://crates.io/api/v1/crates/wasip2/1.0.1+wasi-0.2.4/download -> wasip2-1.0.1+wasi-0.2.4.crate https://crates.io/api/v1/crates/wasm-bindgen/0.2.103/download -> wasm-bindgen-0.2.103.crate https://crates.io/api/v1/crates/wasm-bindgen-backend/0.2.103/download -> wasm-bindgen-backend-0.2.103.crate https://crates.io/api/v1/crates/wasm-bindgen-macro/0.2.103/download -> wasm-bindgen-macro-0.2.103.crate https://crates.io/api/v1/crates/wasm-bindgen-macro-support/0.2.103/download -> wasm-bindgen-macro-support-0.2.103.crate https://crates.io/api/v1/crates/wasm-bindgen-shared/0.2.103/download -> wasm-bindgen-shared-0.2.103.crate https://crates.io/api/v1/crates/web-time/1.1.0/download -> web-time-1.1.0.crate https://crates.io/api/v1/crates/windows_aarch64_gnullvm/0.52.6/download -> windows_aarch64_gnullvm-0.52.6.crate https://crates.io/api/v1/crates/windows_aarch64_gnullvm/0.53.0/download -> windows_aarch64_gnullvm-0.53.0.crate https://crates.io/api/v1/crates/windows_aarch64_msvc/0.52.6/download -> windows_aarch64_msvc-0.52.6.crate https://crates.io/api/v1/crates/windows_aarch64_msvc/0.53.0/download -> windows_aarch64_msvc-0.53.0.crate https://crates.io/api/v1/crates/windows_i686_gnu/0.52.6/download -> windows_i686_gnu-0.52.6.crate https://crates.io/api/v1/crates/windows_i686_gnu/0.53.0/download -> windows_i686_gnu-0.53.0.crate https://crates.io/api/v1/crates/windows_i686_gnullvm/0.52.6/download -> windows_i686_gnullvm-0.52.6.crate https://crates.io/api/v1/crates/windows_i686_gnullvm/0.53.0/download -> windows_i686_gnullvm-0.53.0.crate https://crates.io/api/v1/crates/windows_i686_msvc/0.52.6/download -> windows_i686_msvc-0.52.6.crate https://crates.io/api/v1/crates/windows_i686_msvc/0.53.0/download -> windows_i686_msvc-0.53.0.crate https://crates.io/api/v1/crates/windows-link/0.1.3/download -> windows-link-0.1.3.crate https://crates.io/api/v1/crates/windows-link/0.2.0/download -> windows-link-0.2.0.crate https://crates.io/api/v1/crates/windows-sys/0.59.0/download -> windows-sys-0.59.0.crate https://crates.io/api/v1/crates/windows-sys/0.60.2/download -> windows-sys-0.60.2.crate https://crates.io/api/v1/crates/windows-sys/0.61.0/download -> windows-sys-0.61.0.crate https://crates.io/api/v1/crates/windows-targets/0.52.6/download -> windows-targets-0.52.6.crate https://crates.io/api/v1/crates/windows-targets/0.53.3/download -> windows-targets-0.53.3.crate https://crates.io/api/v1/crates/windows_x86_64_gnu/0.52.6/download -> windows_x86_64_gnu-0.52.6.crate https://crates.io/api/v1/crates/windows_x86_64_gnu/0.53.0/download -> windows_x86_64_gnu-0.53.0.crate https://crates.io/api/v1/crates/windows_x86_64_gnullvm/0.52.6/download -> windows_x86_64_gnullvm-0.52.6.crate https://crates.io/api/v1/crates/windows_x86_64_gnullvm/0.53.0/download -> windows_x86_64_gnullvm-0.53.0.crate https://crates.io/api/v1/crates/windows_x86_64_msvc/0.52.6/download -> windows_x86_64_msvc-0.52.6.crate https://crates.io/api/v1/crates/windows_x86_64_msvc/0.53.0/download -> windows_x86_64_msvc-0.53.0.crate https://crates.io/api/v1/crates/wit-bindgen/0.46.0/download -> wit-bindgen-0.46.0.crate https://crates.io/api/v1/crates/zerocopy/0.8.27/download -> zerocopy-0.8.27.crate https://crates.io/api/v1/crates/zerocopy-derive/0.8.27/download -> zerocopy-derive-0.8.27.crate
_eclasses_=cargo	aeba1d272f6c2e000912e11d8cf626eb	distutils-r1	776bbaa25e9fa0dc9ac6f039ca37f701	flag-o-matic	ed1eda86c8f746a2a75f7f9f7478f5f4	multibuild	83db098a835f71feb71761f72a694aa2	multilib	404b65bd5212cff83b3935c685419d08	multiprocessing	1e32df7deee68372153dca65f4a7c21f	ninja-utils	3a59a39e97af0f7c03f49cf3c22f262b	out-of-source-utils	8e5093bd27b7f541a1913271f25d633a	pypi	d2befa41d10011e03e2158e065d5d6da	python-r1	0e15b2ab9cfc87d7474678201d6bca38	python-utils-r1	dbb8c4d794033ad7e7221eaf567a6c90	rust	dd082fc51cbae6f9afa90dd1668d71a9	rust-toolchain	76468983281b0a7fc167ca224f84ecfd	toolchain-funcs	7e440a20ac11dece04e3334622e6e5da
_md5_=2e282c9764c98eb45b16e981e9d9569e