BDEPEND=|| ( >=dev-lang/rust-bin-1.74.1:* >=dev-lang/rust-1.74.1:* ) python_targets_python3_11? ( dev-lang/python:3.11 ) python_targets_python3_12? ( dev-lang/python:3.12 ) python_targets_python3_13? ( dev-lang/python:3.13 ) >=dev-python/gpep517-16[python_targets_python3_11(-)?,python_targets_python3_12(-)?,python_targets_python3_13(-)?] >=dev-util/maturin-1.8.2[python_targets_python3_11(-)?,python_targets_python3_12(-)?,python_targets_python3_13(-)?]
DEFINED_PHASES=compile configure install prepare setup test unpack
DEPEND=>=dev-python/huggingface-hub-0.16.4[python_targets_python3_11(-)?,python_targets_python3_12(-)?,python_targets_python3_13(-)?] python_targets_python3_11? ( dev-lang/python:3.11 ) python_targets_python3_12? ( dev-lang/python:3.12 ) python_targets_python3_13? ( dev-lang/python:3.13 )
DESCRIPTION=Fast State-of-the-Art Tokenizers optimized for Research and Production
EAPI=8
HOMEPAGE=https://github.com/huggingface/tokenizers
IUSE=debug python_targets_python3_11 python_targets_python3_12 python_targets_python3_13 debug
KEYWORDS=~amd64
LICENSE=Apache-2.0 Apache-2.0 MIT Unicode-3.0
RDEPEND=>=dev-python/huggingface-hub-0.16.4[python_targets_python3_11(-)?,python_targets_python3_12(-)?,python_targets_python3_13(-)?] python_targets_python3_11? ( dev-lang/python:3.11 ) python_targets_python3_12? ( dev-lang/python:3.12 ) python_targets_python3_13? ( dev-lang/python:3.13 )
REQUIRED_USE=|| ( python_targets_python3_11 python_targets_python3_12 python_targets_python3_13 )
SLOT=0
SRC_URI=https://files.pythonhosted.org/packages/source/t/tokenizers/tokenizers-0.22.2.tar.gz https://crates.io/api/v1/crates/ahash/0.8.12/download -> ahash-0.8.12.crate https://crates.io/api/v1/crates/aho-corasick/1.1.4/download -> aho-corasick-1.1.4.crate https://crates.io/api/v1/crates/anstream/0.6.21/download -> anstream-0.6.21.crate https://crates.io/api/v1/crates/anstyle/1.0.13/download -> anstyle-1.0.13.crate https://crates.io/api/v1/crates/anstyle-parse/0.2.7/download -> anstyle-parse-0.2.7.crate https://crates.io/api/v1/crates/anstyle-query/1.1.5/download -> anstyle-query-1.1.5.crate https://crates.io/api/v1/crates/anstyle-wincon/3.0.11/download -> anstyle-wincon-3.0.11.crate https://crates.io/api/v1/crates/autocfg/1.5.0/download -> autocfg-1.5.0.crate https://crates.io/api/v1/crates/base64/0.13.1/download -> base64-0.13.1.crate https://crates.io/api/v1/crates/bitflags/2.10.0/download -> bitflags-2.10.0.crate https://crates.io/api/v1/crates/bumpalo/3.19.0/download -> bumpalo-3.19.0.crate https://crates.io/api/v1/crates/castaway/0.2.4/download -> castaway-0.2.4.crate https://crates.io/api/v1/crates/cc/1.2.48/download -> cc-1.2.48.crate https://crates.io/api/v1/crates/cfg-if/1.0.4/download -> cfg-if-1.0.4.crate https://crates.io/api/v1/crates/colorchoice/1.0.4/download -> colorchoice-1.0.4.crate https://crates.io/api/v1/crates/compact_str/0.9.0/download -> compact_str-0.9.0.crate https://crates.io/api/v1/crates/console/0.16.1/download -> console-0.16.1.crate https://crates.io/api/v1/crates/crossbeam-deque/0.8.6/download -> crossbeam-deque-0.8.6.crate https://crates.io/api/v1/crates/crossbeam-epoch/0.9.18/download -> crossbeam-epoch-0.9.18.crate https://crates.io/api/v1/crates/crossbeam-utils/0.8.21/download -> crossbeam-utils-0.8.21.crate https://crates.io/api/v1/crates/darling/0.20.11/download -> darling-0.20.11.crate https://crates.io/api/v1/crates/darling_core/0.20.11/download -> darling_core-0.20.11.crate https://crates.io/api/v1/crates/darling_macro/0.20.11/download -> darling_macro-0.20.11.crate https://crates.io/api/v1/crates/dary_heap/0.3.8/download -> dary_heap-0.3.8.crate https://crates.io/api/v1/crates/derive_builder/0.20.2/download -> derive_builder-0.20.2.crate https://crates.io/api/v1/crates/derive_builder_core/0.20.2/download -> derive_builder_core-0.20.2.crate https://crates.io/api/v1/crates/derive_builder_macro/0.20.2/download -> derive_builder_macro-0.20.2.crate https://crates.io/api/v1/crates/either/1.15.0/download -> either-1.15.0.crate https://crates.io/api/v1/crates/encode_unicode/1.0.0/download -> encode_unicode-1.0.0.crate https://crates.io/api/v1/crates/env_filter/0.1.4/download -> env_filter-0.1.4.crate https://crates.io/api/v1/crates/env_logger/0.11.8/download -> env_logger-0.11.8.crate https://crates.io/api/v1/crates/errno/0.3.14/download -> errno-0.3.14.crate https://crates.io/api/v1/crates/esaxx-rs/0.1.10/download -> esaxx-rs-0.1.10.crate https://crates.io/api/v1/crates/fastrand/2.3.0/download -> fastrand-2.3.0.crate https://crates.io/api/v1/crates/find-msvc-tools/0.1.5/download -> find-msvc-tools-0.1.5.crate https://crates.io/api/v1/crates/fnv/1.0.7/download -> fnv-1.0.7.crate https://crates.io/api/v1/crates/futures/0.3.31/download -> futures-0.3.31.crate https://crates.io/api/v1/crates/futures-channel/0.3.31/download -> futures-channel-0.3.31.crate https://crates.io/api/v1/crates/futures-core/0.3.31/download -> futures-core-0.3.31.crate https://crates.io/api/v1/crates/futures-executor/0.3.31/download -> futures-executor-0.3.31.crate https://crates.io/api/v1/crates/futures-io/0.3.31/download -> futures-io-0.3.31.crate https://crates.io/api/v1/crates/futures-macro/0.3.31/download -> futures-macro-0.3.31.crate https://crates.io/api/v1/crates/futures-sink/0.3.31/download -> futures-sink-0.3.31.crate https://crates.io/api/v1/crates/futures-task/0.3.31/download -> futures-task-0.3.31.crate https://crates.io/api/v1/crates/futures-util/0.3.31/download -> futures-util-0.3.31.crate https://crates.io/api/v1/crates/getrandom/0.3.4/download -> getrandom-0.3.4.crate https://crates.io/api/v1/crates/heck/0.5.0/download -> heck-0.5.0.crate https://crates.io/api/v1/crates/ident_case/1.0.1/download -> ident_case-1.0.1.crate https://crates.io/api/v1/crates/indicatif/0.18.3/download -> indicatif-0.18.3.crate https://crates.io/api/v1/crates/indoc/2.0.7/download -> indoc-2.0.7.crate https://crates.io/api/v1/crates/is_terminal_polyfill/1.70.2/download -> is_terminal_polyfill-1.70.2.crate https://crates.io/api/v1/crates/itertools/0.14.0/download -> itertools-0.14.0.crate https://crates.io/api/v1/crates/itoa/1.0.15/download -> itoa-1.0.15.crate https://crates.io/api/v1/crates/jiff/0.2.16/download -> jiff-0.2.16.crate https://crates.io/api/v1/crates/jiff-static/0.2.16/download -> jiff-static-0.2.16.crate https://crates.io/api/v1/crates/js-sys/0.3.83/download -> js-sys-0.3.83.crate https://crates.io/api/v1/crates/libc/0.2.177/download -> libc-0.2.177.crate https://crates.io/api/v1/crates/linux-raw-sys/0.11.0/download -> linux-raw-sys-0.11.0.crate https://crates.io/api/v1/crates/log/0.4.28/download -> log-0.4.28.crate https://crates.io/api/v1/crates/macro_rules_attribute/0.2.2/download -> macro_rules_attribute-0.2.2.crate https://crates.io/api/v1/crates/macro_rules_attribute-proc_macro/0.2.2/download -> macro_rules_attribute-proc_macro-0.2.2.crate https://crates.io/api/v1/crates/matrixmultiply/0.3.10/download -> matrixmultiply-0.3.10.crate https://crates.io/api/v1/crates/memchr/2.7.6/download -> memchr-2.7.6.crate https://crates.io/api/v1/crates/memoffset/0.9.1/download -> memoffset-0.9.1.crate https://crates.io/api/v1/crates/minimal-lexical/0.2.1/download -> minimal-lexical-0.2.1.crate https://crates.io/api/v1/crates/mio/1.1.0/download -> mio-1.1.0.crate https://crates.io/api/v1/crates/monostate/0.1.18/download -> monostate-0.1.18.crate https://crates.io/api/v1/crates/monostate-impl/0.1.18/download -> monostate-impl-0.1.18.crate https://crates.io/api/v1/crates/ndarray/0.16.1/download -> ndarray-0.16.1.crate https://crates.io/api/v1/crates/nom/7.1.3/download -> nom-7.1.3.crate https://crates.io/api/v1/crates/num-complex/0.4.6/download -> num-complex-0.4.6.crate https://crates.io/api/v1/crates/num-integer/0.1.46/download -> num-integer-0.1.46.crate https://crates.io/api/v1/crates/numpy/0.26.0/download -> numpy-0.26.0.crate https://crates.io/api/v1/crates/num-traits/0.2.19/download -> num-traits-0.2.19.crate https://crates.io/api/v1/crates/once_cell/1.21.3/download -> once_cell-1.21.3.crate https://crates.io/api/v1/crates/once_cell_polyfill/1.70.2/download -> once_cell_polyfill-1.70.2.crate https://crates.io/api/v1/crates/onig/6.5.1/download -> onig-6.5.1.crate https://crates.io/api/v1/crates/onig_sys/69.9.1/download -> onig_sys-69.9.1.crate https://crates.io/api/v1/crates/paste/1.0.15/download -> paste-1.0.15.crate https://crates.io/api/v1/crates/pin-project-lite/0.2.16/download -> pin-project-lite-0.2.16.crate https://crates.io/api/v1/crates/pin-utils/0.1.0/download -> pin-utils-0.1.0.crate https://crates.io/api/v1/crates/pkg-config/0.3.32/download -> pkg-config-0.3.32.crate https://crates.io/api/v1/crates/portable-atomic/1.11.1/download -> portable-atomic-1.11.1.crate https://crates.io/api/v1/crates/portable-atomic-util/0.2.4/download -> portable-atomic-util-0.2.4.crate https://crates.io/api/v1/crates/ppv-lite86/0.2.21/download -> ppv-lite86-0.2.21.crate https://crates.io/api/v1/crates/proc-macro2/1.0.103/download -> proc-macro2-1.0.103.crate https://crates.io/api/v1/crates/pyo3/0.26.0/download -> pyo3-0.26.0.crate https://crates.io/api/v1/crates/pyo3-async-runtimes/0.26.0/download -> pyo3-async-runtimes-0.26.0.crate https://crates.io/api/v1/crates/pyo3-build-config/0.26.0/download -> pyo3-build-config-0.26.0.crate https://crates.io/api/v1/crates/pyo3-ffi/0.26.0/download -> pyo3-ffi-0.26.0.crate https://crates.io/api/v1/crates/pyo3-macros/0.26.0/download -> pyo3-macros-0.26.0.crate https://crates.io/api/v1/crates/pyo3-macros-backend/0.26.0/download -> pyo3-macros-backend-0.26.0.crate https://crates.io/api/v1/crates/quote/1.0.42/download -> quote-1.0.42.crate https://crates.io/api/v1/crates/rand/0.9.2/download -> rand-0.9.2.crate https://crates.io/api/v1/crates/rand_chacha/0.9.0/download -> rand_chacha-0.9.0.crate https://crates.io/api/v1/crates/rand_core/0.9.3/download -> rand_core-0.9.3.crate https://crates.io/api/v1/crates/rawpointer/0.2.1/download -> rawpointer-0.2.1.crate https://crates.io/api/v1/crates/rayon/1.11.0/download -> rayon-1.11.0.crate https://crates.io/api/v1/crates/rayon-cond/0.4.0/download -> rayon-cond-0.4.0.crate https://crates.io/api/v1/crates/rayon-core/1.13.0/download -> rayon-core-1.13.0.crate https://crates.io/api/v1/crates/r-efi/5.3.0/download -> r-efi-5.3.0.crate https://crates.io/api/v1/crates/regex/1.12.2/download -> regex-1.12.2.crate https://crates.io/api/v1/crates/regex-automata/0.4.13/download -> regex-automata-0.4.13.crate https://crates.io/api/v1/crates/regex-syntax/0.8.8/download -> regex-syntax-0.8.8.crate https://crates.io/api/v1/crates/rustc-hash/2.1.1/download -> rustc-hash-2.1.1.crate https://crates.io/api/v1/crates/rustix/1.1.2/download -> rustix-1.1.2.crate https://crates.io/api/v1/crates/rustversion/1.0.22/download -> rustversion-1.0.22.crate https://crates.io/api/v1/crates/ryu/1.0.20/download -> ryu-1.0.20.crate https://crates.io/api/v1/crates/serde/1.0.228/download -> serde-1.0.228.crate https://crates.io/api/v1/crates/serde_core/1.0.228/download -> serde_core-1.0.228.crate https://crates.io/api/v1/crates/serde_derive/1.0.228/download -> serde_derive-1.0.228.crate https://crates.io/api/v1/crates/serde_json/1.0.145/download -> serde_json-1.0.145.crate https://crates.io/api/v1/crates/shlex/1.3.0/download -> shlex-1.3.0.crate https://crates.io/api/v1/crates/signal-hook-registry/1.4.7/download -> signal-hook-registry-1.4.7.crate https://crates.io/api/v1/crates/slab/0.4.11/download -> slab-0.4.11.crate https://crates.io/api/v1/crates/smallvec/1.15.1/download -> smallvec-1.15.1.crate https://crates.io/api/v1/crates/spm_precompiled/0.1.4/download -> spm_precompiled-0.1.4.crate https://crates.io/api/v1/crates/static_assertions/1.1.0/download -> static_assertions-1.1.0.crate https://crates.io/api/v1/crates/strsim/0.11.1/download -> strsim-0.11.1.crate https://crates.io/api/v1/crates/syn/2.0.111/download -> syn-2.0.111.crate https://crates.io/api/v1/crates/target-lexicon/0.13.3/download -> target-lexicon-0.13.3.crate https://crates.io/api/v1/crates/tempfile/3.23.0/download -> tempfile-3.23.0.crate https://crates.io/api/v1/crates/thiserror/2.0.17/download -> thiserror-2.0.17.crate https://crates.io/api/v1/crates/thiserror-impl/2.0.17/download -> thiserror-impl-2.0.17.crate https://crates.io/api/v1/crates/tokio/1.48.0/download -> tokio-1.48.0.crate https://crates.io/api/v1/crates/tokio-macros/2.6.0/download -> tokio-macros-2.6.0.crate https://crates.io/api/v1/crates/unicode_categories/0.1.1/download -> unicode_categories-0.1.1.crate https://crates.io/api/v1/crates/unicode-ident/1.0.22/download -> unicode-ident-1.0.22.crate https://crates.io/api/v1/crates/unicode-normalization-alignments/0.1.12/download -> unicode-normalization-alignments-0.1.12.crate https://crates.io/api/v1/crates/unicode-segmentation/1.12.0/download -> unicode-segmentation-1.12.0.crate https://crates.io/api/v1/crates/unicode-width/0.2.2/download -> unicode-width-0.2.2.crate https://crates.io/api/v1/crates/unindent/0.2.4/download -> unindent-0.2.4.crate https://crates.io/api/v1/crates/unit-prefix/0.5.2/download -> unit-prefix-0.5.2.crate https://crates.io/api/v1/crates/utf8parse/0.2.2/download -> utf8parse-0.2.2.crate https://crates.io/api/v1/crates/version_check/0.9.5/download -> version_check-0.9.5.crate https://crates.io/api/v1/crates/wasi/0.11.1+wasi-snapshot-preview1/download -> wasi-0.11.1+wasi-snapshot-preview1.crate https://crates.io/api/v1/crates/wasip2/1.0.1+wasi-0.2.4/download -> wasip2-1.0.1+wasi-0.2.4.crate https://crates.io/api/v1/crates/wasm-bindgen/0.2.106/download -> wasm-bindgen-0.2.106.crate https://crates.io/api/v1/crates/wasm-bindgen-macro/0.2.106/download -> wasm-bindgen-macro-0.2.106.crate https://crates.io/api/v1/crates/wasm-bindgen-macro-support/0.2.106/download -> wasm-bindgen-macro-support-0.2.106.crate https://crates.io/api/v1/crates/wasm-bindgen-shared/0.2.106/download -> wasm-bindgen-shared-0.2.106.crate https://crates.io/api/v1/crates/web-time/1.1.0/download -> web-time-1.1.0.crate https://crates.io/api/v1/crates/windows-link/0.2.1/download -> windows-link-0.2.1.crate https://crates.io/api/v1/crates/windows-sys/0.61.2/download -> windows-sys-0.61.2.crate https://crates.io/api/v1/crates/wit-bindgen/0.46.0/download -> wit-bindgen-0.46.0.crate https://crates.io/api/v1/crates/zerocopy/0.8.31/download -> zerocopy-0.8.31.crate https://crates.io/api/v1/crates/zerocopy-derive/0.8.31/download -> zerocopy-derive-0.8.31.crate
_eclasses_=cargo	53bab5a62cfc3526d8ed02c0a38a6033	distutils-r1	62add15ed646bfe511af53e282621b39	eapi9-pipestatus	d2c134036ac31c3085aebc9147f572bd	flag-o-matic	fd3558f73f6503093adee69adf41020d	multibuild	83db098a835f71feb71761f72a694aa2	multilib	fc812e8eb2170022c28272d80a1f9e77	multiprocessing	45d0620f25c2f063ad6276ecd9ed9399	ninja-utils	d063ea1900a793f93eb095010d6d9156	out-of-source-utils	8e5093bd27b7f541a1913271f25d633a	python-r1	5fdca77e34fc9af7d6eb47689f33f6bf	python-utils-r1	9c4180f103a0e8340bc993fc85dbdadd	rust	c5ae4c9b5eca97bd7fc77fede663873f	rust-toolchain	76468983281b0a7fc167ca224f84ecfd	toolchain-funcs	58a918e3a42d0156f010dcfd43725be5
_md5_=13e71340f03f7cd21325d1f085116f23