BDEPEND=test? ( sci-ml/datasets[python_single_target_python3_11(-)?,python_single_target_python3_12(-)?,python_single_target_python3_13(-)?] ) python_single_target_python3_11? ( dev-python/setuptools-rust[python_targets_python3_11(-)] ) python_single_target_python3_12? ( dev-python/setuptools-rust[python_targets_python3_12(-)] ) python_single_target_python3_13? ( dev-python/setuptools-rust[python_targets_python3_13(-)] ) test? ( dev-libs/oniguruma python_single_target_python3_11? ( >=dev-python/pytest-7.4.4[python_targets_python3_11(-)] ) python_single_target_python3_12? ( >=dev-python/pytest-7.4.4[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/pytest-7.4.4[python_targets_python3_13(-)] ) ) || ( >=dev-lang/rust-bin-1.74.1:* >=dev-lang/rust-1.74.1:* ) python_single_target_python3_11? ( dev-lang/python:3.11 ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 ) python_single_target_python3_11? ( >=dev-python/gpep517-16[python_targets_python3_11(-)] >=dev-util/maturin-1.8.2[python_targets_python3_11(-)] ) python_single_target_python3_12? ( >=dev-python/gpep517-16[python_targets_python3_12(-)] >=dev-util/maturin-1.8.2[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/gpep517-16[python_targets_python3_13(-)] >=dev-util/maturin-1.8.2[python_targets_python3_13(-)] )
DEFINED_PHASES=compile configure install prepare setup test unpack
DEPEND=python_single_target_python3_11? ( dev-lang/python:3.11 ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
DESCRIPTION=Implementation of today's most used tokenizers
EAPI=8
HOMEPAGE=https://github.com/huggingface/tokenizers
IUSE=test debug python_single_target_python3_11 python_single_target_python3_12 python_single_target_python3_13 debug
KEYWORDS=~amd64
LICENSE=Apache-2.0 Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0 Unicode-DFS-2016
RDEPEND=dev-libs/oniguruma python_single_target_python3_11? ( dev-lang/python:3.11 ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
REQUIRED_USE=^^ ( python_single_target_python3_11 python_single_target_python3_12 python_single_target_python3_13 )
RESTRICT=!test? ( test )
SLOT=0
SRC_URI=https://github.com/huggingface/tokenizers/archive/refs/tags/v0.21.1.tar.gz -> tokenizers-0.21.1.gh.tar.gz https://crates.io/api/v1/crates/aho-corasick/1.1.3/download -> aho-corasick-1.1.3.crate https://crates.io/api/v1/crates/autocfg/1.4.0/download -> autocfg-1.4.0.crate https://crates.io/api/v1/crates/base64/0.13.1/download -> base64-0.13.1.crate https://crates.io/api/v1/crates/bit-set/0.8.0/download -> bit-set-0.8.0.crate https://crates.io/api/v1/crates/bit-vec/0.8.0/download -> bit-vec-0.8.0.crate https://crates.io/api/v1/crates/bitflags/1.3.2/download -> bitflags-1.3.2.crate https://crates.io/api/v1/crates/cc/1.2.24/download -> cc-1.2.24.crate https://crates.io/api/v1/crates/cfg-if/1.0.0/download -> cfg-if-1.0.0.crate https://crates.io/api/v1/crates/console/0.15.11/download -> console-0.15.11.crate https://crates.io/api/v1/crates/crossbeam-deque/0.8.6/download -> crossbeam-deque-0.8.6.crate https://crates.io/api/v1/crates/crossbeam-epoch/0.9.18/download -> crossbeam-epoch-0.9.18.crate https://crates.io/api/v1/crates/crossbeam-utils/0.8.21/download -> crossbeam-utils-0.8.21.crate https://crates.io/api/v1/crates/darling/0.20.11/download -> darling-0.20.11.crate https://crates.io/api/v1/crates/darling_core/0.20.11/download -> darling_core-0.20.11.crate https://crates.io/api/v1/crates/darling_macro/0.20.11/download -> darling_macro-0.20.11.crate https://crates.io/api/v1/crates/derive_builder/0.20.2/download -> derive_builder-0.20.2.crate https://crates.io/api/v1/crates/derive_builder_core/0.20.2/download -> derive_builder_core-0.20.2.crate https://crates.io/api/v1/crates/derive_builder_macro/0.20.2/download -> derive_builder_macro-0.20.2.crate https://crates.io/api/v1/crates/either/1.15.0/download -> either-1.15.0.crate https://crates.io/api/v1/crates/esaxx-rs/0.1.10/download -> esaxx-rs-0.1.10.crate https://crates.io/api/v1/crates/fancy-regex/0.14.0/download -> fancy-regex-0.14.0.crate https://crates.io/api/v1/crates/fnv/1.0.7/download -> fnv-1.0.7.crate https://crates.io/api/v1/crates/getrandom/0.2.16/download -> getrandom-0.2.16.crate https://crates.io/api/v1/crates/heck/0.5.0/download -> heck-0.5.0.crate https://crates.io/api/v1/crates/ident_case/1.0.1/download -> ident_case-1.0.1.crate https://crates.io/api/v1/crates/indicatif/0.17.11/download -> indicatif-0.17.11.crate https://crates.io/api/v1/crates/indoc/2.0.6/download -> indoc-2.0.6.crate https://crates.io/api/v1/crates/itertools/0.11.0/download -> itertools-0.11.0.crate https://crates.io/api/v1/crates/itertools/0.12.1/download -> itertools-0.12.1.crate https://crates.io/api/v1/crates/itertools/0.13.0/download -> itertools-0.13.0.crate https://crates.io/api/v1/crates/itoa/1.0.15/download -> itoa-1.0.15.crate https://crates.io/api/v1/crates/lazy_static/1.5.0/download -> lazy_static-1.5.0.crate https://crates.io/api/v1/crates/libc/0.2.172/download -> libc-0.2.172.crate https://crates.io/api/v1/crates/log/0.4.27/download -> log-0.4.27.crate https://crates.io/api/v1/crates/macro_rules_attribute-proc_macro/0.2.0/download -> macro_rules_attribute-proc_macro-0.2.0.crate https://crates.io/api/v1/crates/macro_rules_attribute/0.2.0/download -> macro_rules_attribute-0.2.0.crate https://crates.io/api/v1/crates/matrixmultiply/0.3.10/download -> matrixmultiply-0.3.10.crate https://crates.io/api/v1/crates/memchr/2.7.4/download -> memchr-2.7.4.crate https://crates.io/api/v1/crates/memoffset/0.9.1/download -> memoffset-0.9.1.crate https://crates.io/api/v1/crates/minimal-lexical/0.2.1/download -> minimal-lexical-0.2.1.crate https://crates.io/api/v1/crates/monostate-impl/0.1.14/download -> monostate-impl-0.1.14.crate https://crates.io/api/v1/crates/monostate/0.1.14/download -> monostate-0.1.14.crate https://crates.io/api/v1/crates/ndarray/0.16.1/download -> ndarray-0.16.1.crate https://crates.io/api/v1/crates/nom/7.1.3/download -> nom-7.1.3.crate https://crates.io/api/v1/crates/num-complex/0.4.6/download -> num-complex-0.4.6.crate https://crates.io/api/v1/crates/num-integer/0.1.46/download -> num-integer-0.1.46.crate https://crates.io/api/v1/crates/num-traits/0.2.19/download -> num-traits-0.2.19.crate https://crates.io/api/v1/crates/number_prefix/0.4.0/download -> number_prefix-0.4.0.crate https://crates.io/api/v1/crates/numpy/0.25.0/download -> numpy-0.25.0.crate https://crates.io/api/v1/crates/once_cell/1.21.3/download -> once_cell-1.21.3.crate https://crates.io/api/v1/crates/onig/6.4.0/download -> onig-6.4.0.crate https://crates.io/api/v1/crates/onig_sys/69.8.1/download -> onig_sys-69.8.1.crate https://crates.io/api/v1/crates/paste/1.0.15/download -> paste-1.0.15.crate https://crates.io/api/v1/crates/pkg-config/0.3.32/download -> pkg-config-0.3.32.crate https://crates.io/api/v1/crates/portable-atomic/1.11.0/download -> portable-atomic-1.11.0.crate https://crates.io/api/v1/crates/ppv-lite86/0.2.21/download -> ppv-lite86-0.2.21.crate https://crates.io/api/v1/crates/proc-macro2/1.0.95/download -> proc-macro2-1.0.95.crate https://crates.io/api/v1/crates/pyo3-build-config/0.25.0/download -> pyo3-build-config-0.25.0.crate https://crates.io/api/v1/crates/pyo3-ffi/0.25.0/download -> pyo3-ffi-0.25.0.crate https://crates.io/api/v1/crates/pyo3-macros-backend/0.25.0/download -> pyo3-macros-backend-0.25.0.crate https://crates.io/api/v1/crates/pyo3-macros/0.25.0/download -> pyo3-macros-0.25.0.crate https://crates.io/api/v1/crates/pyo3/0.25.0/download -> pyo3-0.25.0.crate https://crates.io/api/v1/crates/quote/1.0.40/download -> quote-1.0.40.crate https://crates.io/api/v1/crates/rand/0.8.5/download -> rand-0.8.5.crate https://crates.io/api/v1/crates/rand_chacha/0.3.1/download -> rand_chacha-0.3.1.crate https://crates.io/api/v1/crates/rand_core/0.6.4/download -> rand_core-0.6.4.crate https://crates.io/api/v1/crates/rawpointer/0.2.1/download -> rawpointer-0.2.1.crate https://crates.io/api/v1/crates/rayon-cond/0.3.0/download -> rayon-cond-0.3.0.crate https://crates.io/api/v1/crates/rayon-core/1.12.1/download -> rayon-core-1.12.1.crate https://crates.io/api/v1/crates/rayon/1.10.0/download -> rayon-1.10.0.crate https://crates.io/api/v1/crates/regex-automata/0.4.9/download -> regex-automata-0.4.9.crate https://crates.io/api/v1/crates/regex-syntax/0.8.5/download -> regex-syntax-0.8.5.crate https://crates.io/api/v1/crates/regex/1.11.1/download -> regex-1.11.1.crate https://crates.io/api/v1/crates/rustc-hash/2.1.1/download -> rustc-hash-2.1.1.crate https://crates.io/api/v1/crates/ryu/1.0.20/download -> ryu-1.0.20.crate https://crates.io/api/v1/crates/serde/1.0.219/download -> serde-1.0.219.crate https://crates.io/api/v1/crates/serde_derive/1.0.219/download -> serde_derive-1.0.219.crate https://crates.io/api/v1/crates/serde_json/1.0.140/download -> serde_json-1.0.140.crate https://crates.io/api/v1/crates/shlex/1.3.0/download -> shlex-1.3.0.crate https://crates.io/api/v1/crates/smallvec/1.15.0/download -> smallvec-1.15.0.crate https://crates.io/api/v1/crates/spm_precompiled/0.1.4/download -> spm_precompiled-0.1.4.crate https://crates.io/api/v1/crates/strsim/0.11.1/download -> strsim-0.11.1.crate https://crates.io/api/v1/crates/syn/2.0.101/download -> syn-2.0.101.crate https://crates.io/api/v1/crates/target-lexicon/0.13.2/download -> target-lexicon-0.13.2.crate https://crates.io/api/v1/crates/thiserror-impl/2.0.12/download -> thiserror-impl-2.0.12.crate https://crates.io/api/v1/crates/thiserror/2.0.12/download -> thiserror-2.0.12.crate https://crates.io/api/v1/crates/unicode-ident/1.0.18/download -> unicode-ident-1.0.18.crate https://crates.io/api/v1/crates/unicode-normalization-alignments/0.1.12/download -> unicode-normalization-alignments-0.1.12.crate https://crates.io/api/v1/crates/unicode-segmentation/1.12.0/download -> unicode-segmentation-1.12.0.crate https://crates.io/api/v1/crates/unicode-width/0.2.0/download -> unicode-width-0.2.0.crate https://crates.io/api/v1/crates/unicode_categories/0.1.1/download -> unicode_categories-0.1.1.crate https://crates.io/api/v1/crates/unindent/0.2.4/download -> unindent-0.2.4.crate https://crates.io/api/v1/crates/zerocopy/0.8.25/download -> zerocopy-0.8.25.crate
_eclasses_=cargo	a1af74aa9e34cd73aeac642cd0dd54d8	distutils-r1	37143302bc375f4878f7b5f7497da8cc	flag-o-matic	a7afe42e95fb46ce9691605acfb24672	multibuild	4650a65187015567b4e041bb9bfdb364	multilib	a199af442898963433d6f038a88cce9c	multiprocessing	1e32df7deee68372153dca65f4a7c21f	ninja-utils	2df4e452cea39a9ec8fb543ce059f8d6	out-of-source-utils	dbf9e34ee8964084651e25907fa8f52c	python-single-r1	47d8ac9be330c9366112b78fd4f8ce57	python-utils-r1	0b620dd604ab8af0bc9cd1bcc66d4f7c	rust	67553b9b0296e5b691a6f6efdce65271	rust-toolchain	76468983281b0a7fc167ca224f84ecfd	toolchain-funcs	98d9f464d912ae6b7316fb8a3721f5db
_md5_=40b32664e4243e51f9eb13aa9a708d12