BDEPEND=test? ( sci-ml/datasets[python_single_target_python3_12(-)?,python_single_target_python3_13(-)?] ) python_single_target_python3_12? ( dev-python/setuptools-rust[python_targets_python3_12(-)] ) python_single_target_python3_13? ( dev-python/setuptools-rust[python_targets_python3_13(-)] ) test? ( dev-libs/oniguruma python_single_target_python3_12? ( >=dev-python/pytest-7.4.4[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/pytest-7.4.4[python_targets_python3_13(-)] ) ) || ( >=dev-lang/rust-bin-1.74.1:* >=dev-lang/rust-1.74.1:* ) python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 ) python_single_target_python3_12? ( >=dev-python/gpep517-16[python_targets_python3_12(-)] >=dev-util/maturin-1.8.2[python_targets_python3_12(-)] ) python_single_target_python3_13? ( >=dev-python/gpep517-16[python_targets_python3_13(-)] >=dev-util/maturin-1.8.2[python_targets_python3_13(-)] )
DEFINED_PHASES=compile configure install prepare setup test unpack
DEPEND=python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
DESCRIPTION=Implementation of today's most used tokenizers
EAPI=8
HOMEPAGE=https://github.com/huggingface/tokenizers
IUSE=test debug python_single_target_python3_12 python_single_target_python3_13 debug
KEYWORDS=~amd64
LICENSE=Apache-2.0 Apache-2.0 Apache-2.0-with-LLVM-exceptions BSD-2 BSD ISC MIT MPL-2.0 Unicode-DFS-2016
RDEPEND=dev-libs/oniguruma python_single_target_python3_12? ( dev-lang/python:3.12 ) python_single_target_python3_13? ( dev-lang/python:3.13 )
REQUIRED_USE=^^ ( python_single_target_python3_12 python_single_target_python3_13 )
RESTRICT=!test? ( test )
SLOT=0
SRC_URI=https://github.com/huggingface/tokenizers/archive/refs/tags/v0.21.1.tar.gz -> tokenizers-0.21.1.gh.tar.gz https://static.crates.io/crates/aho-corasick/aho-corasick-1.1.3.crate https://static.crates.io/crates/autocfg/autocfg-1.4.0.crate https://static.crates.io/crates/base64/base64-0.13.1.crate https://static.crates.io/crates/bit-set/bit-set-0.8.0.crate https://static.crates.io/crates/bit-vec/bit-vec-0.8.0.crate https://static.crates.io/crates/bitflags/bitflags-1.3.2.crate https://static.crates.io/crates/cc/cc-1.2.24.crate https://static.crates.io/crates/cfg-if/cfg-if-1.0.0.crate https://static.crates.io/crates/console/console-0.15.11.crate https://static.crates.io/crates/crossbeam-deque/crossbeam-deque-0.8.6.crate https://static.crates.io/crates/crossbeam-epoch/crossbeam-epoch-0.9.18.crate https://static.crates.io/crates/crossbeam-utils/crossbeam-utils-0.8.21.crate https://static.crates.io/crates/darling/darling-0.20.11.crate https://static.crates.io/crates/darling_core/darling_core-0.20.11.crate https://static.crates.io/crates/darling_macro/darling_macro-0.20.11.crate https://static.crates.io/crates/derive_builder/derive_builder-0.20.2.crate https://static.crates.io/crates/derive_builder_core/derive_builder_core-0.20.2.crate https://static.crates.io/crates/derive_builder_macro/derive_builder_macro-0.20.2.crate https://static.crates.io/crates/either/either-1.15.0.crate https://static.crates.io/crates/esaxx-rs/esaxx-rs-0.1.10.crate https://static.crates.io/crates/fancy-regex/fancy-regex-0.14.0.crate https://static.crates.io/crates/fnv/fnv-1.0.7.crate https://static.crates.io/crates/getrandom/getrandom-0.2.16.crate https://static.crates.io/crates/heck/heck-0.5.0.crate https://static.crates.io/crates/ident_case/ident_case-1.0.1.crate https://static.crates.io/crates/indicatif/indicatif-0.17.11.crate https://static.crates.io/crates/indoc/indoc-2.0.6.crate https://static.crates.io/crates/itertools/itertools-0.11.0.crate https://static.crates.io/crates/itertools/itertools-0.12.1.crate https://static.crates.io/crates/itertools/itertools-0.13.0.crate https://static.crates.io/crates/itoa/itoa-1.0.15.crate https://static.crates.io/crates/lazy_static/lazy_static-1.5.0.crate https://static.crates.io/crates/libc/libc-0.2.172.crate https://static.crates.io/crates/log/log-0.4.27.crate https://static.crates.io/crates/macro_rules_attribute-proc_macro/macro_rules_attribute-proc_macro-0.2.0.crate https://static.crates.io/crates/macro_rules_attribute/macro_rules_attribute-0.2.0.crate https://static.crates.io/crates/matrixmultiply/matrixmultiply-0.3.10.crate https://static.crates.io/crates/memchr/memchr-2.7.4.crate https://static.crates.io/crates/memoffset/memoffset-0.9.1.crate https://static.crates.io/crates/minimal-lexical/minimal-lexical-0.2.1.crate https://static.crates.io/crates/monostate-impl/monostate-impl-0.1.14.crate https://static.crates.io/crates/monostate/monostate-0.1.14.crate https://static.crates.io/crates/ndarray/ndarray-0.16.1.crate https://static.crates.io/crates/nom/nom-7.1.3.crate https://static.crates.io/crates/num-complex/num-complex-0.4.6.crate https://static.crates.io/crates/num-integer/num-integer-0.1.46.crate https://static.crates.io/crates/num-traits/num-traits-0.2.19.crate https://static.crates.io/crates/number_prefix/number_prefix-0.4.0.crate https://static.crates.io/crates/numpy/numpy-0.25.0.crate https://static.crates.io/crates/once_cell/once_cell-1.21.3.crate https://static.crates.io/crates/onig/onig-6.4.0.crate https://static.crates.io/crates/onig_sys/onig_sys-69.8.1.crate https://static.crates.io/crates/paste/paste-1.0.15.crate https://static.crates.io/crates/pkg-config/pkg-config-0.3.32.crate https://static.crates.io/crates/portable-atomic/portable-atomic-1.11.0.crate https://static.crates.io/crates/ppv-lite86/ppv-lite86-0.2.21.crate https://static.crates.io/crates/proc-macro2/proc-macro2-1.0.95.crate https://static.crates.io/crates/pyo3-build-config/pyo3-build-config-0.25.0.crate https://static.crates.io/crates/pyo3-ffi/pyo3-ffi-0.25.0.crate https://static.crates.io/crates/pyo3-macros-backend/pyo3-macros-backend-0.25.0.crate https://static.crates.io/crates/pyo3-macros/pyo3-macros-0.25.0.crate https://static.crates.io/crates/pyo3/pyo3-0.25.0.crate https://static.crates.io/crates/quote/quote-1.0.40.crate https://static.crates.io/crates/rand/rand-0.8.5.crate https://static.crates.io/crates/rand_chacha/rand_chacha-0.3.1.crate https://static.crates.io/crates/rand_core/rand_core-0.6.4.crate https://static.crates.io/crates/rawpointer/rawpointer-0.2.1.crate https://static.crates.io/crates/rayon-cond/rayon-cond-0.3.0.crate https://static.crates.io/crates/rayon-core/rayon-core-1.12.1.crate https://static.crates.io/crates/rayon/rayon-1.10.0.crate https://static.crates.io/crates/regex-automata/regex-automata-0.4.9.crate https://static.crates.io/crates/regex-syntax/regex-syntax-0.8.5.crate https://static.crates.io/crates/regex/regex-1.11.1.crate https://static.crates.io/crates/rustc-hash/rustc-hash-2.1.1.crate https://static.crates.io/crates/ryu/ryu-1.0.20.crate https://static.crates.io/crates/serde/serde-1.0.219.crate https://static.crates.io/crates/serde_derive/serde_derive-1.0.219.crate https://static.crates.io/crates/serde_json/serde_json-1.0.140.crate https://static.crates.io/crates/shlex/shlex-1.3.0.crate https://static.crates.io/crates/smallvec/smallvec-1.15.0.crate https://static.crates.io/crates/spm_precompiled/spm_precompiled-0.1.4.crate https://static.crates.io/crates/strsim/strsim-0.11.1.crate https://static.crates.io/crates/syn/syn-2.0.101.crate https://static.crates.io/crates/target-lexicon/target-lexicon-0.13.2.crate https://static.crates.io/crates/thiserror-impl/thiserror-impl-2.0.12.crate https://static.crates.io/crates/thiserror/thiserror-2.0.12.crate https://static.crates.io/crates/unicode-ident/unicode-ident-1.0.18.crate https://static.crates.io/crates/unicode-normalization-alignments/unicode-normalization-alignments-0.1.12.crate https://static.crates.io/crates/unicode-segmentation/unicode-segmentation-1.12.0.crate https://static.crates.io/crates/unicode-width/unicode-width-0.2.0.crate https://static.crates.io/crates/unicode_categories/unicode_categories-0.1.1.crate https://static.crates.io/crates/unindent/unindent-0.2.4.crate https://static.crates.io/crates/zerocopy/zerocopy-0.8.25.crate
_eclasses_=cargo	983bdde2432d878045ca2524b9ff935d	distutils-r1	8b4cb6626db7331e3187d652f4bbf9d4	eapi9-pipestatus	d2c134036ac31c3085aebc9147f572bd	flag-o-matic	fd3558f73f6503093adee69adf41020d	multibuild	83db098a835f71feb71761f72a694aa2	multilib	fc812e8eb2170022c28272d80a1f9e77	multiprocessing	45d0620f25c2f063ad6276ecd9ed9399	ninja-utils	d063ea1900a793f93eb095010d6d9156	out-of-source-utils	8e5093bd27b7f541a1913271f25d633a	python-single-r1	47d8ac9be330c9366112b78fd4f8ce57	python-utils-r1	18a01bee4e486884d7457cfacde74099	rust	c5ae4c9b5eca97bd7fc77fede663873f	rust-toolchain	76468983281b0a7fc167ca224f84ecfd	toolchain-funcs	58a918e3a42d0156f010dcfd43725be5
_md5_=40b32664e4243e51f9eb13aa9a708d12