From 8c35516a1f20fee8608401b1aea694063458c061 Mon Sep 17 00:00:00 2001 From: mh-northlander Date: Mon, 8 Jul 2024 11:56:02 +0900 Subject: [PATCH] sync pyi and rs --- python/py_src/sudachipy/sudachipy.pyi | 104 ++++++++++++++++---------- python/src/dictionary.rs | 54 ++++++------- python/src/morpheme.rs | 48 +++++++----- python/src/pos_matcher.rs | 10 ++- python/src/tokenizer.rs | 10 +-- 5 files changed, 136 insertions(+), 90 deletions(-) diff --git a/python/py_src/sudachipy/sudachipy.pyi b/python/py_src/sudachipy/sudachipy.pyi index 16c416f6..705b62af 100644 --- a/python/py_src/sudachipy/sudachipy.pyi +++ b/python/py_src/sudachipy/sudachipy.pyi @@ -1,6 +1,20 @@ +# Copyright (c) 2024 Works Applications Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import ClassVar, Iterator, List, Tuple, Union, Callable, Iterable, Optional, Literal, Set from .config import Config +# Part Of Speech POS = Tuple[str, str, str, str, str, str] # POS element PE = Optional[str] @@ -14,6 +28,8 @@ PartialPOS = Union[ Tuple[()], ] +# Fields that can be specified for partial dictionary loading. +# See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. FieldSet = Optional[Set[Literal["surface", "pos", "normalized_form", "dictionary_form", "reading_form", "word_structure", "split_a", "split_b", "synonym_group_id"]]] @@ -23,9 +39,7 @@ class SplitMode: Unit to split text. A == short mode - B == middle mode - C == long mode """ @@ -36,8 +50,9 @@ class SplitMode: @classmethod def __init__(cls, mode: str = "C") -> None: """ - Creates a split mode from a string value - :param mode: string representation of the split mode + Creates a split mode from a string value. + + :param mode: string representation of the split mode. One of [A,B,C] in captital or lower case. """ ... @@ -54,14 +69,15 @@ class Dictionary: Creates a sudachi dictionary. If both config.systemDict and dict are not given, `sudachidict_core` is used. - If both config.systemDict and dict are given, dict_type is used. + If both config.systemDict and dict are given, dict is used. + If dict is an absolute path to a file, it is used as a dictionary. - :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.config.Config] object - :param config: alias to config_path, only one of them can be specified at the same time - :param resource_dir: path to the resource directory folder + :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.Config] object. + :param config: alias to config_path, only one of them can be specified at the same time. + :param resource_dir: path to the resource directory folder. :param dict: type of pre-packaged system dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. Also, can be an _absolute_ path to a compiled dictionary file. - :param dict_type: deprecated alias to dict + :param dict_type: deprecated alias to dict. """ ... @@ -77,11 +93,11 @@ class Dictionary: *, projection: str = None) -> Tokenizer: """ - Creates a Sudachi Tokenizer. + Creates a sudachi tokenizer. :param mode: sets the analysis mode for this Tokenizer :param fields: load only a subset of fields. - See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html + See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. :param projection: Projection override for created Tokenizer. See Config.projection for values. """ ... @@ -91,21 +107,21 @@ class Dictionary: Creates a new POS matcher. If target is a function, then it must return whether a POS should match or not. - If target a list, it should contain partially specified POS. - By partially specified it means that it is possible to omit POS fields or - use None as a sentinel value that matches any POS. + If target is a list, it should contain partially specified POS. + By partially specified it means that it is possible to omit POS fields or use None as a sentinel value that matches any POS. For example, ('名詞',) will match any noun and (None, None, None, None, None, '終止形') will match any word in 終止形 conjugation form. - :param target: can be either a function or a list of POS tuples. + :param target: can be either a list of POS partial tuples or a callable which maps POS to bool. """ ... def pre_tokenizer(self, mode: Union[SplitMode, Literal["A", "B", "C"]] = "C", fields: FieldSet = None, - handler: Optional[Callable[[int, object, MorphemeList], list]] = None, + handler: Optional[Callable[[ + int, object, MorphemeList], list]] = None, *, projection: str = None) -> object: """ @@ -113,10 +129,10 @@ class Dictionary: Requires package `tokenizers` to be installed. :param mode: Use this split mode (C by default) - :param fields: ask Sudachi to load only a subset of fields. See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html - :param handler: custom callable to transform MorphemeList into list of tokens. See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py - First two parameters are the index (int) and HuggingFace NormalizedString. - The handler must return a List[NormalizedString]. By default, just segment the tokens. + :param fields: ask Sudachi to load only a subset of fields. See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. + :param handler: a custom callable to transform MorphemeList into list of tokens. If None, simply use surface as token representations. + It should be a `function(index: int, original: NormalizedString, morphemes: MorphemeList) -> List[NormalizedString]`. + See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py. :param projection: Projection override for created Tokenizer. See Config.projection for values. """ ... @@ -126,7 +142,7 @@ class Dictionary: Returns POS with the given id. :param pos_id: POS id - :return: POS tuple with the given id. + :return: POS tuple with the given id or None for non existing id. """ ... @@ -197,7 +213,8 @@ class Morpheme: def part_of_speech(self) -> POS: """ - Returns the part of speech. + Returns the part of speech as a six-element tuple. + Tuple elements are four POS levels, conjugation type and conjugation form. """ ... @@ -217,8 +234,8 @@ class Morpheme: """ Returns sub-morphemes in the provided split mode. - :param mode: mode of new split - :param out: write results to this MorhpemeList instead of creating new one + :param mode: mode of new split. + :param out: write results to this MorhpemeList instead of creating new one. See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for more information on output parameters. Returned MorphemeList will be invalidated if this MorphemeList is used as an output parameter. @@ -230,6 +247,7 @@ class Morpheme: def surface(self) -> str: """ Returns the substring of input text corresponding to the morpheme, or a projection if one is configured. + See `Config.projection`. """ ... @@ -237,6 +255,7 @@ class Morpheme: def raw_surface(self) -> str: """ Returns the substring of input text corresponding to the morpheme regardless the configured projection. + See `Config.projection`. """ ... @@ -255,7 +274,7 @@ class Morpheme: def __len__(self) -> int: """ - Returns morpheme length in codepoints + Returns morpheme length in codepoints. """ @@ -293,6 +312,11 @@ class MorphemeList: class Tokenizer: + """ + A sudachi tokenizer + + Create using Dictionary.create method. + """ SplitMode: ClassVar[SplitMode] = ... @classmethod def __init__(cls) -> None: ... @@ -303,13 +327,12 @@ class Tokenizer: """ Break text into morphemes. - SudachiPy 0.5.* had logger parameter, it is accepted, but ignored. - - :param text: text to analyze + :param text: text to analyze. :param mode: analysis mode. This parameter is deprecated. Pass the analysis mode at the Tokenizer creation time and create different tokenizers for different modes. If you need multi-level splitting, prefer using :py:meth:`Morpheme.split` method instead. + :param logger: Arg for v0.5.* compatibility. Ignored. :param out: tokenization results will be written into this MorphemeList, a new one will be created instead. See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for details. """ @@ -342,41 +365,44 @@ class WordInfo: class PosMatcher: + """ + A part-of-speech matcher which checks if a morpheme belongs to a set of part of speech. + + Create using Dictionary.pos_matcher method. + """ + def __iter__(self) -> Iterator[POS]: ... def __len__(self) -> int: ... def __call__(self, m: Morpheme) -> bool: """ - Checks whether a morpheme has matching POS - :param m: morpheme - :return: if morpheme has matching POS + Checks whether a morpheme has matching POS. + + :param m: morpheme. + :return: if morpheme has matching POS. """ ... def __or__(self, other: PosMatcher) -> PosMatcher: """ - Returns a POS matcher which matches a POS if any of two matchers would match it - :return: PosMatcher + Returns a POS matcher which matches a POS if any of two matchers would match it. """ ... def __and__(self, other: PosMatcher) -> PosMatcher: """ - Returns a POS matcher which matches a POS if both matchers would match it at the same time - :return: PosMatcher + Returns a POS matcher which matches a POS if both matchers would match it at the same time. """ ... def __sub__(self, other: PosMatcher) -> PosMatcher: """ - Returns a POS matcher which matches a POS if self would match the POS and other would not match the POS - :return: PosMatcher + Returns a POS matcher which matches a POS if self would match the POS and other would not match the POS. """ ... def __invert__(self) -> PosMatcher: """ - Returns a POS matcher which matches all POS tags except ones defined in the current POS matcher - :return: PosMatcher + Returns a POS matcher which matches all POS tags except ones defined in the current POS matcher. """ ... diff --git a/python/src/dictionary.rs b/python/src/dictionary.rs index e208492f..5f1e8f65 100644 --- a/python/src/dictionary.rs +++ b/python/src/dictionary.rs @@ -80,11 +80,12 @@ impl PyDicData { /// A sudachi dictionary. /// -/// If both config.systemDict and dict_type are not given, `sudachidict_core` is used. -/// If both config.systemDict and dict_type are given, dict_type is used. +/// If both config.systemDict and dict are not given, `sudachidict_core` is used. +/// If both config.systemDict and dict are given, dict is used. /// If dict is an absolute path to a file, it is used as a dictionary. /// -/// :param config_path: path to the configuration JSON file. +/// :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.Config] object. +/// :param config: alias to config_path, only one of them can be specified at the same time. /// :param resource_dir: path to the resource directory folder. /// :param dict: type of pre-packaged dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. /// Also, can be an _absolute_ path to a compiled dictionary file. @@ -100,11 +101,12 @@ pub struct PyDictionary { impl PyDictionary { /// Creates a sudachi dictionary. /// - /// If both config.systemDict and dict_type are not given, `sudachidict_core` is used. - /// If both config.systemDict and dict_type are given, dict_type is used. + /// If both config.systemDict and dict are not given, `sudachidict_core` is used. + /// If both config.systemDict and dict are given, dict is used. /// If dict is an absolute path to a file, it is used as a dictionary. /// - /// :param config_path: path to the configuration JSON file. + /// :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.Config] object. + /// :param config: alias to config_path, only one of them can be specified at the same time. /// :param resource_dir: path to the resource directory folder. /// :param dict: type of pre-packaged dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. /// Also, can be an _absolute_ path to a compiled dictionary file. @@ -229,11 +231,12 @@ impl PyDictionary { /// Creates a sudachi tokenizer. /// - /// :param mode: tokenizer's default split mode (C by default). + /// :param mode: sets the analysis mode for this Tokenizer /// :param fields: load only a subset of fields. - /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html + /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. + /// :param projection: Projection override for created Tokenizer. See Config.projection for values. #[pyo3( - text_signature="(self, /, mode=None, fields=None, *, projection=None) -> Tokenizer", + text_signature="(self, /, mode=SplitMode.C, fields=None, *, projection=None) -> Tokenizer", signature=(mode=None, fields=None, *, projection=None) )] fn create<'py>( @@ -267,14 +270,13 @@ impl PyDictionary { /// Creates a POS matcher object /// /// If target is a function, then it must return whether a POS should match or not. - /// If target a list, it should contain partially specified POS. - /// By partially specified it means that it is possible to omit POS fields or - /// use None as a sentinel value that matches any POS. + /// If target is a list, it should contain partially specified POS. + /// By partially specified it means that it is possible to omit POS fields or use None as a sentinel value that matches any POS. /// /// For example, ('名詞',) will match any noun and /// (None, None, None, None, None, '終止形‐一般') will match any word in 終止形‐一般 conjugation form. /// - /// :param target: can be either a callable or list of POS partial tuples + /// :param target: can be either a list of POS partial tuples or a callable which maps POS to bool. #[pyo3(text_signature="(self, /, target) -> PosMatcher")] fn pos_matcher<'py>(&'py self, py: Python<'py>, target: &PyAny) -> PyResult { PyPosMatcher::create(py, self.dictionary.as_ref().unwrap(), target) @@ -285,15 +287,13 @@ impl PyDictionary { /// /// :param mode: Use this split mode (C by default) /// :param fields: ask Sudachi to load only a subset of fields. - /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html - /// :param handler: a custom callable to transform MorphemeList into list of tokens. - /// It should be should be a `function(index: int, original: NormalizedString, morphemes: MorphemeList) -> List[NormalizedString]`. - /// See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py - /// If nothing was passed, simply use surface as token representations. - /// :param projection: projection mode for a created PreTokenizer. - /// See :class:`sudachipy.config.Config` object documentation for supported projections. + /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. + /// :param handler: a custom callable to transform MorphemeList into list of tokens. If None, simply use surface as token representations. + /// It should be a `function(index: int, original: NormalizedString, morphemes: MorphemeList) -> List[NormalizedString]`. + /// See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py. + /// :param projection: Projection override for created Tokenizer. See Config.projection for values. /// - /// :type mode: sudachipy.SplitMode + /// :type mode: SplitMode /// :type fields: Set[str] #[pyo3( text_signature="(self, /, mode=None, fields=None, handler=None, *, projection=None) -> tokenizers.PreTokenizer", @@ -350,8 +350,9 @@ impl PyDictionary { /// :param surface: find all morphemes with the given surface /// :param out: if passed, reuse the given morpheme list instead of creating a new one. /// See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for details. + /// /// :type surface: str - /// :type out: sudachipy.MorphemeList + /// :type out: MorphemeList #[pyo3(text_signature="(self, /, surface, out=None) -> MorphemeList")] fn lookup<'p>( &'p self, @@ -379,14 +380,17 @@ impl PyDictionary { Ok(l) } - /// Close this dictionary + /// Close this dictionary. #[pyo3(text_signature="(self, /) -> ()")] fn close(&mut self) { self.dictionary = None; } - /// Get POS Tuple by its id - #[pyo3(text_signature="(self, /, pos_id: int) -> tuple[str, str, str, str, str, str]")] + /// Returns POS with the given id. + /// + /// :param pos_id: POS id + /// :return: POS tuple with the given id or None for non existing id. + #[pyo3(text_signature="(self, /, pos_id: int) -> tuple[str, str, str, str, str, str] | None")] fn pos_of<'p>(&'p self, py: Python<'p>, pos_id: usize) -> Option<&'p PyTuple> { let dic = self.dictionary.as_ref().unwrap(); dic.pos.get(pos_id).map(|x| x.as_ref(py)) diff --git a/python/src/morpheme.rs b/python/src/morpheme.rs index f1aa204d..0a18f6c4 100644 --- a/python/src/morpheme.rs +++ b/python/src/morpheme.rs @@ -31,7 +31,10 @@ use crate::word_info::PyWordInfo; pub(crate) type PyMorphemeList = MorphemeList>; pub(crate) type PyProjector = Option>; -/// A list of morphemes +/// A list of morphemes. +/// +/// An object can not be instantiated manually. +/// Use Tokenizer.tokenize("") to create an empty morpheme list. #[pyclass(module = "sudachipy.morphemelist", name = "MorphemeList")] pub struct PyMorphemeListWrapper { /// use `internal()` function instead @@ -89,7 +92,7 @@ impl PyMorphemeListWrapper { #[pymethods] impl PyMorphemeListWrapper { - /// Returns an empty morpheme list with dictionary + /// Returns an empty morpheme list with dictionary. #[classmethod] #[pyo3(text_signature="(dict: Dictionary) -> MorphemeList")] fn empty(_cls: &PyType, py: Python, dict: &PyDictionary) -> PyResult { @@ -109,7 +112,7 @@ impl PyMorphemeListWrapper { }) } - /// Returns the total cost of the path + /// Returns the total cost of the path. #[pyo3(text_signature="(self, /) -> int")] fn get_internal_cost(&self, py: Python) -> i32 { self.internal(py).get_internal_cost() @@ -278,21 +281,23 @@ impl PyMorpheme { #[pymethods] impl PyMorpheme { - /// Returns the begin index of this in the input text + /// Returns the begin index of this in the input text. #[pyo3(text_signature="(self, /) -> int")] fn begin(&self, py: Python) -> usize { // call codepoint version self.morph(py).begin_c() } - /// Returns the end index of this in the input text + /// Returns the end index of this in the input text. #[pyo3(text_signature="(self, /) -> int")] fn end(&self, py: Python) -> usize { // call codepoint version self.morph(py).end_c() } - /// Returns the substring of input text corresponding to the morpheme, or a projection if one is configured + /// Returns the substring of input text corresponding to the morpheme, or a projection if one is configured. + /// + /// See `Config.projection`. #[pyo3(text_signature="(self, /) -> str")] fn surface<'py>(&'py self, py: Python<'py>) -> &'py PyString { let list = self.list(py); @@ -303,14 +308,16 @@ impl PyMorpheme { } } - /// Returns the substring of input text corresponding to the morpheme regardless the configured projection + /// Returns the substring of input text corresponding to the morpheme regardless the configured projection. + /// + /// See `Config.projection`. #[pyo3(text_signature="(self, /) -> str")] fn raw_surface<'py>(&'py self, py: Python<'py>) -> &'py PyString { PyString::new(py, self.morph(py).surface().deref()) } /// Returns the part of speech as a six-element tuple. - /// Tuple elements are four POS levels, conjugation type and conjugation form. + /// Tuple elements are four POS levels, conjugation type and conjugation form. #[pyo3(text_signature="(self, /) -> tuple[str, str, str, str, str, str]")] fn part_of_speech<'py>(&'py self, py: Python<'py>) -> Py { let pos_id = self.part_of_speech_id(py); @@ -321,25 +328,25 @@ impl PyMorpheme { .clone_ref(py) } - /// Returns the id of the part of speech in the dictionary + /// Returns the id of the part of speech in the dictionary. #[pyo3(text_signature="(self, /) -> int")] pub fn part_of_speech_id(&self, py: Python) -> u16 { self.morph(py).part_of_speech_id() } - /// Returns the dictionary form + /// Returns the dictionary form. #[pyo3(text_signature="(self, /) -> str")] fn dictionary_form<'py>(&'py self, py: Python<'py>) -> PyObject { self.morph(py).get_word_info().dictionary_form().into_py(py) } - /// Returns the normalized form + /// Returns the normalized form. #[pyo3(text_signature="(self, /) -> str")] fn normalized_form<'py>(&'py self, py: Python<'py>) -> PyObject { self.morph(py).get_word_info().normalized_form().into_py(py) } - /// Returns the reading form + /// Returns the reading form. #[pyo3(text_signature="(self, /) -> str")] fn reading_form<'py>(&'py self, py: Python<'py>) -> PyObject { self.morph(py).get_word_info().reading_form().into_py(py) @@ -347,13 +354,14 @@ impl PyMorpheme { /// Returns sub-morphemes in the provided split mode. /// - /// :param mode: mode of new split - /// :param out: write results to this MorhpemeList instead of creating new one + /// :param mode: mode of new split. + /// :param out: write results to this MorhpemeList instead of creating new one. /// See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for /// more information on output parameters. /// Returned MorphemeList will be invalidated if this MorphemeList is used as an output parameter. /// :param add_single: return lists with the current morpheme if the split hasn't produced any elements. /// When False is passed, empty lists are returned instead. + /// /// :type mode: sudachipy.SplitMode /// :type out: Optional[sudachipy.MorphemeList] /// :type add_single: bool @@ -401,19 +409,19 @@ impl PyMorpheme { Ok(out_cell) } - /// Returns whether if this is out of vocabulary word + /// Returns whether if this is out of vocabulary word. #[pyo3(text_signature="(self, /) -> bool")] fn is_oov(&self, py: Python) -> bool { self.morph(py).is_oov() } - /// Returns word id of this word in the dictionary + /// Returns word id of this word in the dictionary. #[pyo3(text_signature="(self, /) -> int")] fn word_id(&self, py: Python) -> u32 { self.morph(py).word_id().as_raw() } - /// Returns the dictionary id which this word belongs + /// Returns the dictionary id which this word belongs. #[pyo3(text_signature="(self, /) -> int")] fn dictionary_id(&self, py: Python) -> i32 { let word_id = self.morph(py).word_id(); @@ -424,7 +432,7 @@ impl PyMorpheme { } } - /// Returns the list of synonym group ids + /// Returns the list of synonym group ids. #[pyo3(text_signature="(self, /) -> List[int]")] fn synonym_group_ids<'py>(&'py self, py: Python<'py>) -> &'py PyList { let mref = self.morph(py); @@ -432,7 +440,7 @@ impl PyMorpheme { PyList::new(py, ids) } - /// Returns the word info + /// Returns the word info. #[pyo3(text_signature="(self, /) -> WordInfo")] fn get_word_info(&self, py: Python) -> PyResult { let cat = PyModule::import(py, "builtins")?.getattr("DeprecationWarning")?; @@ -441,7 +449,7 @@ impl PyMorpheme { Ok(self.morph(py).get_word_info().clone().into()) } - /// Returns morpheme length in codepoints + /// Returns morpheme length in codepoints. pub fn __len__(&self, py: Python) -> usize { let m = self.morph(py); m.end_c() - m.begin_c() diff --git a/python/src/pos_matcher.rs b/python/src/pos_matcher.rs index 586c7d90..16d1fa56 100644 --- a/python/src/pos_matcher.rs +++ b/python/src/pos_matcher.rs @@ -30,7 +30,7 @@ use crate::morpheme::PyMorpheme; /// /// Create using Dictionary.pos_matcher method. /// -/// Use `__call__(m: Morpheme) -> bool` to check if given morpheme matches the PosMatcher. +/// Use `__call__(m: Morpheme) -> bool` to check whether a morpheme has matching POS. #[pyclass(module = "sudachipy.pos_matcher", name = "PosMatcher")] pub struct PyPosMatcher { matcher: PosMatcher, @@ -123,6 +123,10 @@ impl PyPosMatcher { #[pymethods] impl PyPosMatcher { + /// Checks whether a morpheme has matching POS. + /// + /// :param m: morpheme. + /// :return: if morpheme has matching POS. pub fn __call__<'py>(&'py self, py: Python<'py>, m: &'py PyMorpheme) -> bool { let pos_id = m.part_of_speech_id(py); self.matcher.matches_id(pos_id) @@ -140,6 +144,7 @@ impl PyPosMatcher { self.matcher.num_entries() } + /// Returns a POS matcher which matches a POS if any of two matchers would match it. pub fn __or__(&self, other: &Self) -> Self { assert_eq!( Arc::as_ptr(&self.dic), @@ -153,6 +158,7 @@ impl PyPosMatcher { } } + /// Returns a POS matcher which matches a POS if both matchers would match it at the same time. pub fn __and__(&self, other: &Self) -> Self { assert_eq!( Arc::as_ptr(&self.dic), @@ -166,6 +172,7 @@ impl PyPosMatcher { } } + /// Returns a POS matcher which matches a POS if self would match the POS and other would not match the POS. pub fn __sub__(&self, other: &Self) -> Self { assert_eq!( Arc::as_ptr(&self.dic), @@ -179,6 +186,7 @@ impl PyPosMatcher { } } + /// Returns a POS matcher which matches all POS tags except ones defined in the current POS matcher. pub fn __invert__(&self) -> Self { let max_id = self.dic.pos.len(); // map -> filter chain is needed to handle exactly u16::MAX POS entries diff --git a/python/src/tokenizer.rs b/python/src/tokenizer.rs index 16f2482a..8c7c1c84 100644 --- a/python/src/tokenizer.rs +++ b/python/src/tokenizer.rs @@ -35,7 +35,7 @@ use crate::morpheme::{PyMorphemeListWrapper, PyProjector}; /// B == middle mode /// C == long mode /// -/// :param mode: str to parse. One of [A,B,C] in captital or lower case. +/// :param mode: string representation of the split mode. One of [A,B,C] in captital or lower case. #[pyclass(module = "sudachipy.tokenizer", name = "SplitMode", frozen)] #[derive(Clone, PartialEq, Eq, Copy, Debug)] #[repr(u8)] @@ -88,7 +88,7 @@ impl PySplitMode { } } -/// Sudachi Tokenizer +/// A sudachi tokenizer /// /// Create using Dictionary.create method. #[pyclass(module = "sudachipy.tokenizer", name = "Tokenizer")] @@ -123,15 +123,15 @@ impl PyTokenizer { /// Break text into morphemes. /// - /// SudachiPy 0.5.* had logger parameter, it is accepted, but ignored. - /// - /// :param text: text to analyze + /// :param text: text to analyze. /// :param mode: analysis mode. /// This parameter is deprecated. /// Pass the analysis mode at the Tokenizer creation time and create different tokenizers for different modes. /// If you need multi-level splitting, prefer using :py:meth:`Morpheme.split` method instead. + /// :param logger: Arg for v0.5.* compatibility. Ignored. /// :param out: tokenization results will be written into this MorphemeList, a new one will be created instead. /// See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for details. + /// /// :type text: str /// :type mode: sudachipy.SplitMode /// :type out: sudachipy.MorphemeList