Skip to content

Tokenizer Classes¤

Three tokenizers, each with different procedures for dividing the text into tokens.

Tokenizer pydantic-model ¤

Bases: BaseModel

A class for tokenizing text using spaCy.

Config:

  • arbitrary_types_allowed: True
  • json_schema_extra: DocJSONSchema.schema()
  • validate_assignment: True

Fields:

Source code in lexos/tokenizer/__init__.py 
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
class Tokenizer(BaseModel):
    """A class for tokenizing text using spaCy."""

    model: Optional[str] = Field(
        default="xx_sent_ud_sm",
        description="The name of the spaCy model to be used for tokenization.",
    )
    max_length: Optional[int] = Field(
        default=2000000,
        description="The maximum length of the doc.",
    )
    disable: Optional[list[str]] = Field(
        default=[],
        description="A list of spaCy pipeline components to disable.",
    )
    stopwords: Optional[list[str] | str] = Field(
        default=[],
        description="A list of stop words to apply to docs.",
    )
    nlp: Optional[Language] = Field(
        default=default_model,
        description="The spaCy language object.",
    )

    model_config = ConfigDict(
        arbitrary_types_allowed=True,
        json_schema_extra=DocJSONSchema.schema(),
        validate_assignment=True,
    )

    def __init__(self, **data) -> None:
        """Initialise the Tokenizer class."""
        super().__init__(**data)
        try:
            self.nlp = spacy.load(self.model)
            self.nlp.max_length = self.max_length
        except OSError:
            raise LexosException(
                f"Error loading model {self.model}. Please check the name and try again. You may need to install the model on your system."
            )

    @validate_call
    def __call__(self, texts: str | Iterable[str]) -> Doc | Iterable[Doc]:
        """Tokenize a string or an iterable of strings.

        Args:
            texts (str | Iterable[str]): The text(s) to be tokenized.

        Returns:
            Doc | Iterable[Doc]: The tokenized doc(s).
        """
        if isinstance(texts, str):
            return self.make_doc(texts)
        elif isinstance(texts, Iterable):
            return self.make_docs(texts)

    @property
    def pipeline(self) -> list[str]:
        """Return the spaCy pipeline components."""
        return self.nlp.pipe_names

    @property
    def components(self) -> list[str]:
        """Return the spaCy pipeline components."""
        return self.nlp.components

    @property
    def disabled(self) -> list[str]:
        """Return the disabled spaCy pipeline components."""
        return self.nlp.disabled

    @validate_call
    def add_extension(self, name: str, default: str) -> None:
        """Add an extension to the spaCy Token class.

        Args:
            name (str): The name of the extension.
            default (str): The default value of the extension.
        """
        if not Token.has_extension(name):
            Token.set_extension(name, default=default, force=True)

    @validate_call
    def add_stopwords(self, stopwords: str | list[str]) -> None:
        """Add stopwords to the tokenizer.

        Args:
            stopwords (str | Iterable[str]): A list of stopwords to add to the model.
        """
        stopwords = ensure_list(stopwords)
        for term in stopwords:
            self.nlp.vocab[term].is_stop = True
        self.stopwords.extend(stopwords)

    @validate_call
    def make_doc(
        self, text: str, max_length: int = None, disable: list[str] = [], **kwargs: Any
    ) -> Doc:
        """Return a doc from a text.

        Args:
            text (str): The text to be parsed.
            max_length (int): The maximum length of the doc.
            disable (list[str]): A list of spaCy pipeline components to disable.
            kwargs (Any): Additional keyword arguments. Accepts any keyword arguments that
                can be passed to spaCy's `Language.pipe` method, such as `batch_size`.

        Returns:
            Doc: A spaCy doc object.
        """
        # Override instance settings with keyword arguments
        if max_length:
            self.max_length = max_length
            self.nlp.max_length = max_length
        if disable:
            # self.nlp.disabled.extend(disable)
            self.nlp.select_pipes(disable=disable)
        return next(self.nlp.pipe([text], disable=disable, **kwargs))

    @validate_call
    def make_docs(
        self,
        texts: Iterable[str],
        max_length: int = None,
        disable: Iterable[str] = [],
        **kwargs: Any,
    ) -> Iterable[Doc]:
        """Return a generator of docs from an iterable of texts.

        Args:
            texts (Iterable[str]): The texts to be parsed.
            max_length (int): The maximum length of the docs.
            kwargs (Any): Additional keyword arguments. Accepts any keyword arguments that
                can be passed to spaCy's `Language.pipe` method, such as `batch_size`.

        Yields:
            Iterable[Doc]: A generator of spaCy doc objects.
        """
        # Override instance settings with keyword arguments
        if max_length:
            self.max_length = max_length
            self.nlp.max_length = max_length
        if disable:
            # self.nlp.disabled.extend(disable)
            self.nlp.select_pipes(disable=disable)
        return self.nlp.pipe(texts, disable=disable, **kwargs)

    @validate_call
    def remove_extension(self, name: str) -> None:
        """Remove an extension from the spaCy Token class.

        Args:
            name (str): The name of the extension.
        """
        if Token.has_extension(name):
            Token.remove_extension(name)

    @validate_call
    def remove_stopwords(self, stopwords: str | list[str]) -> None:
        """Remove stopwords from the tokenizer.

        Args:
            stopwords (str | list[str]): A list of stopwords to remove from the model.
        """
        stopwords = ensure_list(stopwords)
        for term in stopwords:
            self.nlp.vocab[term].is_stop = False
        self.stopwords = [word for word in self.stopwords if word not in stopwords]

components: list[str] property ¤

Return the spaCy pipeline components.

disable: Optional[list[str]] = [] pydantic-field ¤

A list of spaCy pipeline components to disable.

disabled: list[str] property ¤

Return the disabled spaCy pipeline components.

max_length: Optional[int] = 2000000 pydantic-field ¤

The maximum length of the doc.

model: Optional[str] = 'xx_sent_ud_sm' pydantic-field ¤

The name of the spaCy model to be used for tokenization.

pipeline: list[str] property ¤

Return the spaCy pipeline components.

stopwords: Optional[list[str] | str] = [] pydantic-field ¤

A list of stop words to apply to docs.

__call__(texts: str | Iterable[str]) -> Doc | Iterable[Doc] ¤

Tokenize a string or an iterable of strings.

Parameters:

Name Type Description Default
texts str | Iterable[str]

The text(s) to be tokenized.

 required

Returns:

Type Description
Doc | Iterable[Doc]

Doc | Iterable[Doc]: The tokenized doc(s).
Source code in lexos/tokenizer/__init__.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
@validate_call
def __call__(self, texts: str | Iterable[str]) -> Doc | Iterable[Doc]:
    """Tokenize a string or an iterable of strings.

    Args:
        texts (str | Iterable[str]): The text(s) to be tokenized.

    Returns:
        Doc | Iterable[Doc]: The tokenized doc(s).
    """
    if isinstance(texts, str):
        return self.make_doc(texts)
    elif isinstance(texts, Iterable):
        return self.make_docs(texts)

__init__(**data) -> None ¤

Initialise the Tokenizer class.

Source code in lexos/tokenizer/__init__.py 
76
77
78
79
80
81
82
83
84
85
def __init__(self, **data) -> None:
    """Initialise the Tokenizer class."""
    super().__init__(**data)
    try:
        self.nlp = spacy.load(self.model)
        self.nlp.max_length = self.max_length
    except OSError:
        raise LexosException(
            f"Error loading model {self.model}. Please check the name and try again. You may need to install the model on your system."
        )

add_extension(name: str, default: str) -> None ¤

Add an extension to the spaCy Token class.

Parameters:

Name Type Description Default
name str

The name of the extension.

 required
default str

The default value of the extension.

 required
Source code in lexos/tokenizer/__init__.py 
117
118
119
120
121
122
123
124
125
126
@validate_call
def add_extension(self, name: str, default: str) -> None:
    """Add an extension to the spaCy Token class.

    Args:
        name (str): The name of the extension.
        default (str): The default value of the extension.
    """
    if not Token.has_extension(name):
        Token.set_extension(name, default=default, force=True)

add_stopwords(stopwords: str | list[str]) -> None ¤

Add stopwords to the tokenizer.

Parameters:

Name Type Description Default
stopwords str | Iterable[str]

A list of stopwords to add to the model.

 required
Source code in lexos/tokenizer/__init__.py 
128
129
130
131
132
133
134
135
136
137
138
@validate_call
def add_stopwords(self, stopwords: str | list[str]) -> None:
    """Add stopwords to the tokenizer.

    Args:
        stopwords (str | Iterable[str]): A list of stopwords to add to the model.
    """
    stopwords = ensure_list(stopwords)
    for term in stopwords:
        self.nlp.vocab[term].is_stop = True
    self.stopwords.extend(stopwords)

make_doc(text: str, max_length: int = None, disable: list[str] = [], **kwargs: Any) -> Doc ¤

Return a doc from a text.

Parameters:

Name Type Description Default
text str

The text to be parsed.

 required
max_length int

The maximum length of the doc.

 None
disable list[str]

A list of spaCy pipeline components to disable.

 []
kwargs Any

Additional keyword arguments. Accepts any keyword arguments that can be passed to spaCy's Language.pipe method, such as batch_size.

 {}

Returns:

Name Type Description
Doc Doc

A spaCy doc object.
Source code in lexos/tokenizer/__init__.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
@validate_call
def make_doc(
    self, text: str, max_length: int = None, disable: list[str] = [], **kwargs: Any
) -> Doc:
    """Return a doc from a text.

    Args:
        text (str): The text to be parsed.
        max_length (int): The maximum length of the doc.
        disable (list[str]): A list of spaCy pipeline components to disable.
        kwargs (Any): Additional keyword arguments. Accepts any keyword arguments that
            can be passed to spaCy's `Language.pipe` method, such as `batch_size`.

    Returns:
        Doc: A spaCy doc object.
    """
    # Override instance settings with keyword arguments
    if max_length:
        self.max_length = max_length
        self.nlp.max_length = max_length
    if disable:
        # self.nlp.disabled.extend(disable)
        self.nlp.select_pipes(disable=disable)
    return next(self.nlp.pipe([text], disable=disable, **kwargs))

make_docs(texts: Iterable[str], max_length: int = None, disable: Iterable[str] = [], **kwargs: Any) -> Iterable[Doc] ¤

Return a generator of docs from an iterable of texts.

Parameters:

Name Type Description Default
texts Iterable[str]

The texts to be parsed.

 required
max_length int

The maximum length of the docs.

 None
kwargs Any

Additional keyword arguments. Accepts any keyword arguments that can be passed to spaCy's Language.pipe method, such as batch_size.

 {}

Yields:

Type Description
Iterable[Doc]

Iterable[Doc]: A generator of spaCy doc objects.
Source code in lexos/tokenizer/__init__.py 
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
@validate_call
def make_docs(
    self,
    texts: Iterable[str],
    max_length: int = None,
    disable: Iterable[str] = [],
    **kwargs: Any,
) -> Iterable[Doc]:
    """Return a generator of docs from an iterable of texts.

    Args:
        texts (Iterable[str]): The texts to be parsed.
        max_length (int): The maximum length of the docs.
        kwargs (Any): Additional keyword arguments. Accepts any keyword arguments that
            can be passed to spaCy's `Language.pipe` method, such as `batch_size`.

    Yields:
        Iterable[Doc]: A generator of spaCy doc objects.
    """
    # Override instance settings with keyword arguments
    if max_length:
        self.max_length = max_length
        self.nlp.max_length = max_length
    if disable:
        # self.nlp.disabled.extend(disable)
        self.nlp.select_pipes(disable=disable)
    return self.nlp.pipe(texts, disable=disable, **kwargs)

remove_extension(name: str) -> None ¤

Remove an extension from the spaCy Token class.

Parameters:

Name Type Description Default
name str

The name of the extension.

 required
Source code in lexos/tokenizer/__init__.py 
193
194
195
196
197
198
199
200
201
@validate_call
def remove_extension(self, name: str) -> None:
    """Remove an extension from the spaCy Token class.

    Args:
        name (str): The name of the extension.
    """
    if Token.has_extension(name):
        Token.remove_extension(name)

remove_stopwords(stopwords: str | list[str]) -> None ¤

Remove stopwords from the tokenizer.

Parameters:

Name Type Description Default
stopwords str | list[str]

A list of stopwords to remove from the model.

 required
Source code in lexos/tokenizer/__init__.py 
203
204
205
206
207
208
209
210
211
212
213
@validate_call
def remove_stopwords(self, stopwords: str | list[str]) -> None:
    """Remove stopwords from the tokenizer.

    Args:
        stopwords (str | list[str]): A list of stopwords to remove from the model.
    """
    stopwords = ensure_list(stopwords)
    for term in stopwords:
        self.nlp.vocab[term].is_stop = False
    self.stopwords = [word for word in self.stopwords if word not in stopwords]

model: Optional[str] = 'xx_sent_ud_sm' pydantic-field ¤

The name of the spaCy model to be used for tokenization.

max_length: Optional[int] = 2000000 pydantic-field ¤

The maximum length of the doc.

disable: Optional[list[str]] = [] pydantic-field ¤

A list of spaCy pipeline components to disable.

stopwords: Optional[list[str] | str] = [] pydantic-field ¤

A list of stop words to apply to docs.

nlp: Optional[Language] pydantic-field ¤

model_config = ConfigDict(arbitrary_types_allowed=True, json_schema_extra=(DocJSONSchema.schema()), validate_assignment=True) class-attribute instance-attribute ¤

__init__(**data) -> None ¤

Initialise the Tokenizer class.

Source code in lexos/tokenizer/__init__.py 
76
77
78
79
80
81
82
83
84
85
def __init__(self, **data) -> None:
    """Initialise the Tokenizer class."""
    super().__init__(**data)
    try:
        self.nlp = spacy.load(self.model)
        self.nlp.max_length = self.max_length
    except OSError:
        raise LexosException(
            f"Error loading model {self.model}. Please check the name and try again. You may need to install the model on your system."
        )

__call__(texts: str | Iterable[str]) -> Doc | Iterable[Doc] ¤

Tokenize a string or an iterable of strings.

Parameters:

Name Type Description Default
texts str | Iterable[str]

The text(s) to be tokenized.

 required

Returns:

Type Description
Doc | Iterable[Doc]

Doc | Iterable[Doc]: The tokenized doc(s).
Source code in lexos/tokenizer/__init__.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
@validate_call
def __call__(self, texts: str | Iterable[str]) -> Doc | Iterable[Doc]:
    """Tokenize a string or an iterable of strings.

    Args:
        texts (str | Iterable[str]): The text(s) to be tokenized.

    Returns:
        Doc | Iterable[Doc]: The tokenized doc(s).
    """
    if isinstance(texts, str):
        return self.make_doc(texts)
    elif isinstance(texts, Iterable):
        return self.make_docs(texts)

pipeline: list[str] property ¤

Return the spaCy pipeline components.

components: list[str] property ¤

Return the spaCy pipeline components.

disabled: list[str] property ¤

Return the disabled spaCy pipeline components.

add_extension(name: str, default: str) -> None ¤

Add an extension to the spaCy Token class.

Parameters:

Name Type Description Default
name str

The name of the extension.

 required
default str

The default value of the extension.

 required
Source code in lexos/tokenizer/__init__.py 
117
118
119
120
121
122
123
124
125
126
@validate_call
def add_extension(self, name: str, default: str) -> None:
    """Add an extension to the spaCy Token class.

    Args:
        name (str): The name of the extension.
        default (str): The default value of the extension.
    """
    if not Token.has_extension(name):
        Token.set_extension(name, default=default, force=True)

add_stopwords(stopwords: str | list[str]) -> None ¤

Add stopwords to the tokenizer.

Parameters:

Name Type Description Default
stopwords str | Iterable[str]

A list of stopwords to add to the model.

 required
Source code in lexos/tokenizer/__init__.py 
128
129
130
131
132
133
134
135
136
137
138
@validate_call
def add_stopwords(self, stopwords: str | list[str]) -> None:
    """Add stopwords to the tokenizer.

    Args:
        stopwords (str | Iterable[str]): A list of stopwords to add to the model.
    """
    stopwords = ensure_list(stopwords)
    for term in stopwords:
        self.nlp.vocab[term].is_stop = True
    self.stopwords.extend(stopwords)

make_doc(text: str, max_length: int = None, disable: list[str] = [], **kwargs: Any) -> Doc ¤

Return a doc from a text.

Parameters:

Name Type Description Default
text str

The text to be parsed.

 required
max_length int

The maximum length of the doc.

 None
disable list[str]

A list of spaCy pipeline components to disable.

 []
kwargs Any

Additional keyword arguments. Accepts any keyword arguments that can be passed to spaCy's Language.pipe method, such as batch_size.

 {}

Returns:

Name Type Description
Doc Doc

A spaCy doc object.
Source code in lexos/tokenizer/__init__.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
@validate_call
def make_doc(
    self, text: str, max_length: int = None, disable: list[str] = [], **kwargs: Any
) -> Doc:
    """Return a doc from a text.

    Args:
        text (str): The text to be parsed.
        max_length (int): The maximum length of the doc.
        disable (list[str]): A list of spaCy pipeline components to disable.
        kwargs (Any): Additional keyword arguments. Accepts any keyword arguments that
            can be passed to spaCy's `Language.pipe` method, such as `batch_size`.

    Returns:
        Doc: A spaCy doc object.
    """
    # Override instance settings with keyword arguments
    if max_length:
        self.max_length = max_length
        self.nlp.max_length = max_length
    if disable:
        # self.nlp.disabled.extend(disable)
        self.nlp.select_pipes(disable=disable)
    return next(self.nlp.pipe([text], disable=disable, **kwargs))

make_docs(texts: Iterable[str], max_length: int = None, disable: Iterable[str] = [], **kwargs: Any) -> Iterable[Doc] ¤

Return a generator of docs from an iterable of texts.

Parameters:

Name Type Description Default
texts Iterable[str]

The texts to be parsed.

 required
max_length int

The maximum length of the docs.

 None
kwargs Any

Additional keyword arguments. Accepts any keyword arguments that can be passed to spaCy's Language.pipe method, such as batch_size.

 {}

Yields:

Type Description
Iterable[Doc]

Iterable[Doc]: A generator of spaCy doc objects.
Source code in lexos/tokenizer/__init__.py 
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
@validate_call
def make_docs(
    self,
    texts: Iterable[str],
    max_length: int = None,
    disable: Iterable[str] = [],
    **kwargs: Any,
) -> Iterable[Doc]:
    """Return a generator of docs from an iterable of texts.

    Args:
        texts (Iterable[str]): The texts to be parsed.
        max_length (int): The maximum length of the docs.
        kwargs (Any): Additional keyword arguments. Accepts any keyword arguments that
            can be passed to spaCy's `Language.pipe` method, such as `batch_size`.

    Yields:
        Iterable[Doc]: A generator of spaCy doc objects.
    """
    # Override instance settings with keyword arguments
    if max_length:
        self.max_length = max_length
        self.nlp.max_length = max_length
    if disable:
        # self.nlp.disabled.extend(disable)
        self.nlp.select_pipes(disable=disable)
    return self.nlp.pipe(texts, disable=disable, **kwargs)

remove_extension(name: str) -> None ¤

Remove an extension from the spaCy Token class.

Parameters:

Name Type Description Default
name str

The name of the extension.

 required
Source code in lexos/tokenizer/__init__.py 
193
194
195
196
197
198
199
200
201
@validate_call
def remove_extension(self, name: str) -> None:
    """Remove an extension from the spaCy Token class.

    Args:
        name (str): The name of the extension.
    """
    if Token.has_extension(name):
        Token.remove_extension(name)

remove_stopwords(stopwords: str | list[str]) -> None ¤

Remove stopwords from the tokenizer.

Parameters:

Name Type Description Default
stopwords str | list[str]

A list of stopwords to remove from the model.

 required
Source code in lexos/tokenizer/__init__.py 
203
204
205
206
207
208
209
210
211
212
213
@validate_call
def remove_stopwords(self, stopwords: str | list[str]) -> None:
    """Remove stopwords from the tokenizer.

    Args:
        stopwords (str | list[str]): A list of stopwords to remove from the model.
    """
    stopwords = ensure_list(stopwords)
    for term in stopwords:
        self.nlp.vocab[term].is_stop = False
    self.stopwords = [word for word in self.stopwords if word not in stopwords]

SliceTokenizer pydantic-model ¤

Bases: BaseModel

Simple slice tokenizer.

Fields:

Source code in lexos/tokenizer/__init__.py 
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
class SliceTokenizer(BaseModel, validate_assignment=True):
    """Simple slice tokenizer."""

    n: int = Field(description="The size of the tokens in characters.")
    drop_ws: Optional[bool] = Field(
        default=True, description="Whether to drop whitespace from the tokens."
    )

    @validate_call
    def __call__(self, text: str) -> list[str]:
        """Slice the text into tokens of n characters.

        Args:
            text (str): The text to tokenize.

        Returns:
            list[str]: A list of tokens.
        """
        if self.drop_ws:
            text = text.replace(" ", "")
        return ["".join(t) for t in batched(text, self.n)]

drop_ws: Optional[bool] = True pydantic-field ¤

Whether to drop whitespace from the tokens.

n: int pydantic-field ¤

The size of the tokens in characters.

__call__(text: str) -> list[str] ¤

Slice the text into tokens of n characters.

Parameters:

Name Type Description Default
text str

The text to tokenize.

 required

Returns:

Type Description
list[str]

list[str]: A list of tokens.
Source code in lexos/tokenizer/__init__.py 
224
225
226
227
228
229
230
231
232
233
234
235
236
@validate_call
def __call__(self, text: str) -> list[str]:
    """Slice the text into tokens of n characters.

    Args:
        text (str): The text to tokenize.

    Returns:
        list[str]: A list of tokens.
    """
    if self.drop_ws:
        text = text.replace(" ", "")
    return ["".join(t) for t in batched(text, self.n)]

n: int pydantic-field ¤

The size of the tokens in characters.

drop_ws: Optional[bool] = True pydantic-field ¤

Whether to drop whitespace from the tokens.

__call__(text: str) -> list[str] ¤

Slice the text into tokens of n characters.

Parameters:

Name Type Description Default
text str

The text to tokenize.

 required

Returns:

Type Description
list[str]

list[str]: A list of tokens.
Source code in lexos/tokenizer/__init__.py 
224
225
226
227
228
229
230
231
232
233
234
235
236
@validate_call
def __call__(self, text: str) -> list[str]:
    """Slice the text into tokens of n characters.

    Args:
        text (str): The text to tokenize.

    Returns:
        list[str]: A list of tokens.
    """
    if self.drop_ws:
        text = text.replace(" ", "")
    return ["".join(t) for t in batched(text, self.n)]

WhitespaceTokenizer pydantic-model ¤

Bases: BaseModel

Simple whitespace tokenizer.

Source code in lexos/tokenizer/__init__.py 
239
240
241
242
243
244
245
246
247
248
249
250
251
252
class WhitespaceTokenizer(BaseModel):
    """Simple whitespace tokenizer."""

    @validate_call
    def __call__(self, text: str) -> list[str]:
        """Split the text into tokens on whitespace.

        Args:
            text (str): The text to tokenize.

        Returns:
            list[str]: A list of tokens.
        """
        return text.split()

__call__(text: str) -> list[str] ¤

Split the text into tokens on whitespace.

Parameters:

Name Type Description Default
text str

The text to tokenize.

 required

Returns:

Type Description
list[str]

list[str]: A list of tokens.
Source code in lexos/tokenizer/__init__.py 
242
243
244
245
246
247
248
249
250
251
252
@validate_call
def __call__(self, text: str) -> list[str]:
    """Split the text into tokens on whitespace.

    Args:
        text (str): The text to tokenize.

    Returns:
        list[str]: A list of tokens.
    """
    return text.split()

__call__(text: str) -> list[str] ¤

Split the text into tokens on whitespace.

Parameters:

Name Type Description Default
text str

The text to tokenize.

 required

Returns:

Type Description
list[str]

list[str]: A list of tokens.
Source code in lexos/tokenizer/__init__.py 
242
243
244
245
246
247
248
249
250
251
252
@validate_call
def __call__(self, text: str) -> list[str]:
    """Split the text into tokens on whitespace.

    Args:
        text (str): The text to tokenize.

    Returns:
        list[str]: A list of tokens.
    """
    return text.split()