Skip to content

Filters¤

BaseFilter pydantic-model ¤

Bases: BaseModel

BaseFilter class.

Config:

  • arbitrary_types_allowed: True
  • json_schema_extra: DocJSONSchema.schema()

Fields:

  • doc (Optional[Doc])
  • matcher (Optional[Matcher])
  • matches (Optional[list[tuple[int, int, int]]])
Source code in lexos/filter/filters.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
class BaseFilter(BaseModel):
    """BaseFilter class."""

    id: ClassVar[str] = "base_filter"
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc.")
    matcher: Optional[Matcher] = Field(default=None, description="A spaCy matcher.")
    matches: Optional[list[tuple[int, int, int]]] = Field(
        default=None, description="List of matches."
    )

    model_config = ConfigDict(
        arbitrary_types_allowed=True, json_schema_extra=DocJSONSchema.schema()
    )

    @validate_call(config=model_config)
    def __call__(self, doc: Optional[Doc], matcher: Optional[Matcher] = None) -> Doc:
        """Call the filter function.

        Args:
            doc (Optional[Doc]): A spaCy doc.
            matcher (Optional[Matcher]): A spaCy matcher.

        Returns:
            Doc: The filtered doc.
        """
        # Validate the inputs
        if not doc and not self.doc:
            raise LexosException("No doc has been assigned to the filter.")
        if not matcher and not self.matcher:
            raise LexosException("No matcher has been assigned to the filter.")
        if doc:
            self.doc = doc
        if matcher:
            self.matcher = matcher
        # Get the matches
        self.matches = self.matcher(doc)

    @property
    def matched_token_ids(self) -> set[int]:
        """A list of matched token ids after the filter was applied."""
        if not self.matches:
            return None
        token_ids = set()
        for _, start, end in self.matches:
            for i in range(start, end):
                token_ids.add(i)
        return token_ids

    @property
    def matched_tokens(self) -> list[int]:
        """A list of matched tokens after the filter was applied."""
        return [self.doc[i] for i in self.matched_token_ids]

    @property
    def filtered_tokens(self) -> list[int]:
        """A list of filtered tokens after the filter was applied."""
        return [self.doc[i] for i in self.filtered_token_ids]

    @property
    def filtered_token_ids(self) -> set[int]:
        """A list of filtered token ids after the filter was applied."""
        if not self.matches:
            return None
        return set(range(len(self.doc))) - self.matched_token_ids

    def _set_extensions(self, attr: str, default: Any):
        """Set the extensions."""
        if not Token.has_extension(attr):
            Token.set_extension(attr, default=default, force=True)

    def get_matched_doc(self, add_spaces: bool = False) -> Doc:
        """Get a new doc from the matched tokens.

        Args:
            add_spaces (bool): If True, add a space after every token.
                             If False, preserve original whitespace. Default is False.

        Returns:
            Doc: A new spaCy Doc containing only the matched tokens.
        """
        words = [t.text for t in self.matched_tokens]
        if add_spaces:
            spaces = [" " for _ in self.matched_tokens]
        else:
            spaces = [t.whitespace_ for t in self.matched_tokens]
        return Doc(self.doc.vocab, words=words, spaces=spaces)

    def get_filtered_doc(self, add_spaces: bool = False) -> Doc:
        """Get a new doc from the filtered tokens.

        Args:
            add_spaces (bool): If True, add a space after every token.
                             If False, preserve original whitespace. Default is False.

        Returns:
            Doc: A new spaCy Doc containing only the filtered tokens.
        """
        words = [t.text for t in self.filtered_tokens]
        if add_spaces:
            spaces = [" " for _ in self.filtered_tokens]
        else:
            spaces = [t.whitespace_ for t in self.filtered_tokens]
        return Doc(self.doc.vocab, words=words, spaces=spaces)

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

filtered_token_ids: set[int] property ¤

A list of filtered token ids after the filter was applied.

filtered_tokens: list[int] property ¤

A list of filtered tokens after the filter was applied.

matched_token_ids: set[int] property ¤

A list of matched token ids after the filter was applied.

matched_tokens: list[int] property ¤

A list of matched tokens after the filter was applied.

matcher: Optional[Matcher] = None pydantic-field ¤

A spaCy matcher.

matches: Optional[list[tuple[int, int, int]]] = None pydantic-field ¤

List of matches.

__call__(doc: Optional[Doc], matcher: Optional[Matcher] = None) -> Doc ¤

Call the filter function.

Parameters:

Name Type Description Default
doc Optional[Doc]

A spaCy doc.

 required
matcher Optional[Matcher]

A spaCy matcher.

 None

Returns:

Name Type Description
Doc Doc

The filtered doc.
Source code in lexos/filter/filters.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
@validate_call(config=model_config)
def __call__(self, doc: Optional[Doc], matcher: Optional[Matcher] = None) -> Doc:
    """Call the filter function.

    Args:
        doc (Optional[Doc]): A spaCy doc.
        matcher (Optional[Matcher]): A spaCy matcher.

    Returns:
        Doc: The filtered doc.
    """
    # Validate the inputs
    if not doc and not self.doc:
        raise LexosException("No doc has been assigned to the filter.")
    if not matcher and not self.matcher:
        raise LexosException("No matcher has been assigned to the filter.")
    if doc:
        self.doc = doc
    if matcher:
        self.matcher = matcher
    # Get the matches
    self.matches = self.matcher(doc)

get_filtered_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the filtered tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the filtered tokens.
Source code in lexos/filter/filters.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def get_filtered_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the filtered tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the filtered tokens.
    """
    words = [t.text for t in self.filtered_tokens]
    if add_spaces:
        spaces = [" " for _ in self.filtered_tokens]
    else:
        spaces = [t.whitespace_ for t in self.filtered_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

get_matched_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the matched tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the matched tokens.
Source code in lexos/filter/filters.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def get_matched_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the matched tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the matched tokens.
    """
    words = [t.text for t in self.matched_tokens]
    if add_spaces:
        spaces = [" " for _ in self.matched_tokens]
    else:
        spaces = [t.whitespace_ for t in self.matched_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

id: str = 'base_filter' class-attribute ¤

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

matcher: Optional[Matcher] = None pydantic-field ¤

A spaCy matcher.

matches: Optional[list[tuple[int, int, int]]] = None pydantic-field ¤

List of matches.

model_config = ConfigDict(arbitrary_types_allowed=True, json_schema_extra=(DocJSONSchema.schema())) class-attribute instance-attribute ¤

__call__(doc: Optional[Doc], matcher: Optional[Matcher] = None) -> Doc ¤

Call the filter function.

Parameters:

Name Type Description Default
doc Optional[Doc]

A spaCy doc.

 required
matcher Optional[Matcher]

A spaCy matcher.

 None

Returns:

Name Type Description
Doc Doc

The filtered doc.
Source code in lexos/filter/filters.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
@validate_call(config=model_config)
def __call__(self, doc: Optional[Doc], matcher: Optional[Matcher] = None) -> Doc:
    """Call the filter function.

    Args:
        doc (Optional[Doc]): A spaCy doc.
        matcher (Optional[Matcher]): A spaCy matcher.

    Returns:
        Doc: The filtered doc.
    """
    # Validate the inputs
    if not doc and not self.doc:
        raise LexosException("No doc has been assigned to the filter.")
    if not matcher and not self.matcher:
        raise LexosException("No matcher has been assigned to the filter.")
    if doc:
        self.doc = doc
    if matcher:
        self.matcher = matcher
    # Get the matches
    self.matches = self.matcher(doc)

matched_token_ids: set[int] property ¤

A list of matched token ids after the filter was applied.

matched_tokens: list[int] property ¤

A list of matched tokens after the filter was applied.

filtered_token_ids: set[int] property ¤

A list of filtered token ids after the filter was applied.

filtered_tokens: list[int] property ¤

A list of filtered tokens after the filter was applied.

_set_extensions(attr: str, default: Any) ¤

Set the extensions.

Source code in lexos/filter/filters.py 
84
85
86
87
def _set_extensions(self, attr: str, default: Any):
    """Set the extensions."""
    if not Token.has_extension(attr):
        Token.set_extension(attr, default=default, force=True)

get_matched_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the matched tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the matched tokens.
Source code in lexos/filter/filters.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def get_matched_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the matched tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the matched tokens.
    """
    words = [t.text for t in self.matched_tokens]
    if add_spaces:
        spaces = [" " for _ in self.matched_tokens]
    else:
        spaces = [t.whitespace_ for t in self.matched_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

get_filtered_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the filtered tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the filtered tokens.
Source code in lexos/filter/filters.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def get_filtered_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the filtered tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the filtered tokens.
    """
    words = [t.text for t in self.filtered_tokens]
    if add_spaces:
        spaces = [" " for _ in self.filtered_tokens]
    else:
        spaces = [t.whitespace_ for t in self.filtered_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

IsRomanFilter pydantic-model ¤

Bases: BaseFilter

A filter for Roman numerals.

Config:

  • arbitrary_types_allowed: True
  • json_schema_extra: DocJSONSchema.schema()

Fields:

  • matcher (Optional[Matcher])
  • matches (Optional[list[tuple[int, int, int]]])
  • doc (Optional[Doc])
  • attr (Optional[str])
  • default (Optional[Any])
Source code in lexos/filter/filters.py 
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
class IsRomanFilter(BaseFilter):
    """A filter for Roman numerals."""

    id: ClassVar[str] = "is_roman"
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc.")
    attr: Optional[str] = Field(
        default="is_roman",
        description="The name of the attribute to add to the tokens.",
    )
    default: Optional[Any] = Field(
        default=None, description="The default value of the attribute."
    )

    model_config = ConfigDict(
        arbitrary_types_allowed=True, json_schema_extra=DocJSONSchema.schema()
    )

    def __init__(self, **data):
        """Initialise the filter object and set custom attribute extensions."""
        super().__init__(**data)
        if self.attr:
            self._set_extensions(self.attr, self.default)

    @validate_call(config=model_config)
    def __call__(
        self,
        doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
        attr: Optional[str] = Field(
            default=None, description="The name of the attribute to add to the tokens."
        ),
        default: Optional[Any] = Field(
            default=None, description="The default value of the attribute."
        ),
    ) -> Doc:
        """Apply the filter.

        Returns:
            Doc: The filtered doc.
        """
        # Validation
        if doc:
            self.doc = doc
        if attr:
            self.attr = attr
        if default is not None:
            self.default = default

        # Use instance attributes if we have them
        working_doc = self.doc if self.doc is not None else doc
        working_attr = (
            self.attr if hasattr(self, "attr") and self.attr is not None else attr
        )
        working_default = (
            self.default
            if hasattr(self, "default") and self.default is not None
            else default
        )

        # Set custom extensions
        if working_attr:
            self._set_extensions(working_attr, working_default)

        # Apply the filter only if we have a valid doc
        if working_doc is not None:
            # Store matches for tokens that are Roman numerals
            self.matches = []
            for i, token in enumerate(working_doc):
                is_roman_result = self.is_roman(token)
                setattr(working_doc[i]._, working_attr, is_roman_result)
                if is_roman_result:
                    self.matches.append((0, i, i + 1))

        return working_doc if working_doc is not None else doc

    @validate_call(config=model_config)
    def is_roman(self, token: Token) -> bool:
        """Detect Roman numerals (capitals only).

        Args:
            token (Token): A spaCy token.

        Returns:
            bool: True if the token is a Roman numeral.
        """
        if token.text == "":
            return False
        pattern = r"^M{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$"
        return bool(re.search(pattern, token.text))

attr: Optional[str] = 'is_roman' pydantic-field ¤

The name of the attribute to add to the tokens.

default: Optional[Any] = None pydantic-field ¤

The default value of the attribute.

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

filtered_token_ids: set[int] property ¤

A list of filtered token ids after the filter was applied.

filtered_tokens: list[int] property ¤

A list of filtered tokens after the filter was applied.

matched_token_ids: set[int] property ¤

A list of matched token ids after the filter was applied.

matched_tokens: list[int] property ¤

A list of matched tokens after the filter was applied.

matcher: Optional[Matcher] = None pydantic-field ¤

A spaCy matcher.

matches: Optional[list[tuple[int, int, int]]] = None pydantic-field ¤

List of matches.

__call__(doc: Optional[Doc] = Field(default=None, description='A spaCy doc.'), attr: Optional[str] = Field(default=None, description='The name of the attribute to add to the tokens.'), default: Optional[Any] = Field(default=None, description='The default value of the attribute.')) -> Doc ¤

Apply the filter.

Returns:

Name Type Description
Doc Doc

The filtered doc.
Source code in lexos/filter/filters.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@validate_call(config=model_config)
def __call__(
    self,
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
    attr: Optional[str] = Field(
        default=None, description="The name of the attribute to add to the tokens."
    ),
    default: Optional[Any] = Field(
        default=None, description="The default value of the attribute."
    ),
) -> Doc:
    """Apply the filter.

    Returns:
        Doc: The filtered doc.
    """
    # Validation
    if doc:
        self.doc = doc
    if attr:
        self.attr = attr
    if default is not None:
        self.default = default

    # Use instance attributes if we have them
    working_doc = self.doc if self.doc is not None else doc
    working_attr = (
        self.attr if hasattr(self, "attr") and self.attr is not None else attr
    )
    working_default = (
        self.default
        if hasattr(self, "default") and self.default is not None
        else default
    )

    # Set custom extensions
    if working_attr:
        self._set_extensions(working_attr, working_default)

    # Apply the filter only if we have a valid doc
    if working_doc is not None:
        # Store matches for tokens that are Roman numerals
        self.matches = []
        for i, token in enumerate(working_doc):
            is_roman_result = self.is_roman(token)
            setattr(working_doc[i]._, working_attr, is_roman_result)
            if is_roman_result:
                self.matches.append((0, i, i + 1))

    return working_doc if working_doc is not None else doc

__init__(**data) ¤

Initialise the filter object and set custom attribute extensions.

Source code in lexos/filter/filters.py 
141
142
143
144
145
def __init__(self, **data):
    """Initialise the filter object and set custom attribute extensions."""
    super().__init__(**data)
    if self.attr:
        self._set_extensions(self.attr, self.default)

get_filtered_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the filtered tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the filtered tokens.
Source code in lexos/filter/filters.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def get_filtered_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the filtered tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the filtered tokens.
    """
    words = [t.text for t in self.filtered_tokens]
    if add_spaces:
        spaces = [" " for _ in self.filtered_tokens]
    else:
        spaces = [t.whitespace_ for t in self.filtered_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

get_matched_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the matched tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the matched tokens.
Source code in lexos/filter/filters.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def get_matched_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the matched tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the matched tokens.
    """
    words = [t.text for t in self.matched_tokens]
    if add_spaces:
        spaces = [" " for _ in self.matched_tokens]
    else:
        spaces = [t.whitespace_ for t in self.matched_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

is_roman(token: Token) -> bool ¤

Detect Roman numerals (capitals only).

Parameters:

Name Type Description Default
token Token

A spaCy token.

 required

Returns:

Name Type Description
bool bool

True if the token is a Roman numeral.
Source code in lexos/filter/filters.py 
198
199
200
201
202
203
204
205
206
207
208
209
210
211
@validate_call(config=model_config)
def is_roman(self, token: Token) -> bool:
    """Detect Roman numerals (capitals only).

    Args:
        token (Token): A spaCy token.

    Returns:
        bool: True if the token is a Roman numeral.
    """
    if token.text == "":
        return False
    pattern = r"^M{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$"
    return bool(re.search(pattern, token.text))

id: str = 'is_roman' class-attribute ¤

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

attr: Optional[str] = 'is_roman' pydantic-field ¤

The name of the attribute to add to the tokens.

default: Optional[Any] = None pydantic-field ¤

The default value of the attribute.

model_config = ConfigDict(arbitrary_types_allowed=True, json_schema_extra=(DocJSONSchema.schema())) class-attribute instance-attribute ¤

__init__(**data) ¤

Initialise the filter object and set custom attribute extensions.

Source code in lexos/filter/filters.py 
141
142
143
144
145
def __init__(self, **data):
    """Initialise the filter object and set custom attribute extensions."""
    super().__init__(**data)
    if self.attr:
        self._set_extensions(self.attr, self.default)

__call__(doc: Optional[Doc] = Field(default=None, description='A spaCy doc.'), attr: Optional[str] = Field(default=None, description='The name of the attribute to add to the tokens.'), default: Optional[Any] = Field(default=None, description='The default value of the attribute.')) -> Doc ¤

Apply the filter.

Returns:

Name Type Description
Doc Doc

The filtered doc.
Source code in lexos/filter/filters.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@validate_call(config=model_config)
def __call__(
    self,
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
    attr: Optional[str] = Field(
        default=None, description="The name of the attribute to add to the tokens."
    ),
    default: Optional[Any] = Field(
        default=None, description="The default value of the attribute."
    ),
) -> Doc:
    """Apply the filter.

    Returns:
        Doc: The filtered doc.
    """
    # Validation
    if doc:
        self.doc = doc
    if attr:
        self.attr = attr
    if default is not None:
        self.default = default

    # Use instance attributes if we have them
    working_doc = self.doc if self.doc is not None else doc
    working_attr = (
        self.attr if hasattr(self, "attr") and self.attr is not None else attr
    )
    working_default = (
        self.default
        if hasattr(self, "default") and self.default is not None
        else default
    )

    # Set custom extensions
    if working_attr:
        self._set_extensions(working_attr, working_default)

    # Apply the filter only if we have a valid doc
    if working_doc is not None:
        # Store matches for tokens that are Roman numerals
        self.matches = []
        for i, token in enumerate(working_doc):
            is_roman_result = self.is_roman(token)
            setattr(working_doc[i]._, working_attr, is_roman_result)
            if is_roman_result:
                self.matches.append((0, i, i + 1))

    return working_doc if working_doc is not None else doc

is_roman(token: Token) -> bool ¤

Detect Roman numerals (capitals only).

Parameters:

Name Type Description Default
token Token

A spaCy token.

 required

Returns:

Name Type Description
bool bool

True if the token is a Roman numeral.
Source code in lexos/filter/filters.py 
198
199
200
201
202
203
204
205
206
207
208
209
210
211
@validate_call(config=model_config)
def is_roman(self, token: Token) -> bool:
    """Detect Roman numerals (capitals only).

    Args:
        token (Token): A spaCy token.

    Returns:
        bool: True if the token is a Roman numeral.
    """
    if token.text == "":
        return False
    pattern = r"^M{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$"
    return bool(re.search(pattern, token.text))

IsStopwordFilter pydantic-model ¤

Bases: BaseFilter

A filter to detect stop words in a spaCy doc.

Config:

  • arbitrary_types_allowed: True
  • json_schema_extra: DocJSONSchema.schema()

Fields:

Source code in lexos/filter/filters.py 
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
class IsStopwordFilter(BaseFilter):
    """A filter to detect stop words in a spaCy doc."""

    id: ClassVar[str] = "is_stopword"
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc.")
    stopwords: Optional[list | str] = Field(
        default=None,
        description="A list or string containing the stop word(s) to add or remove.",
    )
    remove: Optional[bool] = Field(
        default=False,
        description="If True, the stop word(s) will be removed from the model.",
    )
    case_sensitive: Optional[bool] = Field(
        default=False,
        description="If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.",
    )

    model_config = ConfigDict(
        arbitrary_types_allowed=True, json_schema_extra=DocJSONSchema.schema()
    )

    def __init__(self, **data: Any):
        """Initialise the filter object with configuration.

        Args:
            **data (Any): Configuration data
        """
        super().__init__(**data)
        self.stopwords = ensure_list(self.stopwords)

    @validate_call(config=model_config)
    def __call__(
        self,
        doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
        stopwords: Optional[list | str] = Field(
            default=None,
            description="A list or string containing the stop word(s) to add or remove.",
        ),
        remove: Optional[bool] = Field(
            default=None,
            description="If True, the stop word(s) will be removed from the model.",
        ),
        case_sensitive: Optional[bool] = Field(
            default=None,
            description="If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.",
        ),
    ) -> Doc:
        """Apply the filter.

        Returns:
            Doc: The filtered doc.

        Note:
            This filter modifies the model defaults. If you need the model's original default stop words.
            you will need to re-load the model.
        """
        # Validation
        if doc:
            self.doc = doc
        if stopwords is not None:
            self.stopwords = ensure_list(stopwords)
        if remove is not None:
            self.remove = remove
        if case_sensitive is not None:
            self.case_sensitive = case_sensitive

        # Use instance attributes if parameters are None
        working_doc = self.doc if self.doc is not None else doc
        # Handle stopwords carefully - convert to list if it's a pydantic ValidatorIterator
        if (
            stopwords is None
            and hasattr(self, "stopwords")
            and self.stopwords is not None
        ):
            try:
                working_stopwords = list(self.stopwords)
            except (TypeError, AttributeError):
                working_stopwords = self.stopwords
        else:
            working_stopwords = stopwords
        working_remove = (
            self.remove
            if hasattr(self, "remove") and self.remove is not None
            else remove
        )
        working_case_sensitive = (
            self.case_sensitive
            if hasattr(self, "case_sensitive") and self.case_sensitive is not None
            else case_sensitive
            if case_sensitive is not None
            else False
        )

        # Apply the filter only if we have valid inputs
        if working_doc is not None and working_stopwords is not None:
            # Ensure stopwords is iterable and properly formatted
            if not isinstance(working_stopwords, list):
                working_stopwords = ensure_list(working_stopwords)

            # Get the vocab (shared across all docs from the same model)
            vocab = working_doc.vocab

            if working_remove:
                for item in working_stopwords:
                    if item is not None:  # Skip None values
                        if working_case_sensitive:
                            # Only modify the exact case provided
                            vocab[item].is_stop = False
                        else:
                            # Modify common case variations
                            vocab[item.lower()].is_stop = False
                            vocab[item].is_stop = False
                            vocab[item.capitalize()].is_stop = False
            else:
                for item in working_stopwords:
                    if item is not None:  # Skip None values
                        if working_case_sensitive:
                            # Only modify the exact case provided
                            vocab[item].is_stop = True
                        else:
                            # Modify common case variations
                            vocab[item.lower()].is_stop = True
                            vocab[item].is_stop = True
                            vocab[item.capitalize()].is_stop = True

            # Store matches for stopwords
            self.matches = []
            for i, token in enumerate(working_doc):
                if token.is_stop:
                    self.matches.append((0, i, i + 1))

        return working_doc if working_doc is not None else doc

case_sensitive: Optional[bool] = False pydantic-field ¤

If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

filtered_token_ids: set[int] property ¤

A list of filtered token ids after the filter was applied.

filtered_tokens: list[int] property ¤

A list of filtered tokens after the filter was applied.

matched_token_ids: set[int] property ¤

A list of matched token ids after the filter was applied.

matched_tokens: list[int] property ¤

A list of matched tokens after the filter was applied.

matcher: Optional[Matcher] = None pydantic-field ¤

A spaCy matcher.

matches: Optional[list[tuple[int, int, int]]] = None pydantic-field ¤

List of matches.

remove: Optional[bool] = False pydantic-field ¤

If True, the stop word(s) will be removed from the model.

__call__(doc: Optional[Doc] = Field(default=None, description='A spaCy doc.'), stopwords: Optional[list | str] = Field(default=None, description='A list or string containing the stop word(s) to add or remove.'), remove: Optional[bool] = Field(default=None, description='If True, the stop word(s) will be removed from the model.'), case_sensitive: Optional[bool] = Field(default=None, description='If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.')) -> Doc ¤

Apply the filter.

Returns:

Name Type Description
Doc Doc

The filtered doc.
Note

This filter modifies the model defaults. If you need the model's original default stop words. you will need to re-load the model.

Source code in lexos/filter/filters.py 
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
@validate_call(config=model_config)
def __call__(
    self,
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
    stopwords: Optional[list | str] = Field(
        default=None,
        description="A list or string containing the stop word(s) to add or remove.",
    ),
    remove: Optional[bool] = Field(
        default=None,
        description="If True, the stop word(s) will be removed from the model.",
    ),
    case_sensitive: Optional[bool] = Field(
        default=None,
        description="If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.",
    ),
) -> Doc:
    """Apply the filter.

    Returns:
        Doc: The filtered doc.

    Note:
        This filter modifies the model defaults. If you need the model's original default stop words.
        you will need to re-load the model.
    """
    # Validation
    if doc:
        self.doc = doc
    if stopwords is not None:
        self.stopwords = ensure_list(stopwords)
    if remove is not None:
        self.remove = remove
    if case_sensitive is not None:
        self.case_sensitive = case_sensitive

    # Use instance attributes if parameters are None
    working_doc = self.doc if self.doc is not None else doc
    # Handle stopwords carefully - convert to list if it's a pydantic ValidatorIterator
    if (
        stopwords is None
        and hasattr(self, "stopwords")
        and self.stopwords is not None
    ):
        try:
            working_stopwords = list(self.stopwords)
        except (TypeError, AttributeError):
            working_stopwords = self.stopwords
    else:
        working_stopwords = stopwords
    working_remove = (
        self.remove
        if hasattr(self, "remove") and self.remove is not None
        else remove
    )
    working_case_sensitive = (
        self.case_sensitive
        if hasattr(self, "case_sensitive") and self.case_sensitive is not None
        else case_sensitive
        if case_sensitive is not None
        else False
    )

    # Apply the filter only if we have valid inputs
    if working_doc is not None and working_stopwords is not None:
        # Ensure stopwords is iterable and properly formatted
        if not isinstance(working_stopwords, list):
            working_stopwords = ensure_list(working_stopwords)

        # Get the vocab (shared across all docs from the same model)
        vocab = working_doc.vocab

        if working_remove:
            for item in working_stopwords:
                if item is not None:  # Skip None values
                    if working_case_sensitive:
                        # Only modify the exact case provided
                        vocab[item].is_stop = False
                    else:
                        # Modify common case variations
                        vocab[item.lower()].is_stop = False
                        vocab[item].is_stop = False
                        vocab[item.capitalize()].is_stop = False
        else:
            for item in working_stopwords:
                if item is not None:  # Skip None values
                    if working_case_sensitive:
                        # Only modify the exact case provided
                        vocab[item].is_stop = True
                    else:
                        # Modify common case variations
                        vocab[item.lower()].is_stop = True
                        vocab[item].is_stop = True
                        vocab[item.capitalize()].is_stop = True

        # Store matches for stopwords
        self.matches = []
        for i, token in enumerate(working_doc):
            if token.is_stop:
                self.matches.append((0, i, i + 1))

    return working_doc if working_doc is not None else doc

__init__(**data: Any) ¤

Initialise the filter object with configuration.

Parameters:

Name Type Description Default
**data Any

Configuration data

 {}
Source code in lexos/filter/filters.py 
236
237
238
239
240
241
242
243
def __init__(self, **data: Any):
    """Initialise the filter object with configuration.

    Args:
        **data (Any): Configuration data
    """
    super().__init__(**data)
    self.stopwords = ensure_list(self.stopwords)

get_filtered_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the filtered tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the filtered tokens.
Source code in lexos/filter/filters.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def get_filtered_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the filtered tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the filtered tokens.
    """
    words = [t.text for t in self.filtered_tokens]
    if add_spaces:
        spaces = [" " for _ in self.filtered_tokens]
    else:
        spaces = [t.whitespace_ for t in self.filtered_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

get_matched_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the matched tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the matched tokens.
Source code in lexos/filter/filters.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def get_matched_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the matched tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the matched tokens.
    """
    words = [t.text for t in self.matched_tokens]
    if add_spaces:
        spaces = [" " for _ in self.matched_tokens]
    else:
        spaces = [t.whitespace_ for t in self.matched_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

id: str = 'is_stopword' class-attribute ¤

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

stopwords: Optional[list | str] pydantic-field ¤

remove: Optional[bool] = False pydantic-field ¤

If True, the stop word(s) will be removed from the model.

model_config = ConfigDict(arbitrary_types_allowed=True, json_schema_extra=(DocJSONSchema.schema())) class-attribute instance-attribute ¤

__init__(**data: Any) ¤

Initialise the filter object with configuration.

Parameters:

Name Type Description Default
**data Any

Configuration data

 {}
Source code in lexos/filter/filters.py 
236
237
238
239
240
241
242
243
def __init__(self, **data: Any):
    """Initialise the filter object with configuration.

    Args:
        **data (Any): Configuration data
    """
    super().__init__(**data)
    self.stopwords = ensure_list(self.stopwords)

__call__(doc: Optional[Doc] = Field(default=None, description='A spaCy doc.'), stopwords: Optional[list | str] = Field(default=None, description='A list or string containing the stop word(s) to add or remove.'), remove: Optional[bool] = Field(default=None, description='If True, the stop word(s) will be removed from the model.'), case_sensitive: Optional[bool] = Field(default=None, description='If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.')) -> Doc ¤

Apply the filter.

Returns:

Name Type Description
Doc Doc

The filtered doc.
Note

This filter modifies the model defaults. If you need the model's original default stop words. you will need to re-load the model.

Source code in lexos/filter/filters.py 
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
@validate_call(config=model_config)
def __call__(
    self,
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
    stopwords: Optional[list | str] = Field(
        default=None,
        description="A list or string containing the stop word(s) to add or remove.",
    ),
    remove: Optional[bool] = Field(
        default=None,
        description="If True, the stop word(s) will be removed from the model.",
    ),
    case_sensitive: Optional[bool] = Field(
        default=None,
        description="If False (default), stop word changes apply to all case variations. If True, only the exact case provided is modified.",
    ),
) -> Doc:
    """Apply the filter.

    Returns:
        Doc: The filtered doc.

    Note:
        This filter modifies the model defaults. If you need the model's original default stop words.
        you will need to re-load the model.
    """
    # Validation
    if doc:
        self.doc = doc
    if stopwords is not None:
        self.stopwords = ensure_list(stopwords)
    if remove is not None:
        self.remove = remove
    if case_sensitive is not None:
        self.case_sensitive = case_sensitive

    # Use instance attributes if parameters are None
    working_doc = self.doc if self.doc is not None else doc
    # Handle stopwords carefully - convert to list if it's a pydantic ValidatorIterator
    if (
        stopwords is None
        and hasattr(self, "stopwords")
        and self.stopwords is not None
    ):
        try:
            working_stopwords = list(self.stopwords)
        except (TypeError, AttributeError):
            working_stopwords = self.stopwords
    else:
        working_stopwords = stopwords
    working_remove = (
        self.remove
        if hasattr(self, "remove") and self.remove is not None
        else remove
    )
    working_case_sensitive = (
        self.case_sensitive
        if hasattr(self, "case_sensitive") and self.case_sensitive is not None
        else case_sensitive
        if case_sensitive is not None
        else False
    )

    # Apply the filter only if we have valid inputs
    if working_doc is not None and working_stopwords is not None:
        # Ensure stopwords is iterable and properly formatted
        if not isinstance(working_stopwords, list):
            working_stopwords = ensure_list(working_stopwords)

        # Get the vocab (shared across all docs from the same model)
        vocab = working_doc.vocab

        if working_remove:
            for item in working_stopwords:
                if item is not None:  # Skip None values
                    if working_case_sensitive:
                        # Only modify the exact case provided
                        vocab[item].is_stop = False
                    else:
                        # Modify common case variations
                        vocab[item.lower()].is_stop = False
                        vocab[item].is_stop = False
                        vocab[item.capitalize()].is_stop = False
        else:
            for item in working_stopwords:
                if item is not None:  # Skip None values
                    if working_case_sensitive:
                        # Only modify the exact case provided
                        vocab[item].is_stop = True
                    else:
                        # Modify common case variations
                        vocab[item.lower()].is_stop = True
                        vocab[item].is_stop = True
                        vocab[item.capitalize()].is_stop = True

        # Store matches for stopwords
        self.matches = []
        for i, token in enumerate(working_doc):
            if token.is_stop:
                self.matches.append((0, i, i + 1))

    return working_doc if working_doc is not None else doc

IsWordFilter pydantic-model ¤

Bases: BaseFilter

A filter to detect words in a spaCy doc.

Config:

  • arbitrary_types_allowed: True
  • json_schema_extra: DocJSONSchema.schema()

Fields:

Source code in lexos/filter/filters.py 
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
class IsWordFilter(BaseFilter):
    """A filter to detect words in a spaCy doc."""

    id: ClassVar[str] = "is_word"
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc.")
    attr: Optional[str] = Field(
        default="is_word", description="The name of the attribute to add to the tokens."
    )
    default: Optional[bool] = Field(
        default=False, description="The default value of the attribute."
    )
    exclude: Optional[Optional[list[str] | str]] = Field(
        default=[" ", "\n"],
        description="A string/regex or list of strings/regex patterns to exclude.",
    )
    exclude_digits: Optional[Optional[bool]] = Field(
        default=False, description="If True, digits will not be treated as words."
    )
    exclude_roman_numerals: Optional[Optional[bool]] = Field(
        default=False,
        description="Same as above for Roman numerals, but only works on capital letters.",
    )
    exclude_pattern: Optional[list[str] | str] = Field(
        default=None,
        description="Additional patterns to add to the default exclude list.",
    )

    model_config = ConfigDict(
        arbitrary_types_allowed=True, json_schema_extra=DocJSONSchema.schema()
    )

    def __init__(self, **data: Any):
        """Initialise the filter object with configuration.

        Args:
            **data (Any): Configuration data
        """
        super().__init__(**data)
        if self.attr:
            self._set_extensions(self.attr, self.default)

    @validate_call(config=model_config)
    def __call__(
        self,
        doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
        attr: Optional[str] = Field(
            default=None,
            description="The name of the attribute to add to the tokens.",
        ),
        default: Optional[bool] = Field(
            default=None, description="The default value of the attribute."
        ),
        exclude: Optional[list[str] | str] = Field(
            default=None,
            description="A string/regex or list of strings/regex patterns to exclude.",
        ),
        exclude_digits: Optional[bool] = Field(
            default=None, description="If True, digits will not be treated as words."
        ),
        exclude_roman_numerals: Optional[bool] = Field(
            default=None,
            description="Same as above for Roman numerals, but only works on capital letters.",
        ),
        exclude_pattern: Optional[list[str] | str] = Field(
            default=None,
            description="Additional patterns to add to the default exclude list.",
        ),
    ) -> Doc:
        """Apply the filter.

        Returns:
            Doc: The filtered doc.
        """
        # Assign keyword variables to the instance attributes
        if doc:
            self.doc = doc
        if exclude:
            self.exclude = ensure_list(exclude)
        if exclude_digits is not None:
            self.exclude_digits = exclude_digits
        if exclude_roman_numerals is not None:
            self.exclude_roman_numerals = exclude_roman_numerals
        if exclude_pattern:
            self.exclude_pattern = ensure_list(exclude_pattern)
        if attr:
            self.attr = attr
        if default is not None:
            self.default = default

        # Use instance attributes if we have them
        working_doc = self.doc if self.doc is not None else doc
        working_attr = (
            self.attr if hasattr(self, "attr") and self.attr is not None else attr
        )
        working_default = (
            self.default
            if hasattr(self, "default") and self.default is not None
            else default
        )

        # Set ._is_word extension
        if working_attr:
            self._set_extensions(working_attr, working_default)

        # Apply the filter only if we have a valid doc
        if working_doc is not None:
            # Store matches for tokens that are words
            self.matches = []
            for i, token in enumerate(working_doc):
                is_word_result = self.is_word(token)
                setattr(working_doc[i]._, working_attr, is_word_result)
                if is_word_result:
                    self.matches.append((0, i, i + 1))

        return working_doc if working_doc is not None else doc

    def _is_roman_numeral(self, string: str) -> bool:
        """Check if a string is a Roman numeral.

        Args:
            string (str): A string.

        Returns:
            bool: True if the string is a Roman numeral.
        """
        if string == "":
            return False
        pattern = r"^M{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$"
        return bool(re.search(pattern, string))

    @validate_call(config=model_config)
    def is_word(self, token: Token) -> bool:
        """Detect words.

        Args:
            token (Token): A spaCy token.

        Returns:
            bool: True if the token is a word.
        """
        predicates = []
        if self.exclude_digits:
            predicates.append(lambda token: token.text.isalpha())
        else:
            predicates.append(
                lambda token: token.text.isalpha() or token.text.isdigit()
            )
        if self.exclude_roman_numerals:
            predicates.append(lambda token: not self._is_roman_numeral(token.text))
        if self.exclude_pattern:
            self.exclude += self.exclude_pattern
        if len(self.exclude) > 0:
            exclude_pat = "|".join(self.exclude)
            predicates.append(lambda token: re.search(exclude_pat, token.text) is None)
        return all([f(token) for f in predicates])

attr: Optional[str] = 'is_word' pydantic-field ¤

The name of the attribute to add to the tokens.

default: Optional[bool] = False pydantic-field ¤

The default value of the attribute.

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

exclude: Optional[Optional[list[str] | str]] = [' ', '\n'] pydantic-field ¤

A string/regex or list of strings/regex patterns to exclude.

exclude_digits: Optional[Optional[bool]] = False pydantic-field ¤

If True, digits will not be treated as words.

exclude_pattern: Optional[list[str] | str] = None pydantic-field ¤

Additional patterns to add to the default exclude list.

exclude_roman_numerals: Optional[Optional[bool]] = False pydantic-field ¤

Same as above for Roman numerals, but only works on capital letters.

filtered_token_ids: set[int] property ¤

A list of filtered token ids after the filter was applied.

filtered_tokens: list[int] property ¤

A list of filtered tokens after the filter was applied.

matched_token_ids: set[int] property ¤

A list of matched token ids after the filter was applied.

matched_tokens: list[int] property ¤

A list of matched tokens after the filter was applied.

matcher: Optional[Matcher] = None pydantic-field ¤

A spaCy matcher.

matches: Optional[list[tuple[int, int, int]]] = None pydantic-field ¤

List of matches.

__call__(doc: Optional[Doc] = Field(default=None, description='A spaCy doc.'), attr: Optional[str] = Field(default=None, description='The name of the attribute to add to the tokens.'), default: Optional[bool] = Field(default=None, description='The default value of the attribute.'), exclude: Optional[list[str] | str] = Field(default=None, description='A string/regex or list of strings/regex patterns to exclude.'), exclude_digits: Optional[bool] = Field(default=None, description='If True, digits will not be treated as words.'), exclude_roman_numerals: Optional[bool] = Field(default=None, description='Same as above for Roman numerals, but only works on capital letters.'), exclude_pattern: Optional[list[str] | str] = Field(default=None, description='Additional patterns to add to the default exclude list.')) -> Doc ¤

Apply the filter.

Returns:

Name Type Description
Doc Doc

The filtered doc.
Source code in lexos/filter/filters.py 
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
@validate_call(config=model_config)
def __call__(
    self,
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
    attr: Optional[str] = Field(
        default=None,
        description="The name of the attribute to add to the tokens.",
    ),
    default: Optional[bool] = Field(
        default=None, description="The default value of the attribute."
    ),
    exclude: Optional[list[str] | str] = Field(
        default=None,
        description="A string/regex or list of strings/regex patterns to exclude.",
    ),
    exclude_digits: Optional[bool] = Field(
        default=None, description="If True, digits will not be treated as words."
    ),
    exclude_roman_numerals: Optional[bool] = Field(
        default=None,
        description="Same as above for Roman numerals, but only works on capital letters.",
    ),
    exclude_pattern: Optional[list[str] | str] = Field(
        default=None,
        description="Additional patterns to add to the default exclude list.",
    ),
) -> Doc:
    """Apply the filter.

    Returns:
        Doc: The filtered doc.
    """
    # Assign keyword variables to the instance attributes
    if doc:
        self.doc = doc
    if exclude:
        self.exclude = ensure_list(exclude)
    if exclude_digits is not None:
        self.exclude_digits = exclude_digits
    if exclude_roman_numerals is not None:
        self.exclude_roman_numerals = exclude_roman_numerals
    if exclude_pattern:
        self.exclude_pattern = ensure_list(exclude_pattern)
    if attr:
        self.attr = attr
    if default is not None:
        self.default = default

    # Use instance attributes if we have them
    working_doc = self.doc if self.doc is not None else doc
    working_attr = (
        self.attr if hasattr(self, "attr") and self.attr is not None else attr
    )
    working_default = (
        self.default
        if hasattr(self, "default") and self.default is not None
        else default
    )

    # Set ._is_word extension
    if working_attr:
        self._set_extensions(working_attr, working_default)

    # Apply the filter only if we have a valid doc
    if working_doc is not None:
        # Store matches for tokens that are words
        self.matches = []
        for i, token in enumerate(working_doc):
            is_word_result = self.is_word(token)
            setattr(working_doc[i]._, working_attr, is_word_result)
            if is_word_result:
                self.matches.append((0, i, i + 1))

    return working_doc if working_doc is not None else doc

__init__(**data: Any) ¤

Initialise the filter object with configuration.

Parameters:

Name Type Description Default
**data Any

Configuration data

 {}
Source code in lexos/filter/filters.py 
380
381
382
383
384
385
386
387
388
def __init__(self, **data: Any):
    """Initialise the filter object with configuration.

    Args:
        **data (Any): Configuration data
    """
    super().__init__(**data)
    if self.attr:
        self._set_extensions(self.attr, self.default)

get_filtered_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the filtered tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the filtered tokens.
Source code in lexos/filter/filters.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def get_filtered_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the filtered tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the filtered tokens.
    """
    words = [t.text for t in self.filtered_tokens]
    if add_spaces:
        spaces = [" " for _ in self.filtered_tokens]
    else:
        spaces = [t.whitespace_ for t in self.filtered_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

get_matched_doc(add_spaces: bool = False) -> Doc ¤

Get a new doc from the matched tokens.

Parameters:

Name Type Description Default
add_spaces bool

If True, add a space after every token. If False, preserve original whitespace. Default is False.

 False

Returns:

Name Type Description
Doc Doc

A new spaCy Doc containing only the matched tokens.
Source code in lexos/filter/filters.py 
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def get_matched_doc(self, add_spaces: bool = False) -> Doc:
    """Get a new doc from the matched tokens.

    Args:
        add_spaces (bool): If True, add a space after every token.
                         If False, preserve original whitespace. Default is False.

    Returns:
        Doc: A new spaCy Doc containing only the matched tokens.
    """
    words = [t.text for t in self.matched_tokens]
    if add_spaces:
        spaces = [" " for _ in self.matched_tokens]
    else:
        spaces = [t.whitespace_ for t in self.matched_tokens]
    return Doc(self.doc.vocab, words=words, spaces=spaces)

is_word(token: Token) -> bool ¤

Detect words.

Parameters:

Name Type Description Default
token Token

A spaCy token.

 required

Returns:

Name Type Description
bool bool

True if the token is a word.
Source code in lexos/filter/filters.py 
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
@validate_call(config=model_config)
def is_word(self, token: Token) -> bool:
    """Detect words.

    Args:
        token (Token): A spaCy token.

    Returns:
        bool: True if the token is a word.
    """
    predicates = []
    if self.exclude_digits:
        predicates.append(lambda token: token.text.isalpha())
    else:
        predicates.append(
            lambda token: token.text.isalpha() or token.text.isdigit()
        )
    if self.exclude_roman_numerals:
        predicates.append(lambda token: not self._is_roman_numeral(token.text))
    if self.exclude_pattern:
        self.exclude += self.exclude_pattern
    if len(self.exclude) > 0:
        exclude_pat = "|".join(self.exclude)
        predicates.append(lambda token: re.search(exclude_pat, token.text) is None)
    return all([f(token) for f in predicates])

id: str = 'is_word' class-attribute ¤

doc: Optional[Doc] = None pydantic-field ¤

A spaCy doc.

attr: Optional[str] = 'is_word' pydantic-field ¤

The name of the attribute to add to the tokens.

default: Optional[bool] = False pydantic-field ¤

The default value of the attribute.

exclude: Optional[Optional[list[str] | str]] = [' ', '\n'] pydantic-field ¤

A string/regex or list of strings/regex patterns to exclude.

exclude_digits: Optional[Optional[bool]] = False pydantic-field ¤

If True, digits will not be treated as words.

exclude_roman_numerals: Optional[Optional[bool]] = False pydantic-field ¤

Same as above for Roman numerals, but only works on capital letters.

exclude_pattern: Optional[list[str] | str] = None pydantic-field ¤

Additional patterns to add to the default exclude list.

model_config = ConfigDict(arbitrary_types_allowed=True, json_schema_extra=(DocJSONSchema.schema())) class-attribute instance-attribute ¤

__init__(**data: Any) ¤

Initialise the filter object with configuration.

Parameters:

Name Type Description Default
**data Any

Configuration data

 {}
Source code in lexos/filter/filters.py 
380
381
382
383
384
385
386
387
388
def __init__(self, **data: Any):
    """Initialise the filter object with configuration.

    Args:
        **data (Any): Configuration data
    """
    super().__init__(**data)
    if self.attr:
        self._set_extensions(self.attr, self.default)

__call__(doc: Optional[Doc] = Field(default=None, description='A spaCy doc.'), attr: Optional[str] = Field(default=None, description='The name of the attribute to add to the tokens.'), default: Optional[bool] = Field(default=None, description='The default value of the attribute.'), exclude: Optional[list[str] | str] = Field(default=None, description='A string/regex or list of strings/regex patterns to exclude.'), exclude_digits: Optional[bool] = Field(default=None, description='If True, digits will not be treated as words.'), exclude_roman_numerals: Optional[bool] = Field(default=None, description='Same as above for Roman numerals, but only works on capital letters.'), exclude_pattern: Optional[list[str] | str] = Field(default=None, description='Additional patterns to add to the default exclude list.')) -> Doc ¤

Apply the filter.

Returns:

Name Type Description
Doc Doc

The filtered doc.
Source code in lexos/filter/filters.py 
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
@validate_call(config=model_config)
def __call__(
    self,
    doc: Optional[Doc] = Field(default=None, description="A spaCy doc."),
    attr: Optional[str] = Field(
        default=None,
        description="The name of the attribute to add to the tokens.",
    ),
    default: Optional[bool] = Field(
        default=None, description="The default value of the attribute."
    ),
    exclude: Optional[list[str] | str] = Field(
        default=None,
        description="A string/regex or list of strings/regex patterns to exclude.",
    ),
    exclude_digits: Optional[bool] = Field(
        default=None, description="If True, digits will not be treated as words."
    ),
    exclude_roman_numerals: Optional[bool] = Field(
        default=None,
        description="Same as above for Roman numerals, but only works on capital letters.",
    ),
    exclude_pattern: Optional[list[str] | str] = Field(
        default=None,
        description="Additional patterns to add to the default exclude list.",
    ),
) -> Doc:
    """Apply the filter.

    Returns:
        Doc: The filtered doc.
    """
    # Assign keyword variables to the instance attributes
    if doc:
        self.doc = doc
    if exclude:
        self.exclude = ensure_list(exclude)
    if exclude_digits is not None:
        self.exclude_digits = exclude_digits
    if exclude_roman_numerals is not None:
        self.exclude_roman_numerals = exclude_roman_numerals
    if exclude_pattern:
        self.exclude_pattern = ensure_list(exclude_pattern)
    if attr:
        self.attr = attr
    if default is not None:
        self.default = default

    # Use instance attributes if we have them
    working_doc = self.doc if self.doc is not None else doc
    working_attr = (
        self.attr if hasattr(self, "attr") and self.attr is not None else attr
    )
    working_default = (
        self.default
        if hasattr(self, "default") and self.default is not None
        else default
    )

    # Set ._is_word extension
    if working_attr:
        self._set_extensions(working_attr, working_default)

    # Apply the filter only if we have a valid doc
    if working_doc is not None:
        # Store matches for tokens that are words
        self.matches = []
        for i, token in enumerate(working_doc):
            is_word_result = self.is_word(token)
            setattr(working_doc[i]._, working_attr, is_word_result)
            if is_word_result:
                self.matches.append((0, i, i + 1))

    return working_doc if working_doc is not None else doc

_is_roman_numeral(string: str) -> bool ¤

Check if a string is a Roman numeral.

Parameters:

Name Type Description Default
string str

A string.

 required

Returns:

Name Type Description
bool bool

True if the string is a Roman numeral.
Source code in lexos/filter/filters.py 
465
466
467
468
469
470
471
472
473
474
475
476
477
def _is_roman_numeral(self, string: str) -> bool:
    """Check if a string is a Roman numeral.

    Args:
        string (str): A string.

    Returns:
        bool: True if the string is a Roman numeral.
    """
    if string == "":
        return False
    pattern = r"^M{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$"
    return bool(re.search(pattern, string))

is_word(token: Token) -> bool ¤

Detect words.

Parameters:

Name Type Description Default
token Token

A spaCy token.

 required

Returns:

Name Type Description
bool bool

True if the token is a word.
Source code in lexos/filter/filters.py 
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
@validate_call(config=model_config)
def is_word(self, token: Token) -> bool:
    """Detect words.

    Args:
        token (Token): A spaCy token.

    Returns:
        bool: True if the token is a word.
    """
    predicates = []
    if self.exclude_digits:
        predicates.append(lambda token: token.text.isalpha())
    else:
        predicates.append(
            lambda token: token.text.isalpha() or token.text.isdigit()
        )
    if self.exclude_roman_numerals:
        predicates.append(lambda token: not self._is_roman_numeral(token.text))
    if self.exclude_pattern:
        self.exclude += self.exclude_pattern
    if len(self.exclude) > 0:
        exclude_pat = "|".join(self.exclude)
        predicates.append(lambda token: re.search(exclude_pat, token.text) is None)
    return all([f(token) for f in predicates])