mainegeo.patterns

  1"""Regex patterns and helpers for parsing Maine election results and place names.
  2"""
  3
  4__docformat__ = 'google'
  5
  6import re
  7from mainegeo.lookups import (
  8    CountyData,
  9    TownshipData,
 10    Overrides
 11)
 12
 13# Base character sets for patterns
 14FUZZY = f"[^,\\w]{{0,3}}"
 15"""@private"""
 16
 17PUNCTUATION = f'[^\\s\\w]'
 18"""@private"""
 19
 20## Other
 21KNOWN_TYPOS: dict[str, str] = {
 22    group['original']: group['replacement']
 23    for group in Overrides.get_lookup().known_typos
 24}
 25""" Errors and replacements for known typos in election results files.
 26
 27These errors represent one-off typos rather than confusion about how the
 28town name should be spelled. Misspellings that are the result of confusion
 29and might reoccur in the future are logged as aliases in 
 30`mainegeo.matching.TownDatabase`.
 31
 32See `lookups.Overrides` to add new entries.
 33
 34Used in `mainegeo.elections.ResultString.normalized_string`.
 35"""
 36
 37AMBIGUOUS_GROUPS: dict[str, str] = {
 38    group['pattern']: group['replacement']
 39    for group in Overrides.get_lookup().ambiguous_groups
 40}
 41""" Errors and replacements for ambiguously-named unspecified groups.
 42
 43These typos follow patterns, but it simplifies unspecified group name 
 44detection significantly if they are corrected early in processing.
 45
 46See `mainegeo.lookups.Overrides` to add new entries.
 47    
 48Used in `mainegeo.elections.ResultString.normalized_string`."""
 49
 50## Townships
 51# Constants
 52REGIONS: list[str] = ['ED','MD','ND','SD','TS','BKP','BPP','EKR','NWP','WKR','NBKP','NBPP','WBKP','WELS']
 53""" Valid region codes that can appear in township names.
 54    
 55These are two- to four-letter codes like 'WELS' (West of the
 56Easterly Line of the State) or 'BPP' (Bingham's Penobscot Purchase).
 57
 58Township names can contain zero, one, or two regions."""
 59
 60# Building blocks
 61REGION: str = f"(?:(?<![a-z])(?:{'|'.join(REGIONS)})(?![a-z]))"
 62""" Uncompiled regex building block representing a region designator."""
 63
 64RANGE: str = f"(?:R.?[\\d]{{1,2}})"
 65""" Uncompiled regex building block representing a range designator.
 66
 67Ranges are counted from the easterly line toward the west,
 68numbered 1-19 (e.g., R1, R19)."""
 69
 70TOWNSHIP_STANDARD: str = "(?:T.?\\d{1,2})"
 71TOWNSHIP_ALTERNATE: str = f"(?<!\\w)T[ABCDX](?![a-z])"
 72TOWNSHIP: str = f"(?:{TOWNSHIP_STANDARD}|{TOWNSHIP_ALTERNATE})"
 73""" Uncompiled regex building block representing a township designator.
 74
 75Townships are designated by:
 76    - Numbers 1-19 from south to north (e.g., T1, T19)
 77    - Occasional letter designations (TA, TB, TC, TD, TX)"""
 78
 79UNNAMED: str = f"((?:{TOWNSHIP})(?:{FUZZY}{RANGE})?(?:{FUZZY}{REGION}){{0,2}})"
 80UNNAMED_ELEMENTS: str = f"(?:{'|'.join([TOWNSHIP, RANGE, REGION])})"
 81
 82# Patterns
 83UNNAMED_PATTERN: re.Pattern = re.compile(UNNAMED, re.I)
 84""" Compiled regex matching a full unnamed township name, with tolerance for formatting variation.
 85
 86Used in `mainegeo.townships.is_unnamed_township` and `mainegeo.townships.clean_codes`."""
 87
 88UNNAMED_ELEMENTS_PATTERN: re.Pattern = re.compile(UNNAMED_ELEMENTS, re.I)
 89""" Compiled regex matching any unnamed township name element.
 90
 91Used in `mainegeo.townships.clean_code`."""
 92
 93LAST_REGION_PATTERN: re.Pattern = re.compile(f" {REGION}$", re.I)
 94""" Compiled regex matching the last region code in an unnamed township name.
 95
 96Used in `mainegeo.townships.strip_region`."""
 97
 98## Result parsing
 99# Building blocks
100LEADING_ZERO: str = '(?<=[^\\d])0(?=\\d)'
101NOT_REPORTING: str = f'(?!AND|&)'
102PARENTHETICAL: str = f'\\({NOT_REPORTING}[^\\(]+\\)'
103PRECEDES_DASH: str = f'^[^-]+--'
104
105STANDARD_DELIMITER: str = ", "
106""" Preferred delimiter character; all other delimiters will be replaced by this."""
107
108NONSTANDARD_DELIMITERS: str = ["&", "/", "(AND ", "(INC "]
109""" Substrings that are sometimes used by the SoS to delimit reporting towns."""
110
111MEANINGFUL_CHARACTERS: list[str] = ["(", ")", "-", "&", "/", ","]
112""" Punctuation characters used by the SoS to communicate info about a reporting unit.
113
114Examples:
115    * BENEDICTA/SILVER RIDGE TWPS: forward slashes delimit towns.
116    * T15 R6 TWP (EAGLE LAKE): parentheses indicate registration town
117    * BLAINE -- E TWP: double hyphens indicate registration town
118
119These characters should not be stripped until this info has been parsed out."""
120
121# Patterns
122REGISTRATION_PATTERN: re.Pattern = re.compile(f'{PARENTHETICAL}|{PRECEDES_DASH}', re.I)
123""" Compiled regex matching a substring of non-reporting registration towns.
124
125Used in `mainegeo.elections.ResultString`.
126"""
127
128CLEAN_TOWNSHIP_PATTERN: re.Pattern = re.compile(f"[^\\w]|{LEADING_ZERO}")
129""" Compiled regex matching non-word characters and leading zeroes.
130
131Used in `mainegeo.townships.clean_code`."""
132
133NON_ALIAS_PATTERN: re.Pattern = re.compile(
134    f'{UNNAMED}(?: twps?)?|{PUNCTUATION}',
135    re.I
136    )
137""" Compiled regex matching substrings that are not an unnamed township code or punctuation.
138
139Used in `mainegeo.townships.has_alias` and `mainegeo.townships.extract_alias`."""
140
141DROP_CHARACTERS_PATTERN: re.Pattern = re.compile(
142    f"[^\\w\\s{''.join(map(re.escape, MEANINGFUL_CHARACTERS))}]"
143    )
144""" Matches all characters except word characters, whitespace, and meaningful punctuation.
145
146Meaningful punctuation characters are those used by the SoS to communicate information
147about a reporting unit (e.g. forward slashes to separate reporting towns). The full list
148of meanginful characters is defined in `mainegeo.patterns.MEANINGFUL_CHARACTERS`.
149
150Used in `mainegeo.elections.ResultString._drop_non_meaningful_characters`."""
151
152NONSTANDARD_DELIMITER_PATTERN: re.Pattern = re.compile(
153    '|'.join(map(re.escape, NONSTANDARD_DELIMITERS)),
154    re.I
155    )
156""" Compiled regex matching non-standard result string delimiters used occasionally by the SoS.
157
158Used in `mainegeo.elections.ResultString`."""
159
160ORPHAN_PARENTHESIS_PATTERN: re.Pattern = re.compile(
161    f'^(?P<result>[^(]+)(?P<orphan_parenthesis>[)])$'
162    )
163""" Compiled regex matching the orphaned closing parenthesis left after delimiter normalization.
164
165Capture groups:
166    * result
167    * orphan_parenthesis
168
169Used in `mainegeo.elections.ResultString`."""
170
171## Name standardization
172# Constants
173GNIS_GEOTYPES: list[str] = ["CITY", "PLANTATION", "TOWNSHIP", "TOWN"]
174""" Geotypes used by the Geographic Names Information System (GNIS)."""
175
176ABBREVIATIONS: dict[str, str] = {
177    "PLANTATION":       "PLT",
178    "TOWNSHIP":         "TWP",
179    "VOTING DISTRICT":  "VOTING DIST",
180    "RESERVATION":      "RES"
181}
182""" Geotype suffixes used by the Maine SoS and their abbreviations."""
183
184JUNIOR_SUFFIXES: list[str] = ['GORE', 'GRANT', 'ISLAND']
185""" Geotypes which may precede another geotype suffix or be used alone.
186
187For example, Moxie Gore and Moxie Gore Twp have subtly different meanings,
188but refer to the same place and are often used interchangeably by the SoS."""
189
190DIRECTIONS: list[str] = ['NORTH', 'SOUTH', 'EAST', 'WEST']
191""" Direction words which may modify place names."""
192
193CONTAINS_FALSE_SUFFIX: list[str] = ['INDIAN TOWNSHIP', 'INDIAN RESERVATION']
194""" Canonical place names that contain a word that is normally a suffix.
195
196For example: Indian Township is the name of a town, not a township.
197These false suffixes should be treated differently than true 
198suffixes during processing."""
199
200AMBIGUOUS_SUFFIXES: list[str] = ['RES']
201""" Full or abbreviated suffixes that may occur as substrings in other contexts. """
202
203# Factory functions
204def generate_suffixes() -> dict[str, str]:
205    replacements = []
206    for name, abbr in ABBREVIATIONS.items():
207        
208        precedes_false = []
209        for town in CONTAINS_FALSE_SUFFIX:
210            if town.endswith((name, abbr)):
211                precedes_false.append(re.sub(f' {name}|{abbr}$', '', town)) 
212        
213        if len(precedes_false) > 0:
214            ignore_false = f"(?<!{'|'.join(precedes_false)} )"
215        else:
216            ignore_false = ''
217        
218        for suffix in (name, abbr):
219            required_whitespace = ' ' if suffix in AMBIGUOUS_SUFFIXES else ' ?'
220            terminator = '$' if suffix in AMBIGUOUS_SUFFIXES else 's?$'
221            
222            replacements.append({
223                'suffix': suffix,
224                'replacement': ' ' + abbr,
225                'strip_pattern': required_whitespace + ignore_false + suffix + terminator,
226                'clean_pattern': required_whitespace + suffix + f"(?={terminator})",
227                'terminator': terminator
228            })
229    
230    return replacements
231
232def generate_valid_punctuation(char: str, template: str) -> str:
233    pattern = re.compile(f'(?P<leading>\\w+ ?){char}(?P<trailing> ?\\w+)')
234    matches = map(pattern.match, TownshipData.get_lookup().town)
235    valid_contexts = [match.expand(template) for match in matches if match]
236    return '|'.join(valid_contexts)
237
238# Templates
239AMPERSANDS_TEMPLATE: str = '(?:(?<=\g<leading>)&(?=\g<trailing>))'
240HYPHENS_TEMPLATE: str = '\g<leading>-(?=\g<trailing>)'
241
242# Building blocks
243GNIS_NAME = f"(?P<geotype>{'|'.join(GNIS_GEOTYPES)}) of (?P<town>.+)"
244VALID_AMPERSANDS: str = generate_valid_punctuation('&', AMPERSANDS_TEMPLATE)
245VALID_HYPHENS: str = generate_valid_punctuation('-', HYPHENS_TEMPLATE)
246JUNIOR_SUFFIX: str = f"\\b({'|'.join(JUNIOR_SUFFIXES)})"
247SUFFIXES: list[dict] = generate_suffixes()
248SUFFIX_REPLACEMENTS: dict[str, str] = {
249    suffix['clean_pattern']: suffix['replacement']
250    for suffix in SUFFIXES
251}
252
253# Patterns
254GNIS_PATTERN: re.Pattern = re.compile(GNIS_NAME, re.I)
255""" Matches place names with Geographic Names Information System (GNIS) formatting.
256
257Capture groups:
258    * geotype
259    * town
260
261Used in `mainegeo.townships.normalize_suffix`."""
262
263SUFFIX_PATTERN: re.Pattern = re.compile(
264    f"{'|'.join([suffix['strip_pattern'] for suffix in SUFFIXES])}", re.I
265    )
266""" Matches all valid terminal suffixes and suffix abbreviations.
267
268Used in `mainegeo.townships.strip_suffix`."""
269
270VALID_AMPERSANDS_PATTERN: re.Pattern = re.compile(VALID_AMPERSANDS, re.I)
271""" Matches ampersands that are part of canonical town names and should
272not be interpreted as SoS result string formatting.
273
274Used in `mainegeo.townships.strip_town` (a helper for `mainegeo.townships.clean_town`)."""
275
276INVALID_PUNCTUATION_PATTERN: re.Pattern = re.compile(
277    f"{PUNCTUATION}(?<!{VALID_HYPHENS})", re.I
278    )
279""" Matches all punctuation except hyphens that are part of canonical town names.
280
281Used in `mainegeo.townships.strip_town` (a helper for `mainegeo.townships.clean_town`)."""
282
283ENDSWITH_JUNIOR_SUFFIX_PATTERN: re.Pattern = re.compile(f".+{JUNIOR_SUFFIX}$", re.I)
284""" Matches town name that ends with a junior suffix (gore, grant, island, etc).
285
286Junior suffixes are listed in `mainegeo.patterns.JUNIOR_SUFFIXES`.
287
288Used in `mainegeo.townships.toggle_suffix`."""
289
290CONTAINS_JUNIOR_SUFFIX_PATTERN: re.Pattern = re.compile(f".+{JUNIOR_SUFFIX} TWP$", re.I)
291""" Matches town name that contains a junior suffix (gore, grant, island, etc).
292
293Junior suffixes are listed in `mainegeo.patterns.JUNIOR_SUFFIXES`.
294
295Used in `mainegeo.townships.toggle_suffix`."""
296
297
298## Unspecified groups
299# Constants
300UNSPECIFIED_FLAG: str = 'TWPS'
301""" Substring that indicates an unspecified group may be present in a raw election result.
302
303Used in multiple functions in `mainegeo.elections`. Most important use is in
304`mainegeo.elections.ReportingUnit.has_unspecified_group`, which uses it in 
305combination with other context clues to detect unspecified groups."""
306
307STANDARD_FLAG: str = 'UNSPECIFIED'
308""" Substring that will be applied to unspecified groups during formatting.
309
310Chosen to avoid overlap with words that may occur naturally in election 
311result strings."""
312
313MULTI_COUNTY_REGISTRATION_TOWNS: list[str] = ['MILLINOCKET']
314""" Towns that host unspecified township groups from multiple counties.
315
316As of 2025, the only town that is typically reported this way is Millinocket,
317e.g. Millinocket Penobscot Twps and Millinocket Piscataquis Twps."""
318
319# Building blocks
320PLURAL = UNSPECIFIED_FLAG
321SINGULAR = re.sub('S$', '', UNSPECIFIED_FLAG)
322UNSPECIFIED_REGTOWN = f"(?P<regtown>{'|'.join(MULTI_COUNTY_REGISTRATION_TOWNS)})"
323UNSPECIFIED_COUNTY = f"(?P<cty>{'|'.join(CountyData.get_lookup().sos_county)})"
324SOS_FLAG = f"(?P<sos_flag>{UNSPECIFIED_FLAG})"
325
326# Patterns
327PLURAL_PATTERN: re.Pattern = re.compile(f'\\b{PLURAL}\\b')
328SINGULAR_PATTERN: re.Pattern = re.compile(f'\\b{SINGULAR}\\b')
329MULTI_COUNTY_PATTERN: re.Pattern = re.compile(
330    f"(?i){UNSPECIFIED_REGTOWN} {UNSPECIFIED_COUNTY}\\w* {SOS_FLAG}", re.I
331)
332""" Matches raw, unformatted unspecified groups that contain a county.
333
334Capture groups:
335    * regtown
336    * cty
337    * sos_flag
338"""
339
340MULTI_COUNTY_FORMAT: str = r"\g<regtown> \g<sos_flag> [\g<cty>]"
341""" Standardized format to apply to all multi-county unspecified groups."""
342
343FORMATTED_GROUP_PATTERN: re.Pattern = re.compile(
344    f'{STANDARD_FLAG} (?P<regtown>.+) {UNSPECIFIED_FLAG}( \\[{UNSPECIFIED_COUNTY}\\])?',
345    re.I
346)
347""" Matches all formatted unspecified groups, including multi-county groups."""