Configuration

Customize PySET behavior with configuration options.

Language Support

detector = TokenBoundaryDetector(language='en')   # English (default)
detector = TokenBoundaryDetector(language='de')   # German
detector = TokenBoundaryDetector(language='fr')  # French
# 52+ languages supported

Abbreviations

# Add custom abbreviations
detector.set_abbreviations({'CEO', 'CFO', 'CTO', 'ACME', 'NASA'})

# Or use aggressive mode
detector = TokenBoundaryDetector(aggressive_abbreviations=True)

Sentence Length

# Minimum sentence length
detector = TokenBoundaryDetector(min_sentence_length=5)

# Merge short sentences
detector = TokenBoundaryDetector(merge_short_sentences=True)

Rule Configuration

# Include specific rules
detector = TokenBoundaryDetector(include_rules=[1, 2, 3, 10])

# Exclude specific rules
detector = TokenBoundaryDetector(exclude_rules=[17, 18])

Debug Mode

detector = TokenBoundaryDetector(debug=True)
explanations = detector.explain(text)
# Shows rule-by-rule breakdown

Recommended Configurations

RAG / Document Chunking

detector = TokenBoundaryDetector(
    min_sentence_length=10,
    merge_short_sentences=True,
    aggressive_abbreviations=True
)

Legal Documents

detector = TokenBoundaryDetector(
    aggressive_abbreviations=True,
    min_sentence_length=5
)

High Accuracy

detector = TokenBoundaryDetector(
    aggressive_abbreviations=True,
    debug=False
)