Table of Contents

August 12, 2026 · View on GitHub

yasbd

register_spacy_component

def register_spacy_component()

Register the yasbd spaCy pipeline component on demand.

Call this to add the yasbd component factory to spaCy's registry. Requires spaCy v3+ to be installed.

Examples

import spacy from yasbd import register_spacy_component register_spacy_component() nlp = spacy.blank("en") nlp.add_pipe("yasbd", first=True, config={"lang": "en"})

yasbd.boundary_detector

HookContext Objects

class HookContext(TypedDict)

Per-paragraph context passed to a post-processing hook.

Keys: text: The paragraph text being segmented. lang: ISO language code of the active rule set. boundaries: Paragraph-relative boundary offsets. Mutate this list in place to add or remove sentence boundaries; reassigning ctx["boundaries"] to a new list also works, though it is not recommended. paragraph_index: Zero-based index of the paragraph in the stream.

BoundaryDetector Objects

class BoundaryDetector()

__init__

@validate_input
def __init__(lang: str | None = None,
             *,
             preserve_quote_and_paren: bool = True,
             verbose: bool = False,
             hook: Callable[[HookContext], None] | None = None)

Initialize the boundary detector.

Arguments:

  • lang - Two chars ISO language code (e.g., 'en', 'fr', ...). Use 'auto' for automatic language detection via py3langid. Explicit is faster and more reliable; use auto if you don't mind a slight decrease in both.
  • preserve_quote_and_paren - Do not split on terminators inside quoted or parenthesised text.
  • verbose - Enable verbose logging.
  • hook - Optional per-paragraph post-processing callback. Receives a dict with text, lang, boundaries and paragraph_index keys; mutate boundaries in place to add or remove sentence boundaries. Reassigning ctx["boundaries"] to a new list also works, though in-place mutation is recommended.

lang

@property
def lang() -> str

ISO language code of the active rule set.

detect

@validate_input
def detect(source: str | TextIOBase | StreamCleanerStub,
           *,
           relative: bool = False) -> Generator[int, None, None]

Detect sentence boundaries in the source text.

Arguments:

  • source - Plain text string, an open text stream (e.g., StringIO), or a StreamCleaner instance.
  • relative - If False (default), yields absolute character offsets from the beginning of the entire stream. If True, offsets reset at each paragraph break, yielding indices relative to the start of the current paragraph.

Notes:

When relative=True, a ParagraphEOF sentinel is yielded between distinct paragraphs to signal the boundary of the local coordinate system. Import via: from yasbd import ParagraphEOF.

Yields:

Integer boundary offsets or ParagraphEOF sentinels.

segment

@validate_input
def segment(source: str | TextIOBase | StreamCleanerStub,
            *,
            preserve_whitespace: bool = False) -> Generator[str, None, None]

Split text into sentences.

Arguments:

  • source - Plain text string or TextIOBase stream (e.g., StringIO, opened file).
  • preserve_whitespace - If False (default), strip leading and trailing whitespace from each sentence.

Yields:

Individual sentences as strings.

yasbd.cli

segment

@cli.command(
    "segment",
    text=Arg(help="Text to split. Use --file to read from a file instead."),
    file=Arg("--file", "-f", help="Read input from a text file."),
    destination=Arg("--destination", "-d", help="Write output to a file."),
    lang=Arg("--lang", "-l", help="Language code (e.g., 'en', 'fr', 'de')."),
    from_pack=Arg("--from-pack",
                  help="Load external language pack (repeatable)."),
    preserve_whitespace=Arg("--preserve-whitespace",
                            "-w",
                            help="Preserve original whitespace in output."),
    verbose=Arg("--verbose", "-v", help="Enable verbose logging."),
)
def segment(text: Optional[str] = None,
            file: Optional[str] = None,
            destination: Optional[str] = None,
            lang: Optional[str] = None,
            from_pack: Optional[list[str]] = None,
            preserve_whitespace: bool = False,
            verbose: bool = False)

Split text into sentences.

Reads from a positional string, --file, or stdin pipe. Writes enumerated sentences to stdout or JSONL to --destination.

detect

@cli.command(
    "detect",
    text=Arg(
        help=
        "Text to detect boundaries in. Use --file to read from a file instead."
    ),
    file=Arg("--file", "-f", help="Read input from a text file."),
    destination=Arg("--destination",
                    "-d",
                    help="Write boundary offsets to a file."),
    lang=Arg("--lang", "-l", help="Language code (e.g., 'en', 'fr', 'de')."),
    from_pack=Arg("--from-pack",
                  help="Load external language pack (repeatable)."),
    relative=Arg("--relative", "-r", help="Yield paragraph-relative offsets."),
    verbose=Arg("--verbose", "-v", help="Enable verbose logging."),
)
def detect(text: Optional[str] = None,
           file: Optional[str] = None,
           destination: Optional[str] = None,
           lang: Optional[str] = None,
           from_pack: Optional[list[str]] = None,
           relative: bool = False,
           verbose: bool = False)

Detect sentence boundary offsets (character positions).

Reads from a positional string, --file, or stdin pipe. Writes boundary offsets to stdout or JSONL to --destination. Use --relative for per-paragraph offsets (ParagraphEOF marks gaps).

clean

@cli.command(
    "clean",
    text=Arg(help="Text to clean. Use --file to read from a file instead."),
    file=Arg("--file", "-f", help="Read input from a text file."),
    destination=Arg("--destination",
                    "-d",
                    help="Write cleaned text to a file."),
    steps_to_skip=Arg("--steps-to-skip",
                      "--skip",
                      help="Comma-separated cleaning steps to skip."),
    extra_step=Arg(
        "--extra-step",
        "-e",
        help=
        "External shell command to run as an extra cleaning step (repeatable).",
    ),
    verbose=Arg("--verbose", "-v", help="Enable verbose logging."),
)
def clean(text: Optional[str] = None,
          file: Optional[str] = None,
          destination: Optional[str] = None,
          steps_to_skip: Optional[str] = None,
          extra_step: Optional[list[str]] = None,
          verbose: bool = False)

Clean and normalize noisy text paragraphs.

Applies ftfy mojibake fixing, OCR cleanup, HTML tag stripping, slash normalization, and whitespace normalization. Use --skip to omit specific steps (comma-separated). Use --extra-step to run external shell commands as extra cleaning steps.

langs

@cli.command("langs")
def langs()

List supported language codes.

main

def main()

CLI entry point. Handles --version, --help, and dispatches to radicli.

yasbd.exceptions

YasbdError Objects

class YasbdError(Exception)

Base exception for all yasbd errors.

UnsupportedLanguageError Objects

class UnsupportedLanguageError(YasbdError, ValueError)

Raised when an unsupported language code is provided.

InvalidInputError Objects

class InvalidInputError(YasbdError, TypeError)

Raised when invalid input(s) are encountered.

LangPackError Objects

class LangPackError(YasbdError)

Raised when a language pack module fails validation or handshake.

CleanStepError Objects

class CleanStepError(YasbdError, TypeError)

Raised when a StreamCleaner extra step fails (non-callable or non-str return).

HookError Objects

class HookError(YasbdError, RuntimeError)

Raised when a post-processing hook fails or leaves invalid boundaries.

yasbd.rules

register_lang_packs

@validate_input
def register_lang_packs(names: list[str]) -> list[str]

Import and validate external language pack modules.

Each module must expose a PROFILES list of Rules subclasses. All validated profiles are stored in _LANG_PACK_REGISTRY.

Caution: This function imports arbitrary Python modules by name. Only load lang packs from sources you trust — an untrusted module can execute arbitrary code at import time.

Arguments:

  • names - Module names resolvable from the Python path (e.g. ["yasbd_indic", "yasbd_legal"]).

Returns:

List of registered language codes (e.g. ["xx", "eo"]).

Raises:

  • LangPackError - If a language pack module cannot be imported.

clear_lang_packs

def clear_lang_packs() -> None

Remove all registered language packs and reset the supported-languages cache.

get_supported_langs

@cache
def get_supported_langs() -> list[str]

Discover and cache supported language codes.

Returns a sorted list of auto plus all language codes from the built-in rules directory and any registered language packs.

load_rule

def load_rule(lang: str, *, verbose: bool = False) -> Rules

Import and instantiate the rule module for lang.

Checks the language pack registry first; falls back to the built-in rules directory.

Returns:

The instantiated rule object.

Raises:

  • UnsupportedLanguageError - If no rule module exists for lang.

yasbd.rules.af

yasbd.rules.am

yasbd.rules.ar

yasbd.rules.base

CJK Objects

class CJK()

Mixin for CJK languages sharing full-width geopolitical abbreviation patterns.

Rules Objects

class Rules()

__init__

def __init__()

Initialize rule instance with lazy-compiled regex patterns.

Patterns are compiled once per class and cached via _REGEX_CACHED. Subclasses can override data constants (abbreviation sets, terminators, etc.) and the classmethod _compile_regex_dynamically will pick them up.

post_process_boundaries

def post_process_boundaries(sentence_boundaries: set[int], text: str) -> None

Hook for language-specific boundary filtering.

Override in subclasses to remove false-positive boundaries that the regex passes cannot catch. Mutate sentence_boundaries in place; do not touch any other engine state.

apply

def apply(text: str, preserve_quote_and_paren: bool) -> list[int]

Detect sentence boundaries in text.

Two-pass algorithm:

  1. Collect boundary candidates from punctuation positions.
  2. Remove false alarms (mid-sentence abbreviations, ellipsis, quote/paren spans, list markers).

Arguments:

  • text - A string to find sentence boundaries in.
  • preserve_quote_and_paren - If True, suppress boundaries inside quote and parenthesis spans.

Returns:

Sorted list of character offsets at which sentences end.

yasbd.rules.bg

yasbd.rules.bn

yasbd.rules.cs

yasbd.rules.da

yasbd.rules.de

yasbd.rules.el

yasbd.rules.en

yasbd.rules.es

yasbd.rules.fa

yasbd.rules.fr

yasbd.rules.hi

yasbd.rules.ht

yasbd.rules.hy

yasbd.rules.id

yasbd.rules.it

yasbd.rules.ja

yasbd.rules.kk

yasbd.rules.ko

yasbd.rules.lt

yasbd.rules.ml

yasbd.rules.mr

yasbd.rules.my

yasbd.rules.nl

yasbd.rules.pl

yasbd.rules.pt

yasbd.rules.ro

yasbd.rules.ru

yasbd.rules.sk

yasbd.rules.sv

yasbd.rules.sw

yasbd.rules.th

yasbd.rules.tr

yasbd.rules.uk

yasbd.rules.ur

yasbd.rules.vi

yasbd.rules.zh

yasbd.utils

yasbd.utils.cleaner

normalize_newlines

def normalize_newlines(text: str) -> str

Normalize Windows ( ) and Classic Mac ( ) line endings to Unix ( ).

unwrap_htmls

def unwrap_htmls(text: str) -> str

Strip HTML tags only when the text actually contains angle brackets.

normalize_spaces

def normalize_spaces(text: str) -> str

Collapse repeated spaces when present; skip the regex otherwise.

StreamCleaner Objects

class StreamCleaner(StreamCleanerStub)

Normalize line endings, clean noisy text by applying ftfy, HTML sanitization, and various regex cleanup rules across paragraphs.

Examples:

list(StreamCleaner("x < 5 and y > 3")) ['x < 5 and y > 3'] list(StreamCleaner("Hello world. How are you?")) ['Hello world. How are you?'] list(StreamCleaner("clean text")) ['clean text'] list(StreamCleaner("Hello world", steps_to_skip=["unwrap_htmls"])) ['Hello world'] list(StreamCleaner("W\nO\nR\nD")) ['W O R D'] list(StreamCleaner("I am a good\nman")) ['I am a good man'] list(StreamCleaner("An hyphe-\nnated sentence")) ['An hyphenated sentence'] list(StreamCleaner("state-of-the-\nart")) ['state-of-the-art'] list(StreamCleaner("")) [] StreamCleaner("Hello world", steps_to_skip=["nothing"]) Traceback (most recent call last): ...

  • yasbd.exceptions.InvalidInputError - 🧩 Oops! Unknown step(s): 'nothing'...

    list(StreamCleaner("Hello™ world", extra_steps=[lambda t: t.replace("™", "")])) ['Hello world'] list(StreamCleaner("hello", extra_steps=[lambda t: 1/0])) Traceback (most recent call last): ...

  • yasbd.exceptions.CleanStepError - extra step '' raised an error.
  • Details - division by zero

__init__

@validate_input
def __init__(source: str | TextIOBase,
             steps_to_skip: Collection[str] | None = None,
             extra_steps: Collection[Callable[[str], str]] | None = None,
             *,
             verbose: bool = False) -> None

Implements the iterator protocol. Yields cleaned paragraph strings.

Arguments:

  • source - Plain text string or open text stream (e.g., StringIO).
  • steps_to_skip - A collection of steps to ignore. All steps will run if not provided. choices are:
    • normalize_newlines
    • fix_mojibake
    • fix_ocr_text
    • unwrap_htmls
    • normalize_spaces
  • extra_steps - Optional user-defined cleaning functions, run after built-in steps. Each function must accept and return str.
  • verbose - Enable verbose logging.

yasbd.utils.input_validator

validate_input

def validate_input(fn: F) -> F

Validate function arguments and return values using beartype.

yasbd.utils.lang_code_normalizer

normalize_lang

@validate_input
def normalize_lang(lang_code: str) -> str

Normalize a language tag to an ISO-639-1 language code.

The helper is explicit and opt-in. It does not alter the core BoundaryDetector language handling.

Examples:

normalize_lang("EN") 'en' normalize_lang("en-US") 'en' normalize_lang("en-Latn") 'en' normalize_lang("pt-BR") 'pt' normalize_lang("") '' normalize_lang(" ") '' normalize_lang("not-a-language-code") # doctest: +ELLIPSIS Traceback (most recent call last): ...

  • yasbd.exceptions.InvalidInputError - ...

    normalize_lang("akk") # doctest: +ELLIPSIS Traceback (most recent call last): ...

  • yasbd.exceptions.InvalidInputError - ...

Arguments:

  • lang_code - A language code or tag, such as "EN", "en-US", or "en-Latn".

Returns:

A two-letter ISO-639-1 language code, or an empty string if empty input.

Raises:

  • ImportError - If the optional langcodes dependency is missing.
  • InvalidInputError - If the tag cannot be parsed or does not resolve to a two-letter ISO-639-1 language code.

yasbd.utils.language_classifier

classify_language

@lru_cache(maxsize=12)
def classify_language(text: str) -> tuple[str, float]

Classify text with a preference for expected languages.

This function avoids the explicit py3langid.LanguageIdentifier initialization and its associated cold-start cost by relying on this convenience API.

The algorithm works as follows:

  1. Obtain the ranked language predictions.
  2. Look for preferred languages within the top TOP_K results.
  3. Only consider preferred languages whose score is within MAX_GAP of the top prediction.
  4. If no suitable preferred language is found, fall back to all ranked candidates.
  5. Compute a normalized confidence score using a softmax over the selected candidates.

Arguments:

  • text - The text to classify.

Returns:

A tuple containing:

  • The predicted ISO 639-1 language code.
  • A confidence score between 0.0 and 1.0.

Examples:

language, confidence = classify_language("kiyès ?") language 'ht' 0.0 <= confidence <= 1.0 True

Raises:

  • ImportError - If py3langid is not installed.
  • ValueError - If the detector returns no language scores.

yasbd.utils.logger

log_info

def log_info(verbose: bool, *args, **kwargs) -> None

Log an info message if verbose is enabled.

This is a convenience function that only logs when verbose mode is enabled, avoiding unnecessary log output in production.

Arguments:

  • verbose - If True, logs the message; if False, does nothing.
  • *args - Positional arguments passed to logger.info().
  • **kwargs - Keyword arguments passed to logger.info().

Example:

log_info(True, "hello {}", "world") log_info(False, "This will not be logged")

yasbd.utils.paragraph_stream

ParagraphStream Objects

class ParagraphStream()

An iterator that groups lines of text into paragraph blocks.

This class implements Python's Iterator Protocol (iter and next), retaining state across calls and yielding reconstructed paragraph blocks.

Examples:

streamer = ParagraphStream('Hello\n\nWorld', skip_empty_lines=False) list(streamer) ['Hello\n\n', 'World']

streamer = ParagraphStream('Hello\n\nWorld', skip_empty_lines=True) list(streamer) ['Hello\n', 'World']

__init__

@validate_input
def __init__(source: str | TextIOBase | StreamCleanerStub,
             *,
             skip_empty_lines: bool = False) -> None

Initialize ParagraphStream.

Arguments:

  • source - Input text as a string, TextIOBase stream, or StreamCleaner.
  • skip_empty_lines - If True, blank separator lines are omitted from paragraph blocks.

__next__

def __next__() -> str

Advance the stream and return the next paragraph.

Yields paragraphs reconstructed as strings, preserving original line endings.

Returns:

The next complete paragraph string.

Raises:

  • StopIteration - When there are no more paragraphs to return.

close

def close() -> None

Close the underlying source stream if applicable.

yasbd.utils.pysbd_adapter

TextSpan Objects

class TextSpan()

A sentence with its character-offset span in the original text.

Arguments:

  • sent - Sentence text.
  • start - Start character offset of a sentence in original text.
  • end - End character offset of a sentence in original text.

Segmenter Objects

class Segmenter()

__init__

@validate_input
def __init__(language: str = "en",
             clean: bool = False,
             doc_type: str | None = None,
             char_span: bool = False)

Initializes the Segmenter.

Arguments:

  • language - Two-character ISO 639-1 language code. Defaults to "en".
  • clean - Whether to clean the original text. Defaults to False.
  • doc_type - Normal text or OCRed text (e.g. "pdf"). Defaults to None.
  • char_span - Whether to return character offset spans. Defaults to False.

sentences_with_char_spans

@validate_input
def sentences_with_char_spans(sentences: list[str]) -> list[TextSpan]

Map sentences to their char offsets using cumulative lengths.

Pysbd compatibility method

segment

@validate_input
def segment(text: str) -> list[str | TextSpan]

Segments text into sentences.

Arguments:

  • text - Raw text to be segmented into sentences.

Returns:

A list of sentences (strings) by default, or a list of TextSpan objects if char_span was set to True.

yasbd.utils.spacy_component

YasbdComponent Objects

class YasbdComponent()

A pipeline component for spaCy.

__call__

def __call__(doc: Doc) -> Doc

Assign sentence sent_ends using yasbd.

create_yasbd

@Language.factory(
    "yasbd",
    default_config={
        "lang": None,
        "preserve_quote_and_paren": True,
        "verbose": False,
    },
)
def create_yasbd(nlp: Language, name: str, lang: str | None,
                 preserve_quote_and_paren: bool, verbose: bool)

Create a spaCy component powered by yasbd.

yasbd.utils.trie

build_optimized_pattern

def build_optimized_pattern(options: set[str]) -> str

Build an optimised and escaped regex alternation pattern.

Returns a never-match pattern if no valid options exist. Ref: https://stackoverflow.com/questions/1723182/a-regex-that-will-never-be-matched-by-anything?