force_small_spacy

View Source

1from intelli3text import PipelineBuilder, Intelli3Config
2
3cfg = Intelli3Config(
4    nlp_model_pref="sm",  # try sm → fallback to blank if missing
5    export=None,
6)
7pipeline = PipelineBuilder(cfg).build()
8res = pipeline.process("docs/mixed_language.txt")
9print("Global language:", res["language_global"])

cfg = Intelli3Config(cleaners=['ftfy', 'ocr_tilde_fix', 'pdf_breaks', 'pt_diacritics_repair', 'clean_text', 'strip_accents'], lid_primary='fasttext', lid_fallback=None, languages_supported={'es', 'pt', 'en'}, nlp_model_pref='sm', paragraph_min_chars=30, lid_min_chars=60, lid_threshold=0.65, lid_max_chars=2500, lid_use_cld3=True, lid_cld3_weight=0.4, export=None)

pipeline = <intelli3text.pipeline.Pipeline object>

res = {'language_global': 'pt', 'language_mixed': False, 'language_distribution': {}, 'raw': '', 'cleaned': '', 'normalized': '', 'paragraphs': []}