offline_no_lid

View Source

 1from intelli3text import PipelineBuilder, Intelli3Config
 2
 3cfg = Intelli3Config(export=None)
 4pipeline = PipelineBuilder(cfg).build()
 5
 6# Disable language detectors to avoid fastText/CLD3 downloads
 7pipeline.lid_primary = None
 8pipeline.lid_fallback = None
 9
10res = pipeline.process("docs/short.txt")
11print("Global language (fallback when LID disabled):", res["language_global"])

cfg = Intelli3Config(cleaners=['ftfy', 'ocr_tilde_fix', 'pdf_breaks', 'pt_diacritics_repair', 'clean_text', 'strip_accents'], lid_primary='fasttext', lid_fallback=None, languages_supported={'es', 'pt', 'en'}, nlp_model_pref='lg', paragraph_min_chars=30, lid_min_chars=60, lid_threshold=0.65, lid_max_chars=2500, lid_use_cld3=True, lid_cld3_weight=0.4, export=None)

pipeline = <intelli3text.pipeline.Pipeline object>

res = {'language_global': 'pt', 'language_mixed': False, 'language_distribution': {}, 'raw': '', 'cleaned': '', 'normalized': '', 'paragraphs': []}