paragraph_language_distribution
1from collections import Counter 2from intelli3text import PipelineBuilder, Intelli3Config 3 4cfg = Intelli3Config(export=None) 5pipeline = PipelineBuilder(cfg).build() 6res = pipeline.process("docs/bilingual.txt") 7 8counts = Counter(p["language"] for p in res["paragraphs"]) 9print("Language distribution:", dict(counts)) 10for i, p in enumerate(res["paragraphs"], 1): 11 print(f"{i:03d} {p['language']} ({p['score']:.3f}) :: {p['normalized'][:80]}…")
cfg =
Intelli3Config(cleaners=['ftfy', 'ocr_tilde_fix', 'pdf_breaks', 'pt_diacritics_repair', 'clean_text', 'strip_accents'], lid_primary='fasttext', lid_fallback=None, languages_supported={'es', 'pt', 'en'}, nlp_model_pref='lg', paragraph_min_chars=30, lid_min_chars=60, lid_threshold=0.65, lid_max_chars=2500, lid_use_cld3=True, lid_cld3_weight=0.4, export=None)
pipeline =
<intelli3text.pipeline.Pipeline object>
res =
{'language_global': 'pt', 'language_mixed': False, 'language_distribution': {}, 'raw': '', 'cleaned': '', 'normalized': '', 'paragraphs': []}
counts =
Counter()