export_json_and_jsonl

Custom cleaners example.

This script shows how to plug custom cleaners into the pipeline. It is safe to import (pdoc won't execute it), and only runs when executed as a script.

Run:

python -m examples.custom_cleaners

View Source

 1"""Custom cleaners example.
 2
 3This script shows how to plug custom cleaners into the pipeline.
 4It is safe to import (pdoc won't execute it), and only runs when executed as a script.
 5
 6Run:
 7    python -m examples.custom_cleaners
 8"""
 9
10from intelli3text import PipelineBuilder, Intelli3Config
11
12def main() -> None:
13    from intelli3text import PipelineBuilder, Intelli3Config
14    from intelli3text.export.json_export import JSONExporter, JSONLExporter
15
16    cfg = Intelli3Config(export=None)
17    pipeline = PipelineBuilder(cfg).build()
18
19    # Add exporters programmatically
20    pipeline.exporters["json"] = JSONExporter(path="report.json", pretty=True)
21    pipeline.exporters["jsonl"] = JSONLExporter(path="paragraphs.jsonl")
22
23    res = pipeline.process("https://pt.wikipedia.org/wiki/Howard_Gardner")
24    print("Wrote report.json and paragraphs.jsonl")

def main() -> None: View Source

13def main() -> None:
14    from intelli3text import PipelineBuilder, Intelli3Config
15    from intelli3text.export.json_export import JSONExporter, JSONLExporter
16
17    cfg = Intelli3Config(export=None)
18    pipeline = PipelineBuilder(cfg).build()
19
20    # Add exporters programmatically
21    pipeline.exporters["json"] = JSONExporter(path="report.json", pretty=True)
22    pipeline.exporters["jsonl"] = JSONLExporter(path="paragraphs.jsonl")
23
24    res = pipeline.process("https://pt.wikipedia.org/wiki/Howard_Gardner")
25    print("Wrote report.json and paragraphs.jsonl")