token_preprocessor.py

[Chip (OP)]

WinSCP Preferences for &Binary added a mask for "*.py; *.ai

Code: [Select]

bash file:
/python/my36project/bin/python3 $1 $2 $3 $4

run:
python3 token_preprocessor.py transcript.txt > query2.ai


Code: [Select]

#!/usr/bin/env python3
"""
token_preprocessor.py

Cleans forum transcript text (BBCode + PDF-extraction artifacts) before
pasting into an LLM conversation, to reduce token count without losing
meaning.

Usage:
    python3 token_preprocessor.py input.txt output.txt
    python3 token_preprocessor.py input.txt          # prints to stdout
    cat input.txt | python3 token_preprocessor.py -   # read from stdin

What it does:
    1. Strips BBCode tags ([b], [color=...], [/color], etc.) - pure
       formatting overhead, zero semantic content.
    2. Fixes broken-hyphenation artifacts from PDF text extraction
       (e.g. "the- kind- of" -> "the kind of").
    3. Collapses "===== Page NN =====" page-break markers.
    4. Replaces common verbose phrases with shorter equivalents
       (e.g. "in order to" -> "to", "utilize" -> "use").
    5. Collapses excess whitespace/blank lines.

What it deliberately does NOT do:
    - Touch precise technical terms (won't blindly simplify vocabulary
      that carries real information, e.g. "positional encoding").
    - Rewrite sentence structure or meaning.
"""

import re
import sys


# ---------------------------------------------------------------------
# 0. Fix double-encoded UTF-8 (mojibake), e.g. "—" instead of "—"
#    This happens when UTF-8 bytes get misread as cp1252/Latin-1 and
#    then re-saved as UTF-8. Common with copy/paste through some PDF
#    extraction and terminal pipelines.
# ---------------------------------------------------------------------
def fix_mojibake(text: str) -> str:
    try:
        repaired = text.encode('cp1252').decode('utf-8')
        return repaired
    except (UnicodeEncodeError, UnicodeDecodeError):
        # Original text wasn't double-encoded this way; leave it alone.
        return text

# ---------------------------------------------------------------------
# 1. BBCode stripping
# ---------------------------------------------------------------------
BBCODE_TAG_RE = re.compile(r'\[/?(?:color|b|i|u|size|url|quote|img)[^\]]*\]', re.IGNORECASE)

def strip_bbcode(text: str) -> str:
    return BBCODE_TAG_RE.sub('', text)


# ---------------------------------------------------------------------
# 2. Fix "word- broken- like- this" hyphenation artifacts
#    Pattern: letter + "-" + optional space + letter, repeated, where
#    the hyphen was inserted by PDF line-wrap extraction rather than
#    being a genuine compound word.
# ---------------------------------------------------------------------
BROKEN_HYPHEN_RE = re.compile(r'(\w)-\s+(\w)')

def fix_broken_hyphenation(text: str) -> str:
    # Repeatedly collapse until no more matches (handles multi-part breaks
    # like "the- kind- of" -> "the kind of")
    prev = None
    while prev != text:
        prev = text
        text = BROKEN_HYPHEN_RE.sub(r'\1 \2', text)
    return text


# ---------------------------------------------------------------------
# 3. Remove page-break markers
# ---------------------------------------------------------------------
PAGE_MARKER_RE = re.compile(r'=+\s*Page\s*\d+\s*=+', re.IGNORECASE)

def strip_page_markers(text: str) -> str:
    return PAGE_MARKER_RE.sub('', text)


# ---------------------------------------------------------------------
# 4. Verbose -> plain phrase dictionary
#    Ordered longest-first so multi-word phrases match before single
#    words that might be substrings.
# ---------------------------------------------------------------------
PHRASE_MAP = [
    ("in order to", "to"),
    ("due to the fact that", "because"),
    ("in the event that", "if"),
    ("with regard to", "about"),
    ("with respect to", "about"),
    ("in spite of the fact that", "although"),
    ("for the purpose of", "to"),
    ("at this point in time", "now"),
    ("in the process of", "while"),
    ("a large number of", "many"),
    ("the majority of", "most"),
    ("utilize", "use"),
    ("utilization", "use"),
    ("commence", "start"),
    ("terminate", "end"),
    ("facilitate", "help"),
    ("subsequently", "then"),
    ("approximately", "about"),
    ("in conjunction with", "with"),
    ("prior to", "before"),
    ("subsequent to", "after"),
]

def shorten_phrases(text: str) -> str:
    for verbose, plain in PHRASE_MAP:
        pattern = re.compile(r'\b' + re.escape(verbose) + r'\b', re.IGNORECASE)
        text = pattern.sub(plain, text)
    return text


# ---------------------------------------------------------------------
# 5. Whitespace cleanup
# ---------------------------------------------------------------------
def collapse_whitespace(text: str) -> str:
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r' +\n', '\n', text)
    return text.strip()


def preprocess(text: str) -> str:
    text = fix_mojibake(text)
    text = strip_bbcode(text)
    text = strip_page_markers(text)
    text = fix_broken_hyphenation(text)
    text = shorten_phrases(text)
    text = collapse_whitespace(text)
    return text


def main():
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(1)

    src = sys.argv[1]
    if src == '-':
        raw = sys.stdin.read()
    else:
        with open(src, 'r', encoding='utf-8') as f:
            raw = f.read()

    cleaned = preprocess(raw)

    if len(sys.argv) >= 3:
        with open(sys.argv[2], 'w', encoding='utf-8') as f:
            f.write(cleaned)
        orig_words = len(raw.split())
        new_words = len(cleaned.split())
        print(f"Original words: {orig_words}")
        print(f"Cleaned words:  {new_words}")
        print(f"Reduction:      {orig_words - new_words} words "
              f"({100*(orig_words-new_words)/max(orig_words,1):.1f}%)")
        print(f"Written to {sys.argv[2]}")
    else:
        print(cleaned)


if __name__ == '__main__':
    main()