Robust Semantic Entropy Calculator (Python 3)

[smfadmin (OP)]

Manual rewrite version:

Code: [Select]

#!/usr/bin/env python3
"""
semantic_entropy.py

A robust, modular semantic entropy calculator.
Measures:
    1. Lexical ambiguity entropy (polysemy)
    2. Syntactic branching entropy (dependency complexity)
    3. Intent dispersion entropy (multiple plausible intents)

Requires:
    pip install nltk spacy
    python3 -m spacy download en_core_web_sm
"""

import math
import nltk
from nltk.corpus import wordnet as wn
import spacy

# Load spaCy model once
nlp = spacy.load("en_core_web_sm")


# ------------------------------------------------------------
# 1. Lexical Ambiguity Entropy
# ------------------------------------------------------------
def lexical_entropy(text: str) -> float:
    tokens = [t.lower() for t in nltk.word_tokenize(text) if t.isalpha()]
    if not tokens:
        return 0.0

    polysemous = 0
    for token in tokens:
        senses = wn.synsets(token)
        if len(senses) > 1:
            polysemous += 1

    return polysemous / len(tokens)


# ------------------------------------------------------------
# 2. Syntactic Branching Entropy
# ------------------------------------------------------------
def branching_entropy(text: str) -> float:
    doc = nlp(text)
    if len(list(doc.sents)) == 0:
        return 0.0

    total_branches = 0
    for sent in doc.sents:
        for token in sent:
            # Count dependency children as branching points
            total_branches += len(list(token.children))

    return total_branches / len(list(doc.sents))


# ------------------------------------------------------------
# 3. Intent Dispersion Entropy
# ------------------------------------------------------------
INTENT_KEYWORDS = {
    "memory": ["AI memory", "biological memory", "computer memory", "short-term memory"],
    "model": ["AI model", "statistical model", "mental model"],
    "system": ["computer system", "biological system", "social system"],
    "process": ["biological process", "computational process", "legal process"],
}

def intent_entropy(text: str) -> float:
    tokens = set([t.lower() for t in nltk.word_tokenize(text) if t.isalpha()])
    intents = 0

    for word, interpretations in INTENT_KEYWORDS.items():
        if word in tokens:
            intents += len(interpretations)

    return float(intents)


# ------------------------------------------------------------
# 4. Composite Semantic Entropy Score
# ------------------------------------------------------------
def semantic_entropy(text: str) -> dict:
    L = lexical_entropy(text)
    B = branching_entropy(text)
    I = intent_entropy(text)

    # Weighted composite score
    score = (0.4 * L) + (0.4 * B) + (0.2 * I)

    return {
        "lexical_entropy": L,
        "branching_entropy": B,
        "intent_entropy": I,
        "composite_entropy": score,
    }


# ------------------------------------------------------------
# Demo
# ------------------------------------------------------------
if __name__ == "__main__":
    sample = "Before we proceed, can you clarify what you meant earlier about memory?"
    result = semantic_entropy(sample)

    print("Semantic Entropy Analysis:")
    for k, v in result.items():
        print(f"{k:20s}: {v:.4f}")




This is a real canonical rewriter:

Code: [Select]

#!/usr/bin/env python3
"""
canonical_rewriter.py

Uses spaCy to rewrite user queries into a low-entropy,
canonical form suitable for transformer reasoning.
"""

import spacy

nlp = spacy.load("en_core_web_sm")

def extract_main_verb(doc):
    for token in doc:
        if token.pos_ == "VERB" and token.dep_ in ("ROOT", "advcl", "ccomp"):
            return token
    return None

def extract_object(verb):
    objs = [child for child in verb.children if child.dep_ in ("dobj", "pobj", "attr")]
    return objs[0] if objs else None

def extract_entities(doc):
    return [ent.text for ent in doc.ents]

def canonical_rewrite(text: str) -> str:
    doc = nlp(text)

    verb = extract_main_verb(doc)
    if not verb:
        return text  # fallback

    obj = extract_object(verb)
    entities = extract_entities(doc)

    # Build canonical form
    parts = []

    # Canonical verb
    canonical_verb = {
        "clarify": "explain",
        "describe": "explain",
        "tell": "explain",
        "show": "demonstrate",
        "give": "provide",
        "list": "list",
        "identify": "identify",
    }.get(verb.lemma_, verb.lemma_)

    parts.append(canonical_verb.capitalize())

    # Object
    if obj:
        parts.append(obj.text)

    # Entities (disambiguation)
    if entities:
        parts.append("about " + ", ".join(entities))

    return " ".join(parts)

if __name__ == "__main__":
    sample = "Before we proceed, can you clarify what you meant earlier about memory?"
    print("Original:", sample)
    print("Canonical:", canonical_rewrite(sample))




This version does not require editing for each query.
It rewrites anything you feed it.

Code: [Select]

#!/usr/bin/env python3
"""
canonical_rewriter.py

General-purpose canonical query rewriter.
No per-query editing. No hard-coded samples.
"""

import spacy

nlp = spacy.load("en_core_web_sm")

# Universal polite → command verb mapping
VERB_MAP = {
    "clarify": "explain",
    "describe": "explain",
    "tell": "explain",
    "show": "demonstrate",
    "give": "provide",
    "list": "list",
    "identify": "identify",
    "explain": "explain",
    "define": "define",
    "summarize": "summarize",
    "compare": "compare",
    "analyze": "analyze",
}

# Universal filler removal
FILLER_PHRASES = [
    "before we proceed",
    "at this point",
    "if possible",
    "could you",
    "can you",
    "would you",
    "please",
    "kindly",
    "i was wondering",
    "i want to know",
    "i need to know",
]

def remove_filler(text):
    lowered = text.lower()
    for phrase in FILLER_PHRASES:
        lowered = lowered.replace(phrase, "")
    return lowered.strip()

def canonical_rewrite(text: str) -> str:
    cleaned = remove_filler(text)
    doc = nlp(cleaned)

    # Extract main verb
    verb = None
    for token in doc:
        if token.pos_ == "VERB" and token.dep_ in ("ROOT", "ccomp", "advcl"):
            verb = token
            break

    if not verb:
        return cleaned  # fallback

    # Canonical verb
    canonical_verb = VERB_MAP.get(verb.lemma_, verb.lemma_)

    # Extract object
    obj = None
    for child in verb.children:
        if child.dep_ in ("dobj", "pobj", "attr", "oprd"):
            obj = child
            break

    # Extract entities
    entities = [ent.text for ent in doc.ents]

    # Build canonical form
    parts = [canonical_verb.capitalize()]

    if obj:
        parts.append(obj.text)

    if entities:
        parts.append("about " + ", ".join(entities))

    return " ".join(parts)