Text Clustering: Representation Over Algorithm

import os, json, time, datetime, urllib.parse, urllib.request
import numpy as np, pandas as pd, torch
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD, NMF
from sklearn.preprocessing import Normalizer
from sklearn.pipeline import make_pipeline
from sklearn.cluster import KMeans, AgglomerativeClustering, SpectralClustering, DBSCAN
from sklearn.mixture import GaussianMixture
from sklearn.neighbors import NearestNeighbors
from sklearn.metrics import adjusted_rand_score, normalized_mutual_info_score
from sentence_transformers import SentenceTransformer
import umap, hdbscan

SEED = 0
np.random.seed(SEED)
# the clustering here is all CPU (scikit-learn); only the E5 embedding step uses a
# GPU when one is around, and just for speed — the numbers are identical either way
device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
e5 = SentenceTransformer("intfloat/multilingual-e5-base", device=device)   # one embedding model, both experiments

def ari_nmi(pred, true):
    return round(adjusted_rand_score(true, pred), 3), round(normalized_mutual_info_score(true, pred), 3)

print("executed:", datetime.date.today().isoformat())

groups = {
  "login": ["I can't sign in", "my password won't work", "locked out and can't get in",
            "authentication keeps failing", "the portal rejects my credentials",
            "it says my details are incorrect every time", "two-factor never sends me a code",
            "the reset link in my email does nothing", "it keeps logging me out immediately",
            "my session expires the second I connect", "the app won't let me through the front door",
            "my credentials aren't recognized anymore", "stuck on the verification screen",
            "can't get past the security check on my new phone", "my identity won't validate"],
  "billing": ["I was charged twice for one order", "there is a duplicate transaction on my card",
              "you billed me again this month", "an unexpected fee showed up on my statement",
              "my invoice is higher than the agreed price", "please refund the extra payment",
              "the amount deducted is wrong", "I see two identical debits",
              "my bank shows a repeated withdrawal", "overcharged compared to the quote",
              "the receipt total does not match", "money left my account twice",
              "there is a surcharge I never agreed to", "the plan cost jumped without notice",
              "I want the second charge reversed"],
  "delivery": ["where is my package", "the order has not arrived yet", "tracking shows no movement for days",
               "my parcel is running late", "it was supposed to be here Monday", "the courier never showed up",
               "shipment stuck at the depot", "nothing has been dispatched", "the estimated date keeps slipping",
               "I still have not received my items", "marked delivered but I have nothing",
               "no update since it left the warehouse", "seems lost in transit",
               "when will my goods get here", "the carrier cannot locate my parcel"],
  "cancel": ["I want to cancel my subscription", "please close my account for good", "stop the recurring charge",
             "end my membership immediately", "unsubscribe me from the plan", "I no longer want this service",
             "terminate my contract", "take me off the monthly billing", "shut down my profile permanently",
             "opt me out of renewals", "I would like to discontinue everything", "wind down my account",
             "remove my recurring payment", "I am done, delete my registration", "cease all future charges"],
}
tickets = [t for g in groups.values() for t in g]
t_true = np.array([i for i, g in enumerate(groups.values()) for _ in g])

X_tf = TfidfVectorizer(stop_words="english").fit_transform(tickets)
X_e5 = e5.encode(["query: " + t for t in tickets], normalize_embeddings=True, show_progress_bar=False)

print("cluster 60 paraphrased tickets into 4 intents — same algorithm, two representations:")
for name, run in [("K-means", lambda Z: KMeans(4, n_init=10, random_state=SEED).fit_predict(Z)),
                  ("Agglomerative", lambda Z: AgglomerativeClustering(4).fit_predict(np.asarray(Z.todense()) if hasattr(Z, "todense") else Z))]:
    lt, le = run(X_tf), run(X_e5)
    print(f"  {name:14s}  TF-IDF ARI {ari_nmi(lt, t_true)[0]:.3f}   E5 ARI {ari_nmi(le, t_true)[0]:.3f}")

CACHE, CATS = "clustering-corpus.json", ["cs.CV", "cs.CL", "cs.CR", "cs.LG", "stat.ML"]
SHORT = {"cs.CV": "vision", "cs.CL": "NLP", "cs.CR": "security", "cs.LG": "ML", "stat.ML": "stats-ML"}

def fetch_arxiv(cat, want=110):
    import feedparser
    seen, out, start = set(), [], 0
    while len(out) < want and start < 1600:
        q = urllib.parse.urlencode({"search_query": f"cat:{cat}", "start": start, "max_results": 100,
                                    "sortBy": "submittedDate", "sortOrder": "descending"})
        req = urllib.request.Request("http://export.arxiv.org/api/query?" + q, headers={"User-Agent": "lena-notes/1.0"})
        for e in feedparser.parse(urllib.request.urlopen(req, timeout=40).read()).entries:
            if e.get("arxiv_primary_category", {}).get("term") != cat:
                continue
            t = " ".join(e.summary.split())
            if len(t.split()) >= 40 and e.id not in seen:
                seen.add(e.id); out.append(t)
            if len(out) >= want:
                break
        start += 100; time.sleep(3)                            # arXiv etiquette
    return out

if os.path.exists(CACHE):
    with open(CACHE, encoding="utf-8") as f:                    # committed snapshot → reproducible
        data = json.load(f)
else:
    texts, labels = [], []
    for ci, c in enumerate(CATS):
        for t in fetch_arxiv(c): texts.append(t); labels.append(ci)
    order = np.random.RandomState(SEED).permutation(len(texts))
    data = {"names": [SHORT[c] for c in CATS], "labels": [int(labels[i]) for i in order],
            "texts": [texts[i] for i in order]}
    with open(CACHE, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False)

texts, true, names = data["texts"], np.array(data["labels"]), data["names"]
K_TRUE = len(names)
print(f"{len(texts)} arXiv abstracts · {K_TRUE} areas")
for i, nm in enumerate(names):
    print(f"  {nm:9s} {(true == i).sum():>3}")

tfidf = TfidfVectorizer(max_features=20000, stop_words="english", min_df=5).fit(texts)
X_tfidf = tfidf.transform(texts)
lexical = make_pipeline(TruncatedSVD(200, random_state=SEED), Normalizer(copy=False)).fit_transform(X_tfidf)
emb = e5.encode(["passage: " + t for t in texts], normalize_embeddings=True,
                batch_size=32, show_progress_bar=False).astype("float32")
print("lexical (TF-IDF→SVD):", lexical.shape, "  semantic (E5):", emb.shape)

km_lex = KMeans(K_TRUE, n_init=10, random_state=SEED).fit_predict(lexical)
km_emb = KMeans(K_TRUE, n_init=10, random_state=SEED).fit_predict(emb)
print("K-means, same algorithm, two representations:")
print(f"  lexical  (TF-IDF)  →  ARI {ari_nmi(km_lex, true)[0]:.3f}   NMI {ari_nmi(km_lex, true)[1]:.3f}")
print(f"  semantic (E5)      →  ARI {ari_nmi(km_emb, true)[0]:.3f}   NMI {ari_nmi(km_emb, true)[1]:.3f}")

proj2 = umap.UMAP(n_components=2, n_neighbors=15, min_dist=0.1, metric="cosine",
                  random_state=SEED).fit_transform(emb)                       # 2-D, display only
red5 = umap.UMAP(n_components=5, n_neighbors=15, min_dist=0.0, metric="cosine",
                 random_state=SEED).fit_transform(emb)                        # 5-D, for density methods

agg_lab = AgglomerativeClustering(n_clusters=K_TRUE, linkage="ward").fit_predict(emb)
gmm_lab = GaussianMixture(n_components=K_TRUE, covariance_type="diag", random_state=SEED).fit_predict(emb)
sc_lab = SpectralClustering(n_clusters=K_TRUE, affinity="nearest_neighbors", n_neighbors=15,
                            assign_labels="kmeans", random_state=SEED).fit_predict(emb)
nmf_lab = NMF(n_components=K_TRUE, init="nndsvd", max_iter=600, random_state=SEED).fit_transform(X_tfidf).argmax(1)
hdb_raw = hdbscan.HDBSCAN(min_cluster_size=20).fit_predict(emb)               # raw 768-D
hdb_u5 = hdbscan.HDBSCAN(min_cluster_size=20).fit_predict(red5)               # E5 → UMAP-5
ms = 10
eps = float(np.percentile(NearestNeighbors(n_neighbors=ms).fit(red5).kneighbors(red5)[0][:, -1], 65))
db_lab = DBSCAN(eps=eps, min_samples=ms).fit_predict(red5)

def row(rep, algo, pred):
    a, n = ari_nmi(pred, true)
    return {"representation": rep, "algorithm": algo, "ARI": a, "NMI": n,
            "clusters": len(set(pred.tolist()) - {-1}), "noise%": float(round(100 * np.mean(pred == -1), 1))}

table = pd.DataFrame([
    row("TF-IDF→SVD", "K-means", km_lex),      row("TF-IDF", "NMF (argmax)", nmf_lab),
    row("E5 768-D", "K-means", km_emb),        row("E5 768-D", "Agglomerative", agg_lab),
    row("E5 768-D", "GMM", gmm_lab),           row("E5 768-D", "Spectral", sc_lab),
    row("E5 768-D", "HDBSCAN", hdb_raw),       row("E5→UMAP-5", "HDBSCAN", hdb_u5),
    row("E5→UMAP-5", "DBSCAN", db_lab),
]).sort_values("ARI", ascending=False).reset_index(drop=True)
table

rows = []
for k in range(2, 11):
    a, n = ari_nmi(KMeans(k, n_init=10, random_state=SEED).fit_predict(emb), true)
    rows.append({"k": k, "ARI": a, "NMI": n})
pd.DataFrame(rows)

print("HDBSCAN, same algorithm, two geometries:")
print(f"  raw 768-D   →  {row('E5 768-D',  'HDBSCAN', hdb_raw)}")
print(f"  E5 → UMAP-5 →  {row('E5→UMAP-5', 'HDBSCAN', hdb_u5)}")

def contingency(pred):
    cl = sorted(set(pred.tolist()) - {-1})
    return pd.DataFrame([[int(((pred == c) & (true == t)).sum()) for t in range(K_TRUE)] for c in cl],
                        index=[f"cluster {c}" for c in cl], columns=names)
contingency(km_emb)

fig, ax = plt.subplots(1, 2, figsize=(7.2, 3.3))
pal = np.array(["#2c5142", "#c4521e", "#3b6ea5", "#b59410", "#7a4fa3"])
ax[0].scatter(proj2[:, 0], proj2[:, 1], c=pal[true], s=6, alpha=.8)
ax[0].set_title("hidden areas", fontsize=9)
ax[1].scatter(proj2[:, 0], proj2[:, 1], c=pal[km_emb % len(pal)], s=6, alpha=.8)
ax[1].set_title("E5 + K-means clusters", fontsize=9)
for a in ax:
    a.set_xticks([]); a.set_yticks([])
plt.tight_layout(); plt.show()

def to_area(pred):                                            # cluster id → its majority hidden area
    m = {c: np.bincount(true[pred == c], minlength=K_TRUE).argmax() for c in set(pred.tolist()) if c != -1}
    return np.array([m.get(c, -1) for c in pred])

lex_area, sem_area = to_area(km_lex), to_area(km_emb)
disagree = np.where(lex_area != sem_area)[0]
print(f"{len(disagree)} abstracts land in different areas under TF-IDF vs E5 — the review pile:\n")
for i in disagree[:3]:
    print(f"  TF-IDF says {names[lex_area[i]]:8s} · E5 says {names[sem_area[i]]:8s} · truly {names[true[i]]}")
    print(f"    {texts[i][:150]}…\n")