Embeddings: How Geometry Pretends to Be Meaning

import os, warnings, datetime, numpy as np, pandas as pd, torch
os.environ["TOKENIZERS_PARALLELISM"] = "false"  # silence the tokenizers fork warning
warnings.filterwarnings("ignore")
import transformers; transformers.logging.set_verbosity_error()
from sentence_transformers import SentenceTransformer

np.random.seed(0); torch.manual_seed(0)

def cos(a, b) -> float:
    a, b = np.asarray(a, float), np.asarray(b, float)
    return float(a @ b / (np.linalg.norm(a) * np.linalg.norm(b)))

mini = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")

def emb(texts, model=mini, normalize=True):
    return model.encode(texts, normalize_embeddings=normalize, show_progress_bar=False)

print("executed:", datetime.date.today().isoformat())
print("torch", torch.__version__, "· transformers", transformers.__version__)
print("workhorse model: all-MiniLM-L6-v2 · dim =", mini.get_sentence_embedding_dimension())

texts = [
    "overheating",
    "My car's engine keeps overheating on the highway after about twenty minutes "
    "of driving in heavy summer traffic, especially when the air conditioning is on.",
]
V = emb(texts)
print("short text  ->", V[0].shape)
print("long text   ->", V[1].shape, "(same width — everything is squeezed through it)")

words = ["cat", "kitten", "small domestic animal",
         "car engine overheating", "vehicle temperature problem",
         "quarterly revenue forecast"]
E = emb(words)
pd.DataFrame(np.round(E @ E.T, 2), index=words, columns=[w[:13] for w in words])

from transformers import AutoTokenizer, AutoModel
tok = AutoTokenizer.from_pretrained("distilbert-base-uncased")
bert = AutoModel.from_pretrained("distilbert-base-uncased").eval()

@torch.no_grad()
def bert_mean(texts):
    enc = tok(list(texts), padding=True, truncation=True, return_tensors="pt")
    h = bert(**enc).last_hidden_state
    m = enc["attention_mask"].unsqueeze(-1).float()
    return ((h * m).sum(1) / m.sum(1)).numpy()

unrelated = [
    "The recipe calls for two cups of flour and a pinch of salt.",
    "Quarterly revenue rose across all three regions last year.",
    "The hiking trail closes at dusk during the winter months.",
    "She tuned the guitar before the evening rehearsal.",
    "Photosynthesis converts sunlight into chemical energy in plants.",
    "The train to the airport leaves every fifteen minutes.",
    "He repainted the fence a pale shade of green on Sunday.",
    "The committee postponed the vote until next Thursday.",
    "Saturn's rings are made mostly of ice and rock.",
    "The cafe on the corner serves oat-milk lattes.",
    "Interest rates were left unchanged at the latest meeting.",
    "The novel is set in a coastal town in the 1950s.",
    "Add the onions and cook until translucent.",
    "The marathon route passes three city bridges.",
    "Gravity bends the path of light near massive objects.",
    "The printer is out of cyan ink again.",
    "Their flight was delayed by a thunderstorm.",
    "The museum's new wing opens to the public in spring.",
    "Bees communicate the location of flowers by dancing.",
    "He filed the tax return a day before the deadline.",
    "The orchestra rehearsed the symphony's final movement.",
    "Sea turtles return to the beach where they hatched.",
    "The software update fixed the battery-drain bug.",
    "A cold front will move through the valley overnight.",
    "The library extended its hours during exam week.",
    "The bridge was repainted to prevent corrosion.",
    "Volcanic soil is unusually fertile for growing coffee.",
    "She swapped the summer tires for winter ones.",
    "The startup raised a modest seed round in March.",
    "The choir sang two encores to a full hall.",
    "Migrating geese fly in a V to save energy.",
    "The contract was signed after weeks of negotiation.",
    "He grilled the vegetables over a low flame.",
    "The telescope captured a faint distant galaxy.",
    "The ferry crosses the strait twice a day.",
    "The class debated the ethics of the proposal.",
]

def random_pair_cos(vecs):
    vn = vecs / np.linalg.norm(vecs, axis=1, keepdims=True)
    S = vn @ vn.T
    return S[np.triu_indices(len(vecs), k=1)]

raw = random_pair_cos(bert_mean(unrelated))
trained = random_pair_cos(emb(unrelated, normalize=False))
print(f"raw distilbert (mean-pooled): mean cosine of random pairs = {raw.mean():.2f}")
print(f"all-MiniLM-L6 (contrastive) : mean cosine of random pairs = {trained.mean():.2f}")

import matplotlib.pyplot as plt
plt.figure()
plt.hist(raw, bins=18, alpha=.7, color="#9a6f2f", label=f"raw distilbert  (μ={raw.mean():.2f})")
plt.hist(trained, bins=18, alpha=.7, color="#2c5142", label=f"all-MiniLM  (μ={trained.mean():.2f})")
plt.xlabel("cosine of random, unrelated pairs"); plt.ylabel("count"); plt.legend()
plt.tight_layout(); plt.show()

a, b = emb(["how do I reset my password?", "steps to recover a forgotten login"])
c = cos(a, b)
print(f"cosine            = {c:.4f}")
print(f"‖a-b‖²            = {np.sum((a-b)**2):.4f}")
print(f"2 - 2·cos         = {2 - 2*c:.4f}   (identical)")

# ranking equivalence over a small candidate set
q = emb(["how do I reset my password?"])[0]
cands = emb(["reset your account password", "the cat sat on the mat",
             "recover a forgotten login", "quarterly revenue forecast"])
by_cos = np.argsort(-(cands @ q))
by_euc = np.argsort(np.sum((cands - q)**2, axis=1))
print("rank by cosine  :", by_cos.tolist())
print("rank by euclid  :", by_euc.tolist(), "  -> same order:", (by_cos == by_euc).all())

pairs = [
    ("the treatment is safe for patients",      "the treatment is not safe for patients", "negation"),
    ("John sued Mary",                          "Mary sued John",                          "entity roles"),
    ("revenue grew 5% this quarter",            "revenue grew 50% this quarter",           "quantity"),
    ("the outage may happen tomorrow",          "the outage did happen yesterday",         "tense / modality"),
]
rows = [(k, round(cos(*emb([a, b])), 3)) for a, b, k in pairs]
pd.DataFrame(rows, columns=["distinction the model should make", "cosine(A, B)"])

q = emb(["is this treatment safe for patients?"])[0]
print(f"cos(query, '...is safe...')     = {cos(q, emb(['the treatment is safe for patients'])[0]):.3f}")
print(f"cos(query, '...is NOT safe...') = {cos(q, emb(['the treatment is not safe for patients'])[0]):.3f}")
print("Both land high — the opposite statement is only a hair behind, easily close enough to outrank real evidence.")

e5 = SentenceTransformer("intfloat/e5-base-v2")
queries = ["how do I reset my password?", "what is the refund window?", "how long does shipping take?",
           "how do I update my billing card?", "is my customer data encrypted?"]
passages = [
    "To reset your password, open Settings, choose Security, and follow the reset link emailed to you.",
    "Our refund policy allows returns within thirty days of the original purchase date for a full refund.",
    "Standard shipping is delivered within five to seven business days after the order is dispatched.",
    "Update the credit card on file under Billing, then Payment methods, then Edit card.",
    "All customer data is encrypted at rest with AES-256 and in transit over TLS 1.3.",
    # hard near-misses: same surface, wrong intent
    "To change your account email, open Settings, choose Profile, and follow the verification link emailed to you.",
    "Our privacy policy explains how account data is retained for thirty days before permanent deletion.",
    "Express shipping is delivered within one to two business days for an additional fee at checkout.",
    "Update the mailing address on file under Profile, then Addresses, then Edit address.",
    "All account passwords are hashed with bcrypt before they are written to the database.",
]
def mrr(model, qpre, dpre):
    Q = model.encode([qpre + q for q in queries], normalize_embeddings=True, show_progress_bar=False)
    D = model.encode([dpre + p for p in passages], normalize_embeddings=True, show_progress_bar=False)
    S = Q @ D.T
    return float(np.mean([1.0 / (1 + int(np.sum(S[i] > S[i, i]))) for i in range(len(queries))]))

print("mean reciprocal rank of the correct passage (1.0 = always ranked first):")
print(f"  E5  · correct prefixes (query:/passage:) = {mrr(e5, 'query: ', 'passage: '):.3f}")
print(f"  E5  · no prefix                          = {mrr(e5, '', ''):.3f}")
print(f"  E5  · swapped (passage:/query:)          = {mrr(e5, 'passage: ', 'query: '):.3f}")
print(f"  all-MiniLM · no prefix (its correct mode)= {mrr(mini, '', ''):.3f}")

@torch.no_grad()
def bert_pool(texts, how):
    enc = tok(list(texts), padding=True, truncation=True, return_tensors="pt")
    h = bert(**enc).last_hidden_state
    if how == "cls":
        return h[:, 0].numpy()
    m = enc["attention_mask"].unsqueeze(-1).float()
    return ((h * m).sum(1) / m.sum(1)).numpy()

t = ["how do I reset my password", "steps to recover a forgotten login", "the cat sat on the mat"]
cls, mean = bert_pool(t, "cls"), bert_pool(t, "mean")
print(f"CLS  : cos(reset, recover)={cos(cls[0],cls[1]):.3f}   cos(reset, cat)={cos(cls[0],cls[2]):.3f}")
print(f"mean : cos(reset, recover)={cos(mean[0],mean[1]):.3f}   cos(reset, cat)={cos(mean[0],mean[2]):.3f}")

big = SentenceTransformer("intfloat/multilingual-e5-large")  # 1024-dim
docs = ["passage: " + d for d in [
    "To reset your password, open Settings and choose Security, then Reset.",
    "Change your account password from the Security tab after email verification.",
    "Our refund policy allows returns within thirty days of purchase.",
    "Refunds are issued to the original payment method within five business days.",
    "The vacation policy grants twenty paid days off per calendar year.",
    "Unused vacation days roll over up to a maximum of five days.",
    "Contact support if your account is locked after failed logins.",
    "Accounts lock automatically after five failed login attempts.",
    "Invoices are issued on the first business day of each month.",
    "Download past invoices from the Billing page under Invoice history.",
    "Two-factor authentication can be enabled in the Security settings.",
    "Standard shipping is delivered within five to seven business days.",
    "Express shipping arrives in one to two business days for a fee.",
    "Update the credit card on file under Billing, then Payment methods.",
    "All customer data is encrypted at rest with AES-256 and in transit over TLS.",
    "Export your account data as a CSV from the Privacy settings.",
    "Team members can be invited from the Members page by an admin.",
    "Notification preferences are managed under Account, then Notifications.",
    "The API rate limit is one thousand requests per minute per key.",
    "Webhooks can be configured in the Integrations panel.",
    "Cancel your subscription anytime from the Billing page before renewal.",
    "Upgrading your plan takes effect immediately and is prorated.",
    "Set your time zone under Account preferences to fix timestamp display.",
    "The mobile app supports offline mode for previously loaded content.",
    "Single sign-on is available on the enterprise plan via SAML.",
]]
D = big.encode(docs, normalize_embeddings=True, show_progress_bar=False)
qv = big.encode(["query: how do I change my account password?"],
                normalize_embeddings=True, show_progress_bar=False)[0]

def renorm(M, dim):
    Mt = M[..., :dim]
    return Mt / np.linalg.norm(Mt, axis=-1, keepdims=True)

def ranks(x):                        # dense ranks (argsort of argsort) — no scipy needed
    return np.argsort(np.argsort(x))

full = renorm(D, 1024) @ renorm(qv, 1024)   # full-dimensional ranking — the reference order

rows = []
for dim in [1024, 512, 256, 128, 64, 32, 16]:
    sims = renorm(D, dim) @ renorm(qv, dim)
    top = np.sort(sims)[::-1]
    rank_corr = float(np.corrcoef(ranks(sims), ranks(full))[0, 1])  # retrieval order vs full-1024
    rows.append([dim, round(float(top[0] - top[1]), 3), round(rank_corr, 2),
                 f"{4*dim/1024:.1f} KB", f"{4*dim*1_000_000/1e9:.1f} GB"])
pd.DataFrame(rows, columns=["dim", "top1–top2 margin", "order vs full-1024 (rank corr)",
                            "per vector", "per 1M vectors"])

rng = np.random.default_rng(0)

# A single-domain knowledge base: many specific, *similar* help-center lines.
acts   = ["reset", "update", "change", "verify", "cancel", "renew", "export", "enable",
          "disable", "upgrade", "transfer", "delete", "restore", "configure", "schedule"]
objs   = ["password", "billing card", "email address", "subscription plan", "API key",
          "two-factor login", "notification settings", "account data", "shipping address",
          "team seats", "invoice history", "display name", "payment method", "time zone", "webhook"]
places = ["Settings", "the Billing page", "your Profile", "the Security tab",
          "the Admin console", "Account preferences", "the Integrations panel"]
tails  = ["after you confirm by email.", "once the change is verified.",
          "and it takes effect immediately.", "before the next billing cycle.",
          "and a confirmation is sent to you."]
combos = [(a, o, p, t) for a in acts for o in objs for p in places for t in tails]
rng.shuffle(combos)
kb = [f"To {a} your {o}, open {p} and save {t}" for a, o, p, t in combos[:1500]]

# Generic filler: same domain words, no specific content.
filler = ["For more information, please contact our support team.",
          "Please refer to the documentation for further details.",
          "Was this article helpful? Let us know your feedback.",
          "Thank you for using our product and services.",
          "See the related articles below for more assistance.",
          "These settings can be managed from your account at any time."]
corpus = kb + filler
n_kb = len(kb)

C = emb(corpus, normalize=True)                       # all-MiniLM, the workhorse
S = C @ C.T; np.fill_diagonal(S, -1.0)
k = 10
indeg = np.bincount(np.argpartition(-S, k, axis=1)[:, :k].ravel(), minlength=len(corpus))
med = float(np.median(indeg[:n_kb]))

print(f"corpus = {len(corpus)} lines ({n_kb} help articles + {len(filler)} generic filler) · fair share k = {k}")
print(f"help-article in-degree:  median {med:.0f} · mean {indeg[:n_kb].mean():.1f} · "
      f"p99 {np.percentile(indeg[:n_kb], 99):.0f} · max {indeg[:n_kb].max()}   <- heavy head = hubness\n")
print("heaviest hubs — the densest near-duplicate articles, neighbor to many:")
for i in np.argsort(-indeg)[:4]:
    print(f"  {indeg[i]:3}×   {corpus[i][:56]}")
print("\ngeneric filler — far BELOW the median: an anti-hub. It shares the words, not the geometry:")
for i in range(n_kb, len(corpus)):
    print(f"  {indeg[i]:3}×   {corpus[i][:56]}")

plt.figure()
plt.hist(indeg[:n_kb], bins=range(0, int(indeg[:n_kb].max()) + 2), color="#4a6b8a", alpha=.85, label="help articles")
plt.axvline(k, color="#9a6f2f", linestyle="--", label=f"fair share (k={k})")
plt.axvline(indeg[n_kb:].max(), color="#c4521e", linestyle=":", label=f"heaviest filler ({indeg[n_kb:].max()}×)")
plt.xlabel(f"times a line appears in others' top-{k}"); plt.ylabel("number of lines"); plt.legend()
plt.tight_layout(); plt.show()

from sklearn.feature_extraction.text import TfidfVectorizer
corpus2 = [
    "To resolve ERR_TIMEOUT_4012, raise the gateway read timeout in the proxy config.",   # correct
    "To resolve ERR_TIMEOUT_4093, restart the upstream authentication service.",          # same shape, wrong code
    "High network latency causes request timeouts when the upstream service is slow.",
    "Restart the service and verify the firewall rules for any blocked ports.",
]
query = "ERR_TIMEOUT_4012"

dsims = emb(corpus2) @ emb([query])[0]
vec = TfidfVectorizer().fit(corpus2 + [query])
ssims = (vec.transform(corpus2) @ vec.transform([query]).T).toarray().ravel()

print(f"query: {query}   (correct = #0; #1 is a different code of the same shape)\n")
print(f"dense  (MiniLM)  #0 {dsims[0]:.3f} vs #1 {dsims[1]:.3f}   gap {abs(dsims[0]-dsims[1]):.3f}  <- can it tell the codes apart?")
print(f"sparse (TF-IDF)  #0 {ssims[0]:.3f} vs #1 {ssims[1]:.3f}   gap {abs(ssims[0]-ssims[1]):.3f}")
print(f"\ndense top-1 = #{int(np.argmax(dsims))} · sparse top-1 = #{int(np.argmax(ssims))} (sparse locks onto the exact string)")

big_chunk = ("Vision correction is covered under the plan. "
             "This benefit applies only after 12 months of continuous coverage.")
small_a = "Vision correction is covered under the plan."
small_b = "This benefit applies only after 12 months of continuous coverage."
qv = emb(["is vision correction covered?"])[0]

for name, txt in [("large chunk (answer + condition)", big_chunk),
                  ("small chunk A (answer only)",      small_a),
                  ("small chunk B (condition only)",   small_b)]:
    print(f"  cos = {cos(qv, emb([txt])[0]):.3f}   {name}")
print("\nThe small answer-only chunk retrieves confidently — and silently drops the 12-month condition.")