Create app.py

app.py

@@ -0,0 +1,31 @@
+import re
+import torch
+import gradio as gr
+from sentence_transformers import SentenceTransformer
+
+# Load SentenceTransformer model
+model = SentenceTransformer('all-MiniLM-L6-v2')
+
+def process_text(text):
+    # Remove ASCII characters and lowercase
+    cleaned = re.sub(r'[^\x00-\x7F]+', '', text).lower()
+
+    # Get token embeddings
+    token_embeddings = model.encode(cleaned, output_value='token_embeddings', convert_to_tensor=True)
+    tokens = model.tokenizer.tokenize(cleaned)
+
+    # Pair each token with its embedding (truncated for display)
+    result = []
+    for token, emb in zip(tokens, token_embeddings):
+        result.append([token, str(emb[:5].tolist()) + '...'])  # truncate vector
+
+    return result
+
+# Gradio interface
+gr.Interface(
+    fn=process_text,
+    inputs=gr.Textbox(lines=4, placeholder="Enter text here..."),
+    outputs=gr.Dataframe(headers=["Token", "Embedding (truncated)"]),
+    title="SentenceTransformer Token Embeddings",
+    description="Removes ASCII, lowercases input, tokenizes and embeds with SentenceTransformer."
+).launch()