usage.py

# -*- coding: utf-8 -*-
#
# maya2vec: Word embeddings for Yucatec Maya.
#
# maya2vec embeddings use 512 dimensions and were trained
# using the Skip-gram with Negative Sampling algorithm (SGNS)
# on data from La Jornada Maya (collaboration agreement),
# CENTROGEO - SEDECULTA phrases (Agreement SEDECULTA-DASJ-149-04-2024),
# and the T'aantsil corpus project (https://taantsil.com.mx/info).
#
# Writen by: Alejandro Molina Villegas
# Contact: amolina@centrogeo.org.mx
# Date: April 2025


import re
from gensim.models import Word2Vec


# Simple tokenizer for Yucatec Maya
def my_simple_tokenizer(text):
    """Tokenizes text while preserving punctuation as separate tokens."""
    return [x.lower() for x in re.split(r"([.,;:¡!¿?]+)?\s+", text) if x]


# Load the trained Word2Vec model
# <class 'gensim.models.doc2vec.Doc2Vec'>
maya2vec_path = "./model_512_60_5_-0.25_0.7308_3.35E-05"
model = Word2Vec.load(maya2vec_path)
print("Model loaded successfully.",type(model))


# Generate word embedding for a single word
# <class 'numpy.ndarray'> (512,)
word = "meyaj"
if word in model.wv:
    vector = model.wv[word]
    print(f"Semantic encoded word '{word}' in", type(vector), vector.shape)
else:
    print(f"The word '{word}' is out-of-vocabulary (OOV).")


# Generate document embedding
# <class 'numpy.ndarray'> (512,)
text = "Bix a bel Táan in bin ich kool Tene' ooxolen"
tokens = my_simple_tokenizer(text)
try:
    vector = model.wv.get_mean_vector(tokens)
    print("Semantic encoded text in", type(vector), vector.shape)
except KeyError:
    print("Some words in the input text are OOV, affecting the embedding computation.")


# Compute cosine similarity between two words
# Similarity between 'peek'' and 'waalak'': 0.9583
word1, word2 = "peek'", "waalak'"
if word1 in model.wv and word2 in model.wv:
    similarity = model.wv.similarity(word1, word2)
    print(f"Similarity between '{word1}' and '{word2}': {similarity:.4f}")
else:
    print(f"One or both words ('{word1}', '{word2}') are OOV.")


# Handling OOV words
unknown_word = "furnance"
if unknown_word in model.wv:
    vector = model.wv[unknown_word]
else:
    print(f"The word '{unknown_word}' is OOV.")