GensimによるWord2Vecの学習と使用

#!pip install --upgrade gensim
#!pip install mecab-python3
#!pip install unidic-lite

from gensim.models import Word2Vec

sample_sents = [['you', 'say', 'goodbye', 'and', 'I', "say", "hello" '.']]
model = Word2Vec(sentences=sample_sents, vector_size=5, window=1, min_count=1)

model.wv['you']

with open('./Data/lee_background.cor', 'r') as file:
    corpus = file.readlines()

processed_corpus = [line.lower().split() for line in corpus]

model = Word2Vec(
    sentences=processed_corpus,  # 学習対象のコーパス
    vector_size=100,             # 埋め込みベクトルの次元数
    window=5,                    # コンテキストウィンドウのサイズ
    min_count=5,                 # 登場頻度が5未満の単語は無視
    sg=1,                        # 1でskip-gramを使用（0ならCBOW）
    negative=5                   # negative samplingのサンプル数
)

import MeCab
def tokenize(text):
    """ テキストを形態素解析して、トークンのリストを返す """
    mecab = MeCab.Tagger("-Owakati")
    return mecab.parse(text).strip().split()

documents = ["これはサンプルの文書です。", "Word2Vecの学習を行います。"]

# 形態素解析を行い、単語リストに変換
tokenized_documents = [tokenize(doc) for doc in documents]

# Word2Vecモデルの訓練

model_jp = Word2Vec(sentences=tokenized_documents, vector_size=100, window=5, min_count=1, workers=4)

model.wv['you']

model.wv.similarity("you", "your")

model.wv.most_similar("you")

# モデルの保存と読み込み
#model.save("word2vec.model")
#model = Word2Vec.load("word2vec.model")

import gensim.downloader

print(list(gensim.downloader.info()['models'].keys()))

model = gensim.downloader.load('word2vec-google-news-300')

similarity = model.similarity('woman', 'man')
similarity

model['king']

import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
import numpy as np


# List of words to visualize
words = ['woman', 'women', 'man', 'men', 'king', 'queen', 'prince', 'princess']

# Check if the words are in the model to avoid KeyError
vectors = [model[word] for word in words if word in model]

# Converting list of vectors to a numpy array
vectors_array = np.array(vectors)

# Applying t-SNE for dimensionality reduction
tsne = TSNE(n_components=2, random_state=0, perplexity=3)
vectors_tsne = tsne.fit_transform(vectors_array)

# Visualization
plt.figure(figsize=(4, 4))
for i, word in enumerate(words):
    if word in model:
        plt.scatter(vectors_tsne[i, 0], vectors_tsne[i, 1])
        plt.annotate(word, (vectors_tsne[i, 0], vectors_tsne[i, 1]))

plt.xlabel('t-SNE Feature 0')
plt.ylabel('t-SNE Feature 1')
plt.title('t-SNE Visualization of Word Vectors')
plt.show()

from tensorboardX import SummaryWriter
import torch

# 分散表現・単語のリストを取得
weights = model.vectors
labels = model.index_to_key

weights = weights[:1000]
labels = labels[:1000]

writer = SummaryWriter('runs/google_embeddings')
writer.add_embedding(torch.FloatTensor(weights), metadata=labels)