埋め込みベクトルでSlackのカスタム絵文字を分類・検索

from requests import Session
import yaml


list_emojis_url = (
    "https://edgeapi.slack.com/cache/xxxx/emojis/list?fp=ba&_x_num_retries=0"
) # ブラウザ上コピーした実のfetch句のURLに合わせる
cookie_string = '<ログイン処理が面倒なので、ブラウザのクッキー情報を引っ張ってくる>'
token = "xxxx" # こちらもブラウザの情報を引っ張る
s = Session()
for cookie in cookie_string.split("; "):
    name, value = cookie.split("=", 1)
    s.cookies.set(name, value)

marker = "_doing" # 1個目の絵文字名称
while True:
    json_data = {
        "token": token,
        "enterprise_token": token,
        "count": 100,
        "marker": marker,
    }

    response = s.post(url=list_emojis_url, json=json_data).json()

    with open("results.yaml", "a") as f:
        yaml.dump(response["results"], f)
    marker = response.get("next_marker")
    if marker is None:
        break

import os

import requests
import yaml


def read_yaml_result(filename):
    with open(filename) as input_data:
        return yaml.safe_load(input_data)


def download_image(file_path, url):
    try:
        response = requests.get(url)
        response.raise_for_status()
        with open(file_path, "wb") as file:
            file.write(response.content)
        print(f"Downloaded {file_path}")
    except requests.exceptions.RequestException as e:
        print(f"Failed to download {name}: {e}")


if __name__ == "__main__":
    input_file_name = "results.yaml"

    output_directory = "images"
    os.makedirs(output_directory, exist_ok=True)

    skip = True

    for item in read_yaml_result(input_file_name):
        name = item["name"]
        url = item["value"]
        if skip and name == "simple_smile":
            skip = False
        if skip:
            continue
        _, ext = os.path.splitext(url)
        file_path = os.path.join(output_directory, f"{name}{ext}")
        download_image(file_path, url)

import os

from PIL import Image
from sentence_transformers import SentenceTransformer

# Set the path to your images directory
images_dir = "images"

# Set a maximum number of images to process
max_images = 1000

# Load the model
model = SentenceTransformer("clip-ViT-B-32")

# Get the list of image file paths
image_files = [os.path.join(images_dir, file) for file in os.listdir(images_dir)][
    :max_images
]


# Load images and compute embeddings
images = []
image_labels = []
for image_file in image_files:
    images.append(Image.open(image_file))
    base, ext = os.path.splitext(os.path.basename(image_file))
    image_labels.append(base)

img_emb = model.encode(images)

print(img_emb)
print(img_emb.shape)

import matplotlib.pyplot as plt
from sklearn.manifold import TSNE

# Apply t-SNE to reduce dimensions to 2D
# Choosing a perplexity value appropriate for the dataset size
perplexity_value = min(30, len(img_emb) - 1)
# perplexity_value = 50
tsne = TSNE(n_components=2, perplexity=perplexity_value, random_state=0)
reduced_embeddings = tsne.fit_transform(img_emb)

num_clusters = 5  # Specify the number of clusters
kmeans = KMeans(n_clusters=num_clusters, random_state=0)
cluster_labels = kmeans.fit_predict(reduced_embeddings)


# Plot the reduced embeddings
plt.figure(figsize=(15, 9))
plt.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], alpha=0.5)

# Optional: Annotate points with image labels
for i, label in enumerate(image_labels):
    plt.text(
        reduced_embeddings[i, 0],
        reduced_embeddings[i, 1],
        label,
        fontsize=8,
        alpha=0.6,
        fontname="YuMincho", # 日本語ファイル名を化けないようにフォントの設定
    )

plt.title("t-SNE Visualization of Image Embeddings")
plt.xlabel("t-SNE Dimension 1")
plt.ylabel("t-SNE Dimension 2")
plt.grid(True)
plt.show()

from sklearn.cluster import KMeans
import plotly.express as px


num_clusters = 7  # Specify the number of clusters
kmeans = KMeans(n_clusters=num_clusters, random_state=0)
cluster_labels = kmeans.fit_predict(reduced_embeddings)


fig = px.scatter_3d(
    x=reduced_embeddings[:, 0],
    y=reduced_embeddings[:, 1],
    z=reduced_embeddings[:, 2],
    color=cluster_labels,  # Color points by cluster
    text=image_labels,  # Labels for each point
    labels={"x": "Component 1", "y": "Component 2", "z": "Component 3"},  # Axis labels
    title="3D t-SNE Visualization of Image Embeddings with Clusters",
    color_continuous_scale=px.colors.qualitative.G10,  # A color scale for discrete colors
)

query_emb = model.encode(Image.open("images/白旗.png"))
# query_emb = model.encode(Image.open("images/天才.png"))

# Compute cosine similarity between the query embedding and each image embedding
similarities = model.similarity(query_emb, img_emb).reshape((-1,))

# Get the indices of the most similar images
k = 5  # Number of nearest neighbors you want to find
top_k_indices = reversed(np.argsort(similarities)[-k:])

# Retrieve the top k similarities and corresponding images
top_k_similarities = similarities[top_k_indices]

# Print the results
for i, (index, score) in enumerate(zip(top_k_indices, top_k_similarities)):
    print(
        f"Rank {i + 1}: Image Name: {image_files[index]}, Similarity Score {score:.4f}"
    )

model2 = SentenceTransformer("clip-ViT-B-32-multilingual-v1")

query_emb = model2.encode("ポケモンボール")
# query_emb = model2.encode("上は赤、下は白、球体")
# query_emb = model2.encode("ビーチボール")
# query_emb = model2.encode("ビール")

# Compute cosine similarity between the query embedding and each image embedding
similarities = model.similarity(query_emb, img_emb).reshape((-1,))