万字长文讲解文本嵌入及其高阶应用

import collectionsbag_of_words = collections.Counter(stemmed_words)print(bag_of_words)# {'we': 2, 'are': 2, 'in': 2, 'lucki': 1, 'to': 1, 'live': 1, # 'an': 1, 'age': 1, 'which': 1, 'still': 1, 'make': 1, 'discoveri': 1}

from openai import OpenAIclient = OpenAI()
def get_embedding(text, model="text-embedding-3-small"):   text = text.replace("\n", " ")   return client.embeddings.create(input = [text], model=model)\       .data[0].embedding
get_embedding("We are lucky to live in an age in which we are still making discoveries.")

import numpy as np
sum(list(map(lambda x, y: (x - y) ** 2, vector1, vector2))) ** 0.5# 2.2361
np.linalg.norm((np.array(vector1) - np.array(vector2)), ord = 2)# 2.2361

sum(list(map(lambda x, y: abs(x - y), vector1, vector2)))# 3
np.linalg.norm((np.array(vector1) - np.array(vector2)), ord = 1)# 3.0

sum(list(map(lambda x, y: x*y, vector1, vector2)))# 11
np.dot(vector1, vector2)# 11

dot_product = sum(list(map(lambda x, y: x*y, vector1, vector2)))norm_vector1 = sum(list(map(lambda x: x ** 2, vector1))) ** 0.5norm_vector2 = sum(list(map(lambda x: x ** 2, vector2))) ** 0.5
dot_product/norm_vector1/norm_vector2
# 0.8575
from sklearn.metrics.pairwise import cosine_similarity
cosine_similarity(  np.array(vector1).reshape(1, -1),   np.array(vector2).reshape(1, -1))[0][0]
# 0.8575

import mathmath.degrees(math.acos(0.8575))
# 30.96

import numpy as npembeddings_array = np.array(df.embedding.values.tolist())print(embeddings_array.shape)# (1400, 1536)

from sklearn.decomposition import PCA
pca_model = PCA(n_components = 2)pca_model.fit(embeddings_array)
pca_embeddings_values = pca_model.transform(embeddings_array)print(pca_embeddings_values.shape)# (1400, 2)

fig = px.scatter(    x = pca_embeddings_values[:,0],     y = pca_embeddings_values[:,1],    color = df.topic.values,    hover_name = df.full_text.values,    title = 'PCA embeddings', width = 800, height = 600,    color_discrete_sequence = plotly.colors.qualitative.Alphabet_r)
fig.update_layout(    xaxis_title = 'first component',     yaxis_title = 'second component')fig.show()

from sklearn.manifold import TSNEtsne_model = TSNE(n_components=2, random_state=42)tsne_embeddings_values = tsne_model.fit_transform(embeddings_array)
fig = px.scatter(    x = tsne_embeddings_values[:,0],     y = tsne_embeddings_values[:,1],    color = df.topic.values,    hover_name = df.full_text.values,    title = 't-SNE embeddings', width = 800, height = 600,    color_discrete_sequence = plotly.colors.qualitative.Alphabet_r)
fig.update_layout(    xaxis_title = 'first component',     yaxis_title = 'second component')fig.show()

embedding1 = df.loc[1].embeddingembedding2 = df.loc[616].embeddingembedding3 = df.loc[749].embedding

import seaborn as snsimport matplotlib.pyplot as pltembed_len_thr = 1536
sns.heatmap(np.array(embedding1[:embed_len_thr]).reshape(-1, embed_len_thr),    cmap = "Greys", center = 0, square = False,     xticklabels = False, cbar = False)plt.gcf().set_size_inches(15,1)plt.yticks([0.5], labels = ['AI'])plt.show()
sns.heatmap(np.array(embedding3[:embed_len_thr]).reshape(-1, embed_len_thr),    cmap = "Greys", center = 0, square = False,     xticklabels = False, cbar = False)plt.gcf().set_size_inches(15,1)plt.yticks([0.5], labels = ['AI'])plt.show()
sns.heatmap(np.array(embedding2[:embed_len_thr]).reshape(-1, embed_len_thr),    cmap = "Greys", center = 0, square = False,     xticklabels = False, cbar = False)plt.gcf().set_size_inches(15,1)plt.yticks([0.5], labels = ['Bioinformatics'])plt.show()

from sklearn.cluster import KMeansfrom sklearn.metrics import silhouette_scoreimport tqdm
silhouette_scores = []for k in tqdm.tqdm(range(2, 51)):    kmeans = KMeans(n_clusters=k,                     random_state=42,                     n_init = 'auto').fit(embeddings_array)    kmeans_labels = kmeans.labels_    silhouette_scores.append(        {            'k': k,            'silhouette_score': silhouette_score(embeddings_array,                 kmeans_labels, metric = 'cosine')        }    )
fig = px.line(pd.DataFrame(silhouette_scores).set_index('k'),       title = '<b>Silhouette scores for K-means clustering</b>',       labels = {'value': 'silhoutte score'},        color_discrete_sequence = plotly.colors.qualitative.Alphabet)fig.update_layout(showlegend = False)

tsne_model = TSNE(n_components=2, random_state=42)tsne_embeddings_values = tsne_model.fit_transform(embeddings_array)
fig = px.scatter(    x = tsne_embeddings_values[:,0],     y = tsne_embeddings_values[:,1],    color = list(map(lambda x: 'cluster %s' % x, kmeans_labels)),    hover_name = df.full_text.values,    title = 't-SNE embeddings for clustering', width = 800, height = 600,    color_discrete_sequence = plotly.colors.qualitative.Alphabet_r)fig.update_layout(    xaxis_title = 'first component',     yaxis_title = 'second component')fig.show()

from sklearn.ensemble import RandomForestClassifierfrom sklearn.model_selection import train_test_splitclass_model = RandomForestClassifier(max_depth = 10)
# defining features and targetX = embeddings_arrayy = df.topic
# splitting data into train and test setsX_train, X_test, y_train, y_test = train_test_split(    X, y, random_state = 42, test_size=0.2, stratify=y)
# fit & predict class_model.fit(X_train, y_train)y_pred = class_model.predict(X_test)

from sklearn.metrics import confusion_matrixcm = confusion_matrix(y_test, y_pred)
fig = px.imshow(  cm, x = class_model.classes_,  y = class_model.classes_, text_auto='d',   aspect="auto",   labels=dict(      x="predicted label", y="true label",       color="cases"),   color_continuous_scale='pubugn',  title = '<b>Confusion matrix</b>', height = 550)
fig.show()