{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Embedding\n",
"\n",
"Описание: NN_Embedding.html"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Необходимые библиотеки"
]
},
{
"cell_type": "code",
"execution_count": 414,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"python: 3.7.6 (tags/v3.7.6:43364a7ae0, Dec 19 2019, 00:42:30) [MSC v.1916 64 bit (AMD64)]\n",
"numpy: 1.18.1\n",
"matplotlib: 3.2.0\n",
"keras: 2.3.1\n"
]
}
],
"source": [
"import numpy as np # работа с тензорами\n",
"import matplotlib.pyplot as plt # постороение графиков\n",
"\n",
"from keras.models import Sequential, Model\n",
"from keras.layers import Input, Embedding, Dense, SimpleRNN, LSTM, Lambda\n",
"from keras.layers import Concatenate, Flatten, Bidirectional, Dot, Activation, Add\n",
"from keras.optimizers import SGD, Adam \n",
"\n",
"import sys, numpy, matplotlib, keras\n",
"print(\"python: \", sys.version)\n",
"print(\"numpy: \", numpy.__version__)\n",
"print(\"matplotlib:\", matplotlib.__version__)\n",
"print(\"keras: \", keras.__version__)\n",
"\n",
"np.set_printoptions(precision=3, suppress=True) # число цифр после точки в print"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Вывод структуры графа\n",
""
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"import os # Надо поставить graphviz c www.graphviz.org\n",
"from IPython.display import SVG, display\n",
"from keras.utils.vis_utils import model_to_dot\n",
"os.environ[\"PATH\"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'\n",
"\n",
"def svg_model(model):\n",
" \"\"\" Выводит модель нейронной сети в форме графа \"\"\"\n",
" gv = model_to_dot(model, show_shapes=True, dpi=65) # граф библиотеки graphviz\n",
" svg = SVG(gv.create(prog='dot', format='svg')) # рисуем как svg\n",
" display(svg) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Слой Embedding в Keras\n",
""
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_5\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_5 (Embedding) (None, 3, 2) 10 \n",
"=================================================================\n",
"Total params: 10\n",
"Trainable params: 10\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
},
{
"data": {
"image/svg+xml": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"VOC_SIZE = 5 # число слов в словаре\n",
"VEC_DIM = 2 # размерность векторного пространства\n",
"\n",
"model = Sequential() \n",
"model.add(Embedding(input_dim = VOC_SIZE, output_dim = VEC_DIM, input_length = 3))\n",
"\n",
"model.summary() # output: (batch_size, inputs, VEC_DIM)\n",
"svg_model(model)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_9\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_9 (Embedding) (None, None, 2) 10 \n",
"=================================================================\n",
"Total params: 10\n",
"Trainable params: 10\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
},
{
"data": {
"image/svg+xml": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"model = Sequential() \n",
"model.add(Embedding(VOC_SIZE, VEC_DIM)) # переменное число батчей и входов\n",
"\n",
"model.summary() # output: (batch_size, inputs, VEC_DIM)\n",
"svg_model(model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Матрица векторов"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(5, 2)\n",
"[[-0.004 -0.022]\n",
" [-0.029 0.025]\n",
" [-0.005 0.03 ]\n",
" [-0.002 0.037]\n",
" [-0.013 -0.039]]\n"
]
}
],
"source": [
"print(model.layers[0].get_weights()[0].shape) # (5,2)\n",
"print(model.layers[0].get_weights()[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#print(model.predict([[0]])) # [ [[0.01 0.025]] ]\n",
"print(model.predict(np.array([[0,4]]) )) # [ [[0.01 0.025]], [[0.035 0.012]] ]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(model.predict(np.array([ [1,2], [3,4] ]) ))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Загружаемая,не тренеруемая матрица"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emb_mat = np.arange(VOC_SIZE*VEC_DIM).reshape(VOC_SIZE,VEC_DIM)\n",
"model = Sequential() \n",
"model.add(Embedding(VOC_SIZE,VEC_DIM, weights=[emb_mat], trainable=False))\n",
"print(model.layers[0].get_weights()[0])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Маскирование входов"
]
},
{
"cell_type": "code",
"execution_count": 396,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[[0.031]\n",
" [0.031]\n",
" [0.027]\n",
" [0.027]\n",
" [0.052]\n",
" [0.052]\n",
" [0.052]]]\n"
]
},
{
"data": {
"text/plain": [
"[array([[0.643],\n",
" [0.95 ]], dtype=float32),\n",
" array([[1.]], dtype=float32),\n",
" array([0.], dtype=float32)]"
]
},
"execution_count": 396,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"m = Sequential()\n",
"m.add( Embedding(5,2, mask_zero=True) )\n",
"m.add( SimpleRNN(1, return_sequences=True) )\n",
"\n",
"print(m.predict(np.array([ [1,0,2,0,3,0,0] ]) ))"
]
},
{
"cell_type": "code",
"execution_count": 400,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_37\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_74 (Embedding) (None, None, 2) 6 \n",
"_________________________________________________________________\n",
"dense_75 (Dense) (None, None, 1) 2 \n",
"=================================================================\n",
"Total params: 8\n",
"Trainable params: 8\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"[[[7.]\n",
" [3.]]]\n"
]
},
{
"data": {
"text/plain": [
"[array([[1.],\n",
" [1.]], dtype=float32)]"
]
},
"execution_count": 400,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"m = Sequential()\n",
"m.add( Embedding(3, 2, weights=[np.array([ [1,2], [3,4], [5,6] ])], mask_zero=True) )\n",
"m.add( Dense(1, use_bias=False, weights=[np.array([[1],[1]]) ] ) )\n",
"m.summary()\n",
"\n",
"print( m.predict( np.array([ [1,0] ])))\n",
"\n",
"m.layers[1].get_weights()"
]
},
{
"cell_type": "code",
"execution_count": 385,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_25\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_62 (Embedding) (None, None, 2) 6 \n",
"_________________________________________________________________\n",
"lambda_44 (Lambda) (None, 2) 0 \n",
"=================================================================\n",
"Total params: 6\n",
"Trainable params: 6\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"[[1. 1.333]]\n"
]
}
],
"source": [
"import keras.backend as K\n",
"\n",
"m = Sequential()\n",
"m.add( Embedding(3, 2, weights=[np.array([ [0,0], [3,4], [5,6] ])], mask_zero=True) )\n",
"m.add( Lambda(lambda t: K.mean(t, axis=1)))\n",
"m.summary()\n",
"\n",
"print(m.predict(np.array([ [1,0,0] ]) ))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Взаимодействие с Dense"
]
},
{
"cell_type": "code",
"execution_count": 117,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_10\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_16 (Embedding) (None, 3, 2) 10 \n",
"_________________________________________________________________\n",
"flatten_5 (Flatten) (None, 6) 0 \n",
"_________________________________________________________________\n",
"dense_22 (Dense) (None, 1) 7 \n",
"=================================================================\n",
"Total params: 17\n",
"Trainable params: 17\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"0: (5, 2) \n",
"1: \n",
"2: (6, 1) (1,) \n",
"[[-0.013]]\n"
]
}
],
"source": [
"VOC_SIZE = 5\n",
"VEC_DIM = 2\n",
"model = Sequential() \n",
"model.add(Embedding(input_dim = VOC_SIZE, output_dim = VEC_DIM, input_length=3))\n",
"model.add(Flatten())\n",
"model.add(Dense(1))\n",
"\n",
"model.summary()\n",
"for i, lr in enumerate(model.layers):\n",
" print(i, end=': ')\n",
" for w in lr.get_weights():\n",
" print(w.shape, end=' ')\n",
" print()\n",
" \n",
"print( model.predict(np.array([[0,1,2]]) ))"
]
},
{
"cell_type": "code",
"execution_count": 118,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_11\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_17 (Embedding) (None, 3, 2) 10 \n",
"_________________________________________________________________\n",
"lstm_1 (LSTM) (None, 1) 16 \n",
"=================================================================\n",
"Total params: 26\n",
"Trainable params: 26\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
},
{
"data": {
"image/svg+xml": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"model = Sequential() \n",
"model.add(Embedding(VOC_SIZE, VEC_DIM, input_length = 3)) # (None, 3, 2)\n",
"#model.add(Dense(1))\n",
"model.add(LSTM(1)) # (None, 1)\n",
"model.summary()\n",
"svg_model(model)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}