\n",
"$$\n",
"\\begin{array}{ll}\n",
" & h_0 = 0 \\\\\n",
"x_0 = 1, & h_1 = x_0 + h_0 = 1 + 0 = 1\\\\\n",
"x_1 = 1, & h_2 = x_1 + h_1 = 1 + 1 = 2\\\\\n",
"x_2 = 1, & h_3 = x_3 + h_2 = 1 + 2 = 3\\\\\n",
"\\end{array}\n",
"$$\n",
"
\n",
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from keras.models import Model # для функционального способа\n",
"from keras.layers import Input, SimpleRNN, RepeatVector\n",
"\n",
"inp = Input( shape=(1,) ) # 1 вход сети (как input_shape)\n",
"out = RepeatVector(3)(inp) # 3 раза его повторяем\n",
"out = SimpleRNN(units=1, return_sequences=True, activation='linear', name='rnn')(out)\n",
"\n",
"model = Model(inputs=inp, outputs=out) # создаём модель\n",
"\n",
"W, H, b = np.array([[1]]), np.array([[1]]), np.array([0])\n",
"model.get_layer('rnn').set_weights(([ W, H, b ])) # меняем веса\n",
"\n",
"res = model.predict(np.array([ 1 ] )) # вычисляем вход\n",
"print(res)\n",
"svg_model(model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## stateful = True\n",
" \n",
"\n",
"* `stateful=False`- каждый следующий batch содержит новые последовательности, независмые от предыдущих.\n",
"* `stateful=True` - каждый следующий batch содержит примеры, продолжающие туже последовательность из предыдущего batch. Это продолжается, пока не вызывают `model.reset_states()`.\n",
"\n",
"\n",
"$$\n",
"\\begin{array}{ll}\n",
"\\mathrm{stateful} & \\mathrm{=~False} \\\\\n",
"x_0 = 1, & h_1 = x_0 + h_0 = 1 + \\mathbf{0} = 1\\\\\n",
"x_1 = 2, & h_2 = x_1 + h_1 = 2 + 1 = 3\\\\\n",
"x_2 = 3, & h_3 = x_3 + h_2 = 3 + 3 = \\mathbf{6}\\\\\n",
"\\hline\n",
"x_0 = 4, & h_4 = x_0 + h_3 = 4 + \\mathbf{0} = 4\\\\\n",
"x_1 = 5, & h_2 = x_1 + h_1 = 5 + 4 = 9\\\\\n",
"x_2 = 6, & h_3 = x_3 + h_2 = 6 + 9 = \\mathbf{15}\\\\\n",
"\\end{array}\n",
"$$\n",
"
\n",
"\n",
"$$\n",
"\\begin{array}{ll}\n",
"\\mathrm{stateful} & \\mathrm{=~True} \\\\\n",
"x_0 = 1, & h_1 = x_0 + h_0 = 1 + \\mathbf{0} = 1\\\\\n",
"x_1 = 2, & h_2 = x_1 + h_1 = 2 + 1 = 3\\\\\n",
"x_2 = 3, & h_3 = x_3 + h_2 = 3 + 3 = \\mathbf{6}\\\\\n",
"\\hline\n",
"x_0 = 4, & h_4 = x_0 + h_3 = 4 + \\mathbf{6} = 10\\\\\n",
"x_1 = 5, & h_2 = x_1 + h_1 = 5 + 10 = 15\\\\\n",
"x_2 = 6, & h_3 = x_3 + h_2 = 6 + 15 = \\mathbf{21}\\\\\n",
"\\end{array}\n",
"$$\n",
"
"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"res: [6.]\n",
"res: [21.]\n"
]
}
],
"source": [
"model = Sequential()\n",
"model.add(SimpleRNN(units=1, batch_input_shape=(1, 3, 1), \n",
" activation=\"linear\", stateful = True, name='rnn') )\n",
"W, H, b = np.array([[1]]), np.array([[1]]), np.array([0])\n",
"model.get_layer('rnn').set_weights(([ W, H, b ])) # меняем веса\n",
"\n",
"res = model.predict(np.array([ [ [1], [2], [3] ] ] )) \n",
"print(\"res:\",np.reshape(res, -1) )\n",
"res = model.predict(np.array([ [ [4], [5], [6] ] ] )) \n",
"print(\"res:\",np.reshape(res, -1) )"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"model_3\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"input_3 (InputLayer) (None, 3, 2) 0 \n",
"_________________________________________________________________\n",
"bidirectional_2 (Bidirection [(None, 3, 8), (None, 4), 224 \n",
"=================================================================\n",
"Total params: 224\n",
"Trainable params: 224\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
},
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"G \n",
"\n",
"\n",
"1903660929608 \n",
"\n",
"input_3: InputLayer \n",
"\n",
"input: \n",
"\n",
"output: \n",
"\n",
"(None, 3, 2) \n",
"\n",
"(None, 3, 2) \n",
" \n",
"\n",
"1903659934344 \n",
"\n",
"bidirectional_2(lstm_2): Bidirectional(LSTM) \n",
"\n",
"input: \n",
"\n",
"output: \n",
"\n",
"(None, 3, 2) \n",
"\n",
"[(None, 3, 8), (None, 4), (None, 4), (None, 4), (None, 4)] \n",
" \n",
"\n",
"1903660929608->1903659934344 \n",
"\n",
"\n",
" \n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"units = 4 # размерность скрытого состояния = dim(h)\n",
"features = 2 # размерность входов = dim(x)\n",
"inputs = 3 # число входов (ячек RNN слоя)\n",
"\n",
"inp = Input(shape=(inputs, features) )\n",
"rnn = LSTM (units, return_sequences = True, return_state=True)\n",
"bid = Bidirectional(rnn)(inp)\n",
"\n",
"model = Model(inp, bid)\n",
"\n",
"model.summary()\n",
"svg_model(model)"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"\n",
"\n",
"G \n",
"\n",
"\n",
"1903660910472 \n",
"\n",
"embedding_1_input: InputLayer \n",
"\n",
"input: \n",
"\n",
"output: \n",
"\n",
"(None, 3) \n",
"\n",
"(None, 3) \n",
" \n",
"\n",
"1903660886664 \n",
"\n",
"embedding_1: Embedding \n",
"\n",
"input: \n",
"\n",
"output: \n",
"\n",
"(None, 3) \n",
"\n",
"(None, 3, 8) \n",
" \n",
"\n",
"1903660910472->1903660886664 \n",
"\n",
"\n",
" \n",
"\n",
"1903659118728 \n",
"\n",
"bidirectional_3(simple_rnn_3): Bidirectional(SimpleRNN) \n",
"\n",
"input: \n",
"\n",
"output: \n",
"\n",
"(None, 3, 8) \n",
"\n",
"(None, 3, 8) \n",
" \n",
"\n",
"1903660886664->1903659118728 \n",
"\n",
"\n",
" \n",
"\n",
"1903605539912 \n",
"\n",
"dense_2: Dense \n",
"\n",
"input: \n",
"\n",
"output: \n",
"\n",
"(None, 3, 8) \n",
"\n",
"(None, 3, 1) \n",
" \n",
"\n",
"1903659118728->1903605539912 \n",
"\n",
"\n",
" \n",
" \n",
" "
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"VOC_SIZE, VEC_DIM, inputs = 100, 8, 3\n",
"\n",
"model = Sequential() # число входов можно не указывать (аналог None)\n",
"model.add(Embedding(input_dim=VOC_SIZE, output_dim=VEC_DIM, input_length=inputs)) \n",
"model.add(Bidirectional(SimpleRNN(units=4, return_sequences=True))) # (None,3,8)\n",
"model.add(Dense(units = 1, activation='sigmoid')) # (None,3,1)\n",
"\n",
"svg_model(model)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential_8\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"simple_rnn_4 (SimpleRNN) (None, None, 4) 28 \n",
"_________________________________________________________________\n",
"dense_3 (Dense) (None, None, 8) 40 \n",
"=================================================================\n",
"Total params: 68\n",
"Trainable params: 68\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"units = 4 # размерность скрытого состояния = dim(h)\n",
"features = 2 # размерность входов = dim(x)\n",
"inputs = 3 # число входов (ячек RNN слоя)\n",
" \n",
"model = Sequential() \n",
"model.add(SimpleRNN(units=4, input_shape=(None,2), return_sequences=True))\n",
"model.add(Dense(units=8))\n",
"\n",
"model.summary()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## LSTM\n",
"\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Число параметров\n",
" \n",
"\n",
"Если выход $h_t$ имеет размерность `(units,)`, а вход $x_t$ размерность `(features,)`, то \n",
"\n",
"* 4 матрицы $\\mathbf{W}_{xi}, \\mathbf{W}_{xf}, \\mathbf{W}_{xo}, \\mathbf{W}_{xc}$ \n",
"Keras упаковывает в одну, получая размерности `(features, 4*units)` и \n",
"* 4 матрицы $\\mathbf{W}_{hi}, \\mathbf{W}_{hf}, \\mathbf{W}_{ho}, \\mathbf{W}_{hc}$ \n",
"Keras упаковывает в одну, получая размерности `(units, 4*units)`. \n",
"* 4 вектора $\\mathbf{b}_o, \\mathbf{b}_f, \\mathbf{b}_i, \\mathbf{b}_c,$ упакованные в один длины `4*units`.\n",
"\n",
"Поэтому get_weight(), как и в SimpleRNN имеет 2 матрицы и 1 вектор.\n",
"Общее число параметров в ячейке LSTM равно `(features+units+1)*4*units`"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(2, 40) (2, 40) \n",
"(10, 40) (10, 40) \n",
"(40,) (40,) \n",
"params: 520\n",
"Model: \"sequential_10\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"rnn (LSTM) (None, 10) 520 \n",
"=================================================================\n",
"Total params: 520\n",
"Trainable params: 520\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"source": [
"units = 10\n",
"features = 2\n",
"\n",
"model = Sequential()\n",
"model.add(LSTM(units=units, input_shape=(4, features), name='rnn') )\n",
"\n",
"shapes, i = [(features, 4*units), (units, 4*units), (4*units,)], 0\n",
"for w in model.get_layer('rnn').get_weights(): # слой и его веса по имени слоя\n",
" print(\"%-10s %-10s\" % (str(w.shape), str(shapes[i])) )\n",
" i += 1\n",
"\n",
"print(\"params:\", (features+units+1)*4*units)\n",
"model.summary()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Активационные функции\n",
" \n",
"\n",
"Активационные функции в Keras LSTM задаются параметрами:\n",
"* `activation`: Default: `tanh`.\n",
"* `recurrent_activation`: to use for the recurrent step. Default: `hard_sigmoid`. \n",
"\n",
"If you pass `None`, no activation is applied (ie. \"linear\" activation: `a(x) = x`)."
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAsIAAAD4CAYAAADmbIA7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdd3xUVf7/8ddJJyTUhJoAofcaYFWqroCgILj2hg1ddX/qfnVx1V117buru651wb4rYKGIig3pogKB0EsoAUJCGoQkpE7m/P7IwIYYYIAkN8m8n4/HPGbuvefc+QzGm0/OnHs+xlqLiIiIiIiv8XM6ABERERERJygRFhERERGfpERYRERERHySEmERERER8UlKhEVERETEJwU49cYRERG2Xbt2Tr29iMg5iYuLy7DWRjodR3XRNVtEarOTXbMdS4TbtWvHmjVrnHp7EZFzYozZ63QM1UnXbBGpzU52zdbUCBERERHxSUqERURERMQnKREWEREREZ/k2BzhihQXF5OUlERBQYHToUgZISEhREVFERgY6HQoIlKD+Mo1W9dAkbqrRiXCSUlJhIeH065dO4wxTocjgLWWzMxMkpKSiImJcTocEalBfOGarWugSN122qkRxph3jDFpxphNJzlujDH/MsbsNMZsMMb0P9tgCgoKaNq0aZ29oNZGxhiaNm1a50d8RGqbc7k2G2PGGGO2e449fLYx+MI1W9dAkbrNmznC7wFjTnH8EqCT5zEFeONcAqrLF9TaSv9NRGqk9ziLa7Mxxh94zXO8O3CtMab72QbhC9cHX/iMIr7qtFMjrLXLjDHtTtFkAvCBtdYCPxljGhljWlprUyopRhE5VyXFEP8hHDngdCQ1T/vh0G6I01GcsbO9NgPtgJ3W2t0AxphZnrZbqjZiEakOJW5LQXEJeUUlFJW4cZW4KS6xuNxuil2WYrcbV4nFVeL2HPccK7G4rQXAbS1ud+mztWCxuK1nvy2dMmTLbbuP7+P4eY6xZbaPvbTltkv32Qr2ndiwXUR9JvWPqqx/rkqZI9wa2F9mO8mz7xeJsDFmCqUjE7Rp06YS3rp63H777fz+97+ne/ezHjQ5rbFjxzJjxgwaNWp0wv4nnniCsLAwHnzwwSp7b/EB3z8JK1/xbPj26JYtt320xI+wWpgIe+Fk1+aK9g+u6AS14ZqdlZXFjBkzuPvuu8+q/4gRI/j73/9ObGxsJUcmcvaOFro4mF1AanYBGblFHMkr4kh+MVl5xWR5nrPzi8krdpFXVEJBUQl5xSXkF5VQ6HI7HX6VMQaGd46scYlwRb9Vy/+uKd1p7TRgGkBsbGyFbWqit956q8rfY8GCBVX+HuKj9iyHla9C/5vgsn+VXklqOWstOYUuDh8t4nBesee59HVW3v9eH8krJqegmJxCF7kFLnILS39plPVQQBfucehzVLGTXZvr1DU7KyuL119//awTYREnWGtJOpzP7oyjJGYcZU/GUXZnHCU5K5/UIwXkFLoq7Bca5E/DeoHHH83CQ6gX6E+9IH/qBfoTGvS/1/WC/AkO8CPAz48Af0OQvx8B/qWvAz37Av39CPQ3BPiVPhtj8DPgZwx+xmBM6a+MY9t+pnSq0P/2cbxP2b7wv181hhO3S/d5no+1LfMZj/erpt9VlZEIJwHRZbajgORKOK8jjh49ylVXXUVSUhIlJSX86U9/4o033jg+YvD222/zwgsv0KpVKzp16kRwcDCvvvoqkydPpl69emzbto29e/fy7rvv8v777/Pjjz8yePBg3nvvPQBmzpzJs88+i7WWcePG8cILLwD/K18aERHBM888wwcffEB0dDSRkZEMGDDAwX8RqdXys2DuXdCkPYx5vlYkwa4SNylHCkg6nM/B7HxSswtJyy4kNaeAtOwC0nIKSc0uoKC44lEPPwMN6wXSODSIRqGBNAoNIqpJKOHBAYQFBxAeEkhYSEDpdkgAXVuEV/MnrDYnuzYHnWR/rfTwww+za9cu+vbty8iRI9mwYQOHDx+muLiYp59+mgkTJpCYmMgll1zCkCFDWLlyJa1bt+azzz6jXr16AHzyySfcfffdZGVl8fbbbzN06FCHP5XUNanZBfy85xAbk7LYeOAIm5OzySn4X7IbFhxAu4hQOkaGMaRjBM0bhNC8QTAtGoQQER5Mo9DSxDc4wN/BT1E3VUYiPB+41zPPbDBwpDLmBz/5+Wa2JGefc3BldW/VgMcv63HKNl9//TWtWrXiyy+/BODIkSO88Ubp/X/Jyck89dRTrF27lvDwcC688EL69OlzvO/hw4dZtGgR8+fP57LLLuOHH37grbfeYuDAgcTHx9OsWTOmTp1KXFwcjRs3ZtSoUcybN4/LL7/8+Dni4uKYNWsW69atw+Vy0b9/fyXCcvYWPAQ5KXDbdxBU3+lojit0lZCYkcfOtFwSM4+SdDiPfYfy2H8on+SsfFzuEwcfQ4P8ad4ghGbhwfSJakTzBsFEhgfTpH4wjT3JbpP6QTQODaRBSCB+fjU/4a8GFV6bjTHpQCdjTAxwALgGuO5c38ypa/bzzz/Ppk2biI+Px+VykZeXR4MGDcjIyOBXv/oV48ePByAhIYGZM2cyffp0rrrqKmbPns0NN9wAgMvlYtWqVSxYsIAnn3yShQsXVurnEN9TUFzCsh3pLEtI58ddmexKPwpAUIAf3VqEM75PK7q3akDHyDBiIusTGRasmzIdctpE2BgzExgBRBhjkoDHgUAAa+2bwAJgLLATyANuqapgq0OvXr148MEHmTp1KpdeeukJIwOrVq1i+PDhNGnSBIArr7ySHTt2HD9+2WWXYYyhV69eNG/enF69egHQo0cPEhMT2bt3LyNGjCAyMhKA66+/nmXLlp2QCC9fvpyJEycSGhoKcPwiLnLGNn4KGz+GEY9AlDN/TJW4LXsyctmcnM2O1Bx2puWSkJbL3sw8Ssoku03rBxHdJJQ+0Y24tHdLopuEEt04lJaNQmjeIISw4Bq15HmNcLbXZmutyxhzL/AN4A+8Y63dXO0foApYa3nkkUdYtmwZfn5+HDhwgNTUVABiYmLo27cvAAMGDCAxMfF4v0mTJlW4X+RMFLpK+H5rGgs2prBoWxp5RSXUD/JnUEwTrh4YzXntI+jaMpxAfxX1rUm8WTXi2tMct1D5U+xONwpQVTp37kxcXBwLFizgj3/8I6NGjTp+zNpTT5ELDg4GwM/P7/jrY9sul4uAAO9+meuvQjlnR5Lgy99D1EAY+n/V8pbWWvZkHGVD0hE2HjjCxqQjbE4+wlHPnFx/P0O7pqF0bhbOuF4t6dgsjA6RYcRE1Ke+Et0zdi7XZmvtAkoT5Urj1DW7rA8//JD09HTi4uIIDAykXbt2x9f/LXtN9vf3Jz8///j2sWP+/v64XBXPzRQ5mX2ZecxYtY9P1uwn82gRTesHcXm/1lzSswW/at9UiW8Np98+5SQnJ9OkSRNuuOEGwsLCjs/tBRg0aBAPPPAAhw8fJjw8nNmzZx8f9fXG4MGDue+++8jIyKBx48bMnDmT3/3udye0GTZsGJMnT+bhhx/G5XLx+eefc+edd1bWxxNf4HbDvN9CiQsm/hv8q+Z/c7fbsu1gDqv2ZLIq8RCr9hwiI7cIgOAAP3q0asBvBkTRK6oRPVs3oH1EGEEB+oUglSs8PJycnBygdCpbs2bNCAwMZPHixezdu9fh6KQu25GawyuLdvLFhmT8jOGirs24bnAbhnaKxF/Ts2oNJcLlbNy4kYceegg/Pz8CAwN54403ji9d1rp1ax555BEGDx5Mq1at6N69Ow0bNvT63C1btuS5555j5MiRWGsZO3YsEyZMOKFN//79ufrqq+nbty9t27bVTRty5n5+A/Ysg8tehqYdKvXUadkFLNmRzpLtaaxIyCDbc7NH60b1GNYpkoExTejXphEdI8MI0CiIVIOmTZtywQUX0LNnTwYOHMi2bduIjY2lb9++dO3a1enwpA5KOpzHc19t48sNKdQP8ufOYR2YfH47WjQMcTo0OQvmdF/3V5XY2Fi7Zs2aE/Zt3bqVbt26ORKPt3JzcwkLC8PlcjFx4kRuvfVWJk6c6HRYVa42/LcRIHULTBsBHS+Ca2ac8yoR1lo2J2fzzeaDLN6exqYDpTdDNQsPZnjnSM7r0JRBMU2IahxaCcHXLsaYOGutzyw+W1uv2ZXFlz6rVKyguIQ3l+7ijSW7MAZuH9Ke24bE0Lh+kNOhiRdOds3WiPAZeuKJJ1i4cCEFBQWMGjXqhBvdRBzlKoQ5d0BIg3NaL9ja0ikPX25I4cuNKezJOIq/n6F/m0Y8NLoLI7s0o1vLcM1lFxGfsenAER74KJ6EtFzG9W7Jo2O70apRPafDkkqgRPgM/f3vf3c6BJGKLXoKUjfBdR9DWOQZd8/ILWTO2iQ+XpPEzrRc/Ayc3yGCKcPaM7pHC5po1ENEfIzbbXlj6S7+8d0OmoYF8f6tgxje+cyvr1JzKREWqQuOVY8bcAt0Hu11txK3ZXlCOh+t3s93W1JxuS0D2jbm6ct7MqZnCyLCgk9/EvFp1to6/+2AU1MIxVlHC138/uN4vtmcyrjeLXnm8p40CtWAQF2jRFiktitbPW70M151ySty8WlcEu+s2ENiZh5N6gdxywXtuHpgNB2b1dlKa1LJQkJCyMzMpGnTpnU2GbbWkpmZSUiIboTyJfsP5XH7+2tISMvhT5d259YL2tXZn3Ffp0RYpLY7g+pxaTkFvPdDIh/+vI8j+cX0jW7EK6O6MLpHCy1tJmcsKiqKpKQk0tPTnQ6lSoWEhBAVFeV0GFJNdqblcMNbq8grcvHBrYMZ0inC6ZCkCikRFqnNNs32qnpcek4h/166i//8tJeiEjeju7fgjmEx9G/TWKMcctYCAwOJiYlxOgyRSrPpwBFuemcVfsbw8V3n0bVFA6dDkiqmRFiktjpyAL544JTV4w4fLeLNpbv44Me9FLpKmNgvit9d2JF2EaceORYR8TU703K48e2fCQ0K4L+3DyZG10mfoO9Cy0lMTKRnz56Vdr6wsLAz7jN27FiysrIqLYaKnH/++RXunzx5Mp9++mmVvrdUArcb5t110upxRS43b6/Yw/C/LWba8t2M7tGchb8fzotX9VESLCJSzoGsfG58exX+fn58qCTYp2hEuBK5XC4CAs79n3TBggWVEM2prVy5ssrfQ6rQSarHWWtZtC2NZ77cyu6MowztFMFj47rTpYVugBMRqUh2QTE3vf0zuYUuPr7zPA0W+Jiamwh/9TAc3Fi552zRCy55/rTNSkpKuOOOO1i5ciWtW7fms88+47///S/Tpk2jqKiIjh078p///IfQ0FAmT55MkyZNWLduHf379+fee+/luuuuw+VyMWbMmFO+T0pKCldffTXZ2dm4XC7eeOMNhg4dSrt27VizZg0RERE89dRTfPjhh0RHRxMREcGAAQN48MEHGTFiBP369SMuLo709HQ++OADnnvuOTZu3MjVV1/N008/DcBLL73EO++8A8Dtt9/O/fffD5SOVOfm5mKt5Xe/+x2LFi0iJiZGywTVBqlbYOGT0GUs9L/5+O6UI/n8ad4mFm5No31Efd6ZHMvILs00B1hE5CRK3Jb7Z8WzNzOP/9w2mG4tNSfY12hqRAUSEhK455572Lx5M40aNWL27NlMmjSJ1atXs379erp168bbb799vP2OHTtYuHAhL774Ivfddx+//e1vWb16NS1atDjl+8yYMYPRo0cTHx/P+vXr6du37wnH16xZw+zZs1m3bh1z5syhfHnToKAgli1bxl133cWECRN47bXX2LRpE++99x6ZmZnExcXx7rvv8vPPP/PTTz8xffp01q1bd8I55s6dy/bt29m4cSPTp0/XSHFNV0H1OLfb8p8fE7n4pWWs2JnBHy/pyjcPDOPCrs2VBIuInMKL325n0bY0Hh/fg/M6NHU6HHFAzR0R9mLktqrExMQcT0oHDBhAYmIimzZt4rHHHiMrK4vc3FxGj/5f0YIrr7wSf39/AH744Qdmz54NwI033sjUqVNP+j4DBw7k1ltvpbi4mMsvv/wXifCKFSuYMGEC9eqVlnG87LLLTjg+fvx4AHr16kWPHj1o2bIlAO3bt2f//v2sWLGCiRMnUr9+6dc8kyZNYvny5fTr1+/4OZYtW8a1116Lv78/rVq14sILLzzzfzCpPoueLq0ed+1HEBbJnoyjPPjJeuL2HmZopwieubwXbZqGOh2liEiNt2hbKq8v2cW1g6K5YXAbp8MRh2hEuALBwf+rpuXv74/L5WLy5Mm8+uqrbNy4kccff5yCgoLjbY4lmsd4Owo3bNgwli1bRuvWrbnxxhv54IMPTjh+umkKx+L08/M7IWY/Pz9cLpfX0xw0alhL7FkOK1+BAbdgO4/mo9X7GPev5exMy+Wlq/rwwa2DlASLiHghLaeAhz7ZQNcW4Tx+WQ/9HvRhSoS9lJOTQ8uWLSkuLubDDz88absLLriAWbNmAZyyHcDevXtp1qwZd9xxB7fddhtr16494fiQIUP4/PPPKSgoIDc3ly+//PKMYh42bBjz5s0jLy+Po0ePMnfuXIYOHfqLNrNmzaKkpISUlBQWL158Ru8h1aRM9bjDQx7nrv/GMXX2RvpGN+Lr+4cyqX+ULuQ+yBgzxhiz3Riz0xjzcAXHHzLGxHsem4wxJcaYJp5jicaYjZ5ja355dpG6ye22PPTJBnILXbxybT9CAv2dDkkcVHOnRtQwTz31FIMHD6Zt27b06tWLnJycCtu9/PLLXHfddbz88stcccUVpzznkiVL+Nvf/kZgYCBhYWG/GBEeOHAg48ePp0+fPrRt25bY2FgaNmzodcz9+/dn8uTJDBo0CCi9Wa7stAiAiRMnsmjRInr16kXnzp0ZPny41+eXauSpHrf90tlMfnMtGbmFPDq2G7cNicHPTwmwLzLG+AOvARcDScBqY8x8a+2WY22stX8D/uZpfxnwgLX2UJnTjLTWZlRj2CKOm7l6H0t3pPPUhB50aq4VdXydcWqVgNjYWFv+5q+tW7fSrVs3R+KpqXJzcwkLCyMvL49hw4Yxbdo0+vfvX+1x6L+NgzbNhk9vZX3H3/KbrcNo0TCEN64fQM/W3v9RJJXPGBNnrY118P3PA56w1o72bP8RwFr73EnazwAWW2une7YTgVhvE+GKrtkitU1qdgG/fnEpvaIa8uHtg/VNmg852TVbI8I13JQpU9iyZQsFBQXcfPPNjiTB4qAjB7BfPMC+et2ZtOl8hnSO4OVr+tIoNMjpyMR5rYH9ZbaTgMEVNTTGhAJjgHvL7LbAt8YYC/zbWjutgn5TgCkAbdroZiKp/Z78fDNFJW6endhLSbAASoSrxcaNG7nxxhtP2BccHMzPP/982r4zZsyoqrCkpnO7KZp9JyWFhdxccBv3XNSV+y7qhL+mQkipin4QTvYV32XAD+WmRVxgrU02xjQDvjPGbLPWLjvhZKXJ8TQoHRGujKBFnLJwSyoLNh7kodFdVDRDjqtxibC1ts79ldarVy/i4+OdDuOsqciGM9K//yeR+5bzWMkdPHz9OMb0bOl0SFKzJAHRZbajgOSTtL0GmFl2h7U22fOcZoyZCwwCllXQV6TWK3K5efrLLXRsFsaUYe2dDkdqkBq1akRISAiZmZlKvGoQay2ZmZmEhIQ4HYpPWR+3kgY/PMtSE8sVtz+iJFgqshroZIyJMcYEUZrszi/fyBjTEBgOfFZmX31jTPix18AoYFO1RC3igA9+TCQxM4/HxnUj0L9GpT7isBo1IhwVFUVSUhLp6elOhyJlhISEEBUV5XQYPuOb9ftoO/9O8vxC6Xjbu7SOauJ0SFIDWWtdxph7gW8Af+Ada+1mY8xdnuNveppOBL611h4t0705MNfz7VsAMMNa+3X1RS9SfQ4fLeJf3ycwrHMkI7o0czocqWFqVCIcGBhITEyM02GIOGbO2iQy5v6R0f77ODrxQxpH6QYlOTlr7QJgQbl9b5bbfg94r9y+3UCfKg5PpEZ4+fsEcgtdPDpWKx/JL9WoRFjEl/3nx0S+mP8pM4O/pLjvzdTvfanTIYmI1GrJWfnM+HkfV8VG06WF1gyWX1IiLFIDTFu2i1cXxLG4/r+hYXsCx1a4FKyIiJyB15fsxGK598KOTociNZQSYRGHvbNiD88u2MbHkR/RJDcTM2kWBGlpHxGRc3EgK5+PVu/nythoohqHOh2O1FC6dVLEQf/9aS9/+WILj7bdwqCchZjhf4CoAU6HJSJS6722eCcA94zUaLCcnBJhEYd8vGY/j83bxBUdDbcfeQVax8LQB50OS0Sk1kvNLuCTNfu5Kjaa1o3qOR2O1GBKhEUc8NXGFKbO3sCwjk34a8CbmJJimDQN/DVbSUTkXL23MpESt1XxDDktrxJhY8wYY8x2Y8xOY8zDFRxvaIz53Biz3hiz2RhzS+WHKlI3rNpziPs+iqdfdCPe6hqHf+JSGPMcNO3gdGgiIrXe0UIXH/60l9E9WtC2qe63kFM7bSJsjPEHXgMuAboD1xpjupdrdg+wxVrbBxgBvOipdCQiZexIzeH291cT3bge744NI2jxX6DLWOh/s9OhiYjUCR+v2U92gYs7NBosXvBmRHgQsNNau9taWwTMAiaUa2OBcFNapigMOAS4KjVSkVou5Ug+N7+zipBAf96/qQ8Nv7obQhrAZf+C0gpfIiJyDlwlbt5esYfYto3p36ax0+FILeBNItwa2F9mO8mzr6xXgW5AMrARuM9a6y5/ImPMFGPMGmPMGpVRFl+SW+jilndXk1vg4r1bBhG17iVI3QTjX4WwSKfDExGpE77ZnErS4XyNBovXvEmEKxqqsuW2RwPxQCugL/CqMabBLzpZO81aG2utjY2M1C9/8Q1ut+WBj+JJSMvl9Rv6071oA6x8BQZMhi5jnA5PRKTO+M9PiUQ1rsevuzV3OhSpJbxJhJOA6DLbUZSO/JZ1CzDHltoJ7AG6Vk6IIrXbi99t57stqfxpXDeGRgfB3LugSQyMesbp0ERE6oydabn8tPsQ1w1ug7+fppuJd7xJhFcDnYwxMZ4b4K4B5pdrsw+4CMAY0xzoAuyuzEBFaqPP4g/w2uJdXDsompvPbwcLHoLsZJg0HYLDnA5PRKTOmPHzPgL9DVfFRp++sYjHaRcttda6jDH3At8A/sA71trNxpi7PMffBJ4C3jPGbKR0KsVUa21GFcYtUuOt35/FHz7dwKCYJjw5vidm8xzY8BGM+CNExTodnohInZFfVMKncfsZ07MlEWHBTocjtYhXq/dbaxcAC8rte7PM62RgVOWGJlJ7ZeYWctd/44gMD+bNGwYQdDQFvnhA1eNERKrAFxuSyS5wcf3gNk6HIrWMKsuJVLISt+X+j+LJPFrEmzcMoEm9AJj3W1D1OKlkXhQ7GmGMOWKMifc8/uxtX5Ha5MOf99Ehsj6DY5o4HYrUMvqNLFLJXlmUwPKEDJ6b1IuerRvCj6/DnqVw6T9VPU4qTZliRxdTelPzamPMfGvtlnJNl1trLz3LviI13s60HOL3Z/HYuG4YrckuZ0gjwiKVaHlCOi9/n8Ck/q25ZmA0pG6BhU9A5zGly6WJVB5vih1VRV+RGmX22gP4+xkm9C1f4kDk9JQIi1SSlCP53Dcrns7Nwnn68p6YkiKYM6W0etz4V1U9TiqbN8WOAM4zxqw3xnxljOlxhn1FarQSt2XeugMM7xxJZLhukpMzp0RYpBKUuC33zYynsLiE12/oT2hQACx+BlI3qnqcVBVvih2tBdpaa/sArwDzzqCvqoFKjffT7kxSjhQwqb/+jpOzo0RYpBK8sWQnqxIP8dTlPekQGQaJK+CHf6l6nFSl0xY7stZmW2tzPa8XAIHGmAhv+nr6qBqo1Giz45IIDwlQJTk5a0qERc5R/P4s/rEwgcv6tGJiv9ZQcETV46Q6nLbYkTGmhfHcPWSMGUTpNT/Tm74iNd3RQhdfbTrIpb1bERLo73Q4Uktp1QiRc5Bb6OK+Weto0SCkdF6wMf+rHnfbt6oeJ1XGy2JHvwF+a4xxAfnANdZaC1TY15EPInKWvt50kPziEq7QtAg5B0qERc7Bk/M3s/9QHrOmnEfDeoGwyVM9bvjDqh4nVc6LYkevAq9621ekNpm77gBtmoQyoG1jp0ORWkxTI0TO0pcbUvgkLom7R3RkUEyT0lHgY9Xjhql6nIhIVcnMLWTlrgzG92mltYPlnCgRFjkLaTkFPDpvI32iGnLfrzuB2+2pHlfkqR4X6HSIIiJ11tebD+K2MK53S6dDkVpOUyNEzpC1lsfmbiKvqIQXr+pDoL9fafW43UtUPU5EpBp8uSGF9pH16doi3OlQpJbTiLDIGZq/Pplvt6Ty+4s707FZuKrHiYhUo/ScQn7ancmlvVpqWoScMyXCImcgPaeQx+dvpm90I+4Y2h5chaXV44LDYfwrqh4nIlLFvt6U4pkW0crpUKQO0NQIES9Za3ls3kbyikr4+5W98fcz8L2nety1syCsmdMhiojUeV9sSKFTszC6aFqEVAKNCIt46fMNKXyzucyUiBOqx13idHgiInVeWnYBqxIP6SY5qTRKhEW8kJFbyOOfbfrflAhVjxMRqXZfbTqItTCulxJhqRyaGiHihae/2EJuoet/UyJUPU5EpNp9u+UgHSLr06m5pkVI5dCIsMhpLE9IZ158Mr8d0bF0SsSx6nHDHlL1OBGRanIkr5ifdh9iVI8WTocidYgSYZFTKCgu4dG5m2gfUZ+7R3QoUz1ugKrHiYhUo0XbUylxW0Z1b+50KFKHaGqEyCn86/sE9h3KY+YdvyLE35SpHjdd1eNERKrRd1tSaRYeTJ+oRk6HInWIRoRFTmLbwWymLdvNbwZEcV6HprDq36XV40Y/q+pxIiLVqKC4hCXb0/l19+b4+Wm9dqk8SoRFKuB2Wx6Zs5EG9QJ5dGy30upx3z2u6nEiIg74cVcmeUUlmhYhlU6JsEgFZqzax9p9WTw2rhuNg62qx4mIOOjbLQcJCw4o/XZOpBJpjrBIORm5hbzw9TbO79CUif1aw8LHVT1ORMQhbrfluy1pDO8SSXCAv9PhSB2jEWGRcl74ahsFxSU8dXlPzN4fSiINQikAACAASURBVKvH9b9Z1eOkxjHGjDHGbDfG7DTGPFzB8euNMRs8j5XGmD5ljiUaYzYaY+KNMWuqN3IR763bn0VGbqGmRUiV0IiwSBlr9x3mk7gk7hzeng7hJfBfT/W40c86HZrICYwx/sBrwMVAErDaGDPfWrulTLM9wHBr7WFjzCXANGBwmeMjrbUZ1Ra0yFn4dstBAvwMI7roGzmpfEqERTxK3JY/f7aJ5g2C+X8XdoIv71H1OKnJBgE7rbW7AYwxs4AJwPFE2Fq7skz7n4Coao1QpBIs3JLKr9o3pWE9LVkplU9TI0Q8Zq3ex6YD2Tw6rjv1E+bDhlmqHic1WWtgf5ntJM++k7kN+KrMtgW+NcbEGWOmVNTBGDPFGLPGGLMmPT39nAMWOVP7MvPYlX6UC7tqNFiqhkaERYDDR4v42zfbGRzThMvaWXhD1eOkxqto+RJbYUNjRlKaCA8ps/sCa22yMaYZ8J0xZpu1dtkJJ7N2GqXTKYiNja3w3CJVacmONABGKhGWKuLViPDpbsjwtBnhueliszFmaeWGKVK1/vbtdnIKXPxlfHfMZ3erepzUBklAdJntKCC5fCNjTG/gLWCCtTbz2H5rbbLnOQ2YS+lUC5EaZfG2NNo1DSUmor7ToUgdddpEuMwNGZcA3YFrjTHdy7VpBLwOjLfW9gCurIJYRarEhqQsZq7ax83ntaPL3hmqHie1xWqgkzEmxhgTBFwDzC/bwBjTBpgD3Git3VFmf31jTPix18AoYFO1RS7ihYLiElbuytRNclKlvJkacdobMoDrgDnW2n1wfIRBpMZzuy1//mwzTesH8/u+JfCeqsdJ7WCtdRlj7gW+AfyBd6y1m40xd3mOvwn8GWgKvG5KC8G4rLWxQHNgrmdfADDDWvu1Ax9D5KR+3JVJocut+cFSpbxJhCu6IWNwuTadgUBjzBIgHHjZWvtB+RN5bsiYAtCmTZuziVekUs1em0T8/iz+MakbYV/epOpxUqtYaxcAC8rte7PM69uB2yvotxvoU36/SE2yeHsa9QL9GRTTxOlQpA7zJhH25oaMAGAAcBFQD/jRGPNT2a/iQDdeSM1ytNDFX7/ZTt/oRlye9R4cVPU4EZGawFrLom1pXNCxKSGBqiYnVcebm+W8uSEjCfjaWnvUszj7MjTaIDXcm0t3kZ5TyAux2ZiVqh4nIlJT7Eo/StLhfM0PlirnTSJ82hsygM+AocaYAGNMKKVTJ7ZWbqgiledAVj7Tlu3myp4N6LLyIVWPExGpQZZs17JpUj1OOzXCmxsyrLVbjTFfAxsAN/CWtVZ3IEuN9devtwHwRMB7pdXjbv1G1eNERGqIxdvT6NI8nNaN6jkditRxXhXUON0NGZ7tvwF/q7zQRKrGun2H+Sw+mZd7J1J/26cw/GGIHuh0WCIiAuQWuli15xC3DolxOhTxASqxLD7FWstTX2yhe1gu4/f9VdXjRERqmBUJGRSXWEZqfrBUA5VYFp/y+YYU1u07xI9R72COqHqciEhNs2R7GuHBAQxo29jpUMQHKBEWn1FQXMILX23j4cZLaZHxE1z6D1WPExGpQay1LN6extDOEQT660trqXr6KROf8faKPYQeSeCOwvc91eNucTokEREpY2tKDqnZhVo2TaqNRoTFJ6TlFDB98Tbmh0/DL6CBqseJiNRAiz3Lpo3oEulwJOIrlAiLT3jp2x381n5Em6KdcIWqx4mI1ERLd6TTs3UDmoWHOB2K+AhNjZA6b0tyNnvivmWK/+eqHiciUkNlFxSzdu9hhnfWaLBUHyXCUqdZa3nx89X8I+gN3I3aqXqciEgNtXJnBi63ZXhnfWMn1UdTI6ROW7g1jbFJL9Ei4DB+V8xS9TgRkRpq6Y50woMD6NemkdOhiA/RiLDUWUUuNz/Mf4sr/Fdgh/yfqseJiNRQ1lqWbk/ngo5aNk2ql37apM6avWQV9+e/xpEmvfEf8QenwxERkZPYmZZL8pEChmu1CKlmSoSlTso6WkC7FQ9Rz89Fg+veVfU4qZOMMWOMMduNMTuNMQ9XcNwYY/7lOb7BGNPf274i1WnpjnQAhulGOalmSoSlTvpx5nOcxwYOD30SE9HR6XBEKp0xxh94DbgE6A5ca4zpXq7ZJUAnz2MK8MYZ9BWpNkt3pNOpWRitG9VzOhTxMUqEpc7Zty2OC/e/xtYG59Ni5F1OhyNSVQYBO621u621RcAsYEK5NhOAD2ypn4BGxpiWXvYVqRZ5RS5+3n1Iy6aJI5QIS93iKoI5d5BLPZpdP03V46Quaw3sL7Od5NnnTRtv+mKMmWKMWWOMWZOenl4pQYuU9/PuQxSVuDU/WByhRFjqlP1zHqNN0S5W9X6Sps2jnQ5HpCpV9Fee9bKNN32x1k6z1sZaa2MjI5WkSNVYuiOdeoH+DGzXxOlQxAdpHWGpM0r2rKD1lmnM97+Y0RNudjockaqWBJT9ay8KSPayTZAXfUWqxdId6ZzXoSkhgf5OhyI+SCPCUjcUHCH/4zvY625G8KXPExygC6rUeauBTsaYGGNMEHANML9cm/nATZ7VI34FHLHWpnjZV6TK7c08yp6Mo5ofLI7RiLDUCcWfP0hI/kGmR/ydZ/p2cDockSpnrXUZY+4FvgH8gXestZuNMXd5jr8JLADGAjuBPOCWU/V14GOIj1vmWTZNibA4RYmw1H6b5xK4+WNedk3imklXYHSDnPgIa+0CSpPdsvveLPPaAvd421ekui3Znk7bpqG0i6jvdCjiozQ1Qmq37GRK5t/PBncH9ve6h95RqlEvIlIbFLpKWLkrU6PB4iiNCEvt5XbDvLtxFRUw1d7Lu2N6Oh2RiIh4aU3iYfKLS5QIi6M0Iiy116ppsHsxTxZdz+jhQ2jRMMTpiERExEtLd6QT5O/Hr9o3dToU8WEaEZbaKW0rduHjrA4cxKLAsSwa1t7piERE5Aws3Z7OwJjG1A9WKiLO0Yiw1D6e6nGFfqHcnXMLf7ikK6FBupCKiNQWKUfy2Z6ao2kR4jglwlL7LH4GDm7kMfcUWke14fK+v6gMKyIiNdj/lk1r5nAk4us0jCa1S+IP8MPLbGw+gU/39ubTG7rj56fl0kREapOlO9Jp0SCEzs3DnA5FfJxGhKX2KDgCc+/C1bAtNydPZFzvlsSqNr2ISK3iKnGzPCGD4Z0jte67OE6JsNQeX02F7AO81vgP5NoQHh7T1emIRETkDMXvzyKnwMXwLpofLM5TIiy1w+a5sH4mB/veyz+2NeK2ITFENwl1OioRETlDi7en4e9nuKBjhNOhiCgRllogOxk+vx/bqj/3Hfg1EWFB3D2ig9NRiYjIWVi0LZ0BbRvTsF6g06GIeJcIG2PGGGO2G2N2GmMePkW7gcaYEmPMbyovRPFpnupxlBSxsNvT/LwvhwdHdSE8RBdQEZHaJjkrn60p2VzUVatFSM1w2kTYGOMPvAZcAnQHrjXGdD9JuxeAbyo7SPFhnupxBRf+hUeX5dM7qiFXxUY7HZWIiJyFxdvTALhQibDUEN6MCA8Cdlprd1tri4BZwIQK2v0OmA2kVWJ84svStsHCx6HTaP55eAhpOYU8Mb6HlksTEamlFm9LI6pxPTo207JpUjN4kwi3BvaX2U7y7DvOGNMamAi8eaoTGWOmGGPWGGPWpKenn2ms4ktcRTDndggKY++QF3j7hz1c0T+K/m0aOx2ZiIichYLiEn7YmclFXZtp2TSpMbxJhCv6abXltv8JTLXWlpzqRNbaadbaWGttbGSklk2RU1jyLBzcCONf4YlF6QQH+DP1ki5ORyUiImfpx92Z5BeXMFLTIqQG8SYRTgLKTsqMApLLtYkFZhljEoHfAK8bYy6vlAjF9yT+ACv+Cf1v4ns7gMXb07nvok40Cw9xOjKRGsEY08QY850xJsHz/IuvSowx0caYxcaYrcaYzcaY+8oce8IYc8AYE+95jK3eTyC+aPG2NOoF+vOr9k2dDkXkOG8S4dVAJ2NMjDEmCLgGmF+2gbU2xlrbzlrbDvgUuNtaO6/So5W6z1M9jsbtKLjoaf7yxRY6RNbn5vPbOR2ZSE3yMPC9tbYT8L1nuzwX8H/W2m7Ar4B7yt3o/A9rbV/PY0HVhyy+zFrLom1pXNAxgpBAf6fDETnutImwtdYF3EvpahBbgY+ttZuNMXcZY+6q6gDFx3iqxzFpOm+vSmdvZh5PjO9BUICWvBYpYwLwvuf1+8AvvoGz1qZYa9d6XudQev1uXb6dSHVISMsl6XC+VouQGifAm0ae0YIF5fZVeGOctXbyuYclPmnzPFg/E4ZPJaVBT15dtJTRPZoztJPmk4uU09xamwKlCa8x5pTZhTGmHdAP+LnM7nuNMTcBaygdOT5cQb8pwBSANm3aVE7k4pMWbStdUGpkV13PpWbRMJvUDNkp8MX90Ko/DHuIp7/YittaHhv3iyWrRXyCMWahMWZTBY+Klq881XnCKF3a8n5rbbZn9xtAB6AvkAK8WFFf3eAslWXRtjS6tWxAy4b1nA5F5ARejQiLVCm3G+b9FlyFMGk6i3ce5suNKTw4qjPRTUKdjk7EEdbaX5/smDEm1RjT0jMa3JKTrN9ujAmkNAn+0Fo7p8y5U8u0mQ58UXmRi5woK6+IuL2HuWt4e6dDEfkFjQiL8zzV4xj1NAUNY/jzZ5voEFmfO4bpoilyEvOBmz2vbwY+K9/AlC7U+jaw1Vr7UrljLctsTgQ2VVGcIny/NY0St2VU9xZOhyLyC0qExVllqscReyuvLEpg/6F8nr68F8EBurNY5CSeBy42xiQAF3u2Mca0MsYcu5/jAuBG4MIKlkn7qzFmozFmAzASeKCa4xcf8u2Wg7RoEEKv1g2dDkXkFzQ1QpxTpnocE14lIS2Xact2M6l/a87roHUmRU7GWpsJXFTB/mRgrOf1CiouiIS19sYqDVDEI7+ohKU70rkqNho/P1WTk5pHI8LinDLV42z9SB6dt4nQoAAeGdvN6chERKQSLE9Ip6DYrWkRUmMpERZn7F15vHocXccye+0BVu05xB8v6UpEWLDT0YmISCX4dksqDUICGNy+idOhiFRIibBUv4IjMOdOaNwORj/H4aNFPLtgKwPaNuaq2OjTdhcRkZrPVeLm+62pXNStOYH+SjekZtIcYal+X02F7CS49RsIDuPJuevIzi/mmYk9NYdMRKSOWJ14mMN5xYzq3tzpUEROSn+iSfU6Vj1u6IMQPYhF21KZF5/MPSM70rVFA6ejExGRSvLtloMEBfgxrLOKsUjNpURYqk/Z6nHD/0B2QTGPzNlEl+bh3DOyo9PRiYhIJbHW8u3mVIZ1iqB+sL58lppLibBUD7cbPrv7ePU4/AN5bsFW0nIK+OtvehMUoB9FEZG6YuOBIxzIytdqEVLj6c80qR6rp8OuRTDuJYjoyMqdGcxctZ8pw9rTJ7qR09GJiEgl+mJDCoH+htE9lAhLzaZhOKl6advguz8frx6XV+Ri6pwNtGsaygO/7ux0dCIiUomstXy5IYWhnSJpGBrodDgip6REWKrW8epx9WH8K2AMf/16O/sP5fPCFb2pF6QyyiIidcnafVkcyMrn0t4tnQ5F5LQ0NUKq1rHqcdfMgPDmLE9I572ViUw+vx2D26uMsohIXfPFhmSCAvy4WMumSS2gEWGpOidUjxtHVl4RD36yng6R9Zk6pqvT0YmISCVzuy0LNqYwonMk4SGaFiE1n0aEpWoUZJ9QPc5ay6PzNpGZW8RbNw3UlAgRkTpozd7DpGYXcmmfVk6HIuIVJcJSNcpVj5sff4AvN6Tw4KjO9Ipq6HR0IiJSBb7YkExIoB8XdW3mdCgiXtHUCKl8m+fB+hnHq8clZ+Xz2LxN9G/TiLuGd3A6OhERqQLFJW4WbEzhwq7NVERDag0lwlK5ylWPK3Fbfv9xPCVuyz+u7kuAv37kRM6VMaaJMeY7Y0yC57nxSdolGmM2GmPijTFrzrS/yJlYtiOdjNwiJvWLcjoUEa8pK5HKU0H1uH99n8BPuw/xxPgetG1a3+kIReqKh4HvrbWdgO892ycz0lrb11obe5b9Rbwye20STesHMbxLpNOhiHhNibBUnmPV40Y9fbx63L8WJTCpf2uuHKARApFKNAF43/P6feDyau4vcoKsvCIWbkljfN9WBOqbP6lF9NMqlaNc9bj0nELu+yie9hH1eWpCT4wxTkcoUpc0t9amAHieT3ZnkgW+NcbEGWOmnEV/Ea98viGFohI3V/TXoIfULprNLufOVQRz7jhePa7EwgMfxZOdX8x/bhukmyZEzoIxZiHQooJDj57BaS6w1iYbY5oB3xljtllrl51BDFOAKQBt2rQ5g7cVXzM7LomuLcLp0aqB06GInBFlKHLuljwLBzccrx732vcJrNiZwfOTetG1hS6KImfDWvvrkx0zxqQaY1paa1OMMS2BtJOcI9nznGaMmQsMApYB3vafBkwDiI2Ntef2iaSu2pWeS/z+LB4d203f/kmto6kRcm6OVY/rdyN0Hceiban8Y+EOJvZrzdUDo52OTqSumg/c7Hl9M/BZ+QbGmPrGmPBjr4FRwCZv+4t466PV+/H3M0zoqyIaUvsoEZazV7Z63Jjn2ZWey30z4+nesgHPTuylkQGRqvM8cLExJgG42LONMaaVMWaBp01zYIUxZj2wCvjSWvv1qfqLnKmC4hI+WbOfUd2b06xBiNPhiJwxTY2Qs1emelyODWbKBz8QGODHv28coBLKIlXIWpsJXFTB/mRgrOf1bqDPmfQXOVNfbzrI4bxirh/c1ulQRM6KRoTl7Gz57Hj1OHfrgfz+4/UkZubx2nX9iWoc6nR0IiJSDT78eS8xEfU5v0NTp0MROSteJcLGmDHGmO3GmJ3GmF8svG6Mud4Ys8HzWGmMqXAUQuqI7BT4/L7j1eP+uXAH321J5dGx3ThPF0MREZ+w/WAOqxMPc92gNvj5aSqc1E6nTYSNMf7Aa8AlQHfgWmNM93LN9gDDrbW9gafw3GUsddCx6nHFBTBpGp+sO8i/Fu3kqtgobrmgndPRiYhINZnx816CAvy4QgWTpBbzZkR4ELDTWrvbWlsEzKK0KtFx1tqV1trDns2fAP1fUVcdqx43+ml+yGrMH+dsZEjHCJ7RzXEiIj4ju6CY2WsPMK5XS5rUD3I6HJGz5k0i3BrYX2Y7ybPvZG4DvqrogDFmijFmjTFmTXp6uvdRSs1wvHrcKLZHXcVd/4mjQ2QYr9/QXyU1RUR8yKxV+8gtdHHbkBinQxE5J95kLxUN81W4sLoxZiSlifDUio5ba6dZa2OttbGRkZHeRynOK1M9LnXk37n1/TXUC/Ln3VsG0iAk0OnoRESkmhSXuHn3h0TOa9+Unq0bOh2OyDnxJhFOAspWRogCkss3Msb0Bt4CJniW5pG6ZMlzcHAD2aNe4rqZe8jOL+adyQNp1aie05GJiEg1+nJDCilHCrhjmEaDpfbzJhFeDXQyxsQYY4KAayitSnScMaYNMAe40Vq7o/LDFEftXQkr/kFR7+u5dllTkg7n8/bkgRoJEBHxMdZapi/fTYfI+ozo3MzpcETO2WkLalhrXcaYe4FvAH/gHWvtZmPMXZ7jbwJ/BpoCr3tumHJZa2OrLmypNp7qce5Gbbnt4CR2pOYw7aZYBsU0cToyERGpZkt2pLM5OZvnJ/XSkmlSJ3hVWc5auwBYUG7fm2Ve3w7cXrmhSY3w1VRsdhJPRb7ED/sLeeXa/ozsolEAERFfY63l5YUJtG5Uj0n9tTiU1A261V9OzlM9bk79a3hvfzNeuKI343q3dDoqERFxwNId6cTvz+KekR0JClD6IHWDVyPC4oOyU7Dz72NXYGf+mDmGl67qw8R+GgEQEfFF1lpe/r50NPg3KqAhdYj+pJNfshbX3LspKsjjrqN38uI1A5UEi4j4sCXb01m3L4vfjuig0WCpU/TTLL+Qvex1AvYs4hnX9fzfteO4rE8rp0MSERGHuErcPLtgK+2ahnJVbPTpO4jUIkqE5QR7t8URvPgJltp+XHzTH7mkl+YEi4j4so/XJJGQlsvDl3TVaLDUOZojLMfF7TpI6KzbyCOE5jdMp2snrQ4hIuLLcgtdvPTdDmLbNmZ0jxZOhyNS6fSnnQDwaVwSq9/7A93Yg2vcy3Tt1MnpkETkJIwxTYwx3xljEjzPjSto08UYE1/mkW2Mud9z7AljzIEyx8ZW/6eQ2uC1xTvJyC3k0XHd8NQJEKlTlAj7OFeJm798voVZn37EFP/PKex1PZEDJzkdloic2sPA99baTsD3nu0TWGu3W2v7Wmv7AgOAPGBumSb/OHbcs1a8yAm2H8xh+rLdXNE/in5tfvG3lkidoKkRPiwrr4h7Z6wjfuc+ljeYjqnfhuBLX3A6LBE5vQnACM/r94ElwNRTtL8I2GWt3Vu1YUld4XZbHpm7kfCQAB4d183pcESqjEaEfdSaxEOMfXk5P+/J5POOX9C4OBUzaToEhzsdmoicXnNrbQqA5/l0E/qvAWaW23evMWaDMeadiqZWABhjphhj1hhj1qSnp5971FJrzFq9n7i9h3lkbDea1A9yOhyRKqNE2Me43ZbXFu/k6mk/EeDvx3ejs4hJmgdD/w+iBzkdnoh4GGMWGmM2VfCYcIbnCQLGA5+U2f0G0AHoC6QAL1bU11o7zVoba62NjYyMPMtPIrXNvsw8nvlyC+e1b6riGVLnaWqED0k5ks8fPt3A8oQMLu3dkucujiD8nVugVT8YfqpvVUWkullrf32yY8aYVGNMS2ttijGmJZB2ilNdAqy11qaWOffx18aY6cAXlRGz1H6uEjcPfByPn5/h71f10Q1yUucpEfYB1lo+WZPEU19sweW2PD+pF1fHRmE+/A0UF8Ck6eAf6HSYIuK9+cDNwPOe589O0fZayk2LOJZEezYnApuqIkipfV5fsou4vYd5+Zq+tG5Uz+lwRKqcEuE6Ljkrn4fnbGTZjnQGxzThb7/pQ5umofDzNNj1PYx7ESK0VJpILfM88LEx5jZgH3AlgDGmFfCWtXasZzsUuBi4s1z/vxpj+gIWSKzguPigFQkZ/HPhDsb3acWEvq2dDkekWigRrqOKXG7eW7mHlxcmYIG/TOjBDYPb4udnIH07fPcn6DQKYm9zOlQROUPW2kxKV4Iovz8ZGFtmOw9oWkG7G6s0QKl19h/K43cz19KxWRjPTerldDgi1UaJcB20PCGdJ+ZvZlf6US7q2ozHL+tROgoM4CqC2bdDUH0Y/ypo/peIiE87Wujizv/E4XJb/n1jLPWDlRqI79BPex2yMy2Hv369nW+3pNK2aShv3xzLRd2an9hoyXNwcANc/SGEN6/4RCIi4hOKXG7u+m8c21NzeOvmWGIi6jsdkki1UiJcByQdzuOfCxOYszaJeoH+PDiqM7cPbU9IoP+JDff+CD/8E/rdCN0udSZYERGpEdxuy4OfrGd5QgZ//U1vRnY53XLUInWPEuFabP+hPKYv382sVfvBwK0XxHD3yI4VL35ekA1zp0CjNjDmueoPVkREaowSt2Xq7A3MX5/M1DFduSo22umQRByhRLgW2nYwmzeX7OLzDSn4GbiifxT/76JOtDrVUjdfPwxHkuDWb1Q9TkTEhxWXuHngo3i+2JDC/b/uxF3D2zsdkohjlAjXEiVuy5Ltafznp70s2Z5OaJA/t5zfjtuGxtCy4WnWetzyGcR/CMMeUvU4EREfdiS/mHtnrGV5QgaPjO3KlGEdnA5JxFFKhGu4tJwCPl69n5mr9nMgK59m4cH8/uLO3HReWxqFelH/PTsFPr9P1eNERHxcYsZRbnt/NXsz83jhil5cPbCN0yGJOE6JcA2UV+Tiuy2pzI9PZumOdFxuy5COETw2rhu/7t6cQH8/705kLXx2j6rHiYj4uM/iD/DY3E34+xv+e/tgftX+F8tLi/gkJcI1REFxCSsSMvhiQzLfbkklr6iElg1DuG1IDFcPjKZ9ZNiZn3TVdFWPExHxYUfyinnyi83MWXuAAW0b88+r+xLdJNTpsERqDCXCDsrILWTRtjS+25LK8oR0CordNAgJYELf0vKWg9o1Ka0EdzZUPU5ExGdZa5m77gDPLtjKoaNF3HdRJ353YUcCvP1GUcRHKBGuRnlFLlbtOcSPuzJZuSuTTclHsBZaN6rH1bHRXNStOb9q35SggHO8ULmKYM4dqh4nIuKDVu7K4MVvdxC39zD92jTi/VsH0aNVQ6fDEqmRlAhXofScQuL3ZxG//zCr9hxi3b4sXG5LkL8f/do04oFfd+aibs3o3rIBpjKT1aXPQ8p6VY8TEfER1lp+2JnJa4t38uPuTFo0COH5Sb24Kjb67L9ZFPEBSoQrSWZuIdsP5rAlJZt1+7OI35fFgax8APz9DD1bNeCOYe05v0NTYts2oV6Q/2nOeJb2/ggr/gH9blD1OBGROu5IXjHz1x/g/R/3sjMtl4iwYP58aXeuG9zml9VFReQXlAifAWsth44WkZiZx+70XLYfzGF7ag5bU3LIyC083q51o3r0jW7E5PPb0bdNI3q2alh1iW9ZJ1SPe77q309ERKpddkExi7el8fn60pWFikssvVo35O9X9uHS3i2VAIucASXC5eQVuTh4pICDRwo4kJXP3sw8EjOPHn/OKXAdbxsc4Efn5uGM6BJJ1xbhdG3RgC4twokMD3YmeFWPExGpcwqKS9icnM0POzNYtiOddfuzKHH///buPrauuo7j+PvT24fbh61du412W/egLgoZREiZEB8gwsxAwqKJRo06QwxBJIFENCj/+CcJxodEoiIhwUiCxCcWxSAYTfQPyAYyA0xlIshGYYyVdg99uG2//nFv1267bW/pbc+9535eyc0553d+55zvL+u+/d7T8xCct7KJ3Zdv5vr3r+PC9e3lvcTOrEbURCE8PjHJwKkcx06O8dbJUQZO5jh2cpS3To7xxtAI/YXC1Z+qAAAACBNJREFUt39whMHh3BnbZurEhlXNbOpq5ZKNHWzqamXz6hY2d7WyqauVTKVce/XCHr89zsysyg2N5Dh45AQHj5zgucOD7H/1bV7oHyI3EUiwbV07X7ni3Vzx3jVcsnFV5fwOMqtSJRXCknYCPwAywH0RcddZ61VYfy1wCvhSRDxT5lh5fXCEoydGOTE6zvGRcY6P5Kanp9um2wdOjXHs5BiDwzkiiu9zdVsTPe1Zejtb2L6lk+72LD3tWbpXNtPTnmX9qubSX2CRlOOv++1xZjVE0qeAbwPnA9sjYt8s/YrmbkmdwC+AzcDLwKcjYmDJA69xE5PB0HCOI8dH6R8cpn9whP63h3ltcITDA8P8580THDk+fZldS2OGiza0c8OHtnBxbweXbu6kqy2hvziapdS8hbCkDHAPsAM4BOyVtCciXpjR7Rpga+HzAeBHhWlZff2X+/nri0eLrmusr2Nltp4V2QbamupZka3n/O6VdLY20tnaSFdb4/R8axOrWhtY1dJY+UXufCLgtzdDbthvjzOrHc8BnwR+MluHeXL3HcCfIuIuSXcUlmvyW3REkJsIJiaD3OQk4xPB+NR0an4yyE1M5vtMBKPjEwyPTTCcm+DU2AQjhen0/DgnRsZ5ezjHwKkcg6fGGDiVY2jk3JMydYK1K7L0dGT58NY1vGdt2+nPxs4Wn/E1W2KlnBHeDhyMiJcAJD0E7AJmFsK7gJ9FRABPSuqQ1BMR/eUM9u7s/bSt2Uud8smjrk6FeZEvZwsZJgKGyX/empF1zshAMXvbGbPvdPuYo63M248OwbXf8dvjzGpERBwA5rsmdK7cvQu4stDvAeAvLEEh/OBTr3D/3/6bz1ZTKYt88ZmfQhD56emUFjP6zVjPVJ+Y7nv2vgrzxHTmLHqsQp+JyBfA5ZRtqKO5IUNbtp5VLY20NzewqbOFjpYGOloa6WhuYPWKJta1Z+npaGbtiqbqPyFjVsVKKYTXA6/OWD7EuWd7i/VZD5xRCEu6EbgRYOPGjQuNle7erdAwMrWzmXueeZBz2+ft+063L/H4ZTnWHNt3vgsu/TJmZjPMlbvPmzpRERH9ktYW28Fic3ZXayPv614JymeuqcI9Pz/dpkKj0Ol2TvcptBU6TW9X6HNGm87Z7ozjnXWs+jqRqRMNGVGfqaO+Lt92ej5TR0Mm36e+bnq+qT5DS2OG5sYMzQ35aUtjhmx9xs/sNasypRTCxf5Xn/0VupQ+RMS9wL0AfX19C/8a/pHbF7yJmVk1kvQE0F1k1Z0R8UgpuyjStqC8u9icvXNbDzu39Sx0MzOzZVNKIXwI6J2xvAF47R30MTOzEkXE1YvcxVx5+Y2py9ck9QBHFnksM7OqVMqFSXuBrZK2SGoEPgPsOavPHuCLyrsMGCz39cFmZrYgc+XuPcDuwvxuoJQzzGZmqTNvIRwR48AtwGPAAeDhiHhe0k2Sbip0exR4CTgI/BS4eYniNTOreZI+IekQcDnwe0mPFdrXSXoUZs/dhV3cBeyQ9CL5p0r4VZRmVpMUsz1gd4n19fXFvn1FH31pZlbxJD0dEX1Jx7FcnLPNrJrNlrP9zBYzMzMzq0kuhM3MzMysJrkQNjMzM7Oa5ELYzMzMzGpSYjfLSXoTeCWRgy/MauBo0kEsEY+teqV5fNUytk0RsSbpIJaLc3ZFSPPYIN3j89iSVzRnJ1YIVwtJ+9J6Z7jHVr3SPL40j82WXpp/ftI8Nkj3+Dy2yuVLI8zMzMysJrkQNjMzM7Oa5EJ4fvcmHcAS8tiqV5rHl+ax2dJL889PmscG6R6fx1ahfI2wmZmZmdUknxE2MzMzs5rkQtjMzMzMapIL4RJJul1SSFqddCzlJOluSf+U9A9Jv5HUkXRMiyVpp6R/SToo6Y6k4ykXSb2S/izpgKTnJd2adEzlJikj6e+Sfpd0LFb90pi3nbOri/N25XMhXAJJvcAO4H9Jx7IEHge2RcRFwL+BbyYcz6JIygD3ANcAFwCflXRBslGVzTjwtYg4H7gM+GqKxjblVuBA0kFY9Utx3nbOri7O2xXOhXBpvgd8A0jdnYUR8ceIGC8sPglsSDKeMtgOHIyIlyJiDHgI2JVwTGUREf0R8Uxh/jj5xLM+2ajKR9IG4OPAfUnHYqmQyrztnF1dnLcrnwvheUi6HjgcEfuTjmUZ3AD8IekgFmk98OqM5UOkKOlMkbQZuBh4KtlIyur75AuXyaQDsepWQ3nbObuKOG9XpvqkA6gEkp4AuousuhP4FvCx5Y2ovOYaX0Q8UuhzJ/k/4Ty4nLEtARVpS9UZIUltwK+A2yJiKOl4ykHSdcCRiHha0pVJx2OVL8152zk7XTkbnLcrmQthICKuLtYu6UJgC7BfEuT/BPWMpO0R8foyhrgos41viqTdwHXAVVH9D5Y+BPTOWN4AvJZQLGUnqYF8Mn0wIn6ddDxl9EHgeknXAllgpaSfR8TnE47LKlSa87ZzdnpyNjhvVzq/UGMBJL0M9EXE0aRjKRdJO4HvAldExJtJx7NYkurJ30ByFXAY2At8LiKeTzSwMlD+t/oDwLGIuC3peJZK4czC7RFxXdKxWPVLW952zq4uztuVz9cI2w+BFcDjkp6V9OOkA1qMwk0ktwCPkb8p4eG0JFTy376/AHy08G/1bOGbuJnVDufs6uK8XeF8RtjMzMzMapLPCJuZmZlZTXIhbGZmZmY1yYWwmZmZmdUkF8JmZmZmVpNcCJuZmZlZTXIhbGZmZmY1yYWwmZmZmdWk/wMp0tidGXTdIgAAAABJRU5ErkJggg==\n",
"text/plain": [
""
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"x = np.linspace(-5,5, 1000)\n",
"\n",
"plt.figure(figsize=(12,4))\n",
"plt.subplot(1, 2, 1)\n",
"plt.plot(x,1/(1+np.exp(-x)))\n",
"plt.plot(x,np.where(x < -2.5, 0, np.where(x < 2.5, 0.2*x + 0.5, 1)))\n",
"plt.legend(['sigmoid', 'hard_sigmoid'])\n",
"\n",
"plt.subplot(1, 2, 2)\n",
"plt.plot(x,np.tanh(x))\n",
"plt.legend(['tanh'])\n",
"plt.show()"
]
},
{
"attachments": {
"image.png": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAAFbCAYAAADGGAZpAAAgAElEQVR4Aey9CbxeVXnv/5yTCkJCgIIyE6JchauYOCItClGwrRUIaGupSqBQraJN6O2tgFqg2tDa1gTFoQqSOFGrJYH6rwNTcLgC6iUBFLAICVNAQUIGEC7k/D/f96znnHV29jvv/b57+K3P5333tMbvWnuv33722mubyYmACIiACIiACIiACIhASQmMlDTfyrYIlJ3ADDObZ2aHmtnzzGxW+O1T9oIp/yIgAqUl8N9mti78fm5m15nZDaUtjTJeGwISs7WpahW0AARmmtkZZvYHQcSSpZvM7D4zezT8HjOzsQLkVVkQARGoF4HfMrPfNrNdzGxXM3uJme1tZhuDqP2ymX21XkhUWhEQAREQAScwamZ/YWa/DEL1djP7WzOb7R60FAEREIGCEeC6xdOjz4UbbW6yr49uxAuWXWVHBERABEQgLwKvMLOfBhF7m5kdn1dCilcEREAEciKwg5l9yMw2hWvZl8xsp5zSUrQiIAIiIAIFIvDucOHHIntqgfKlrIiACIhALwR2N7NPhevaXWZ2SC+RKIwIiIAIiEDxCWxvZowv47Ec1tj9ip9l5VAEREAEOibwJjPbYma/MbM/7TiUPIqACIiACJSCAOPMvhOE7JVmxqwFciIgAiJQNQK8IPZguNadVrXCqTwiIAIiUGcC/xQu7l+vMwSVXQREoBYEmErw3nDNe20tSqxCioAIiEDFCfC4jaEFa8yMFybkREAERKDqBOaa2eNm9mvN0FL1qlb5REAEqk6A+RgZP7bezPasemFVPhEQARGICBxrZlvN7AfRPq2KgAiIgAiUjMAXglX2mJLlW9kVAREQgSwIXBKugQhbOREQAREQgZIReGm4iK8qWb6VXREQARHIigDjZ58wszvMbFpWkSoeERABERCBwRD4YRCz/3MwySkVERABESgkgQ+Ha+HphcydMiUCIiACIpBKAAHLS1/XpB7VThEQARGoD4HdwtjZn9WnyCppEQgwJ6acCIhA7wR+LwTVVFy9M1RIERCBahB4xMwYbnWwme1fjSKpFCIgAkkCRwYrXnI/28ljXBDOTfMY9nEMi2Arx3HiHZabb2ZM29Kp28XMFnXquSD+vh0sEZrBoCAVomyIgAgMlQBDDOh7/nyouVDitSIgy2xxq3tZuMMtbg7b52yFmSFQO3UIWQRwWRyfrT0iTEfDl3DkREAERKDuBC4LYvb36w5C5R8cAYnZwbHuNqVmYradpRVLaDsBeUAHmWmXThwFfjuJ08OQR0RgN66b+LuJtx+/LzAzBO2N/USisCIgAiJQIQLMtc1XwV5coTKpKCIgAhEBRF+zoQHJY8lhBktDWMI/amYrE3EhEO+O/CwJ67EojeMgnnMSeSPN2A/ptBomcHLIC3HxI3337/tY+nCJpH+OsQ/nwyY8XNi9zf44z+5nWEsEOfn9m2FlQOmKgAiIQAEJcIP/cAHzpSyJgAhkQMAFK4Is+fMJpz2ZWMzy6B3R5MIUwbgh7MM/lliEJ9Zc3yZ8HAaxuDYSmx6Hi0nPG3EQHz/iuCnEmVxwnPh9WADbq4PIdr9x+lhW2fb08OPC2f2TR9J0x7ADyukCmSVlKMq42uNDmU7yDGspAiIgAiJgV5jZM+IgAiJQTQIuGBFsyR9CELHnjuNu0cQK60LVj8dCEIFIWASlO4RfLCYRhUkRGItHz1scB/7jPHncLF3MYgH2ML50f3H6HHMx7sddpHu4OD/4Ic/OwMOQJyzARXB/FvhobFgRakN5yJIA5ybn9rWhjXMu+48bXMbDL4jO/SzTrnJcXOvghvECjs7Ul/CG+3Elh/C5ULadS14OZb8kBH6rJPmsWjaToo7ysY8LWZrDqomgjR3i1x3H1wTx5/vi41xAuahwkeTXyiEg3cVx+D5f4u+MEB8CE7/k8YJEPmL/rJM+Qps888OxHVtkw+5Gnt2C7fuKtHxWyMzTRcqU8iICfRDgxphzrnFu7r7Xflv33O95Nn2nXWz6zF3sl/evtS0bN8y9+/Y1nLP+VIYb6/OanPd9ZKVSQbkGLxwZGf2rsbGtMynZ7IPmNJg+d58DYGpbNm2wB++967UPr7+XvmCR2chjZmMLzWx5CUk8GfL8bDN7rIT5V5ZLRkBithwVFgvMZjme1exAtB/x2UqgRl47WqUTw2LMxZeOjfhZ8nnXpHOxfnm4OJMPLvDNBLyHzzrPHq+WIiACkwQQr1hb5+4wfaet80/5Kzv0qPn2vIPnpr4kvHnjBrvhqpV29YplduuN1yG8TjEb46Y2+QRpMoX6rs03G1lmNrbzbnvus/Vtf/l39uJXHWl77Ov38lPAjD5031q74eqV9pWPnzPz8c0b4QnXUzK+dk9JVBsiUHYCqReqsheqgvlnnGjysRPi0J0Lw/jqGB9HDHN3jHiMhzdgXYnHsHp8nSxJCyFK3FhkiYcfcaY58oP1GLHLBZo8x/lNC7Mustp6vvvJc1oa2icCdScw12yE83Huie89xy5edc/oie87FyHblMuMmbvY60842RZ/cZWd/ckVtuOMnbA28ugcq67cJAGE6Ar4LDz/Evv8qntG4dZEyDZCcezYBYvsomvXjVAf49fAEa5/8TV9MgWtiYAImMRsORoB40Znh0f0vEHPoyd/xEcJEJPXBcsKxxG+dCyx46LKlZGw+GHcFo/86cR6cYhYLLBYc4jP80U+Ykde6BXxj/XYt0k/OR6WcPjhGI7jrJNvzzPriHs5ERCB/gnMHR2d9t0dps/YCVGKiEWoduNefdT8hvDisXk4Z3liIzd+/Voy6wWHPHPB5WtGELHdOOqB+li68ia/WcB4IEHbDcTu/GJcoX/hl3RUHvub3+ElQ2h7oAQkZgeKuyHokmLPc4DYi48hMl20seQihqDlMf28IGZj/4hbLJ8c59E8b9pznHhxWEPZxw8/PLbi5x1PMn3CpO0bj238GHkaCfERJ5bUWGQzfhbBy4WAdBj7xT6sDJSB8HEZyCNxkC8uGp5n/HqeCZ8mgj1fWoqACHRG4ACzkVXb77DDjud/+bujiNJeHcLr77+4yl78Ku45GzfM3Sm3XhMubriG+EHgn//l705rZYltVwQs5Iu/dN3IjjNmjpmNYLiQoGoHrbfj9Ef0LfSRSYdxSP1OkkqBthEiciIgAr0ReJeZfcbMjjazq3qLQqFEYGgEuKk8AotsP0I2zj1jac9622ufWffzW6aFm2+/IY+9VX39gJGR0TVYuxkq0K2luxmcu25bbYvmN15HwNCR9l5Cs6DD2H+hmfFZWz7z/dAwMtBDmhhc/IW7+GYMkevvdkgz9QB2EEFkmR0EZaUhAiIgAsUigBn2iGNOWthSyD50/zr7z+UX2OL3ntBYtisCwu2dH/w4QhaXHOoUdld+cS4zFnzgU5dnJmQhhoV2cgxtz+86VB5+HwV0i3dy6J3vj58i9pGMguZBQHcZeVBVnHUhIMtsXWq6euVcu8P0mftfvKq55fDsdxzJTAXblPzVRx9vZ1942Tb74x1LzzzZrlnRmFGKIUJYgOviGHd5N8MteDkua4fl+7R5s8Ye37xpo9lYd4Obs85M6/jKaJllrl8c75fEgpZtbv4Y4sa6XAEJyDJbwEpRlkRABEQgRwJYmmYd9eZTmloO3/KSHRpCdvpOOzcst6eetaSx3G77Z9v1V66wBYfv3TJ7vI0fXO8DcT2Gci0bj6ej8meaeyzff/qX540wzVfi/YRM06lhZAwlcMdwA+5E/OdtOBa47lfLghCQmC1IRSgbIiACIpABgU6sdY3Oudk4WSyyTz35G5t98Fy79McbGlNvHXfyosby6zc/YXvu/3x79FfrbfHpae/JjJeAR+J8cCFlSsEMiji0KDpiu8P0mWPN2KblHN6XfqLzd4sOfb1rK81skMazx30+lIAXkPkASPzzKCVmnUQBlxKzBawUZUkEREAEeiTAC1e8rMKUds3EV8MKdcihsTFqMjWGFmCRvWAlX1s1u3zZUjv7HfPspz/+fmP7w8uuttFp0+z6q5hgpLk77OgT6F947N5uPunmkRTnCLD4jDbjgNPBjed1zmFvOD7X4XvMjBBuFFyAFYfS4HJCm6KBtmrn3eTGWWKN5c7Cfz5WhHnak2J24q6im4TkNx8C+gJYPlyLFiuPTZgEkjE/TO3ijkdiXAyY0mviuaAfLPmSjvzdZrZdyvfP/Tvo/S4PD4zeGN7cJj6mT6MTxyr1TB8/PgfZTf5Ir5V/3irezcyIN/79Jmw/0SZ8q7jzPpZWNvItN5UAHTznN50sgoup9dhm8Kp3yoTYJUyhNTV0EK7sPORQhrqa3bHmBrv4fGb6M2YpsI/9x4124ItfYbNecIjdfdvqhsB90Sv8NGh4m/iL3uInX1WY1QCrHddMfpSHqQNh62VriPY99mksJjjksbLnfs8bfXj9vY150PKIvwRxIj651lIHOJa09dZ3WMFzyiIWs/Fhv3FJClnS8vMr9q/1IRGQmB0S+AEny3QjOB6dxI4TlR9itmru981s8YAKNd7bDygxJZMJAcRw3iK8XfxpIr1dmHbH9wkWRDpnbujSxFfj6wZpFDc8Mj6L0oteOa6Tpr4ANmZ3335zQ8xiucXdeeuPrZmY3W3PfT0Jbqbp+MvsdgjtBeGKWuUXW+8QU490WsDky3VwvvTC8cszMxbwsYRW7umnnvLDWOHbtYlhHP+dkMF/NLPHc8jjDDPjLUTuuqiLuJ0jNDHc+E2Gs2q19HMiKVqbiVz2067lCkJAYrYgFZFjNvxkJInYOsO2H0uewDlmZ2BRfzNc6LCS8tgvj99RZnaqmZ1vZj8zsx3DhRVrMOfWs6Kfb8fL5Drb8Y/wvx1+Dg4LKnPa/rBNmfAfl/3QkD/ijNOI17HcUgZ3D5rZt8xsS5u0YrZxmvH+uqxT/rwYPNsrpoclHT6WLD/nU6PwT9jecPVKY5wsMxcs/+f3N/zy8tdLD39DY/3hB+9rLDnezD31JE214Vwo+HbVlsz5ikHg1k4LdtrZS23LxnF9f9HiRY3xya8/Hj1m9twOLLtjnE3jzi2Hvl20JU/+Bulo5zxlpGECNNnnpeUlZpjsC/188f3Ez1AT/1IllejW4bS4tW9ABOjI5KpNwE/UpPUVq413Mn6iVokEY5w6uZD1U2bMU4jZa3L+aAKClme5f2Bmf2Fmx5gZSuGt/WS+SVgu0oeZ2dvN7A/N7ITwRbbWczE1iUy7B0qAc5qnMHTidLo4znssSP5Y1Pf5uT/uK/y/5o1vtX8640/sZz/5gTHH7D4H/A/7yo8etXU/v9VmveDFjU/dsv/Be+5qjJvdYx+aS7rb9OiEobIK03PBk8+ZMoQDzjgeafvjbbZd+IwfbfHvNw14mT5zF2NoQrMxzGnRPOtZ3C833KSs9T3jS258hnkD+SEze5uZvdbMfp1DXrCU/0l4wdDrg2s+7Zz23k2fhgWXNpr29GD8DmMyPvwwXx39Kk/kurH+Tq0hbWVKQGI2U5yFjMwvsPQ6/hUTMhpfALo58QtZyIpnis7givD7qpl92cz+OFgA/yjjsjMukN+/BaHO47r/CCLp4xmnpeiyJ8DzaeqPeqNTT+tsN9x922oePacKId7Ev/6qlfY3bz3MPvrVHxqC1YcS8BLYOX+GdXbMTvmbf+4092kiodOwRfHHdRRhw80B4wEQsclyNa6jD92fhjzbYgSrbtJAEScy7DHlzubnOX0BjLpwq2/ypiLm0Mk6Fdas0pIGES8X7JPHOklLfnIiIDGbE9gCRetiFvHqVto4exKyMY3ir3MBPdjMvmZmbwnWWQRuHu7iYHG+Ooij75jZ7XkkpDgzIUBHyyPvduf06i2bHjuCz6PGFkLPAZ+3fefRB9qD9/zC/vx1s22vWc+33fbY1x556D5bv+4XDSHLC2QMQ2jlEMRmI0zu3y4/raIpyjEK0wnbdbfccO1+4Uazo7x3+3GFh+5ba3ff3tCxVeDaEaMUT9QHfVraTUWK90x30Y9KyGaKtP/IGNslV20C/jjxlPAohccp/Pyuvt+TkguKP9KsNsnilG6jmZ0YHt99imF2OWYNKx9pYcnDWqtrRo6wM4i6E4GDELBbb2x+6n/2yjvtze8800anjdr6dXc2/LLcbvvtjQ8otBNgk4JrbEUGZSpKFB2x/dUD94xyo5CXi+qtUY95pVPweLlx48mDW0oHmV0MRPlV8CBLUqG01DFVqDJTihJbYrmDpffynw8ziE9K9jEuLA6XEu3ELvwzGF7z7U0gGdgKQw8YP8t42s/nnOqPzOxfwhjrs3NOS9HnT2AVFtPLly1hNoWmbsH/Ot9W/uxp+9w1axsC9oo7xoyPJrSzyBLhNSu43DRc3QRXo+BXLEdn5eO+/PFzxsYt3rIO5kO4ZawYbnhXIu43WwbQwcEQkJgdDOdhpeKiNPmBdUSov7kRn5T4Z8xdc5PNZEm4O2UMLkI2jmPSh9byJsBQgwfCi1rPyTkxZmzAnR6WWpSawNg5WBA7+fIUY2Y7EbCOA6tsmGaK607dxCzXwuuuWbHc4JC1u/qyZfbw+nv4nO2SIVklsy5S2eLzYXvZV27ZSBQsvxKzBauQjLPjJ15SbLrI5e1PPynxe1zYbjUZt1tj+fqKv0DSifjNuGiKLhDwG5UX5EwES/AtZsbHF5jjUa7cBDAdrlt5yceeyfqR+OLTJx7UtJ4stdz8WuW+MZj4/Pceb5vD9FutPHd6jHq6aPEit8rmZ/rtNEP19EdfytCrR7t4gllPUgMutcTsgIEPODkXs0mxmbaft0OZmw+Bmvy4QpxtHrMggpmzEquLj72N/Wh9cATGvzFqlreYpUQ+G8aLB1c8pZQjgflPPvH4Ex94x5FjWYmupWee7C8nMZtC8rqTY1EKFTWC5wzEp389rd/cUT9L3r/gmS2bHsMqi9FhGGNF+y1GFcJj/KEPpJ+sa/suZD1KzBayWjLLlJ90yUd9WEw4GSdMKNHnbDnmltu0jHChxvLASY2/pNU3LYz25UdgkGLWL96DEM75EVPMTmD11q3PvA+BdNbbj+jbQouQ5fF6mH+19VQHnoPqLrGcLmdYwAVnndKXhZbhCh88aZ6tu+PmaWHOZz8Pq0tPJROBLglIzHYJrMLeexnYjoVXF9bhNoq7Q/J5zmjgJfxVWBn/lqnv1bLMBHhh6RSE0tlvP2Lr1eNitKvyILYWzX9pLGR9ovmu4qmgZzg0BC1itJfhHLfcsMoWzZ87FsIySf/Em3UV5KUiiUDPBCRme0ZXuYBujXVLKyI1/qZ3bMWl8L2I38pBq1mBngnlxUIkVx0CCKSXPr550+YLzjzZPnDSPLvhauahb+0QWFgd//z1s12oMbSA64QegU+iQ9A2hhwg+OHViahljl7qgd+WTRs3hSFgGic7yVVrIjCFgD6aMAVHrTewsvrLRIDgMSEve7lzkevbPu42ud+Pa1k9AhKz1atTL9FqszFmODn3lhtWLcQiuOOMmVtf8urXjc4+2E/1ca98feqHV162ldkQQmCuG53OguLp1WmJCMU4sPTqy5YdwdCD5+y9f4Ptc/fBJjDpfnn/Wvvhdy7b+vjmjc52udkYbP1F3UnPWhMBERABEdiGABdbLpqdOr9Ad+q/iv7eFazXRw2xcDuFPPC1rrzdy0Ja7887IcU/VAIoLG5mk09n4ic1iCuuAf5EZ6gZLlHi3BnADX4xz3gdAwH8pyrdEhXSzC4M5dujXNlWbstKQJbZstZc9vlmOi4uqDwW62RcFhdlWWWzr4cixyjLbJFrJ7u8uVBFdOF4QsP5fqCZ/VjnfaDS28KFqr8gB9dXmNmDfJRNFtjeoCqUCPijDJEQAZ+Wq1NrAFab5CwJolhtAhKz1a7fZqVjDCyC9nAJ2WaIet6PuGWqu90lZHtmqIAiIAIiIAJ9EKjbMIODgvW+1TzEfeBU0AIT4GkNT27icfQFzm6pssbNgs/hXKqMt8ishhm0gKND2ROQZTZ7popRBKpKQJbZqtZs63IhYBcEL8lZTVqH1NF2BODJVHeMPe70qVi7OHVcBGpHQGK2dlWuAotAzwQkZntGV+qAsYBdWOqSFC/z8Zy88XrxcqociUCBCUjMFrhylDURKBgBidmCVciAsuMvK5EcLyzJgpgNeCzefJrWnVu/fVtLERCBDglIzHYISt5EQATs6cBAs6DUpzEgXOckihuL28QhbXZBIGmJhbWmOusCoLyKgBOQmHUSWoqACLQjIMtsO0LVO+7C9bFQtHUJa2L1Sjy4EiFmY66knBS4g8uNUhKBEhOQmC1x5SnrIjBgAhKzAwZegOQYL4uA/XnIC7MaYEGMx9EWIJulywLDNbB4+5zezGiwJtwoaMaI0lWnMjxsAnpcOOwaUPoiUB4CErPlqauscopoRbyeGiJEfPFRBc0x3R9h5pdlbm+WvFTHEJ6/DlEibOVEQAS6ICAx2wUseRWBmhPQmNn6NQDEFr8/DkW/N7Im1o9GtiWObwg2h88HZ5uCYhOBmhDQMIOaVLSKKQIZEJBlNgOIikIEREAERCBbAhKz2fJUbCJQZQISs1WuXZVNBERABEpKQGK2pBWnbIvAEAhIzA4BupIUAREQARFoTUBitjUfHRUBEZgkoDGzkyzqtjYSCjxWt4KrvCIgAsUnIDFb/DpSDkWgKARcyEwrSoaUDxEQAREQARGQmFUbEAER6IYA1lmJ2W6Iya8IiIAIiECuBCRmc8WryEWgcgQYNysxW7lqVYFEQAREoLwEJGbLW3fKuQgMgwCWWc1PPQzyw01TY2aHy1+pi4AItCAgMdsCjg6JgAhsQ0CW2W2QaIcIiIAIiMAwCUjMDpO+0haB8hGQmC1fnSnHIiACIlBpAhKzla5eFU4EMicgMZs5UkUoAiIgAiLQDwGJ2X7oKawI1I+AxszWr85VYhEQAREoNAGJ2UJXjzInAoUjIMts4apkIBnSC2ADwaxEREAEeiEgMdsLNYURgfoSkJitb92r5CIgAiJQSAKaYqeQ1aJMiUBhCWiYQWGrpnYZm2tmx4VSs76Lma02sw1heV1YLyKY+WY2J8oY6+eE7ctD/qPDWhUBERABERCBfAi8y8z4xOtR+UTfUaw7hTxc3JHv/j393Mxu6j8axVAyAl8N7WzY2UawIvruDvnh/Gv1W2FmRw470yH9A8yM/LTKbzg2giC/JAj0gmS/q2xcGMq5R1eh5FkEeiQgy2yP4BRMBGpKQMMMalrxBSj2uWYji8zGdt59r/22Hnb0Cfbqo+bb7IPn2oyZaNxxd9dtq+3WG1fZLTesshuuvhwLKL+VZnbKkCy1ZG6JmZ1MDl/8qiPG833QXDvk0EmdvXnjBrs75P36q1bufPfta04eGRk9YWxs68fM7Dwvn5YiIALbEvBB/dse0R4REIF2BLDMfsbMjjazq9p5zuk4ltmNZvZ5Mzs1pzTiaG81M64bL4p3ar3yBP7dzP4o1P2gCzshBhGxb/vLvxt9/QkNXdg2Hw/dt9YuWrwIUUvWHzMbQz0yFGFQbu7o6G99YevWpw9BxJ529lJ73sGMiGjvEONL3n/S1ofX38u7LavM7PghifH2md3WB5bZ081sTzN7aNvD2iMC2RLQC2DZ8lRsIlB1AhozW/UaLlb5ELLXYtV83fEL7ONX3NyxkKUYe+x7gH3gUytt4fmX2A7Tp88YHZ323WCpHUQp55qNrELIkv7iL67qWMiSOay2n191zyjlZqjE6OhvkfdJE/QgSqA0RKAkBCRmS1JRyqYIFISAhhkUpCJqko1lZjb3mJMW2qJ/WDZlOEE35ceSe/6Xvzft2TtOnzEyMrqcOLsJ34PfXRCyCOi//8K11qklOS0dyo0YRhQHYZ/mTftEoNYEJGZrXf0qvAh0TUBitmtkCtAjgaXMVoBl8s8/wGp/jsf7i7903cjY2NaZIyOjjDvI08q5irG97/zgJ6bF42J7LQFiGEEfRHj/MHrNiMKJQEEJSMwWtGKULREoKAGJ2YJWTM7ZYpw0b9oPyvHm/8LZB81pWGSzShRBe9rZS2xsbOv+ZrYoq3gT8RDvnBPfe05fFtlEnA1Bf+jrGzORoWon3xxLetS2CNSQgMRsDStdRRaBPghozGwf8BS0YwLn4pMXptq5K5YvNV706tQdu2CRPXefWTYyMnqGmSGas3QMLziXF9WOWdBcK3/iA6fZia/Y1Y594Ujj95aX7GiL5r/MHrp/Xcu8RDx8TtqW/nVQBOpCQGK2LjWtcopANgRkmc2Go2JpToDH/wt4+7/dI/qlZ55sFy0+w355f+dilmQRhQw38Omymmel6yPzGV7AjAvxdGEeC2J1weF725Vfv9i2bNpg03fa2XbdnRf+x+yu226ydx19oH3vv5jSN93xQpu/EDaAcb/pmdBeESggAYnZAlaKsiQCBSYgMVvgyqlI1pgX1l5/fPPpt7DELj79eLtmBe9yde+Yn3aH6Tttjb4g1n0k6SEaeT/0qMZiGx+LT59vj/5qve25//Ptc9estUt/vMGW/2C9ff3mJxpzz2595mm74Eymw23uyHtwGmrgJLSsPQGJ2do3AQEQga4ISMx2hUueEwQYN8BcU61evmopCInvgrNOts0bH2285Z+Iv+PNw95wAv2ffwa3XTiGI/BFLv98bhP/I/OwKKdZZbG48lEErLGfvfJO22OfWY04nvrNbxrLsz+5oiFon3ryiYZQb5JAw08Q4hOqtplf7ReBuhCQmK1LTaucIpANAY2ZzYZj2WLJ4gWwxktdZsZ0W3yOFnGYZl3chTGtaYLQoS08f1lj3tbn7tP7kNc9JsN2Mk0X+cRUzJfEyDtf9EomvovZ2MxDXpVWJLNvXvrpRvb/5L2N4cB2+bKlduwLR+0tc3aws98xHgZBi7vlBqbWbe6e/6KX0XePq+Hm3nREBGpDQJ+zrU1Vq6AikAkBWWYzwVjZSBCGO7co3V+Z2UuCMEQc8mPAKwKXMemQ0mcAACAASURBVAOsH9BOpDJ2tF8XpTGng5kafmVm7zGzdwZrLm938eNrYlibmeqrpSh++MH7Glk+7uTxF8P+7UJE7fgEEbfeeN1EcbDcbtnER/3auv4htE1CHkSgHAQkZstRT8qlCBSFgMRsUWqiGPlguABWSoRcSzHXIruIMpQdP2YY6MvieMfq6+2XD0zOCvD8F73c9p514DbJb964wfe1nzLBfW67pMwI8cfM7K+3PTy5Z/c997UH7/mF/fTH37cXveJwe+6+sxvDDvAxOm3ahMf/99STtt32209sp62MC/FJAZzmR/tEoE4EJGbrVNsqqwj0T0Bitn+GVYmBMZsMFUDQrgnWSSyVq1oU8BVm9oeJ4QWERRDyQ2FiuuxZ0P7XVz5l117+xYksvPvcT6eK2WgYAwKafLdyM/jCbLDMxhZRLLLkm+EH6eMLQqx77X+gYYG9/JJ/aYjZcz/3Tbvs4n+yX62/x9767g81fCF0n3ryN40XxFplptvZG1rFpWMiUAUCErNVqEWVQQQGR4Axs5NmpMGlq5SGSyAeM4t4RcQiZrFIHh/EXLscIgJ9MCjhXMAmheTaX96/rmcxe8ZHv2D82rlIELYT4ESFwF4c4kyKb08qWQ7f31j+8Xs+ZFevWGbXX3V5Y/qt17zxrXbqmf8y4Ydpuz7yF8c0tv/wbe+d2N9iZdL83MKTDolAHQjoBbA61LLKKALZEcAyi5OgDSBquGA4AEIWqyQCFatkJ47xsBcE8Ysg9jGnybBrEZrdfAghGUEn23fdNqE9J1ZahEN4k/eXhuEUDE2YGKcQwm0wG9l4w9Vg2dYxe8Epf/PPjXGy/3TGibb4vSc0RC3W2C9+7OzGHLPMPctsCD6udttYxveEMbbdTa7bLDLtF4EKEJBltgKVqCKIwAAJxGLW1weYvJIaMgFELJ9TRbGx3q0bf/updSiGKSy49cZVtse+zeeabR1F+6M3X38tb199N0WUpgX24Q9px6J9Yyvuum31AsbjRsMYJo67SL3ko39t11+5ovGbOGjWmHbLZzSI98fr1181ce8wsRIf17oI1JGALLN1rHWVWQR6J8AwA5xuhAOImi0YXsAQgfxUZrD0XrOy/QcRZh881/7+C9cay27c1Zctsye2bGToRNaCsDFe+JoVGHLTHYL2X6/8hZ161pKGFRZL7Jvfeaad/+XvWTshS4wSs+lctbfeBDiZ5URABHoj8C4z+4yZHW1mV/UWRd+hdjIz5vH5vJmd2nds7SO4KKTDp0A3tfcuHxUh8B9heAB9xrw2L3llUWTU4AKEartP2vaS2J8dOWvs4fX3bjIbY2xucrhAL1FOhBkZGb1nh+kz9rno2nWpn7Sd8NjDCkMjFs1npIMxlUHLF856iD7LIBea2elmxrd6H8oyYsUlAmkEZJlNo6J9IiACzQj40AKNmW1GqJr73fCBVbbVbAVZlf7ckZHRTUvPXLA1mkIrk7ivWL7UHl5/z4jZGFOKZSpkyeDY2Na/fXzzxtFLP8HQ4mzdRYsnRmnkaRnPNtOKTQQGQEBidgCQlYQIVICAv3m+QygLL/7wJaRexk1WAEcti4Cg7eRlqSzgrEUU/uqBe0Y/ftYpWcTXiINH9BctZiauxlRi/cwv2ypPnCtr/vMLFxjDGbJyS888uTG1V3gRTS9/ZQVW8VSCgMRsJapRhRCB3Anw9vkCM9saUsJEhKBVp5o7+kIk4F/1GoRV1guM2FyOAL0gA0F7yw2rbOn7F2xlxoFwE5a5VdYzzhCAkZHRe8l3FoIWK+81KxpjiHnxbsI8G6WnVRGoNQGJ2VpXvwovAh0TcBPTcSEES+bbHJSlruOMymMuBJ4VYh2kmCVJHqdfhyD8wEnzrNchBx7+8c2bNpuNHTGAm7ANY2Nbj0U4I2gvvfC8niqF8kbhOd80vKAnkgpUdQISs1WvYZVPBLIhwFvfTNKOhRbH0gVu2KVFhQlMH2LZeNFpOZbV0+bN2tqNMCQMIjhYdtcEITuoG7DV4QWzNVhWeXGL/HTqEOALj5uzNVh2McvCIU9rcqdZkz8RKBwBH9RfuIwpQyJQAgJ1m82AN1rOiepl9gAsXFFyWh0igdvM7KABzWTQrJjzR0ZGPz42tnW/HWfM3HrYG04YffVR8236TrvYAQfNaczrytv+j296zG65cZXdcNVKiz6MgBjk8fwwxGDjAxEjI6N/NTa2dafnHTzXXvyqIxtzylJQpubCArv29jXGRxMYVnHz9ddsZbzw+JCIMeb1LduNo2YzaNaKtV8EREAECkYAMcvE60cNMV9MzUUeLh5AHhgjS1r8OjcxDSBjSiJ3AohZ6r0I00EhSrGueltsshzxT+bSbovgyAeilHHmTfI8sZ/ycfPoT0KKkP9u8oCYpYx7dBNIfkWgVwKa+LxXcgonAvUjQCfMCyiMly2bpah+tVXdEvNiGD/EIeI6TayuNBsb1HCCTklz/viYV77yQN4Rqwg+n4sVyzFDevRiZadU5U8E9BUftQEREIEuCdDRImaz/nJSl9mQ9yERKJLIIi9lvalCaLvYdkuznnYMqVEr2fITkGW2/HWoEojAIAkgHrAqDWPs4SDLqbSmEvD3K4okZqfmsLxbWGsRthKz5a1D5XzIBCRmh1wBSl4ESkAA8cp0RjwSZX13M7skPApF3PBpTYmcElRkH1n0eWb7iEJBUwgwRGJO+A3rBbWUbGmXCJSLgMRsuepLuRWBQRFAuPIWNVajtDGJhycygmWpMcl9Yr82q0HA55mtRmmKUwoErDu+plfWYRNeBi1FYCgEJGaHgl2JikChCTAF0gVjY1v3332v/bYedvQJdsihRzamE5oxc/Ll6ofuW2u33riqMQ3SD79z2Uue2LKJjhjxy/dCfTxgoQuqzHVMYL2Z7dqxb3nslED8OegyTsHVaTnlTwREQAREoKAEqjg1F9bVMebxPO3sJWNX3DHW0e8rP3p07JiTFjamGxoZGWVKJH9ru6BVp2x1SYAX/p7sMoy8tybAbAbJKbrSnoK0jqWYRzU1VzHrpbK50hfAKlu1KpgIdE0Ay+rC2QfNsYuuXTdy7IL4CWjruLDY/vkHltrSlTfZDtNnMPctY2olaFtjK9NRXgBDeMllR8DPD27++OE6P+lCAC1EQATMJGbVCkRABCCARXbB645fYBdcvrrxNaVesPB1I4QwgjgI2vgxai9RKowIVJEA43WY4m6Nmd0afnwuekEVC6syiUDeBCRm8yas+EWg+AQQnA2L7KJ/6P/9E6y0Z39ype04Y+bYyMgonxGtwqPT5ONgth81s2vNbNCCHZ6DTrP4rbhcOUTM8gES//gDdcoXv9g3OTC9XGVSbkVgaAQkZoeGXgmLQCEI7GI2sowXvf7+i9lNc7nHvgfY4i9dNzI2tnVmsNAWorB9ZgJhPi/6HR8eD68Y8GdeueNgirQyOvLd7OaGY/2Wq1X8ReLFVHYMM6Au7ww/1tmnOZyLVFPKSykISMyWopqUSRHIjcAis7Gd3/aXfzcaz1QQp/bTH3/fFhy+t83/n8+yY184Yse+cNROfMWudvkyjErNHUMOGLYQhB4vu5TdIUBQ/PEPCymPh9Mspe0sbM1EXRE5ZTVmlkbjY0WT5eRY60aVDLHtdqv4t/U9bg3VONU0MtonAiUiIDFbospSVkUgBwKLdt9r/7HXn5CuLz7xgdPsrLe91h791XrbYcfptuvue9r0nWbalk0b7OLzz7BF81/WMksnvpcnpw2XnoAfrdYS6+DdYRgCwxEYihALV1gwRAE/fjwWvghjjvvQBiy/fhwhzQcsztEXozJpNHB0tplEqEhEQAQGT0BidvDMlaIIFIUAomvnw44+3j9VOiVfWGSv/PrnG5rq1LOW2KU/3mDLf7C+sTz/y9+z6TvtbHfddpMt/5ezpoSLNxhuMP4y2Eia5TL2Wob1WUFIIib9h9Bkvw82RhghXhn7CFfmZsVyy+wOOEQt6wxR4Pjs8HPVT50QJ3P1enj8uMUSCzdfXDtvwEMbQvYHtoAjjHGIf0QnbJIOnszPyq+ZKKXtET4Zh8dH/fk68RMPL2cRZ7w/mba2RUAECkJAYrYgFaFsiMAQCDQEZjOr7EcX/lFDyP7vJf9mx508/iT2qaeeamTzRa843JZevsZGp02z//jsP7bM+nj8Y3wOtexDDRBV8RADF6wvjT4SASimWfJH14x/ZJ2y83PB1ZjuIXwGmP3uH1GLWHVx7OEZr+FhW/KuyEFEJKxvCmIWQc+2i3qKSfvl+ClhHDPDQBCm7hC6WL8JQ3huIDw+/PjTAsZB+zrpEoabBeqVNP1GxOPVUgREoGAEJGYLViHKjggMkEDj0TdjW9Pcow8/2BhW8Jo3vrVxmHGzbzlk+8bY2Y2/ftj22GeWvWreMQ3B+6Nrv5EWRWPf7IMm4m+kl+LRrWtYJGOxkuJ1qLsQOFhL+SGgEOgIn/hrZxSW/Vhn/Ue5cJQTvxeEcvoQA6yALlRZIsg8LEssirgJkGG7Dgus2oh9xCbiNbaW0la4oYALx/ATtzG/sWAfx/DHS3wuXP0GghsHX+fjENQPfj0MIrhx41cH4CqjCJSRgD5nW8ZaU55FIBsCccc/Jcann366sb3P817YWK7+P1c3xs2ysfWZp+2yz/+TnfzX/2gveuURdv1VK+3u29fYK+e9aUocKRueHoKNR8iIFESC7ycIVskyOAQQ5VgShhHEIhwBhvCNHdtYDnEIJ/x7+VmHAxZCXGyZDbsa8cWi2fcPaomAR3wP2sVcsZLGL9vFwzvIF0KU4+5coPo2vLnRaOY4TpzcoPgQB/yyTf0Qv5wIiEABCUjMFrBSlCURGBSB6TPdIDg1xdHR8Yc2Dz94X+PAHvvEetNs1v84pLF/wyMPNZbP2Tt+ujs1riiNZwdBgCUyzX3FzBjHgCUS4YLjkf0wRVzIRurCRSj5Rej4bAdYD8mzT7EEZB5VMw4Wix/+sfYhiN0qiCjGEYYhCF5+9iGyGGZQVA7jOe/8n/I0c8kyxhwI4zcE3nCbHcev32wgRFlH6MLX6yWZB4+TOkk6/0JXcr+2RUAECkBAwwwKUAnKgggMi8CWjen9OmKW8bC/vH+dPXT/Ottr1vONsbO/8/tvsbct/LDNO+7tjSz/4Ftfayxf8JJXNi1ClMZvwqNbrJRp4uBPw3HGjSJS+DHG0d/qx0KGKETYTVXXTVPP/QCPorFa+rhKhBAvfbGNdQ/xylABxmwixACOBRbxyjEXqm6RxprIfsL7cfwSPq4sjzv3AmacQFyGZNSUN3mcfbFL1ntSGLsgJQx1AWuGhFBHhG1lXfW0qT/8x79kOnGeslhvfjeYReyKQwQqTkBituIVrOKJQAsCSSvYFK+Mh2VIwd+eclRjP2Nnz7zga/bW93ywsc20XQ/e84vGuNp9Zo8PR5gSwbYbbq1ErCIssFT6Y2GEBPv/JAgQLJeIXn6Mc0Ts4RfxiEhB2CISk2Jn21Sz2UP6bhWMY2QfAhTh42KMJduM50SQ88lSrIM4/FMGXi5CqCOuksc5hqAibPI4cVB+xG0rYTaeWvH+KVOaEIchQwCSZXJulARBiejDD/FwQxSLTLj7i3X4ZxtWcZywj523P/bRPokzHp6AOKatEU+eLinS80xLcYtA5QhomEHlqlQFEoGOCTTEGWNeX31UrBnGw5/9yRV24it2sfXr7mx8JOGNf/puO+CgObbhVw/a1SuWN6blwudH//36lgkSf3CxGES88pieHwIDIcsP8frV4H8ioEcQHhcjYMgwFloELaI4b7ERi6YoO41V0o7Tp5zbAp0MRbnSyuY+EFWtwifT83B5LhHnWYyZ9fpGyCNIcQhGhCc3LckbLIZkIHJpL4jM2A/thpfr/DgW2FicwpghH55vmFI38XhYtn2mCOKnDWIJRzRzjHrnxiIWuCHbWoiACBSFgMRsUWpC+RCBwRNoiIlbbliVKmbJDnPLLpz/Urv7ttX2tX89f0oOmWf2g5/5RmNWgykHEhu33kgyIxvNxpJCxX26OEOctBJ5+EfUuBhEGLHuj/QRM3LFJ4BwRViy5IfVmxsSF7dxCbDQ4wdLdPKmhbpnGAFxITg9TkQoDgFKm0OQso/wLGlnCGjaEuv83NGmPAxxYl0nD/iVEwERKCgB7rblREAEeiPwLjP7jJkdbWZX9RZF36F2MrONZsbXDU7tIba1z9l7//0uuHxN08/ZEuf3/uurtvoHV9r6e+60GTN3bcxi4HPPtkrzrttW26L5PDWfMiVSqyDdHkOUIIZ5qQzR0U4Mdxu//I8TYO41hOP0AQFBgGJ1J800kTugbAwkGcqHtbhK/fGFZna6me1pZuNviQ4EpRKpKwFZZuta8yq3CIwTWPqrB+5Z8p/Ll9qJ7+MJa7pjvKzPN5vuI33vRYsnns5i8crDuXUNixsWWoSBrGh5kFacIiACIlBQAnoBrKAVo2yJwIAIIDLXXb5sydaH7vOns9mkzFjZW29svKQfj3PMJvKpsbigxUqbl2iemqK28iZAY2T8dLaNMu9cK34REIGhEJCYHQp2JSoChSKw6PHNG0c/8p7jntncZKqubnPL8IIlf3PSM+NjZRsv1XQbRbf+GV7A+EZe5pHLnkBWL4B1mjNELI8KJGY7JSZ/IlBjAhKzNa58FV0EAgGE4Bnr7rh52gdPmmf9ClqE7AfeceTYE1s2TTMbYyzgoASJv2DWauaBolU6UzJpWqai1YryIwIiUCoCErOlqi5lVgRyI8Dj+eUI0bPefsQzLHtxDC04++1HbN2y6TEsecwu0FtEvSQ++aJQWcQhb+kzh2lZ8ttbrSiUCIiACORMQGI2Z8CKXgRKRIApik7BQssMBJdeeF7HVlrG2y4+/fjG7/HNmzaHjwLEc68OAoMLZ0RiGRxjfOXyIcANwsTbh/kkoVhLRoA2wbzDzF3MclA3kVyPPC2W8TzHaQh5sqRrQxoZ7RMBEciFAFNzMSH7+CeyckmibaRMzUUeLm7rs3MPXEyZFYB4x1591Pyxhf+wbGzxF1eNfe7qu8euuGNsbOnKmxrbp529dOx5B89t+Av+EbB+4e48xex8kpdBTeVEJ8Wk/aTJRwCY55S0XUzTITHDAtZX/LCkI8Xhxz/Vy9LnOo3D4J8J/IvQsf2XmW0KeS/DgvG2g2oH/fLwc63feIoUnqm5aPN7FCRTPHkiP9zwwpsl237e5ZlN0vOpYliSbivH8W6GShF/N/5bpa1jIiACNSRQVTHrVcmF3jtaLrBNfiN8ApRxt0W4oJJH8py3Q2AiNik3whQhy9jguCNC2NNpwgWBD0+OsyQ8lkO2WfoNAHF4nIQjPPOtDtsNQ8zCFSsWrPglLVq+j/1+AwEnWHITwU2Ch3d+Hib278eGtfRzbFjp55FukcQsQpYZT5LXJz8fk/uz5kH9upilbbZLL76GdJKXbv13Emfp/Gie2dJVmTIsAgMjgBjjhwDgAowA+N1gif6cmT0wLhzHBiEeB1boDhPyr07RIdJRIjpxWGrdIUzh53xYxz8dWhyGsPhF1MLa4yQeuD8a2HsaHn+Vl7BiZgrmdkOQOjv/sACMsFrz+VpYuh/GaXOMHw72/uEFRA2Pl4mTtozY1de9AqiKLjifqHOmefPz0ItKm/Lrmh/Dv3/eGH+cl0wt6A5Ryo/zf05oexynDbrj/PZZVZhhJXYco616ehxLxhX7Zz2Oj23i9GuBi2RPz+OlXKSDo737/rCreguNma1enapEIpA1AS7UWAu5cF4TIv902K78RbIJTDoYOpS4E4NR7ODFcYYWxJbC2E+8TpxYWRDEWGP5uTgukiUxznMe6whROng+HUenzKdsEaTu4ARPBApc8OMilm24c4xOHPFLG3WB63ESB8cQuHLVJUA7wiFc0xztwtsAbYKnLdzg0DZod4Tz44T3c5kbTz53TDuLn5yQHnHQzjhO25sVJUxbdQHKbuImDfwSJnk99fjIC3niPOAmjHhixzF3xM+1iDj5+bXbj1dyKctsJatVhRKB3AhMCzE/k1sK1YmYTorOiA4K4YsFJ+7IkiXFKsSQDURY7NjGQlQXh8CAnVuf3MLt443ZdmurM3G/MExzyTi5yaAuECIck6smAQQqrpPzB4GIpT8WirST5I0kwtH9cG5zw4kf2iDilPPc2xTtGPGZ5giD1Thuy7RJLL7uuH5ckHiZkXRIn7Txz3lBOmyzn+04TsQs7Rw/nXDwtEu1lJgtVXUpsyIwdAL+NGfr0HMy3AzQKdDpIJ7cOkvH446Oikd/WHlii613ru4vXhInltlY8BI/HZ6nEfsf5DodOHkbhIMdHXPs2HYx6/sREfDEPyKkleORKz9YJh3hXQwnj2m7PgQQe/xwtAksqpzjsbjkmPth3c9tzlPaImHi47Sr5M1pI4GEIPZ9iOG4nbso5jhClfh3ds8pS/xwreBc9WEGrLOP61NsZU4JXt5d3jGVtwTKuQiIwCAJyDI7TpsOCysqj7vpQI4LYzi9Llx8unils2OMZ/zI0f3QWeKPOHksSJzu6HzOqLJFxQsaLdOEpXPEG7yxNCGwYQMzxEcrh9jlkTBhkr+09FrFpWPlIeB1G7efOPe0G287nKO0K8QfN0+0LdoK+ztxzdJoFpZ4PX/ux68Jvk3eGLZAnrg++BAIP562JM+09fi3pgA3xGl5zWyfxGxmKBWRCNSCgMTsZDUjqug4sM7QcdD5ucPKyjYClo6Il7jwyyNIhC+OjgyrDYKVMXiEwXrCI0LC8CMNfslOLkRRyQVCwl9o8QImLVTshxV+YQOjVg7Wbqlyf4RHvMhVlwDtAxe3n7CrsUAgurXSbyYZn4rQpE15+DhMs/WkMG3mz/fTbl1I+76kcCZ9rhHcsCGWabNcJ1o5rjV+3fAl4SifnAiIgAhsQ6DqU3NtU2AzWxxE1oFpBwuwDwHYTQfUa5bpdJLiiI6D9JMWmuR2J2kSf7Jj6yRcXn6+aWYb84o8ES/lpsNmrCGPXBGcdPywxTlnjvnQAfy7+McPNwds4wf+/IiDoQncTCCW6fRdyIzHPJx/2quXbTg5yD7VIk3NxbAdF3hxSdkPd4QejnqI2wPtkDYY1w3rtL/YxfuIw1/axA9ileM+dMjT5BhtkmO0VXd+nDT8uOcPPx5fnM84fT834jx6mDgeT68yS42ZrUxVqiAiMBACssyOY6ajoePyFy3o+BiPyeO8pOUkud1JRSG86urcYoU1Da5MRYR1yjt9uPvsBfDHIkbnjRCg4+Y4VijWOe5TGbGNH7egs5S1qvqtjDrnfOWmiLZC+2KbH+3Ix73Sbrj5QRzih/bH+Uy76dTRRokHEUxatL9m47m5LpA+w4r8xhirsDuO0+55uuPj8xlmwL44Tz7ciRfFELk8JaKslItyIGJ5IuTl9Pi1FAEREIEGgTpaZv85XOz3L2gboCOiMxmEo+OisyBNT5cOsoruW2GM8CDKBtfkY2E66UHV6yDKGKdBuWg/VXJFssw6V85NhC0/2ljaueo3RS6AuZliH0tcvB52bbMPv8TvaSA8PS2WxBE7T9Nv3pJpEA/58ePJONjmeGx59Rs39ifTi9PWugiIgAhYHcXsx0LHu29B699FZUGzV9psDVLM0ilTjz6MwMcdx511aUGmZBzrsMRsChjtEoFOCWiYQaek5E8ERAACGmagdpA3AR6H8jgVSxSPabF+s6yqZbaXYSh514HiF4FSEZCYLVV1KbMiMHQCPgNK3eeZHXpFVDwDCFqN8at4Jat4IpAVAYnZrEgqHhGoBwFZZjuvZ8ayJaeY8tBY43gpoyyOacWq9ii8LOyVTxEQgTYEJGbbANJhERCBKQQkZqfgaLnhL2E088S32Hm5Q04EREAERKAPAhKzfcBTUBGoIYF+hxnwpm/y85BZY+Rzjz7VTdZxMw1Op5Oj+/Q5TKXDW8XueLuYl5uYyitLMZtXmck3k7ZzI5NnGnDq1MGw1QcPmJ4oZt5pvM38YWWPv97WzF8v+z3ePNlW/QtQtAeedjQbf0z9ce1pdu76dalVGyQNn9qrl3pWGBEQAREoJIE6zmbw+fC4eUaPNUKHwOPqcv5GRngZqVPnUy4lRRUdq5e/07g68edxlnVJ2+jU8XIYfOMfQsbL7jcSncbXzp/XpcdftmWyDbYrb7/HBz01F/XRqowcow6bOb8uNTvOftLopo22iquXY5Shm3aN31Y3fL3kobBhZJktbNUoYyJQSAKZDDM47PQlttuB3VyXO2Px2P132rN2mGE7/vaenQXowtcPL1xkj/xiDVbfTp1b2uJOFCHLJOm4ZpOph8PdL17wewvsBb+PzsvWbXn4fpv2rO3t2Tvvnm3EZvbzby2zn3+76+HDTGcVf/Ag7riJrJkFruf8//bz59jvvDf+8FLPUU0J+P+e2GyP//pB23mf7D+q98idq+2Hn/RvRExJtm4btJWyv1DI05z4WtKuDpnKbpjiu13+Mj0uMZspTkUmApUn0O8wgwYghOzec7O/zuYRp9fodjN4Etmxi5V6mnWE4QqZz5u6054H5MK141L34HH96m7659QEqBg+IcqSr31lr+bNbPsZO4/tPfdIXoSTKy4Bbha5ieQJCm2hleMcxW+zoQVxXM38eBzcPMV+4mELXOgYWsXxdjdZHh/5x78Pm/CLJfFwzOPxPLLkRPI8sB0PX4mHmZAG8RB3vL8Vq8If846p8BlVBkVABApBIBPLbCFKkm8m6DBwiFY6GDoNHB0Rn7Cks/EOKRzSokcCdOLwhHEuQrbHfCnYYAkcF84pzi+s83xS1h3tIjarY6ml3TCfMRZbPswRO2407w6fPsa0nXbHRTiPw9OjHeI4/znGUxjSJR3yE+cheJ1YcMzjowyk7+3Zh1CwP96HH/b5Z5u9zIhf9uEYP+758jwThs/fcqPtx4L3ci4kZstZb8q1CAyLgItZzTPbugZczNJ50LGwhKVWAgAAIABJREFUjaUIiw3riNrYsd87qXi/1lsTQJRgZeKmAc5Jrq1D62iVCNAGEGa0Az/nWCYd+5gyL/ZH2NjRrhCBnKv4Yzt2Pn7V02PJsKGkKMaS73Eginnps5njGNcAzxfps47zJS+M8iM9hh0gkt0/AtzTIr+eZ45z40w41j3P+CXPPuwpJFXOhcRsOetNuRaBYRHwa4bEbOsa8M4ntr76zAV0WnQoscMqk/mwgziBCq4jKBAlErIVrNweioRV029m/LxLnmdEi2CMH/kTJraYcu4yNj7eF68TB+cqYtHTY1/aOeyCkuOeJ9bTHMISgUqb5uaW9t3sBpchAghlbpZx+G83nr9Znv1aFaIq58I7pnLmXrkWAREYNAEss88MOtESpoe1EBd3YHRAPqbNhW3w1rCYxH59v5bpBOjk6fhxrDdjRweeFCIhmBY1IJAmZtP2xaIUYYiL97Edv7DJ+U37i2e18LHxcfwI7E4dbZU0EMCPhmEJ7GvmOEaa5AH/7d70Y3wwN9JpeS69oJWYbdZMtF8ERCCNANcMWWXTyEzuizuGZGfmlhq3vtDx0bnwsgado4TXJMdmazwe9UejjP1z61Saf6xbSVGS5q+w+zY9uNYeWO33QIXNZpky1q49NDvuL1RRVp4GMAwA62jy5y9tdcuEGzIEKnM6M3yANHixMRbHHifXGI7RMPj4CnmIrzvuL14yprxZnpPXqThcKdYlZktRTcqkCBSGgCyz7auCTomXL+hkkg4xyzE6Kxwdn7+oQSeWtNgGb1pEBPyGgF2I2tjSxDoCFusaligsaFit0gRBFGV2q7d8fak9fGczQ3H36TB12TfOaKdTuo+3xiG4+aFduAUWFPHjfK+8eF+yAoiDF85iR7vrVciSF0Q06bAkfr8WxG3X8+z5Ic20/Mb58nX8cRMdO9LoNc9xPENf19RcQ68CZUAESkVAYrZ9ddEZtbJ0JI/RMVVmipz2ePr2Ad9Wpkr4IgB4WxvHjQOPbwfSaTOv65uW+BPnkAMtikSAmyFEHJWEaKStxI/zaV9Y/LlRQvT6djzMwEUkcdDeEJnESbhenF8zsLb60xnyxHXBrxekz9Mb8ksZWMc/ZeBphaftgtfbOy+lMduC55kZDwjTb557KWduYSRmc0OriEWgkgQKO8zgyc0b7JE7ufZv6/aei5GusI6OyK0rhcwklsanNidf+Dbb7cA5tn138+9mUT63SrWLyzvsTv23i6/t8QfCnLmP/GKNMS/x7uHDIAwV2PTguBbabsbOE/uJELYwxL+332Zcx+MfsZ32nGXMKSw3QYDH5y76fGe8j2Mu7jhOm8DyimDkGNuxoEUs4h8/PMLnGOepx8GSCkDA8gSGbZ6weB7YJv3Ype2Lj3t+yAtxI0DjpxDkgR/C19MjjA+1IYwfJ17CIlj5eZhWeY7zUrp1idnSVZkyLAJDJVBIyyyC4BtnzLOd9pjVEAXr10wa7nZ7/hx780WF1op0XlhNCue4QYDrpvV3N77YxhelntoyKWpPvPTuYYjZTjkhPlxcdBqmL38/WTZejT//5iUNZru/d6mx7yfLz7O95ozfUNE2+Urbke8fH/bLl+UQsuynrcLYRkbszZ+7aYpg/c8z5tlTmxgxYXyJzt7w4RV2wOFoF7km5098TiXbAeIOC6hbQYGYvEgQJg6XPE4ccRpxRSA2k8fS9sVhiA8BGgvY+Djpx3lI5g+/ybBx+TjeKs9xWqVbl5gtXZUpwyIwVAKFE7MuuPhE7gvDp1zXfn+lfedDxxtiq+AWLCwlTKkTd1JDreA48e98cL7Z2Jid+G9rG6IV1pf+yQF22HuXTrCO/RdsHTGb7MxzzeIxS1fZZ+eNNPjwNTossghZhh341+m8bb74zQsnLLR8Be1NS1c1tp0xY2VffvKkHjrgd4+zQ96CIdDs2x+cb4zNzVDMorS5S2nWDmFJO202HIZ2zAtSzeLw42Q/LQ6shzzSb3Y8HNJCBNIJ6AWwdC7aKwIikE6gcMMMGpat6TtPEVfeySMmCu7o5HEIBV8Pu4a7QHRhLXzDR1ZOWF95HM6niBvWw+Fmr13qiCPEVTNx1i58Jse5keKGyoVsHOlTmzGSjTuY+pAEZ+zHfOlClm3ii58+uJ8WS59GKq2NMRwDKx9jKcfV8tSIeJTNsTRLoPvkS1QeR1oaWAw9fFoamJj9uMzNTlXLjgnIMtsxKnkUAREws8JZZhGsbpEtYQ0hthgTwfg4hhsUxiFYeTSeZtkewjjZbrk4SwTSUB0cV/3jKY0bAAQsQwmSLk3sJv3E24jfLh1tDNGadneH9dpFP36Sjn0uUJvxZNwmfog/LQ0EMT9/lJ6WhofzvCT9aFsEmhKQmG2KRgdEQARSCBTOMutjEOO8YlXEpQmx2F8B1uncXXgVIDuTWfCXmSb3mPEInPGabvmOjxVs3cUXFkluFFwoDTSbMGS4y8sXnNP4IVq5+br0xNkDzUewejYbcuEW0WZ56mScZXKsZjIu+E+OmUgeHRe5zYTytr61RwQSBCRmE0C0KQIi0JJA4SyzEy/NRNlmPOGs3z2uDGI2ynWxVhFea39w+ZRM3fr1pY0xtLzAVHDn4srf5B5KdhkLi4vHvqbdJAwlc0pUBCpEQGK2QpWpoojAAAgUTswirHjJZvuddm2IV6yyPNot+AwGA6iq/pJwrjwixxKLMOMmgbfoSzDMAGtiM0tkf2DahN5u+s7GXLO84LXX3CPNlp8XZjNgnOsqu+Ob/vGyNhHpsAiIQMcEJGY7RiWPIiACZla4YQa8GMNwgju+tcweuOnaxgtK/va9aqx3AjA94XM3GdbYW762pME4fiu/95irHfLIM5c12uLmMJYb8c9NAG0Tazc3WcxU4I6bhuRwmHgfgvjl7jks8c/QBTkREIFxAhKzagkiIALdECicZZbMYzkswTjObjgXwi9v2CPO5DonkGyLyW1iimcmSHt5Md6HAOYXu4aYjabtio9pXQTqSEBTc9Wx1lVmEeidQCHFbO/FUUgREAERyJwA003wqVt+jCvxuxHGcBd+wHnmNAYQocTsACArCRGoEIHCDTOoEFsVRQREoBoEGK/N94v5QASfuUXUMhcvn3DTPLo51LGGGeQAVVGKQIUJYJl9qsLlU9FEQAREoF8CWF99Ojg+EoGAxVrLvrS5fPtNr/bhJWZr3wQEQAS6IqBhBl3hkmcREIEaEnAh60VHwErEOo0clhpmkANURSkCFSagYQYVrlwVTQREQATKSEBitoy1pjyLwPAIyDI7PPZKWQREoPgEGC/LGNnki168BMZ+TQ+SQx1qmEEOUBWlCFSYQCaWWSaV324GL/aWx/EZ16K7n397uT2w5rqiZ3NK/piPtQzukV/cPPKfZ/AuT3ncU5v5doTcgAkgYnc2swsS6SJm+U39rF7CkzZ7IyAx2xs3hRKBuhLo1zLb6F35Qlcp3cjIzTY2Vsysj4zeu+nBtfttKok4nAJxZGSjjY0VWdWufWrzhiP887RT8l6OjSKzLQfBznJ5QBCy+B7/lvFkOJ+eK7l/0ofWREAERGAIBN5lZiibo4aQtie5U8jDxb4j5+WDZvbdnNPoJ/rko71+4lLYSQKY0TWl0CSPrNeq1m4vDNelPbIGVfD4OEfoE9JuHtjPz0VtwYtSruzJMluu+lJuRWDYBDIZZpBjIc4NHYmsH9lCppNeoDeys4UaYkPILtRYylzYDjpSpt/CIVrj7w3HY6p0bQqQslxIzGZJU3GJQPUJ9DvMIE9CCK5Z4cULdRjZkkZwHcGXg5tYnbJNrV6x0W4RQfxKOv6mXhXWorRudeU84cY66Yo/8D6Z45JsazaDklSUsikCBSFQdDELpuPMLLaEFARdabNBx4yQxVXtcXgo1tAWsKW94sQ2gCjxws+TM8KXv3hjkJ+/9KWblZwqV2I2J7CKVgQqSqCowwwQrzwGx2l8ZwCR0SIWWc44o6hrH03M1kVt7aGUFIAPMSD7TL/F0yH/+c11UsxS/27NLWmxi5Ftidli1INyIQJlIVBUy2wsCmDJo1u5bAjEbLEkqvPNhiuxJNmq3WbHdtAxuZhdZ2bJOdHcYhsPf+I8ukTDdrKpJonZbDgqFhGoC4GiWmZjUUBdYOVCeMn1R4AOl3HIsUuyjo9pvXMCaWwlZjvnVzSfLmZjwUoe45u/2DKLf4Rv2swHRStb4fMjMVv4KlIGRaBQBIpomaVTmBM6BmD5SxYSBv03HReudLo4vsigMckBRp+LNLYM4/BH0n1Gr+ADJuBiNhasZMH3x18zWWlmS8KNIjMfyPVJQGK2T4AKLgJDJuAXQl/mnZ0iilkXrZ8Ohb/ezB5LWETy5lLV+LEqcXOwKRSQsYCILe+gq1ruQZQLtgicmC3piu0g6GefBvU5YmZ8zjZ2bLM/ttByzeK8Oi8ci/1rXQREQAQGSqCOH01ANH9toJQ7S4whBa8K8zv+tYYYdAatQ1+w/XJg+xxZDjuk1pk32F4a2O5aoXZb148mdFbr4zeEXEtjgdtpWPlLIaB5ZlOgaJcIiEAqAX+S80zq0eHuZNzZniELWzUOLdPKiMf0YfF+KtPY6x0ZbP28etTM+MlVn4Bb35NDEqpf8pxK6CdRTtErWhEQgQoRYIgBrohilnz59QwxKycCZSFAux3UMKGyMKl6PrHIMswgOetB1cudW/n84p9bAopYBESgMgT8elFUsVh0sV2ZhqCCZEqA86qo51SmBVVkEwSwzMoqO4Gj/xXvnPqPSTGIgAhUnUDRxaJfzyQMqt4Sq1U+idlq1WcnpWGs9M562a8TVJ358Yt/Z77lSwREoM4EJGbrXPsqe14EJGbzIlvceJnhgBf+fHq24ua0JDnTC2AlqShlUwSaEBikFdJvfgeZZpNip+5m+hvc02GpRfYE9PJX9kwlZrNnWvQYmeKOn1xGBLxzyig6RSMCIjBgAlvCyyMHDiDdoltm9w0MfN7OASCpTRK0r821Ke1gCyoxO1jeSq2CBCRmK1ipKlLtCDxoZq8YQKn9elFUy6wzeGAALOqUBPX+UjOjncllT0BiNnumirFmBLxzqlmxVVwRqBSB/2tmO5rZ83MuVdEts4hZpjjyz9nmjKM20R9sZs8yM9qZXPYEOK+KOt1d9qVVjCKQAwGJ2RygKkoRGDCBb4T03pNzukUWs0x1wxfAbgifss0ZRa2i93bl7axWhR9AYWWZHQBkJVFtAhKz1a5fla4eBL5iZg+b2V+Z2ctyLLJfL4o4zOBLZobY/kKO5a9j1Iea2bvNjKEb/15HAAMos8TsACAriWoT8M6p2qVU6USg2gQ2mtmHQhEvyrGoRbXM/q2ZvcjMfmZmn86x/HWMermZMUvE+83syToCGECZJWYHAFlJVJuAxGy161elqw+Bz5jZZeFFnS+a2Ywcil5EMYvV8LwwtOCPcyhzXaPcycy+ZmYvNLMvmxmWb7l8CEjM5sNVsdaIgMRsjSpbRa08gQVm9hMze7uZ/dTM+P53ls6vF0UYZrC3mX3bzD4Vpoz6vVDmLMtb17hgeYeZvcXM/k9oT3VlMYhyS8wOgrLSqDQBfTSh0tWrwtWMAPOAImCx0B5tZteGR+8sV5nZNWb26z6YDNsy+8rwkhfjOI8Nn4N8yMyOMrNb+yhX3YM+x8zmmdnrwvIFAcg3zeyEusMZQPklZgcAWUmIgAiIgAikE3hXmAoKMVU0d6qZIfSYqko/MeimDdxjZm8rWoOucH6uN7N1FSvfheG6s0fFyqXiFJSALLMFrRhlSwT6ILCLmR1uZs+N4uAzpHwtzIcKRIc6XiUsYyl/08HLQFhxnx1eHmqVwP/rIC7CI8YowxNm9oiZ6Stfrah2d+xxM7vXzH5uZt8zsxu7Cy7ffRLgnNLX1fqEqOAiIAIiIAK9ESiiZZb5Vu8O4m+1mc03swN6K942oeaEeD+wzRHtEAER6JXAo2Z2Za+BCxpOltmCVkxVsyXLbFVrVuWqIwGELONjscxeYGbnmtmGDEG4VbcIL4BlWCxFJQJDI0AfzPmqTzAPrQqUcBUISMxWoRZVBhEY7xBXhI7xeDNbmQOUYb8AlkORFKUIDJWAPzWRmB1qNSjxshNwS0vZy6H8i0DdCSwNwwmYczUPIQtfidm6tzKVP2sCzNCBY0o9OREQgR4JSMz2CE7BRKBABLDuMMfsmjC0IK+s+fVCwwzyIqx460aApyjPmNn/V7eCq7wikCUB75yyjFNxiYAIDJbAySE5xsjm6WSZzZOu4q4bge3M7E1hnDuzdMiJgAj0SEBitkdwCiYCBSLgX/riwwh5OonZPOkq7roROMXMdjCzb9St4CqvCGRNQGI2a6KKTwQGT+CIMOl6ljMXpJXCrxcaZpBGR/tEoHMCzMHM+PaNZvaFzoPJpwiIQBoBzWaQRkX7RKB8BNYOIMsjIQ3G+MmJgAj0TuB9ZsbXsf63mTHPrJwIiEAfBNzS0kcUCioCIlATAv7VrefVpLwqpgjkQWC6mfHhET5hy3zQciIgAn0SkGW2T4AKLgI1IvBwKOshNSqziioCWRLAgPSV8Fno3zczPucsJwIi0CcBidk+ASq4CNSIAEMZHjOzl9SozCqqCGRJ4LNmdqyZvd/Mrs8yYsUlAnUmoGEGda59lV0EuiMwFubD3NfM/qC7oPItArUn8GEzO9XMLjezj9aehgCIQIYEJGYzhKmoRKAGBD4dyvivYVqhGhRZRRSBvgl80cw+aGZ3mNk7+o5NEYiACEwhIDE7BYc2REAE2hD4vpl93cz2M7Nz2vjVYRGoO4G9zGy1mb3dzK4zs8PMzF+krDsblV8EMiMgMZsZSkUkArUh8C4zuyeM+2OKITkREIGpBJ5lZmeY2U/NbI6ZfcrMXq9puKZC0pYIZEVAYjYrkopHBOpD4NdmNt/MNpvZx83s22a2Z32Kr5KKQFMCfNHrj83sdjP7mJk9aGZHm9npZqb5mZti0wER6I+AZjPoj59Ci0BdCdxkZr9jZleZ2RvCnJk3mNk14VvzvKn9ZF3hqNy1IMAQgl3D75VmxlRbvxdK/iszO83MLq4FCRVSBIZMQGJ2yBWg5EtNoAqfdeVTuK8xswPNbH8zO8DMZvdQK9uFeIhLY2l7AKgglSDAzdzXzOzLGhvbqE9ZoyvRrItfCInZ4teRclhcAhtC1rDOlMkxPRCPQhGePBbFPR7G8/Fpze+ZWbdCfZqZ7RImgydOxK1//jYkkcmCx7b85ERgmAR46sCcy1wDWP53eDGSIThyZr8dIPg1UkxEIFcCErO54lXkFSfwUCgf31gvg8MK+0kze1HI7K3BgvQFM3ugDAVQHkVABEpBgDH0j5jZ06XIrTIpAiIgAjUm8EIz40MCTIY+TEceVrXIAFaSy0Je8bvCzF7awr8OiYAIiEA/BH5mZtwsy4mACIiACBScAI/VEYd8onKYrpWYRbTeF/L5IzM7dJgZVdoiIAK1IMBwC14OlRMBERABESgBgbvMjDf7h+maidk/DTMK8KjvzGFmUGmLgAjUhgAvknJN+sfalFgFFQEREIGSE1gaLtzMAjAslyZmTwn5+mWYQmtYeVO6IiAC9SLw/nDt+d16FVulFQEREIHyEnhduHBzAR+WS4rZeSFP95rZPsPKlNIVARGoJQGGMz2c02wmtQSqQouACIhA3gSYkorpZxCOfMJyGC4Ws1iImSqIqbZeMozMKE0REIHaEnhZuJFeVlsCKrgIiIAIlJTA34cL+PuGlP9YzP4wzBH7piHlRcmKgAjUl8APwrVQs6XUtw2o5CIgAiUlMN3MmHOWeRVZH7RzMXt86Eg+P+gMKD0REIHaE3hjuP4wb7WcCIiACIhACQm8K1zIhzFNl4vZtWb2GzPbu4T8lGUREIHyEphhZrr+lLf+lHMREAERaBDg061XBkHLp2IH6RCzfE6T5bA/4DDIcistERCBYhC4Ilx/FhQjO8qFCIiACIhArwR2NrM7zWyLmR3YayQ9hEPEkuZWm/wmeg/RKIgIiIAIdE3gjCBkP9V1SAUQAREQAREoJIGDzWyzmfE5x+cOKIeIWX7XDSg9JSMCIiACEDgqXHt48UtOBERABESgQgT+IFzgHzCzuQMol4vZhQNIS0mIgAiIAATeE65zt5nZc4REBERABESgegSYmub+8Pj/uJyL52J2z5zTUfQiIAIiwNzazJjCdYf3BGYKiQiIgAiIQHUJMMzg/4axrJeaWV6fvKVTeaq6GFUyERCBghBg+q1bgpD9FzMbLUi+lA0REAEREIGcCZwe5qFFdF6Qg6glXl4AkxMBERCBPAi8xsyuDSKWD7McnkciilMEREAERKDYBPiYwjlBdCI+GWeGsP1DM+MTkLPNbJcei0B8fFJXTgREQAT6IcDTpBeY2aFmxjRbXzWzXwcRe7uZndBP5AorAnkRYG5MOREQgcERYHJxBOxbzIxHdjsOLmmlJAIiIAJdEbjXzC4zs/8ws+8HUdtVBPIsAoMgIDE7CMpKQwTSCTw7PK57npnNCj++3tXtOLQjgsX3x+nJaK8IiIAItCXA1wPvMbN14cfTo5+0DSUPIiACIiACIpABAYYZrMogHkUhAiIgAiIgAqUj0K0FqHQFVIZFQAREQAREQAREQASqS0Bitrp1q5KJgAiIgAiIgAiIQOUJSMxWvopVQBEQAREQAREQARGoLgGJ2erWrUomAiIgAiIgAiIgApUnIDFb+SpWAUVABERABERABESgugQkZqtbtyqZCIiACIiACIiACFSegMRs5atYBRQBERABERABERCB6hKQmK1u3apkIiACIiACIiACIlB5AhKzla9iFVAEREAEREAEREAEqktAYra6dauSiYAIiIAIiIAIiEDlCUjMVr6KVUAREAEREAEREAERqC4Bidnq1q1KVh8C68xsVX2Kq5KKgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAiIgAjkSmBarrErchEQgdPMbDszu6+gKE42s+PMbFczuz2RRz92pJmtMbPfJI5rUwREQAREQAREQAREoOIEfmhmZwyhjKvMDBHaziFYx8zsUTPbJfLs+znGupwIiIAIiIAIiIAIiEANCQxLzCJCOxGzVMnaIGjPDfUzP2wTh++rYdWpyCIgAiIgAiIgAiIgArGYRVxi5ZxrZivM7FozW5hAhHjEQrokHGeJf3cHmNk5vhEtCccx0uA4QvSSJn6jYI1Vt8JinSU8S8IvS3rUtgiIgAiIgAiIgAiIQL0IxGIWwXm3ma02M6yfiMgNZrY0QoKIROTiF2HJEj8uaNmHn6RjH8c8jItRwnfi3DpLOH7kUU4EREAEREAEREAERKDmBJJiFqEYj031R/qOieNJAco2AhfXTswGbw1Bit9OnVtnXcjGeew0DvkTAREQAREQgYETGB14ikpQBOpN4LpgaXUKWF2Tjpe3Ytfpy1xxmG7WEa7xcIeViTx2E5f8ioAIiIAIiMBACUjMDhS3EhOBjggkH/H7dl7WUoY5MIzhsZA7hC3jb5OO9Lux9ibDa1sEREAEREAEMicgMZs5UkUoAn0T8PGxHpFvp1lx8dOPyOUlrwUhIYYarAvxJYc64IVxtWki1/OppQiIgAiIgAgMnIDE7MCRK0ERaEtgUcIH28vDPgQlLraQJv0HL20XiFcXsqeYGcMLXMSyPxauCOqd9WJYW6byIAIiIAIiIAIiIAKVIpB8ASw5Hjb5QhcvYGEtZWgBU2vdFNZj6yvDApg+i+m9OI5/wsUCl3SYOcFfHEtCjV/4Sk7B5TMb+H7EMvH7L04nGa+2RUAEREAEREAEREAEKkTgZWa2bygPlk4fMuBFTI5DdVGKX0RkM+FIPBz3+PAXC17i51ir8BxLO07a8TG2sdoisNmfTMfLoqUIiIAIiIAIiIAIiEDNCbiYLRoGLL0+BKFoeVN+REAEREAEakxAY2ZrXPkqugh0QeAIjZftgpa8ioAIiIAIiIAIiEBNCRTxMT7DFbAYM9xATgREQAREQAREQAREQARKRYCxuc2mBStVQZRZERABERCB6hHQMIPq1alKJAJZE8Aiy8wIssxmTVbxiYAIiIAIiIAIiIAI5E5gfrDMMtRATgREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAREQAS6ITDSjWf57YjAUjOb08TncjNbFh1bZGYLzGxu2LfSzC4ws1WRn+Qqfv7VzL6ZPFCCbcq5xMzmhbzuYmYHmNnqsA071mNGJShW7bN4rpmNNaGwLtTn/mZ2kpl9pIm/Iu2mXW7oMkO0Y87l87oMJ+8iIAIiIAJ9EvitEB6RsXOfcQ0r+GORGBpWHuJ0XZimCdLnnGMQAAAcDElEQVS1kUeO4xcBd0bYj7i91syONzNEa5r7HTO7Iu1ASfbFN1CUEQ4uZuHRrYgoSbFzy2Z87q4ZEr9zzAzRGrdvL7DXLWLvw2b2CTPjnC2qI7+ch2nnb6s8Uz5EvcRsK0qdHevlZqKzmOVLBESgsgS4CGNVKfOPMhTF0QnSqbVybsly4Rv7ReDdHe9IrP/SzP4ssa+sm0lWye2ylivPfB8ZrNu0kWbnLDdEC4PVO8+8eNzko12bf23Ib9FvmikLjOWGR4CbiSJd04dHQimLgAh0RADLbOOisdecI2zvueW6hj+wepWtX3MdBaUMaVahjiAMwdPJZsaQA7daxVno9kKOiODxpl/8EcOnBAsd8fMjPXfzzWyFme0a/JCeCx8YYlnyx/zETaNgP2mQ5zgu4qTzjy3JWJqJL7bAEp64WCK0OIZwPSL8SMMbH1YZ8kc+sdKSF7dcexnquIQPFtAGp9332m/rYUefYDNmgmvS3XXbarvh6sudJ3UBd4auFNHizc3cceEJBe2B9uX5pAy0abZpezgfgkObpe3QtpNlo91wDDB+PARvtF3aIMfwQ/yk6U9BYIXz9MgD7Y8wOLYJF1tfOR84zo9wfoy4OQ8oI8dIhzK6Iw/4J07yeXlY+vE6L2EKP+pZTgREQATaEvBhBg0h+/KT/VreNlwxPCw718VsMfIzmYtZodOb3DP+aJVOC8dxXw+7JhbeMU7saLFChXHBpwOno/TOFxFDh0BHjACKHX5dNOCHOFjSobMkLALCO3jG/7Lt41zjuFinEyZO908ecCzJE5055WWdDsod+SZ/cCBNdwgAxDgClngZY0sasRBwv3VZwqkhsI45aaEdu2CR7bHvAaOtCn/9VSvt0k+ca3ffvuZcs5EzzMZcNLUKNshj1C03LYhRhkdwQ0T7o53R3sgvbYEhCZSfdsSNEO2MfYShjbw0tBPyTpuiPeOf47Qj4iFuHPFjGeZminhYkof4Zgx/fiNGHhGdLqpYIr5deHLskpAH2jZpI2bZf5OZcaeNX46Rd8pGO/bjDM1gmxtL/CfzMZ7rev1Tz1xzqCfnXi8CKq0IiEDXBCbEbNchFaAVATpNfrGjY6NjpSPDNROz4XBHC+IgHRd6LOmk6RBwdOqIQRebpI0o8s4dEcnPhaiLBjoR30cYthHZaQ5/LpjxS0dEWUmT/FBmhAXhYzFL3hEt7I9ZILTJB468kTZxeBnDoVos4InYOvLFrzrCFp6/DBHbUcFffdR843f1Zcvssx9534wntmxGLCHunG1H8XThiTbg7SAO5gIu3sc6+UCMxjcytAXas++j/LNDO2H90SBk/dzCP2ISBxh/udDbCml4nF5uhCPnB20PR9tjm3aMGKUM+PVzCfHqjnaIAGVJON/2dfdHfOSXfHo65MMd6SCYCe8Of+TfzzvfX7el1y316detujFQeUVABLokIDHbJbAOvWOdocNKc9650eF5p5vmr5N9dHx0iHTAxEcHwLIx9iJ0pFiG6CDwS+dAZ+wdJtYPOmsenbojDn6xizvieD/rlAFBQbqeNvE3LIkhTRcSybBp28m0kttpYaq6D45HvO74BbboH7pBOInj9SecbLMPnjvtI+8+duvD6++lnmh/Xv+THvtfi29C4tjiGxXfT5ul7XHjEwtgxCpt1MUs7djPF1/G7SFep+3haMtxeybOuD0TxuPCP+uxqByPZfwfTuQPYUr8+KUi8E8e/YYtDsM6ZcZ6zA0EcXCOeJk4TlyMeY7LztML8klacf6ScVd928Us5SyjmKX+aNc42kBa+w+HtRgCAc5Fzr3ksJ+8ssI5zbmNa/aSbDisRT8EJGb7odd7WBq1d77JWNiPhQYrWtxZJ/2x7ZYkTkysnzyypaONO2c6X6x7XGSTFlfiIGw/F1zySNqkSd65WPCjDJzICIu4g0orh/ZtS4B660vIepTPO3iuffyKm0dPmzdr7Iktm5ePjW2lTvqpc486XtIOqPduHFZXfu766fz9fEoOh+Fco+334mBEeEQVbdnbtotQ9qe1bcQo+eF84ykIS/ZhiW7Ua4jXhzN43hDvpJN13Xj8RV/Ck5scd9wQOzvfV9QlbYx2QX5xtGXKQpuhDdT5BiUgGfqCOuLHOcuPaxbnI31gu76228x7P06/6G2BOLip5TpQFAcP1xuDzBPahRv9zK51ErODrL7JtDiB/OWQZGVSyYwD7OTk4sKJgPULKCkkLcI0GDpk/GAxoMPAcXHlJKMxxyIkFqXBa9sFaXg4LhIuAjhxSbuTsrRNpEYeYLmAoQW9WmSTrHhRbPGXrhs5622vmf7Els3c3MQiMuk9723v2GmTcfunA/Bj3ebB44Fd7PqJk3jitk179nPFzzmOJ513mvjhh0DlnEfssES0kl+Pg/CE4fz0ciTjrMN22o0B1yuYFd3RDmhr8bAa6p39jJfmml4nx3kYPyHJsux7mtkLe4xw+3COUVfUD/0lP/ooljzJ7PUa5FkiXuqcvo/rrPd/tG+ejtHvJvtpDzvoJRzIF8azQTquhX4tzSRdidlMMHYdCQ2ZizQNng7NGzvrPPbvtGFx0vEIg46QdcJzAfFhBp4xLqiIZx+76vvpnNnvj124ACF0EMjdOOInbbdEEJaGimWlXVx04Gl57ib9qvldtsP0mWNnf3Jl0nLXVzmx0M4/5X9Nu/TC87jYcgEblkhAsHGh54Lm47dpw5wPtJdeLvS0NzoJrAxu+eBC3e9YYeL1pwy0cxwdHvuw6KQ5zm06Le/IOL8pM2XEESfnHeefn/usc/5RN3V0sPGX61hyrYIFnIbVTjutB84lrmGxkCUsdct1kXZNW4xvVKhrZoKhzcb7KTMsCOvDFWDhAot4WPd243kkPuJxf75/WEvy2ct5PKz8wp3hgVxz+21vlJt6pU7i+iBe0qG9xGyob6/rpPHH65t43E/cHpwX8fpwhmT/H8ePfz8e76f9xvHG8cX7Ce9tjfDN0nR/tHFcHAfhcZSHduxtOc5P7D94b71o+UZ066A62icBKpTOkM6bzo0fnR8XxE5PJk4K7vgZF0ijoeHTkdNIaBju6CjZZhk7TijS50d4OutexIRfRP0kIQ3Kg2tVFo4lO4AQrLYL6nTW/FPOGElOu5UFkWMWLLIdZ8wcGxkZ9SmksoiWOBCmtKG0X1oaCD7aLjdSdPYsuZAn22ha2LR9XOwRDtwIImCJkx/nWKs2mIyLTohwHoZzgpfG4ourt21fJuMgLP49HyzjDozzjnMlPg4PfnV1dJ4wg/eTZrZdWKdNFN1Rb+Q9rT2wD4HkgpVrNG2dNsI5yHEMCO5oJ9wI0TY4zvUY/4TDuTgOm40F6dNmi+Q4jyl3Fr/4SSXtASb0c/Qd3cbPzbPflMKL851rBGnQBv2874cl9UEasZD1+Dj3Sccd9U39Uibqm3Xq2B0c+bGf49xIsx7HQZ79OGWJjyfjJzxtCz1AHG44I26Pkzx6fOQL/97+yBdtjTzRdgmXbMPJNu558jiIH0faLmxhRprOAYFL3jt2NAQiu/blC86xsk3N9ZNl59pPlsNSgqhNjdOIaHA01rQTrE1wHR4gAS6Cx33lR49uM4dsVnlgyq5LL2ycN1zAvZPtJ3q/IDWLg7bnL3x9N/LEBZW2yY984M8dbZXjcf5IJ3kn7207DkdYfoSNw+MX12qf54nzxP0lw7kfjvv55PviMhDO84xft0CEbEyUnTgI53H58bouuSFC7LdrV0Xh43VHh9zOefvFL/Xt7RRxQCfPjxtDxBr7cFwTMFBg7KBNIS7ic9eFU1edf4i76AvKxFMQzi+EDnz6OU8Iy7UIUQe3LMRrzJB8YlxKiubYj69T9wi4uK4pLzczbuShbfl1hGsI8dOGvL24/7g9EAY/HKO87pd0PU3PH+cY4hQtiPPtOD7CY7llH87PT2/DyTBwxfn5QJ7JO6xdyBKHl5E8+c2bt3nyzjrHOqrv3IcZPHznantqMzc/U91uB86x7WdQRrkcCdAQeAxAo/ALXo7JKer+CYzMm33QS3ITsuTv0KPmu5jlghFbAXrNPhfPdo6LQCxk8c9FirBp4ZPCD/9Jfx4+TtvFQrzP17mgJl1yX1qcnfhpFi4ZNk6fY62Ox361XmwC1H87hyjheuwiA/+0Vzp5hlZ4R8+54p06ftjv1jHaC1ZgP3fpRAnrw3Xa5aFMx+GFkEVoufDpN/9wg2Ha9aXfuAnvoqaT9uBiL65r2gLXZASiX+/i6wTxxtcM4uDGL95HGd2hAWLn+YOti874OGGT8ZE/hvsQxtOJ9YTn0+MmT3G6rCdnlYnTxD91nOTANsc6uuHITcw+uXmDfedDx9sj/32T7XbgXHsEUbtlUtS+81qEuVzOBGhE3HVx8ctCtOSc3dpHf6TZ2Ezmh83TMXY2uIkV36GlCIhATwTizjsZAZ08IsQ7excE7g9hhVhwlzzu2x6ezh3/XNO5WPDoPU2YeHxlXFJWn4UnKyELh7w5uUiOxWiSf9weEI5JR1shvDsXi74dL4nL24fv9zywTbvE0s+1nvV2w3bwgxU2TaB5+yPeZJqeNkv8eZqkS5rsmxSAse/xY1jL09JsdV5NiSW3MbPfWHSkPbXpUTvx39baMUtX2ZuWjtfHm5ZcaxKyU+ogzw2gc8dDg+rkTjHPvCjuDglMT3yitsNgXXmbfZC/S9BVMHkWARHYlgACCQEQd/bui31YpRCefg1OdtDJcHTssUv6R8yyDyFLvHkLtDgvg1qnXJSRoQWtxNyg8tNNOghU2kOaw/JJe3BDgr9AlfQbC9LksXibNpVsH7Qn0me/DyFgfCpaIOk3jsvXGeOK3+Sv03rAH0MSGMvG+wak2Ur8ki7Hk+mx7U8rPG9Nl7mI2Tu+tcwe+cWahoD1oQS7HzjXdnv+HFu/ulMeTfOsAyJQVQKNTm32QX6dy6+YQTA3u+Dml7BiFoHqEUBcYnXikXjS8aiUY/ihw2Y9+egFKxxjON1xAYhFBwKIcN55ImDwz37uSuPHsx5H2ZeUDYtex2KmQAWmPqhTyhA7ru9YLP0xPvVJPcdWWLa5Lntdx+HT1vGH//iGCGbcBBAX+7kx8PjadS7cGDFsJY6P/MUvlaXlI95Hm/SbLNoqcbWynpA38hXnjTCkGbOJ09hmPZdhBgjWveYcoTGx2+DWDhFoScAtNy09ZXiQDlJOBESgPwKct3TeLmIQrjjEDIKC8ax+buOPF3ywvmJ9o7OmE0922nTwiGPCE8bfOg9RN8Qxj+EZQtapFc/DFn0JDyyWWAidW9HzHOcPQUjeqWemp6QuEWe0B6yN/iIV+7kpoR5pO96OqFNvQ3G8aev4I16fYQB2DEGhzdEuuMbTjoifY7xslnbdJ6/kmfhob8TneUAck89O2xn5J03CUW7iS0uTfHIcXsTvZYADZSJMO4vuBJNcLLObHlxre8+dem4yhhZrLeNna+aoLH44Ck+FZQmBi507GiyNIE9HA6UM/LJwcf79pMwi3tLGceuNXOPydVs2NvqIji8U+ebm/2/vfkIlO6owgN8ouBpFEIyiYEC3wegmm6BGIeLGKLgTHN2LGVciLg2YlSbGlQjJuMkmZhJ06cSMIhjcPBxlFGXMaEgiCoaYP7qYifx6+nvUFPd193vd/eZ19znQc/vWn3OqvjpV9d1zq9+U9kJg4xGw8VvXEQJnPH189xcI2mMAyslDahAO0Uf1+rmIACh30J9rjE5r/rZJItc9JpvUT3uZ8UPsMo5eu9vvWoKurwgmgssfRFRb8sRf+g2hTaNLeb7mgcdrffb4B/JJP1LINl9SVttCTOlmXx6hr/dj+S2voCv1p9Um+pPGpn6rpy2pH59VR1/Ngez/PQ76o60tVrE1el1LZPZtp945+CsGrfz+8QeHU7d+YLjtLm3eKYlDxSEN4KqEU9OPYB6HcHKvSTha+rOMXfr8SrfFpP2+jO5NrDuZuK9eJ5prbf9f/2itKZmBgLnVbhozilbWChHI5paNcYWq165q0bXY2jlv/bTxt5t/33gbPZIyq0xfZ9PubyQRm9b662M8b5z1KmvNWA/l9dKn2Tc81Iw92Iz5Wj+3+rqz9GnLGN9o0+hvyW/a32Ix5t+zcIiOA69rIbO3f+HM8LOv3z1c/MlDw7s+eMdw5ddPDhcff3C459vt34Y+sE3bltEe5jc5LUKrEgt/65ie5tcpiTCPOepR7EZf6i7lzFGywVf+ceUPv71w0I8CVtK13/x8f//b/7ISxdulxOvBftPYrh6ezN54wCXtujZNqss0ksU3rcH8c+HI1Qai15KfDWx+Nfk4EVgLmXXEwF8t8J8aPPerc4NIrfv+6MFxdnQJWwijaKQr8mWRFWZvnxotLBaY5AurK5fJ6PyI1wfq+C7foj329wHV8aSEaCC+zpWwbdHyiW1lHKpGfBwAF86XxkY24URSQxrp1I4IW0Ll2qTsWN9SVh+1hThiIDpL1GtJtHJtGtvqhcRrnz7oE7uEvmACR30hfV3tUy4LePpLV37MpE/0b6o8c/nS3ul/PP/ccOv7dWv1cvHZuOWJiOoYY/NAZ42vs1MZX2n8gc9mfvnxxH4Hpugolx8t9Pn8kV52zBd+m/vYpYbdzOlEGdikO/Mp7enLu48dZXLej52SwyFg7SAZi+ntzlx63x7ruPWRH8dPx8pUWiGwUwis5cwsBBFXf5LL59P3P7mpRNYG6DW4cygIEiLle3te1IKCVFlc5HtNro66WWzkWaSkIXWu7hHJLN5gc59fMtoU2cnZExu6cy0hgDZY561s3iFvdKlHfNeOnJfRBuXSJmXYsmnQoe30t327run6v9pLB6HHPVtt++X1aQh0zgLpg4VYiN5G3+pzr25IL11spC6bzhUFW/lsw0M9+fqhf4jFpsqEOD32g3aYVtcVJPn8E49cOyE/HImPGjvzxtX4Gk/iCgi/ajX2mXvt+PIXdfiJfP7mxwwRZT2MwjX/fSK7dObcFp+kI2eg4oNsxr/NcXXMER++2dtxrxw76UPaUdf5CFgXPUBs++vzWUjwq1mT3zrJP2eVmaW/8gqBrURgLZHZLULK4ipi026eFpGW8IlK2sASvbEY+YSwggNhDFkLPO4RVboT2bSZIngiUzZR5K9dtOjMBkqncvSw14t6yF3bdmWy4Sb6pVzarh36ZrHsdbKTaEmf19vOvT4gzDZ/9Ykrm+zP0gcLkbS2rnapr08wJwh9+qhd6m0ykdCHC0+fO/vxz54+MzT/wcG0u8tdkOTXX33FQ2x8bjmFy9U2hsYv5JFf6D8fTXSehXYeGH/5SCvxvc2nQxm+H7+mH9GNz6srP+SVHr7Ij+Qp7w94w0h7+LG3C8hv7EpDgK0PSUPI+V7sXG9h/bsoAsYOfh5qSgqBQqAQWBiBtUVmF27ByS4Y4mRzs5mJAiYyKs3HmVgbXivS1Z0nNnORCIKoeu2ZjZFOizsyKLLEruuiQlffrui2EUfadoZcJm/Za8hCbyPps/RrI7Lf1g0Bbtvft3kbiMSEaH7vG6evrvLHYOefeHR4+twkGI7I9b4xayzWlce381ASG+7Nn1bMg0jKK8MPkB8RU7p8PAC5b3Xob+sX/C8+SIe5Yh6bg2OibB8t5Hf0tnZg2toZ01Vp4wjA0Rrr4SZjPF6yUguBQqAQ6BAoMtsB0t3a3ERfkECRIhtaXo13RY90Sy8b2VzbDdMma1Nmz+tOeSJQi4ryvczbaOfl9/rWfX9Qe9rIa0t2192e49KPKH3lyp9+99bvfFUwcHm5fGlv+OH9X7s6DLe80hC55RUvr6Ef49y3YzxmxfwI+fRaXzQvH74fPQfV9QZC9NX8zvGgsbJJ6x+akq4dkW30xfRtnVcRcUePjJu1cNbYLdoOvpEjSIvWqXKFQCGwoQgUmZ09cCJkXk/aWC2yeYWZWoluZVNNutebefWdtLGrRRtZDZlN5FRZtkQpWtu9nTGdSbP59oSAHbKKzWKqavJjl3x3bds4ho98RKIlAW39fNdGkbZWnzx9it6U3cYrHzrrx1pnPveRYZkIrYjsN7/4sav/e+P1N4bhTZiucvyXxb4f39zPI4b8O/3g1yJ77WfWMQrzzJEAR1hc1ZtnT8R3TA4iuWNlK+1GBIw1EuuoSMZhVXhaf/mAgIAHlvyA8MYW1F0hUAhsBQJ1Znb+MIreRCy+/at+ZFRaXjHaWC2c7auybNDR016RFpur15w211Za20hcjiS0ZQ7aZOm1SbhqW9ru1f2qNgwEQLsQUzpdtZENIg0hFyERPSNIukizPCSCaFuIyTRp0m5l+7p+Ka5PuyA25JcvX9q77757P3ztzANn33L7nYFsfvcR4J/++KHhsYfBeMtrUyK7qrGf34D5Jbym92q5fYhDQKS3AoeMuXwRvDzQ+C5NJwlfyllW6WPCh8zPEFh1zKP4bV9H+8xx4McuX/dgkHb1dXbp/q5hGD66YIffMwzDh6Yf6wD5yzAMD0zXQJiuQv489Qm+Y6wyTq7G01nnkkKgENgSBIrMzh5IG57F0NM9EmDxtUEiiTZA4t7i+O8pIRNhQNxCGmyQog8Wz5bgTqvvk2AbazZXeTZnGyfb0hFdC7FNNeSRXWQP6WW3FWURTVFQRFF7kYREZ9uyR/3OBnyQB4JYaHdssOu7dmqje2XCyPTLPey86m0lddlI3WDb4tTW2cbv/GvvXy8+//1vfenutyOzn/z8l4c7P3XvcOodccEbu+1IwbPnnxqeeuS716Y/9rowDG8aB5ieJNE3Ps5/+Ig28vP4R9pq3oTk8DfzK33JfJRPl/z4Yer3V3PRPMo5WXY9dLmP0GGeK8uGIwzmEn9kmx0Psu53XX45sv4cBhPk9keHqbBEWePmYw1xhifr9BIqq2ohUAjcbATyaucX773jExv357Ne2HtmeHFvEihxnnXyZU2AIo+Ywywb8pUbKyPd4pkN+DDNtLGrd9Ciy67PLII3T8dh2jNWdhF8lCFj/dC+MdymVSakfFeOF6TP/dUYI3X70flEad/9vtsG/z3ta/95eXjp75ev/fOFv+X4EILm4eIkEy79Qi5c+TlSG1/mFwgkgqsM4Se9r/CN5NMRwql80nsMEFQ2SfKUTYQ3OrUl+drjQ/ixtkbUVbZvW/K3+QpLDwSLyEvDMPx3GIZcF6lzlDLmAJL8mWac6RFcMI+M3VHW46O0peocDgFz0MNmH6A5nJYqvXMIWNBFvjb5Y+MpKQR2AYGQv5C+ft4iUzbrkK5NxkQf9K+kEDgsAsh15oaHDHOi9onDonhzyns4qgeNm4N9WS0ECoFCoBBYMQJFZlcM6A6pQ1xF1BeNGO8QNNXVQqAQKAQKgUKgEDguBBCSvPY/LptlpxAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQKAQKgUKgECgECoFCoBAoBAqBQqAQmI/A/wEICMuE+GSlSgAAAABJRU5ErkJggg=="
}
},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Gated recurrent unit (GRU)\n",
"\n",
"* **Update gate**: The update gate decides how much of previous memory to keep around.\n",
"* **Reset Input**: The reset gate defines how to combine new input with previous value.\n",
"\n",
"\n",
"\n",
"
\n",
"
\n",
"\n",
"$$\n",
"\\begin{array}{lcl}\n",
"\\mathbf{R}_t &=& \\sigma(\\mathbf{X}_t \\mathbf{W}_{xr} + \\mathbf{H}_{t-1} \\mathbf{W}_{hr} + \\mathbf{b}_r)\\\\\n",
"\\mathbf{Z}_t &=& \\sigma(\\mathbf{X}_t \\mathbf{W}_{xz} + \\mathbf{H}_{t-1} \\mathbf{W}_{hz} + \\mathbf{b}_z)\\\\\n",
"\\tilde{\\mathbf{H}}_t &=& \\tanh(\\mathbf{X}_t \\mathbf{W}_{xh} + \\left(\\mathbf{R}_t \\odot \\mathbf{H}_{t-1}\\right) \\mathbf{W}_{hh} + \\mathbf{b}_h)\\\\\n",
"\\mathbf{H}_t &=& \\mathbf{Z}_t \\odot \\mathbf{H}_{t-1} + (1 - \\mathbf{Z}_t) \\odot \\tilde{\\mathbf{H}}_t.\n",
"\\end{array}\n",
"$$\n",
"\n",
"\n",
"**GRU v/s LSTM**\n",
"* GRU and LSTM have comparable performance.\n",
"* GRU’s are faster to train and need fewer data to generalize.\n",
"* When there is enough data, an LSTM’s greater expressive power may lead to better results.\n",
"* Like LSTMs, GRUs are drop-in replacements for the SimpleRNN cell."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Двунаправленная модель\n",
" \n",
"\n",
"Слой **Bidirectional** создаёт две реккурентные сети, указанные в его аргументе.\n",
"На одну сеть подает прямую временную последовательность, а на вторую - обратную.\n",
"\n",
"\n",
"$$\n",
"\\begin{array}{ll}\n",
"forward \\\\\n",
"x_0 = 1, & h_1 = x_0 + h_0 = 1 + 0 = 1\\\\\n",
"x_1 = 2, & h_2 = x_1 + h_1 = 2 + 1 = 3\\\\\n",
"x_2 = 3, & h_3 = x_3 + h_2 = 3 + 3 = 6\\\\\n",
"\\end{array}\n",
"$$\n",
"
\n",
"\n",
"$$\n",
"\\begin{array}{ll}\n",
"back \\\\\n",
"x_0 = 1, & h_1 = x_0 + h_0 = 3 + 0 = 3\\\\\n",
"x_1 = 2, & h_2 = x_1 + h_1 = 2 + 3 = 5\\\\\n",
"x_2 = 3, & h_3 = x_3 + h_2 = 1 + 5 = 6\\\\\n",
"\\end{array}\n",
"$$\n",
"
\n",
"Теория: \n",
"\n",
" Двунаправленная модель \n",
"\n",
"\n",
"
Примеры: \n",
"
\n",
"
\n",
"
\n",
"\n",
" \n",
"\n",
"## Книги\n",
"* [Dive into Deep Learning](https://d2l.ai/d2l-en.pdf) ( [on-line](https://d2l.ai/index.html) )\n",
"* [\tDeep Learning with Keras](http://gen.lib.rus.ec/book/index.php?md5=DCFAA3D4F9D8702EE3534BEA90FD625C)\n",
"* [Advanced Deep Learning with Keras](http://gen.lib.rus.ec/book/index.php?md5=085D57E97969791851FE9CBB2BFB44A8)\n",
"\n",
"## Полезная информация\n",
"\n",
"* [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/)\n",
"* [Understanding LSTM Networks](http://colah.github.io/posts/2015-08-Understanding-LSTMs/)\n",
"* [Dive into Deep Learning](https://d2l.ai/chapter_recurrent-neural-networks/gru.html)\n",
"* [Time Series Prediction with LSTM Recurrent Neural Networks in Python with Keras](https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/)\n",
"* [A ten-minute introduction to sequence-to-sequence learning in Keras](https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Примеры "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Генерация текстов\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import re\n",
"\n",
"with open(\"Data/NLP/saltan.txt\", \"r\", encoding='utf-8-sig') as file:\n",
" text = file.read()\n",
" \n",
"text = text.lower()\n",
"text = text.translate( {ord(c): \" \" for c in \"…—–-«»xe\"} )\n",
"text = text.translate( {769: ' '} )\n",
"text = re.sub( '\\s+\\.', '.', text )\n",
"text = re.sub( '\\s+', ' ', text ).strip()\n",
"\n",
"chars = sorted(set( [c for c in text] )) # все символы текста\n",
"n_chars = len(chars) # число символов\n",
"ch2id = dict( (c,i) for i,c in enumerate(chars)) # таблица перевода char -> index\n",
"id2ch = dict( (i,c) for i,c in enumerate(chars)) # таблица перевода index -> char \n",
"print()\n",
"print(n_chars, \"\\n\", chars, \"\\n\", len(text), \"\\n\", text[:1000])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"SEQ_LEN = 25 # длина истории (символов перед c_i)\n",
"n_data = len(text) - SEQ_LEN # число примеров\n",
"\n",
"X = np.empty( (n_data, SEQ_LEN) ) # индексы символов истории\n",
"Y = np.zeros( (n_data, n_chars) ) # вероятности символа после него (one-hot)\n",
"for i in range(n_data):\n",
" st, ch = text[i:i+SEQ_LEN], text[i+SEQ_LEN]\n",
" if i < 5: \n",
" print(st,\"->\",ch)\n",
" for j in range(SEQ_LEN): \n",
" X[i,j] = ch2id[st[j]]\n",
" Y[i, ch2id[ch]] = 1"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from keras.models import Sequential \n",
"from keras.layers import SimpleRNN, LSTM, GRU\n",
"\n",
"VEC_DIM = 10\n",
"\n",
"model = Sequential()\n",
"model.add(Embedding(n_chars, VEC_DIM, input_length=SEQ_LEN, name=\"emb\"))\n",
"model.add(GRU ( 128, dropout = 0.2, recurrent_dropout = 0.2 ) )\n",
"#model.add(LSTM ( 128 ) )\n",
"model.add(Dense(n_chars, activation=\"softmax\")) \n",
"model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) \n",
"\n",
"model.summary()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"hist = model.fit(X,Y, validation_split=0.2, batch_size=32, epochs=100, verbose=0)\n",
"\n",
"for label in [\"loss\",\"val_loss\"]: # рисуем ошибку обучения:\n",
" plt.plot(hist.history[label], label=label)\n",
"plt.xlabel(\"epoch\") \n",
"plt.legend()\n",
"plt.show() "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def gen(st, num, end=\"\\n\"): \n",
" if len(st) < SEQ_LEN: \n",
" st = (\" \" * (SEQ_LEN-len(st))) + st \n",
" \n",
" X_tst = np.empty( (1, SEQ_LEN) )\n",
" for i in range( SEQ_LEN ): \n",
" X_tst[0,i] = ch2id[st[len(st)-SEQ_LEN+i]]\n",
"\n",
" for i in range(num):\n",
" id = np.argmax( model.predict(X_tst, verbose=0)[0] ) \n",
" st += id2ch[id]\n",
" for j in range(1,SEQ_LEN): # сдвигаем\n",
" X_tst[0,j-1] = X_tst[0,j]\n",
" X_tst[0,SEQ_LEN-1] = id # последним ставим прогноз\n",
" \n",
" print(st+\"|\",end=end)\n",
"\n",
"for st in [\"княз\", \"зелёны\", \"сексуально\", \"умн\",\"красив\", \"электр\"]:\n",
" gen(st, 2, \" \")\n",
"print()\n",
"for s in [\"положили молодых \", \"мама мыла раму \"]:\n",
" gen(str(s), 256)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vecs = model.get_layer('emb').get_weights()[0]\n",
"\n",
"plt.figure(figsize=(5,5)) \n",
"plt.scatter(vecs[:, 0], vecs[:, 1], s=5 )\n",
"for i, txt in enumerate(ch2id):\n",
" plt.text(vecs[i,0]+0.005, vecs[i,1]+0.005, txt, fontsize=10)\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"2, 64 \n",
" князь | зелёный | сексуальной | умной| красива | электро | \n",
"положили молодых ответа ве во море салтан и в свать боро салтан и в свать боро салтан\n",
"\n",
"10, 128 \n",
"князь | зелёный | сексуальной | умно | красиве | электрой| \n",
"положили молодых подом слогой пристали в свой гости подетель бель присталь с пристали\n",
"\n",
"Epoch 100/100 - 34s - loss: 1.6634 - accuracy: 0.5096 - val_loss: 1.6785 - val_accuracy: 0.5480"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Временные ряды\n",
" \n",
"### Класс TimeSeries\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from keras.models import Sequential \n",
"from keras.layers import Embedding, SimpleRNN, Dense, Dropout\n",
"\n",
"class TimeSeries(): \n",
" def __init__(self, series):\n",
" self.series = series\n",
" \n",
" def prepare_data(self, step=10, lrn_frac=0.8 ):\n",
" \"\"\" \n",
" Подготавливаем обучающие и тестовые данные.\n",
" step - длина истории для предсказания, lrn_frac - доля обучающих примеров\n",
" Будем предсказывать по step предыдущим шагам 𝑥(𝑡−𝑠),...,𝑥(𝑡−1) следующий 𝑥(𝑡)\n",
" \"\"\"\n",
" self.step = step\n",
" data_X, data_Y = np.array([]), np.array([])\n",
" for i in range(step, len(self.series)):\n",
" data_X = np.append(data_X, self.series[i-step:i] )\n",
" data_Y = np.append(data_Y, self.series[i] ) \n",
"\n",
" data_X = data_X.reshape(data_Y.shape[0], step, 1)\n",
" \n",
" lrn_N = int(lrn_frac*len(self.series)) # число обучающих данных \n",
" self.lrn_X = data_X[:lrn_N]; self.tst_X = data_X[lrn_N:]\n",
" self.lrn_Y = data_Y[:lrn_N]; self.tst_Y = data_Y[lrn_N:]\n",
" \n",
" def create_model(self, kind = 0, show = False):\n",
" \"\"\" \n",
" Создаём модель many-to-one: SimpleRNN if simple else LSTM\n",
" kind - различные варианты архитектуры\n",
" \"\"\"\n",
" model = Sequential()\n",
" if kind == 0:\n",
" model.add(SimpleRNN(units=32, input_shape=(self.step,1), activation=\"tanh\")) \n",
" elif kind == 1:\n",
" model.add(LSTM (units=32, input_shape=(self.step,1) ) )\n",
" else:\n",
" model.add(LSTM (units=64, input_shape=(self.step,1), return_sequences = True ) )\n",
" model.add(LSTM (units=32, input_shape=(self.step,32) ) )\n",
" model.add(Dense(16, activation=\"tanh\")) \n",
" model.add(Dropout(0.2))\n",
" model.add(Dense(1)) # \"linear\" activation \n",
" model.compile(loss='mean_squared_error', optimizer='rmsprop') \n",
" \n",
" if show:\n",
" model.summary()\n",
" svg_model(model) \n",
" self.model = model\n",
" \n",
" def train_model(self, epochs=20, batch_size=16, verbose=0):\n",
" \"\"\" Обучение нейронной сети \"\"\"\n",
" hist = self.model.fit(self.lrn_X, self.lrn_Y, \n",
" validation_data=(self.tst_X, self.tst_Y),\n",
" epochs=epochs, batch_size=batch_size, verbose=verbose)\n",
" \n",
" print(\"lrn: %.3E\" % ( self.model.evaluate(self.lrn_X, self.lrn_Y, verbose=0) ))\n",
" print(\"tst: %.3E\" % ( self.model.evaluate(self.tst_X, self.tst_Y, verbose=0) ))\n",
" \n",
" for label in [\"loss\",\"val_loss\"]: # рисуем ошибку обучения:\n",
" plt.plot(hist.history[label], label=label)\n",
" plt.xlabel(\"epoch\") \n",
" plt.legend()\n",
" plt.show() \n",
" \n",
" def show_test1(self, num=100):\n",
" if num + self.step > self.tst_X.shape[0] : num = self.tst_X.shape[0] - self.step\n",
" \n",
" predict = self.model.predict(self.tst_X[:num,:]) \n",
" \n",
" t = np.arange(0, len(self.series)) \n",
" t_tst = len(self.lrn_X) + self.step\n",
" plt.figure(figsize=(16,4)) \n",
" plt.scatter(t[t_tst: t_tst+num], self.series[t_tst: t_tst+num], s=8)\n",
" plt.plot (t[t_tst: t_tst+num], predict)\n",
" plt.title('predict tst series')\n",
" plt.show() \n",
" \n",
" def show_test2(self, num=100):\n",
" \"\"\"\n",
" Начнём с первых step значений ряда и при помощи сети восстановим все точки.\n",
" \"\"\"\n",
" if num + self.step > len(self.series) : num = len(self.series) - self.step\n",
" step = self.step\n",
" hist, fut_x = self.series[:step].copy(), []\n",
" hist = hist.reshape(1,step, 1)\n",
" for i in range(num):\n",
" nxt_x = self.model.predict( hist )[0] \n",
" for j in range(1,step): # сдвигаем\n",
" hist[0,j-1,0] = hist[0,j,0]\n",
" hist[0,step-1,0] = nxt_x # последним ставим прогноз\n",
" fut_x.append(nxt_x)\n",
"\n",
" fut_x = np.array(fut_x)\n",
" print(fut_x.shape)\n",
"\n",
" t = np.arange(0,len(self.series)) \n",
" plt.figure(figsize=(16,4)) \n",
" plt.scatter(t[step:num+step], self.series[step:num+step], s=8)\n",
" plt.plot (t[step:num+step], fut_x)\n",
" plt.title('reconstruction lrn series')\n",
" plt.show() "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Периодическая функция\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"N = 1000 # число точек\n",
"t = np.arange(0,N) # 0,1,2,...,N-1 - ось времени\n",
"x = np.sin(0.1*t)+np.sin(t) +0.1*np.random.rand(N) # значения временного ряда\n",
"\n",
"N_out = 200\n",
"plt.figure(figsize=(16,4)) # расширяем картинку\n",
"plt.plot (t[:N_out], x[:N_out])\n",
"plt.scatter(t[:N_out], x[:N_out], s=8)\n",
"plt.show() "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ts = TimeSeries(x)\n",
"ts.prepare_data(step = 10)\n",
"ts.create_model(kind = 1)\n",
"ts.train_model (epochs=100, batch_size=64)\n",
"ts.show_test1(num=200)\n",
"ts.show_test2(num=200)\n",
"\n",
"print(ts.lrn_X.shape, ts.tst_X.shape, ts.lrn_Y.shape) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Выводы\n",
"* Если число перегибов маленькое: `x = np.sin(0.02*t)`, то сеть не научится предсказывать перегибы. Чтобы она это научилась делать их должно быть много.\n",
"* Чем больше `step`, тем, ественно, лучше предсказание. Например, при `step=10` график выше существенно лучше, чем при `step=5`. Однако можгут возникнуть эффекты переобучения (?)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Курс акций Apple\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pandas_datareader import data\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"\n",
"df = data.DataReader('AAPL', 'yahoo', '2010-01-01', '2019-11-8')\n",
"close = df.loc[:, ['Close']]\n",
"close.plot()\n",
"plt.show()\n",
"\n",
"close = df.to_numpy()[:,1] # цены закрытия"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from scipy.ndimage.interpolation import shift\n",
"close2 = shift(close, 1, cval=np.NaN)[1:] # сдвигаем вперед на 1 элемент \n",
"close = close[1:]\n",
"delta = 100*(close2-close)/close # процентное изменение цены\n",
"plt.plot(delta)\n",
"plt.show()\n",
"\n",
"delta = np.tanh(delta)\n",
"plt.plot(delta)\n",
"plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ts = TimeSeries(delta)\n",
"ts.prepare_data(step=20)\n",
"ts.create_model(kind = 1)\n",
"ts.train_model (epochs=100, batch_size=32)\n",
"ts.show_test1()\n",
"ts.show_test2()\n",
"\n",
"print(ts.lrn_X.shape, ts.tst_X.shape, ts.lrn_Y.shape) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## MNIST\n",
" \n",
"\n",
"Rowel Atienza - \"*Advanced Deep Learning with Keras_ Apply deep learning techniques, autoencoders, GANs, variational autoencoders, deep reinforcement learning, policy gradients, and more.*\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from keras.datasets import mnist\n",
"\n",
"# load mnist dataset\n",
"(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
"\n",
"plt.figure(figsize=(10, 5)) # plot the 50 mnist digits\n",
"for i in range(50):\n",
" plt.subplot(5, 10, i + 1) \n",
" plt.imshow(x_train[i,:,:], cmap='gray')\n",
" plt.axis('off')\n",
"plt.show()\n",
"\n",
"print(x_test.shape, y_test.shape, end='->')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from keras.utils import to_categorical, plot_model\n",
"\n",
"image_size = x_train.shape[1] # размер картинки\n",
"num_labels = len(np.unique(y_train)) # вычисляем число классов\n",
"\n",
"x_train = x_train.astype('float32') / 255 # нормируем на [0...1]\n",
"x_test = x_test.astype ('float32') / 255\n",
"\n",
"y_train = to_categorical(y_train) # переводим в one-hot вектор\n",
"y_test = to_categorical(y_test)\n",
" \n",
"print(x_test.shape, y_test.shape)\n",
"print(\"num_labels: %d\\nimage_size: %d\" % (num_labels,image_size))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* **dropout**: Float between 0 and 1. Fraction of the units to drop for the linear transformation of the inputs."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from keras.models import Sequential\n",
"from keras.layers import Dense, Activation, SimpleRNN\n",
"\n",
"model = Sequential()\n",
"\n",
"model.add(SimpleRNN(units = 256,\n",
" dropout = 0.2 , \n",
" input_shape = (image_size, image_size) )) # timesteps input\n",
"model.add(Dense(num_labels, activation='softmax'))\n",
"\n",
"model.summary()\n",
"\n",
"plot_model(model, to_file='rnn-mnist.png', show_shapes=True)\n",
"svg_model (model)\n",
"\n",
"model.compile(loss='categorical_crossentropy',\n",
" optimizer='sgd',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"batch_size = 128\n",
"\n",
"hist = model.fit(x_train, y_train, validation_data=(x_test, y_test),\n",
" batch_size=batch_size, epochs=20, verbose=0)\n",
"\n",
"for label in [\"loss\",\"val_loss\"]:\n",
" plt.plot(hist.history[label], label=label)\n",
"plt.xlabel(\"epoch\")\n",
"plt.legend()\n",
"plt.show() \n",
"\n",
"loss, acc = model.evaluate(x_test, y_test, batch_size=batch_size)\n",
"print(\"\\nTest accuracy: %.1f%%\" % (100.0 * acc))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Обучение с stateful=True\n",
" \n",
"\n",
"Демонстрируем работу длинной памяти. \n",
"Есть длинная (max_len) последовательность чисел, состоящая из нулей, кроме первого элемента,\n",
"который может быть 0 или 1. Если он 0 - то это класс 0, если 1 - то класс 1. \n",
"Если `x_t = [1,0,0,0,0,0]` `y=1`, разбивать последовательность на временные шаги длиной 3: \n",
"`x = [1,0,0], y=1; x = [0,0,0], y=1`,..., то рекуррентная сеть ни чему не научится. \n",
"В этом случае необходимо использовать `stateful=True`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np \n",
"from numpy.random import choice\n",
"\n",
"n_data = 100 # число примеров\n",
"n_test = int(n_data*0.8) # число обучающих примеров\n",
"max_len = 20 # длина x_t каждого примера\n",
"\n",
"X_data, Y_data = np.zeros((n_data, max_len)), np.zeros((n_data,))\n",
"\n",
"one_indexes = choice(a=n_data, size=int(n_data / 2), replace=False) # половина случ. индексов\n",
"X_data[one_indexes, 0] = 1 # для них первый элемент 1\n",
"Y_data[one_indexes] = 1 # и класс 1\n",
"\n",
"print(X_data[:10,:],\"\\n\", Y_data[:10])\n",
"\n",
"X_lrn, Y_lrn = X_data[:n_test,:], Y_data[:n_test]\n",
"X_tst, Y_tst = X_data[n_test:,:], Y_data[n_test:]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from keras.models import Sequential \n",
"from keras.layers import LSTM, Dense\n",
"\n",
"model = Sequential()\n",
"model.add(LSTM(10, batch_input_shape=(1, 1, 1), return_sequences=False, stateful=True))\n",
"model.add(Dense(1, activation='sigmoid'))\n",
"model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
"\n",
"model.summary()\n",
"svg_model(model)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print('Train...')\n",
"for epoch in range(3):\n",
" lrn_acc, lrn_loss = [], [] # обучаем:\n",
" for i in range(len(X_lrn)): # i-й пример\n",
" for j in range(max_len): # бежим по нему\n",
" loss, acc = model.train_on_batch( np.array([[[ X_lrn[i][j] ]]]), \n",
" np.array( [Y_lrn[i]]))\n",
" lrn_acc.append(acc)\n",
" lrn_loss.append(loss)\n",
" model.reset_states() # сбрасываем память\n",
"\n",
" print('%2d:\\nlrn acc: %.5f, loss:%.5f' % (epoch+1, np.mean(lrn_acc),np.mean(lrn_loss)))\n",
"\n",
" tst_acc, tst_loss, tst_prob = [],[],[] # тестируем:\n",
" for i in range(len(X_tst)): # i-й пример\n",
" for j in range(max_len):\n",
" loss, acc = model.test_on_batch(np.array([[[ X_tst[i][j] ]]]),\n",
" np.array( [Y_tst[i]]))\n",
" tst_acc.append(acc)\n",
" tst_loss.append(loss)\n",
" model.reset_states() # сбрасываем память\n",
"\n",
" for j in range(max_len):\n",
" y_pred = model.predict_on_batch(np.array([[[ X_tst[i][j] ]]]) )[0,0]\n",
" tst_prob.append(float(int(y_pred > 0.5) == int(Y_tst[i])))\n",
" model.reset_states()\n",
"\n",
" print('tst acc: %.5f, loss:%.5f prob:%.5f' % (np.mean(tst_acc), np.mean(tst_loss), np.mean(tst_prob)) ) "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}