{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Word2Vecを用いるセンチメント分析" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import torch\n", "#device = torch.device('mps') # macbook\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## データ準備\n", "\n", "### CSVファイルを読み込む" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
indexbrandsentimenttext
02401BorderlandsPositiveim getting on borderlands and i will murder yo...
12401BorderlandsPositiveI am coming to the borders and I will kill you...
22401BorderlandsPositiveim getting on borderlands and i will kill you ...
32401BorderlandsPositiveim coming on borderlands and i will murder you...
42401BorderlandsPositiveim getting on borderlands 2 and i will murder ...
\n", "
" ], "text/plain": [ " index brand sentiment \\\n", "0 2401 Borderlands Positive \n", "1 2401 Borderlands Positive \n", "2 2401 Borderlands Positive \n", "3 2401 Borderlands Positive \n", "4 2401 Borderlands Positive \n", "\n", " text \n", "0 im getting on borderlands and i will murder yo... \n", "1 I am coming to the borders and I will kill you... \n", "2 im getting on borderlands and i will kill you ... \n", "3 im coming on borderlands and i will murder you... \n", "4 im getting on borderlands 2 and i will murder ... " ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df= pd.read_csv('./Data/twitter_training.csv',names=['index','brand','sentiment','text'])\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Negative 22542\n", "Positive 20832\n", "Neutral 18318\n", "Irrelevant 12990\n", "Name: sentiment, dtype: int64" ] }, "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[\"sentiment\"].value_counts()" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "df[\"label\"]=df[\"sentiment\"].replace({\"Positive\":2,\"Negative\":0,\"Neutral\":1,\"Irrelevant\":np.nan})" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "df=df[['text','label']]\n", "df=df.dropna()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### トレーニング、バリデーション、テストデータに分割" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", "# Split data (70% train, 15% validation, 15% test)\n", "train_df, temp_df = train_test_split(df, test_size=0.3, random_state=42)\n", "val_df, test_df = train_test_split(temp_df, test_size=0.5, random_state=42)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### テキストデータの前処理\n", "\n", "- テキストを小文字に変換\n", "- 句読点を削除\n", "- トークン化" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[nltk_data] Downloading package punkt to /Users/ryozawau/nltk_data...\n", "[nltk_data] Package punkt is already up-to-date!\n" ] } ], "source": [ "import re\n", "import nltk\n", "from nltk.tokenize import word_tokenize\n", "\n", "# Download NLTK data (if not already done)\n", "nltk.download('punkt')\n", "\n", "# Function for preprocessing text\n", "def preprocess_text(text):\n", " text = text.lower() # Lowercasing\n", " text = re.sub(r'\\W+', ' ', text) # Remove punctuation\n", " tokens = word_tokenize(text) # Tokenization\n", " return tokens" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [], "source": [ "# Apply preprocessing\n", "train_df['processed_text'] = train_df['text'].apply(preprocess_text)\n", "val_df['processed_text'] = val_df['text'].apply(preprocess_text)\n", "test_df['processed_text'] = test_df['text'].apply(preprocess_text)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 単語分散表現によって特徴量の作成" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "tags": [ "hide-output" ] }, "outputs": [], "source": [ "import gensim.downloader\n", "word2vec = gensim.downloader.load('word2vec-google-news-300')" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [], "source": [ "def tokens_to_embedding(tokens, model, embedding_size=300):\n", " embeddings = [model[word] for word in tokens if word in model]\n", " if len(embeddings) == 0:\n", " return np.zeros(embedding_size)\n", " else:\n", " return np.mean(embeddings, axis=0)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "train_df['embeddings'] = train_df['processed_text'].apply(lambda x: tokens_to_embedding(x, word2vec))\n", "val_df['embeddings'] = val_df['processed_text'].apply(lambda x: tokens_to_embedding(x, word2vec))\n", "test_df['embeddings'] = test_df['processed_text'].apply(lambda x: tokens_to_embedding(x, word2vec))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 学習用データセットの作成" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [], "source": [ "from torch.utils.data import DataLoader, TensorDataset\n", "\n", "def create_dataset(df):\n", " features = torch.tensor(df['embeddings'].tolist(),dtype=torch.float32).to(device)\n", " labels = torch.tensor(df['label'].values, dtype=torch.long).to(device)\n", " return TensorDataset(features, labels)\n", "\n", "train_dataset = create_dataset(train_df)\n", "val_dataset = create_dataset(val_df)\n", "test_dataset = create_dataset(test_df)\n", "\n", "batch_size = 32\n", "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n", "val_loader = DataLoader(val_dataset, batch_size=batch_size)\n", "test_loader = DataLoader(test_dataset, batch_size=batch_size)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### モデルの作成" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [], "source": [ "import torch.nn as nn\n", "import torch.optim as optim\n", "\n", "# Define a simple Neural Network\n", "class SimpleNN(nn.Module):\n", " def __init__(self, input_size, hidden_size, num_classes):\n", " super(SimpleNN, self).__init__()\n", " self.fc1 = nn.Linear(input_size, hidden_size)\n", " self.relu = nn.ReLU()\n", " self.fc2 = nn.Linear(hidden_size, num_classes)\n", " \n", " def forward(self, x):\n", " out = self.fc1(x)\n", " out = self.relu(out)\n", " out = self.fc2(out)\n", " return out\n", "\n", "# Model, Loss, and Optimizer\n", "embedding_size = 300\n", "model = SimpleNN(input_size=embedding_size, hidden_size=100, num_classes=3).to(device)\n", "criterion = nn.CrossEntropyLoss()\n", "optimizer = optim.Adam(model.parameters(), lr=0.001)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 学習の実行\n" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "tags": [ "hide-output" ] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Epoch 1, Accuracy: 0.6624, F1 Score: 0.6575\n", "New best model saved at Epoch 1 with F1 Score: 0.6575\n", "Epoch 2, Accuracy: 0.6721, F1 Score: 0.6658\n", "New best model saved at Epoch 2 with F1 Score: 0.6658\n", "Epoch 3, Accuracy: 0.6843, F1 Score: 0.6820\n", "New best model saved at Epoch 3 with F1 Score: 0.6820\n", "Epoch 4, Accuracy: 0.6883, F1 Score: 0.6857\n", "New best model saved at Epoch 4 with F1 Score: 0.6857\n", "Epoch 5, Accuracy: 0.6932, F1 Score: 0.6916\n", "New best model saved at Epoch 5 with F1 Score: 0.6916\n", "Epoch 6, Accuracy: 0.6975, F1 Score: 0.6950\n", "New best model saved at Epoch 6 with F1 Score: 0.6950\n", "Epoch 7, Accuracy: 0.7042, F1 Score: 0.6986\n", "New best model saved at Epoch 7 with F1 Score: 0.6986\n", "Epoch 8, Accuracy: 0.7129, F1 Score: 0.7099\n", "New best model saved at Epoch 8 with F1 Score: 0.7099\n", "Epoch 9, Accuracy: 0.7182, F1 Score: 0.7147\n", "New best model saved at Epoch 9 with F1 Score: 0.7147\n", "Epoch 10, Accuracy: 0.7301, F1 Score: 0.7296\n", "New best model saved at Epoch 10 with F1 Score: 0.7296\n", "Epoch 11, Accuracy: 0.7268, F1 Score: 0.7271\n", "Epoch 12, Accuracy: 0.7435, F1 Score: 0.7423\n", "New best model saved at Epoch 12 with F1 Score: 0.7423\n", "Epoch 13, Accuracy: 0.7479, F1 Score: 0.7465\n", "New best model saved at Epoch 13 with F1 Score: 0.7465\n", "Epoch 14, Accuracy: 0.7450, F1 Score: 0.7415\n", "Epoch 15, Accuracy: 0.7540, F1 Score: 0.7529\n", "New best model saved at Epoch 15 with F1 Score: 0.7529\n", "Epoch 16, Accuracy: 0.7576, F1 Score: 0.7561\n", "New best model saved at Epoch 16 with F1 Score: 0.7561\n", "Epoch 17, Accuracy: 0.7480, F1 Score: 0.7492\n", "Epoch 18, Accuracy: 0.7611, F1 Score: 0.7595\n", "New best model saved at Epoch 18 with F1 Score: 0.7595\n", "Epoch 19, Accuracy: 0.7709, F1 Score: 0.7697\n", "New best model saved at Epoch 19 with F1 Score: 0.7697\n", "Epoch 20, Accuracy: 0.7691, F1 Score: 0.7690\n", "Epoch 21, Accuracy: 0.7702, F1 Score: 0.7679\n", "Epoch 22, Accuracy: 0.7717, F1 Score: 0.7722\n", "New best model saved at Epoch 22 with F1 Score: 0.7722\n", "Epoch 23, Accuracy: 0.7798, F1 Score: 0.7800\n", "New best model saved at Epoch 23 with F1 Score: 0.7800\n", "Epoch 24, Accuracy: 0.7800, F1 Score: 0.7797\n", "Epoch 25, Accuracy: 0.7812, F1 Score: 0.7804\n", "New best model saved at Epoch 25 with F1 Score: 0.7804\n", "Epoch 26, Accuracy: 0.7847, F1 Score: 0.7845\n", "New best model saved at Epoch 26 with F1 Score: 0.7845\n", "Epoch 27, Accuracy: 0.7789, F1 Score: 0.7775\n", "Epoch 28, Accuracy: 0.7881, F1 Score: 0.7879\n", "New best model saved at Epoch 28 with F1 Score: 0.7879\n", "Epoch 29, Accuracy: 0.7928, F1 Score: 0.7924\n", "New best model saved at Epoch 29 with F1 Score: 0.7924\n", "Epoch 30, Accuracy: 0.7897, F1 Score: 0.7901\n", "Epoch 31, Accuracy: 0.8004, F1 Score: 0.8004\n", "New best model saved at Epoch 31 with F1 Score: 0.8004\n", "Epoch 32, Accuracy: 0.7960, F1 Score: 0.7948\n", "Epoch 33, Accuracy: 0.8031, F1 Score: 0.8028\n", "New best model saved at Epoch 33 with F1 Score: 0.8028\n", "Epoch 34, Accuracy: 0.8013, F1 Score: 0.8005\n", "Epoch 35, Accuracy: 0.7990, F1 Score: 0.7993\n", "Epoch 36, Accuracy: 0.8054, F1 Score: 0.8047\n", "New best model saved at Epoch 36 with F1 Score: 0.8047\n", "Epoch 37, Accuracy: 0.8013, F1 Score: 0.8013\n", "Epoch 38, Accuracy: 0.8026, F1 Score: 0.8029\n", "Epoch 39, Accuracy: 0.8087, F1 Score: 0.8083\n", "New best model saved at Epoch 39 with F1 Score: 0.8083\n", "Epoch 40, Accuracy: 0.8043, F1 Score: 0.8035\n", "Epoch 41, Accuracy: 0.8075, F1 Score: 0.8070\n", "Epoch 42, Accuracy: 0.8106, F1 Score: 0.8111\n", "New best model saved at Epoch 42 with F1 Score: 0.8111\n", "Epoch 43, Accuracy: 0.8147, F1 Score: 0.8146\n", "New best model saved at Epoch 43 with F1 Score: 0.8146\n", "Epoch 44, Accuracy: 0.8124, F1 Score: 0.8118\n", "Epoch 45, Accuracy: 0.8153, F1 Score: 0.8144\n", "Epoch 46, Accuracy: 0.8091, F1 Score: 0.8088\n", "Epoch 47, Accuracy: 0.8209, F1 Score: 0.8204\n", "New best model saved at Epoch 47 with F1 Score: 0.8204\n", "Epoch 48, Accuracy: 0.8132, F1 Score: 0.8138\n", "Epoch 49, Accuracy: 0.8182, F1 Score: 0.8180\n", "Epoch 50, Accuracy: 0.8195, F1 Score: 0.8190\n", "Epoch 51, Accuracy: 0.8196, F1 Score: 0.8193\n", "Epoch 52, Accuracy: 0.8032, F1 Score: 0.8021\n", "Epoch 53, Accuracy: 0.8260, F1 Score: 0.8258\n", "New best model saved at Epoch 53 with F1 Score: 0.8258\n", "Epoch 54, Accuracy: 0.8221, F1 Score: 0.8220\n", "Epoch 55, Accuracy: 0.8214, F1 Score: 0.8212\n", "Epoch 56, Accuracy: 0.8241, F1 Score: 0.8237\n", "Epoch 57, Accuracy: 0.8253, F1 Score: 0.8253\n", "Epoch 58, Accuracy: 0.8226, F1 Score: 0.8229\n", "Epoch 59, Accuracy: 0.8245, F1 Score: 0.8239\n", "Epoch 60, Accuracy: 0.8266, F1 Score: 0.8261\n", "New best model saved at Epoch 60 with F1 Score: 0.8261\n", "Epoch 61, Accuracy: 0.8222, F1 Score: 0.8213\n", "Epoch 62, Accuracy: 0.8214, F1 Score: 0.8214\n", "Epoch 63, Accuracy: 0.8254, F1 Score: 0.8250\n", "Epoch 64, Accuracy: 0.8315, F1 Score: 0.8315\n", "New best model saved at Epoch 64 with F1 Score: 0.8315\n", "Epoch 65, Accuracy: 0.8309, F1 Score: 0.8306\n", "Epoch 66, Accuracy: 0.8269, F1 Score: 0.8265\n", "Epoch 67, Accuracy: 0.8257, F1 Score: 0.8252\n", "Epoch 68, Accuracy: 0.8296, F1 Score: 0.8293\n", "Epoch 69, Accuracy: 0.8222, F1 Score: 0.8213\n", "Epoch 70, Accuracy: 0.8305, F1 Score: 0.8304\n", "Epoch 71, Accuracy: 0.8243, F1 Score: 0.8239\n", "Epoch 72, Accuracy: 0.8324, F1 Score: 0.8324\n", "New best model saved at Epoch 72 with F1 Score: 0.8324\n", "Epoch 73, Accuracy: 0.8277, F1 Score: 0.8271\n", "Epoch 74, Accuracy: 0.8356, F1 Score: 0.8358\n", "New best model saved at Epoch 74 with F1 Score: 0.8358\n", "Epoch 75, Accuracy: 0.8304, F1 Score: 0.8301\n", "Epoch 76, Accuracy: 0.8308, F1 Score: 0.8306\n", "Epoch 77, Accuracy: 0.8250, F1 Score: 0.8257\n", "Epoch 78, Accuracy: 0.8340, F1 Score: 0.8337\n", "Epoch 79, Accuracy: 0.8297, F1 Score: 0.8293\n", "Epoch 80, Accuracy: 0.8340, F1 Score: 0.8340\n", "Epoch 81, Accuracy: 0.8307, F1 Score: 0.8302\n", "Epoch 82, Accuracy: 0.8357, F1 Score: 0.8356\n", "Epoch 83, Accuracy: 0.8342, F1 Score: 0.8341\n", "Epoch 84, Accuracy: 0.8343, F1 Score: 0.8343\n", "Epoch 85, Accuracy: 0.8285, F1 Score: 0.8282\n", "Epoch 86, Accuracy: 0.8344, F1 Score: 0.8339\n", "Epoch 87, Accuracy: 0.8271, F1 Score: 0.8269\n", "Epoch 88, Accuracy: 0.8346, F1 Score: 0.8345\n", "Epoch 89, Accuracy: 0.8339, F1 Score: 0.8334\n", "Epoch 90, Accuracy: 0.8398, F1 Score: 0.8398\n", "New best model saved at Epoch 90 with F1 Score: 0.8398\n", "Epoch 91, Accuracy: 0.8401, F1 Score: 0.8400\n", "New best model saved at Epoch 91 with F1 Score: 0.8400\n", "Epoch 92, Accuracy: 0.8412, F1 Score: 0.8411\n", "New best model saved at Epoch 92 with F1 Score: 0.8411\n", "Epoch 93, Accuracy: 0.8337, F1 Score: 0.8333\n", "Epoch 94, Accuracy: 0.8365, F1 Score: 0.8362\n", "Epoch 95, Accuracy: 0.8297, F1 Score: 0.8291\n", "Epoch 96, Accuracy: 0.8389, F1 Score: 0.8391\n", "Epoch 97, Accuracy: 0.8387, F1 Score: 0.8386\n", "Epoch 98, Accuracy: 0.8408, F1 Score: 0.8405\n", "Epoch 99, Accuracy: 0.8417, F1 Score: 0.8414\n", "New best model saved at Epoch 99 with F1 Score: 0.8414\n", "Epoch 100, Accuracy: 0.8386, F1 Score: 0.8382\n", "Epoch 101, Accuracy: 0.8360, F1 Score: 0.8354\n", "Epoch 102, Accuracy: 0.8420, F1 Score: 0.8419\n", "New best model saved at Epoch 102 with F1 Score: 0.8419\n", "Epoch 103, Accuracy: 0.8423, F1 Score: 0.8421\n", "New best model saved at Epoch 103 with F1 Score: 0.8421\n", "Epoch 104, Accuracy: 0.8345, F1 Score: 0.8347\n", "Epoch 105, Accuracy: 0.8411, F1 Score: 0.8413\n", "Epoch 106, Accuracy: 0.8294, F1 Score: 0.8286\n", "Epoch 107, Accuracy: 0.8372, F1 Score: 0.8367\n", "Epoch 108, Accuracy: 0.8425, F1 Score: 0.8423\n", "New best model saved at Epoch 108 with F1 Score: 0.8423\n", "Epoch 109, Accuracy: 0.8399, F1 Score: 0.8399\n", "Epoch 110, Accuracy: 0.8348, F1 Score: 0.8347\n", "Epoch 111, Accuracy: 0.8324, F1 Score: 0.8318\n", "Epoch 112, Accuracy: 0.8418, F1 Score: 0.8416\n", "Epoch 113, Accuracy: 0.8391, F1 Score: 0.8392\n", "Epoch 114, Accuracy: 0.8328, F1 Score: 0.8322\n", "Epoch 115, Accuracy: 0.8417, F1 Score: 0.8416\n", "Epoch 116, Accuracy: 0.8339, F1 Score: 0.8333\n", "Epoch 117, Accuracy: 0.8356, F1 Score: 0.8354\n", "Epoch 118, Accuracy: 0.8388, F1 Score: 0.8385\n", "Epoch 119, Accuracy: 0.8373, F1 Score: 0.8374\n", "Epoch 120, Accuracy: 0.8426, F1 Score: 0.8424\n", "New best model saved at Epoch 120 with F1 Score: 0.8424\n", "Epoch 121, Accuracy: 0.8338, F1 Score: 0.8339\n", "Epoch 122, Accuracy: 0.8349, F1 Score: 0.8343\n", "Epoch 123, Accuracy: 0.8387, F1 Score: 0.8386\n", "Epoch 124, Accuracy: 0.8291, F1 Score: 0.8283\n", "Epoch 125, Accuracy: 0.8353, F1 Score: 0.8353\n", "Epoch 126, Accuracy: 0.8452, F1 Score: 0.8450\n", "New best model saved at Epoch 126 with F1 Score: 0.8450\n", "Epoch 127, Accuracy: 0.8370, F1 Score: 0.8370\n", "Epoch 128, Accuracy: 0.8378, F1 Score: 0.8373\n", "Epoch 129, Accuracy: 0.8436, F1 Score: 0.8434\n", "Epoch 130, Accuracy: 0.8351, F1 Score: 0.8344\n", "Epoch 131, Accuracy: 0.8397, F1 Score: 0.8391\n", "Epoch 132, Accuracy: 0.8423, F1 Score: 0.8421\n", "Epoch 133, Accuracy: 0.8388, F1 Score: 0.8384\n", "Epoch 134, Accuracy: 0.8414, F1 Score: 0.8412\n", "Epoch 135, Accuracy: 0.8386, F1 Score: 0.8382\n", "Epoch 136, Accuracy: 0.8459, F1 Score: 0.8461\n", "New best model saved at Epoch 136 with F1 Score: 0.8461\n", "Epoch 137, Accuracy: 0.8432, F1 Score: 0.8430\n", "Epoch 138, Accuracy: 0.8365, F1 Score: 0.8361\n", "Epoch 139, Accuracy: 0.8416, F1 Score: 0.8414\n", "Epoch 140, Accuracy: 0.8459, F1 Score: 0.8458\n", "Epoch 141, Accuracy: 0.8435, F1 Score: 0.8434\n", "Epoch 142, Accuracy: 0.8376, F1 Score: 0.8371\n", "Epoch 143, Accuracy: 0.8410, F1 Score: 0.8407\n", "Epoch 144, Accuracy: 0.8427, F1 Score: 0.8427\n", "Epoch 145, Accuracy: 0.8340, F1 Score: 0.8332\n", "Epoch 146, Accuracy: 0.8360, F1 Score: 0.8364\n", "Epoch 147, Accuracy: 0.8438, F1 Score: 0.8437\n", "Epoch 148, Accuracy: 0.8392, F1 Score: 0.8389\n", "Epoch 149, Accuracy: 0.8357, F1 Score: 0.8361\n", "Epoch 150, Accuracy: 0.8435, F1 Score: 0.8437\n", "Epoch 151, Accuracy: 0.8447, F1 Score: 0.8444\n", "Epoch 152, Accuracy: 0.8452, F1 Score: 0.8450\n", "Epoch 153, Accuracy: 0.8433, F1 Score: 0.8432\n", "Epoch 154, Accuracy: 0.8348, F1 Score: 0.8341\n", "Epoch 155, Accuracy: 0.8422, F1 Score: 0.8421\n", "Epoch 156, Accuracy: 0.8437, F1 Score: 0.8436\n", "Epoch 157, Accuracy: 0.8453, F1 Score: 0.8451\n", "Epoch 158, Accuracy: 0.8404, F1 Score: 0.8407\n", "Epoch 159, Accuracy: 0.8454, F1 Score: 0.8453\n", "Epoch 160, Accuracy: 0.8406, F1 Score: 0.8405\n", "Epoch 161, Accuracy: 0.8463, F1 Score: 0.8461\n", "New best model saved at Epoch 161 with F1 Score: 0.8461\n", "Epoch 162, Accuracy: 0.8440, F1 Score: 0.8441\n", "Epoch 163, Accuracy: 0.8433, F1 Score: 0.8430\n", "Epoch 164, Accuracy: 0.8472, F1 Score: 0.8469\n", "New best model saved at Epoch 164 with F1 Score: 0.8469\n", "Epoch 165, Accuracy: 0.8463, F1 Score: 0.8461\n", "Epoch 166, Accuracy: 0.8397, F1 Score: 0.8391\n", "Epoch 167, Accuracy: 0.8172, F1 Score: 0.8147\n", "Epoch 168, Accuracy: 0.8307, F1 Score: 0.8301\n", "Epoch 169, Accuracy: 0.8411, F1 Score: 0.8414\n", "Epoch 170, Accuracy: 0.8403, F1 Score: 0.8404\n", "Epoch 171, Accuracy: 0.8409, F1 Score: 0.8406\n", "Epoch 172, Accuracy: 0.8404, F1 Score: 0.8401\n", "Epoch 173, Accuracy: 0.8399, F1 Score: 0.8396\n", "Epoch 174, Accuracy: 0.8422, F1 Score: 0.8422\n", "Epoch 175, Accuracy: 0.8459, F1 Score: 0.8460\n", "Epoch 176, Accuracy: 0.8439, F1 Score: 0.8440\n", "Epoch 177, Accuracy: 0.8365, F1 Score: 0.8362\n", "Epoch 178, Accuracy: 0.8447, F1 Score: 0.8447\n", "Epoch 179, Accuracy: 0.8403, F1 Score: 0.8400\n", "Epoch 180, Accuracy: 0.8424, F1 Score: 0.8424\n", "Epoch 181, Accuracy: 0.8387, F1 Score: 0.8389\n", "Epoch 182, Accuracy: 0.8397, F1 Score: 0.8393\n", "Epoch 183, Accuracy: 0.8435, F1 Score: 0.8434\n", "Epoch 184, Accuracy: 0.8306, F1 Score: 0.8296\n", "Epoch 185, Accuracy: 0.8414, F1 Score: 0.8410\n", "Epoch 186, Accuracy: 0.8428, F1 Score: 0.8428\n", "Epoch 187, Accuracy: 0.8447, F1 Score: 0.8446\n", "Epoch 188, Accuracy: 0.8424, F1 Score: 0.8427\n", "Epoch 189, Accuracy: 0.8360, F1 Score: 0.8354\n", "Epoch 190, Accuracy: 0.8435, F1 Score: 0.8434\n", "Epoch 191, Accuracy: 0.8399, F1 Score: 0.8400\n", "Epoch 192, Accuracy: 0.8448, F1 Score: 0.8446\n", "Epoch 193, Accuracy: 0.8398, F1 Score: 0.8395\n", "Epoch 194, Accuracy: 0.8451, F1 Score: 0.8449\n", "Epoch 195, Accuracy: 0.8441, F1 Score: 0.8437\n", "Epoch 196, Accuracy: 0.8438, F1 Score: 0.8437\n", "Epoch 197, Accuracy: 0.8448, F1 Score: 0.8448\n", "Epoch 198, Accuracy: 0.8418, F1 Score: 0.8419\n", "Epoch 199, Accuracy: 0.8400, F1 Score: 0.8397\n", "Epoch 200, Accuracy: 0.8421, F1 Score: 0.8419\n" ] } ], "source": [ "from sklearn.metrics import accuracy_score, f1_score\n", "#from torch.utils.tensorboard import SummaryWriter\n", "# Initialize the SummaryWriter\n", "#writer = SummaryWriter('runs/sentiment')\n", "\n", "num_epochs = 200\n", "best_f1_score = 0.0\n", "# Training loop\n", "# Training and validation loop\n", "for epoch in range(num_epochs):\n", " # Training phase\n", " model.train()\n", " train_loss = 0.0\n", " for inputs, labels in train_loader:\n", " optimizer.zero_grad()\n", " outputs = model(inputs)\n", " loss = criterion(outputs, labels)\n", " loss.backward()\n", " optimizer.step()\n", " train_loss += loss.item()\n", " # Log training loss\n", " #writer.add_scalar('Loss/train', train_loss/len(train_loader), epoch)\n", "\n", " # Validation phase\n", " model.eval()\n", " val_loss = 0.0\n", " val_preds = []\n", " val_labels = []\n", " with torch.no_grad():\n", " for inputs, labels in val_loader:\n", " outputs = model(inputs)\n", " _, predicted = torch.max(outputs, 1)\n", " val_preds.extend(predicted.numpy())\n", " val_labels.extend(labels.numpy())\n", " # Log validation loss\n", " #writer.add_scalar('Loss/val', val_loss/len(val_loader), epoch)\n", " # Calculate accuracy and F1-score\n", " accuracy = accuracy_score(val_labels, val_preds)\n", " f1 = f1_score(val_labels, val_preds, average='weighted')\n", " # Log accuracy and F1-score\n", " #writer.add_scalar('Accuracy/val', accuracy, epoch)\n", " #writer.add_scalar('F1_Score/val', f1, epoch)\n", "\n", " print(f'Epoch {epoch+1}, Accuracy: {accuracy:.4f}, F1 Score: {f1:.4f}')\n", " if f1 > best_f1_score:\n", " best_f1_score = f1\n", " # Save the model\n", " torch.save(model.state_dict(), './Model/best_model.pth')\n", " print(f\"New best model saved at Epoch {epoch+1} with F1 Score: {f1:.4f}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## テストデータでモデルを検証する" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [], "source": [ "#model.load_state_dict(torch.load('./Model/best_model.pth'))\n", "# Testing loop\n", "model.eval()\n", "test_preds = []\n", "test_labels = []\n", "with torch.no_grad():\n", " for inputs, labels in test_loader:\n", " outputs = model(inputs)\n", " _, predicted = torch.max(outputs, 1)\n", " test_preds.extend(predicted.numpy())\n", " test_labels.extend(labels.numpy())" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAGwCAYAAACZ7H64AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAA9hAAAPYQGoP6dpAABD00lEQVR4nO3deViU9f7/8dfIMgLCICCboqlpYZK5lGK5m0u5tRwtO6ZltrinZpmVdiqpfqe0Mj1mpeVyqm/lVkpqi2kuKWmpkStuCeICGIjDdv/+4DQ5bgPF7Y30fFzXXFfc94fPfKYIXvN+f+57bIZhGAIAALBQJasXAAAAQCABAACWI5AAAADLEUgAAIDlCCQAAMByBBIAAGA5AgkAALAcgQQAAFjO2+oFmCGuyWirl4ByZtzbNa1eAsqRLKfN6iWgHHkkfrjpz1FWf5e2/vBKmcxTHlEhAQAAliOQAAAAy1XIlg0AAOUKXUKPCCQAAJjNRiLxhJYNAACwHBUSAADMRoHEIwIJAABmI5B4RMsGAABYjgoJAACmo0TiCYEEAACTGeQRj2jZAAAAy1EhAQDAbFRIPCKQAABgNm6M5hEtGwAAYDkCCQAAsBwtGwAAzEbHxiMCCQAAZmMPiUe0bAAAgOWokAAAYDYKJB4RSAAAMJlh9QIuA7RsAACA5aiQAABgNja1ekQgAQDAbOQRj2jZAAAAy1EhAQDAdJRIPCGQAABgNvKIR7RsAACA5aiQAABgNiokHhFIAAAwmcFlvx7RsgEAAJYjkAAAAMvRsgEAwGy0bDwikAAAYDbyiEe0bAAAgOWokAAAYDLD6gVcBggkAACYjT0kHtGyAQAAlqNCAgCA2SiQeEQgAQDAbLRsPKJlAwAALEeFBAAAk3GVjWcEEgAAzEbHxiNaNgAAmM1mK5tHKSQkJOj6669XYGCgwsPD1atXL+3YscNtzIABA2Sz2dweLVq0cBvjdDo1bNgwhYWFKSAgQD169NChQ4fcxmRkZKhfv35yOBxyOBzq16+fMjMzS7VeAgkAABXQqlWrNGTIEK1fv14rVqxQQUGBOnXqpJycHLdxXbp0UWpqquuxdOlSt/MjR47UggUL9MEHH2jNmjXKzs5Wt27dVFhY6BrTt29fbdmyRYmJiUpMTNSWLVvUr1+/Uq2Xlg0AAJcJp9Mpp9Ppdsxut8tut58zNjEx0e3rWbNmKTw8XElJSWrdurXb90dGRp73+bKysvTOO+9ozpw56tixoyRp7ty5iomJ0cqVK9W5c2clJycrMTFR69evV/PmzSVJM2fOVHx8vHbs2KGrrrqqRK+NCgkAACYzbLYyeSQkJLjaIr8/EhISSrSGrKwsSVJISIjb8W+++Ubh4eGqX7++Bg0apPT0dNe5pKQk5efnq1OnTq5j0dHRatiwodauXStJWrdunRwOhyuMSFKLFi3kcDhcY0qCCgkAAJeJcePGadSoUW7HzlcdOZthGBo1apRuuukmNWzY0HW8a9eu+sc//qFatWopJSVFTz/9tNq3b6+kpCTZ7XalpaXJ19dXVatWdZsvIiJCaWlpkqS0tDSFh4ef85zh4eGuMSVBIAEAwGxldJXNhdozngwdOlQ//fST1qxZ43a8T58+rn9u2LChmjVrplq1aunzzz/X7bfffsH5DMOQ7YxNtrbzbLg9e4wntGwAAKjAhg0bpsWLF+vrr79WjRo1Ljo2KipKtWrV0q5duyRJkZGRysvLU0ZGhtu49PR0RUREuMYcOXLknLmOHj3qGlMSBBIAACogwzA0dOhQffrpp/rqq69Uu3Ztj99z/PhxHTx4UFFRUZKkpk2bysfHRytWrHCNSU1N1bZt29SyZUtJUnx8vLKysvT999+7xmzYsEFZWVmuMSVBywYAAJMZFnyWzZAhQzR//nwtWrRIgYGBrv0cDodDfn5+ys7O1sSJE3XHHXcoKipK+/bt05NPPqmwsDDddtttrrEDBw7U6NGjFRoaqpCQEI0ZM0ZxcXGuq25iY2PVpUsXDRo0SDNmzJAkPfjgg+rWrVuJr7CRCCQAAJjPgju1Tp8+XZLUtm1bt+OzZs3SgAED5OXlpa1bt+r9999XZmamoqKi1K5dO3344YcKDAx0jZ88ebK8vb3Vu3dv5ebmqkOHDpo9e7a8vLxcY+bNm6fhw4e7rsbp0aOHpk6dWqr12gzDqHC32I9rMtrqJaCcGfd2TauXgHIky8l9vPGHR+KHm/4c9btNLJN5dn5WNvOUR1RIAAAwGxnYIwIJAACmI5F4QiABAMBkBnnEIwJJOTbwvvbq2D5Ota8I12lnvn78cb8mv/6Z9u0/6hrj5+erR4ffqvZtG8rhCNDh1BOa99/V+ujjdZKkoCA/DXm4i+Jb1FdkRLAyM3P01TfbNHV6orKzT7vmib26uh4d3k3XXBOjosIirfzqJ738ymLl5uZd8teNklm9MEm/bNyrY4cz5e3rrZj6kep4dwuFRf9xR0XDMLTqk41K+vJnnc5xqvqVEbrlvtYKj/nj1tEF+YVaPnettq3dpYL8AtW+poZuvb+1gkKrWPGy8Bf8+NU2bf1qm04eOylJCqkeouY9r1fta2tJktYt+F47N+zSbyey5eXtpfArqqnlHc0VVffczzExDEMLX/1M+7ceULdhXXVl0zqX9LXg74f7kJRjzZrW1QcfrdU9/V/Xg4/MkJd3Jc2Y9qD8Kvu6xowd3VM3trxaTzw1Xz3veElz5n2rcWNvU7s210iSwqs5VK1akF6ZskS39/m3npr4gW5sebWefaa3a45qYUGaOf1hHTh4TPfc+5oeHjpTdetE6vln77rkrxkltz/5sK7vFKeB/7pD/Z7srqLCIs1NWKK80/muMd8t2ax1S3/ULfe10qAX7lSVYH/NmbRYzjOCZuL7a/TLpr26c/jNum/Cbco7na/5/+9zFRUVWfGy8BcEVg3Qjf9oobsn9tbdE3srJraGlry2VMd/PS5JqhoZrHb9Wqvf83ep9/jbFBQWqAX/XqJTJ3PPmWvz8h9L+2n3uBhbGT0qMAJJOfbI0JlatGSj9uw9op27UvX0hA8UHRWiBg3+uNNeo2trafGSjdqUtEeHUzP08afrtXPXYV3TIEaStHtPmkY99p5WffuzDh06ru837tYbby5V29bXyMur+D9/m9YNVFBQqBde/FT79h/V9p8P6oUXP1Wnjo0UExNqyWuHZ/8c113Xtbla4TEhiqwVpp4Pt1fWsWylphRX0AzD0IZlP6lVr6aKvaGuwmNC1euRDsrPK9DW74rvwnj6lFObv05Wp3+2VJ24GEXVrqbbh3RU+oET2rv1kJUvD39Cnca1VbvRFaoaGayqkcG68c4W8qnso9TdxXfRvDq+vmpeEyNHuEOh1UPV+u6blJebp2OHjrnNc/TAMf3wxY+6+f72VryMCopE4omlgeTQoUMaP3682rVrp9jYWDVo0EDt2rXT+PHjdfDgQSuXVi5VCawsScrKOuU6tnlLitq2uUbh1YIkSdc3q6taNavpu3U7LjxPFT9l55xWYWHxO2BfH2/l5xfqzCvAnc7id9lNrqNMe7lwniquevhVKf6ci8z0k8rOPKW6cTGuMd4+XroiNlqHdhbfICl171EVFRa5jQkMCVB4TIgO7iz5h2Kh/CkqKtKO9btU4MxX1JXntmQKCwq17Zvt8vXzVbWYMNfxfGe+lv1nudr9s5UCggMu5ZLxN2fZHpI1a9aoa9euiomJUadOndSpUycZhqH09HQtXLhQb7zxhpYtW6Ybb7zxovM4nU45nU63Y0VFBapUqeJtj3lsVE8lbd6r3Xv++EOR8PJCTXz6H/ryiwmuUDHhuY+0eUvKeedwOPz10KCO+viTda5jGzbu0phRPTTg3raaO3+1/P18NXzoLZKksLDA886D8sUwDH0x5zvVvCpK4f+ramX/L7hWcfi7jQ1w+Cvr2G+uMV7eleRXpfJZY/xc34/Ly7GDx/Xh8x+rIL9QPnYfdRvWVaHV/9gztHfLPi2b/oXy8woU4AjQ7Y/1kF+gn+v8qv+uUdSVkarbhDcjZYlNrZ5Z9lf70Ucf1QMPPKDJkydf8PzIkSO1cePGi86TkJCgZ5991u1YtcgWiogq+f3zLwfjn7hd9etFqf/97ne+u+fuVro2rpaGjnxHqakZatqkjp564nYdO3pS67/f5TY2IMCuN19/QHv3HtH0t5a7ju/Ze0RPTfivHhvVQyOG3qKiIkPzPlitY8dOqqiowt03r0JaOmu1jhw4rvsn3nbuybN+EZboXohGRS8OV1xVo4J1z7/6yHkqT7s27dHyt7/UnU/c5golMbHVdc+/+ij3t9PatupnLZ32he565k75B/lrz+YUHUr+VX2f7e3hWVBq/A/lkWWBZNu2bZo7d+4Fzz/00EP6z3/+43GecePGadSoUW7H4ls//ZfXV56MG3ub2ra+RgMeeFNH0rNcx+12b40Y2lUjRs/W6jXJkqSdu1J1Vf3q6n9vW7dA4u9v13+mPqjcU06NGD1bBQXuGxaXJm7W0sTNCg2polO5eZIh3XtPG/36v81wKL+WzlqtnUkpGjDhNrcrY36vjGRnnlJg1T9K76dO5rrOVXH4q7CgSLnZp92qJDknc1Wj/rllfpR/Xt5eCo4IliRF1A7XkZR0bV7xozoOaCdJ8rH7KDgiWMERUtSVkZr9+Fxt+zZZN3RrqoM/H1JmepamD37bbc7PpyYqun6U/jHuPIEXKCOWBZKoqCitXbv2gh+8s27dOtenDV6M3W6X3W53O1aR2jVPPn6b2reL0/2DpunXwyfcznl7e8nHx1vGWVWMoqIiVTpje3xAgF0z3nxQeXkFGvbou8rLK7jg8x0/kS1J6tXzBjnz8rVu/c4yfDUoS4ZhaNns1fplY4r6P91TVcOD3M4HhwepSrC/9m49pKja1SQV7xvYl3xYHe+OlyRF1ammSl6VtHfrIV0Tf6Uk6beMHKUfPKGOfeMv7QuCOQxDhfkXvmLKMAwV5hdKkq6/tYkatmngdn7uUx+odd8bVec6z58Ui4uhROKJZX+5x4wZo4cfflhJSUm6+eabFRERIZvNprS0NK1YsUJvv/22pkyZYtXyyoXxT9yuW7o20YhH31XOKadCQ4v3c2Rn58rpLFBOjlMbN+3WqJHddNqZr9TUDDVrWlfdb22m//fqIknFlZEZ0x6SX2UfPfHUfAUEVFZAQPE74YyMbFdL5u4+N2rLj/t06pRT8S2u0qgR3TTljc/12xn3KkH5svTdb7V17S7dNbqr7H6+ys4s3vNh9/eVj6+3bDabmne9VqsXJSkkyqHQSIdWL/xBPr7eiruxniSpsr9djdvFavnc7+QXWFl+AXatmLdW4TVDVCeuxsWeHuXQdx+v0xVxtVQlpIryT+drx4ZdOvTLYfUa3V35znx9v2ST6lxXWwHB/jqd7dSPX21V9okc1b+hriQpIDjgvBtZA0MC5agWdM5xlBx7SDyzLJAMHjxYoaGhmjx5smbMmKHCwuKE7uXlpaZNm+r9999X795/7z7mXb2LN/TOenuI2/GnJnygRUuK99Y8Nm6uRg67RS++cI8cQf5KTc3QG28udd0YrUFsDTWKK74p0rLFT7rN0/nW53U4NUOS1PCamhr8UGf5+9uVsi9d/5r0sT77PMnU14e/ZtPK7ZKk955b5Ha858PtdV2bqyVJN3ZvrIK8Ai1991vl5jhVo26E+j3ZXXa/P+5l06XfjapUqZI+fu0L5ecVqk7D6rr7kVtUqRJ3BbjcnMrKVeJbK3UqK0e+fnaFxYSq1+juqtUwRgV5BTqRmqmf1yTqdHauKleprIja4frHk7cptDqX98N65eLTfvPz83XsWPF18GFhYfLx8flL8/Fpvzgbn/aLM/FpvzjTpfi037p3PFcm8+z5pGLtkTxTudhs4ePjU6L9IgAAXJ4IwZ6Ui0ACAECFRh7xiCYxAACwHBUSAABMxlU2nhFIAAAwG4HEI1o2AADAclRIAAAwHSUSTwgkAACYjD0kntGyAQAAlqNCAgCA2aiQeESFBAAAWI5AAgAALEfLBgAAs9no2XhCIAEAwGRcZeMZLRsAAGA5AgkAALAcLRsAAMxGy8YjAgkAAGZjU6tHtGwAAIDlqJAAAGAyw+oFXAYIJAAAmI2OjUe0bAAAgOWokAAAYDYqJB5RIQEAAJYjkAAAAMvRsgEAwGzch8QjAgkAACbjw/U8o2UDAAAsRyABAACWo2UDAIDZaNl4RCABAMBsBBKPaNkAAADLUSEBAMB0lEg8IZAAAGA28ohHtGwAAIDlqJAAAGA2KiQeEUgAADCZYfUCLgO0bAAAgOWokAAAYDZaNh5RIQEAAJajQgIAgNlslEg8oUICAAAsR4UEAACzUSDxiAoJAACwHIEEAABYjkACAIDZbGX0KIWEhARdf/31CgwMVHh4uHr16qUdO3a4jTEMQxMnTlR0dLT8/PzUtm1bbd++3W2M0+nUsGHDFBYWpoCAAPXo0UOHDh1yG5ORkaF+/frJ4XDI4XCoX79+yszMLNV6CSQAAJjNgkCyatUqDRkyROvXr9eKFStUUFCgTp06KScnxzXm5Zdf1quvvqqpU6dq48aNioyM1M0336zffvvNNWbkyJFasGCBPvjgA61Zs0bZ2dnq1q2bCgsLXWP69u2rLVu2KDExUYmJidqyZYv69etXun9FhmFUuDvaxjUZbfUSUM6Me7um1UtAOZLlZIch/vBI/HDTnyPmkZfLZJ7dU0bI6XS6HbPb7bLb7R6/9+jRowoPD9eqVavUunVrGYah6OhojRw5Uo8//rik4mpIRESEXnrpJT300EPKyspStWrVNGfOHPXp00eSdPjwYcXExGjp0qXq3LmzkpOT1aBBA61fv17NmzeXJK1fv17x8fH65ZdfdNVVV5XotVEhAQDgMpGQkOBqi/z+SEhIKNH3ZmVlSZJCQkIkSSkpKUpLS1OnTp1cY+x2u9q0aaO1a9dKkpKSkpSfn+82Jjo6Wg0bNnSNWbdunRwOhyuMSFKLFi3kcDhcY0qCy34BADBbGRXlxo0bp1GjRrkdK0l1xDAMjRo1SjfddJMaNmwoSUpLS5MkRUREuI2NiIjQ/v37XWN8fX1VtWrVc8b8/v1paWkKDw8/5znDw8NdY0qCQAIAwGWipO2Zsw0dOlQ//fST1qxZc84521l3kTUM45xjZzt7zPnGl2SeM9GyAQDAZDabrUwef8awYcO0ePFiff3116pRo4breGRkpCSdU8VIT093VU0iIyOVl5enjIyMi445cuTIOc979OjRc6ovF0MgAQCgAjIMQ0OHDtWnn36qr776SrVr13Y7X7t2bUVGRmrFihWuY3l5eVq1apVatmwpSWratKl8fHzcxqSmpmrbtm2uMfHx8crKytL333/vGrNhwwZlZWW5xpQELRsAAMxmwYVdQ4YM0fz587Vo0SIFBga6KiEOh0N+fn6y2WwaOXKkJk2apHr16qlevXqaNGmS/P391bdvX9fYgQMHavTo0QoNDVVISIjGjBmjuLg4dezYUZIUGxurLl26aNCgQZoxY4Yk6cEHH1S3bt1KfIWNRCABAKBCmj59uiSpbdu2bsdnzZqlAQMGSJLGjh2r3NxcDR48WBkZGWrevLmWL1+uwMBA1/jJkyfL29tbvXv3Vm5urjp06KDZs2fLy8vLNWbevHkaPny462qcHj16aOrUqaVab4W8D8m8pNesXgLKmSdfz7V6CShHnnrE3+oloBwZ1ML8+5DUHPb/ymSeA288VibzlEfsIQEAAJYjkAAAAMuxhwQAAJP9ySt2/1aokAAAAMsRSAAAgOVo2QAAYDZaNh4RSAAAMBuBxCNaNgAAwHJUSAAAMBkFEs8IJAAAmI3rfj0ikAAAYDLyiGfsIQEAAJYjkAAAAMvRsgEAwGy0bDyiQgIAACxHhQQAAJNRIPGMQAIAgNlIJB7RsgEAAJajQgIAgMm4D4lnVEgAAIDlqJAAAGAyKiSeUSEBAACWI5AAAADL0bIBAMBktGw8o0ICAAAsR4UEAACzUSHxiEACAIDJbCQSj2jZAAAAy1EhAQDAbBRIPCKQAABgMvKIZ7RsAACA5aiQAABgMu5D4hmBBAAAsxFIPCKQAABgMvKIZ+whAQAAlqNCAgCA2SiReEQgAQDAZOQRz2jZAAAAy1EhAQDAZFz26xmBBAAAsxFIPKJlAwAALEeFBAAAk1Eg8YxAAgCAydhD4hktGwAAYDkCCQAAsBwtGwAATEbLxjMCCQAAZiOQeETLBgAAWI4KCQAAJrNRIvHoT1VI5syZoxtvvFHR0dHav3+/JGnKlClatGhRmS4OAICKwGYrm0dFVupAMn36dI0aNUq33HKLMjMzVVhYKEkKDg7WlClTynp9AADgb6DUgeSNN97QzJkzNX78eHl5ebmON2vWTFu3bi3TxQEAgL+HUu8hSUlJUePGjc85brfblZOTUyaLAgCgIqno7ZayUOoKSe3atbVly5Zzji9btkwNGjQoizUBAIC/mVJXSB577DENGTJEp0+flmEY+v777/Xf//5XCQkJevvtt81YIwAAlzUKJJ6VOpDcd999Kigo0NixY3Xq1Cn17dtX1atX12uvvaa77rrLjDUCAHB5I5F49KfuQzJo0CANGjRIx44dU1FRkcLDw8t6XTiPNYuS9MvGvTp2OFPevt6KqRepDne3UFh0VdcYwzC06pON+uGrn3U6x6nqV0ao632tFV4jRJKUefSkXh8x97zz3zm8kxq0uPKSvBaU3uBuLdSl6VWqGxWi0/kFStr1q1786BvtTTvhGtOlaX31bXed4q6IVEigv7o+/a5+PpB+zlxN6kbrsTvb6Lq6UcovKNLPB9LV/5WP5MwvkCQ1rBWhJ3q31bW1o1RkGFq2aYeem/+lTjnzL9nrRelt+XKbtny1TSePnZQkhVYPUXzP61WnUS1Jxb8f1i7cqJ++2S5njlORdSPUsV9rhdUIdc3x49fblbx+p9L3HVXe6XwNnfaAKgfYLXk9+Hv5SzdGCwsLK6t1oAT2Jx9Ws5vjFF03XEWFRfr6ow2a9+ISPfLy3fKt7CNJWrtks9Yv+1E9H2qv0KhgrV6QpLmTFmvIK31l9/NVUGgVjZo2wG3epK+2a+2SzbryuloWvCqUVPOraur9L3/Qjymp8q5USY/d2VpzHuujjuPeVm5ecVDws/to065ftXTjL3rp/lvOO0+TutF6b0xvTftsvZ6Zu0L5BYVqEBMuwzAkSeHBVTRv7F1a8n2ynpmzQlX8fDXhno56ZdCtemTqwkv1cvEnBIYEqHXvFgqOCJYkbV/zixa+tlT3/qu3wmqE6vulm5WUuEVdBnVQ1chgrV+8Sf/3/xZr4Iv3yNfPV5JUkFeg2nE1VTuuplb/33oLX03FwqZWz0odSGrXri3bRf7N7t279y8tCBd2zxPd3b7u8VB7vfLwLKWmHFWt2GgZhqENiT+pVc+mir2hriSp5yMd9Mojs7Rt7S417XCNKlWqpCrB/m7z7NiYomvir3SFGpRP/V/5yO3rMW9/rs1TRyiudqS+33FQkrRg7XZJUo0wxwXnebpvB81ekaTpn//xx2bfkQzXP3e4rq7yC4v09PvL9b+MoqffX65lz92vWuHB2p+eWUavCGWtbuPabl+3urOFfvxqm1L3HFFo9RD98MWPat6jmeo3K/790HVQR00f/q6S1+9Uo3YNJUlNOzeSJB1I/vXSLr6CI494VuqrbEaOHKkRI0a4HoMHD1Z8fLyysrL04IMPmrFGXIDzVJ4kya9KcTk1M/2ksjNPqc61Ma4x3j5eqhUbrYM70847x+G96Urbf0yN28aav2CUqUC///13z84t8feEBvqryZXVdfzkKX361D+16fVh+nBcXzWrV8M1xu7trfyCQlcYkaTTecWtnOvrx5w9JcqpoqIi/bJ+l/Kd+Yq6MlJZR08qJ+uUrmjo/vuhxlXR+nXX+X8/oAzZyuhRSt9++626d++u6Oho2Ww2LVy40O38gAEDZLPZ3B4tWrRwG+N0OjVs2DCFhYUpICBAPXr00KFDh9zGZGRkqF+/fnI4HHI4HOrXr58yMzNLtdZSV0hGjBhx3uNvvvmmNm3aVNrpLurgwYOaMGGC3n333QuOcTqdcjqdbsfy8wrk41uxP6bHMAwtn/udYq6KUnhMcf83O+uUJKmKw70CUiXIX5nHfjvvPFu+SVZY9aqKqR9l7oJR5p7u20Hf7zionb8eK/H31AwPliSNvO0mvfDBV/p5f7puv6mh5j9+lzqNf0f7jmTou+T9euru9nqo6w16d/km+dl9NPbONpKk8OAAM14KytDRg8c1/7mPVZBfKN/KPuo5vKvCqofo112pkqSAIPffDwFB/jp5/Py/H3D5y8nJUaNGjXTffffpjjvuOO+YLl26aNasWa6vfX193c6PHDlSS5Ys0QcffKDQ0FCNHj1a3bp1U1JSkusGqX379tWhQ4eUmJgoSXrwwQfVr18/LVmypMRrLbNP++3atas++eSTsppOknTixAm99957Fx2TkJDgSmS/PxbPWlGm6yiPls1erSMHjuuOoTd7HGvIOG//Mj+vQFvX7qI6chl6rt/NurpGuIZNX1yq76v0vx+EeV9v1v+t3qrtB47ouflfam/aCfVufa0kadevxzR65ud6oMsN+mXmGG16fZgOHM1Uema2CouMi02PciAkKlj3PtdH9zxzpxq1a6hlM7/UsV//2Ph89rtsQ2KDwyVgUYFEXbt21fPPP6/bb7/9gmPsdrsiIyNdj5CQENe5rKwsvfPOO3rllVfUsWNHNW7cWHPnztXWrVu1cuVKSVJycrISExP19ttvKz4+XvHx8Zo5c6Y+++wz7dixo8RrLbMywscff+z2Ikpi8eKL/zItyX6UcePGadSoUW7HPt0+s1TruNwsm71aO5NS1P+Z2xQUWsV1/PfKSHbWKQVW/eOdbM7JXAWcVTWRpOQNe5TvLNC1ra4yf9EoM8/+82Z1bFxPvSfNU1pG6d7ZpmdmS5J2Hz7udnz34eOqHhLk+nrR+p+1aP3PCgvy1ylnvgxDeqDL9Tp4NPMvrx/m8vL2UtX/bWqNrB2utJR0/bD8R91waxNJUk7WKVU5o9J16uQp+Qf5WbHUv5Wyynzn6wrY7XbZ7X/+SqhvvvlG4eHhCg4OVps2bfTCCy+4rp5NSkpSfn6+OnXq5BofHR2thg0bau3atercubPWrVsnh8Oh5s2bu8a0aNFCDodDa9eu1VVXlexvTKkDSePGjd02tRqGobS0NB09elTTpk0r1Vy9evWSzWZz7e4/n4ttoJXO/x+iorZrDMNQ4uzV+mVTiu59qqeqhge5nQ8OD1KVYH/t3XpIUVdUkyQVFhRqf/Jhdbw7/pz5Nn+TrKuaXqEAfhldNv7V72Z1blpffRLm6+CxrFJ//8FjWUrL+E11It3fPNSJDNHXP+05Z/yxk8VtwN6trpUzv0Brtu/7U+uGlQwVFhTJUS1IAQ5/7d92UBG1/vj9cGjHYbXufe7vB5RPCQkJevbZZ92OTZgwQRMnTvxT83Xt2lX/+Mc/VKtWLaWkpOjpp59W+/btlZSUJLvdrrS0NPn6+qpq1apu3xcREaG0tOK9R2lpaee9/Ud4eLhrTEmU+i93r1693L6uVKmSqlWrprZt2+rqq68u1VxRUVF68803z5nzd1u2bFHTpk1Lu8QKa9msb7V17S71Gd1Vdj9fZWcW/7Gw+/vKx9dbNptNzbtcqzWLkhQa6VBIpENrFv0gH19vNWxZz22uE2lZ2v/LYfUd282Kl4I/4fl7O6lHiwYa9Nonyjmdp2qO4ne5J085XfcPcQRUVvXQIEUEF1fOfg8eR7NydDSr+LOmZizdoEdvu0nJB9K1/cAR3XlTnOpGhejhqQtcz9W/YxMl7fpVOafz1KphbT3Zp51e/L9vdPKU+zszlC+r/2+dal9bS4EhVZR3Ol+/bNilg8mHdceY7rLZbGrSuZE2fJakqhHBCo50aMOSJHn7eiu2RX3XHDmZOcrJOqXMI8WB99ih4/Kt7KPA0ED5Vals1Uu7/JVRheR8XYG/Uh3p06eP658bNmyoZs2aqVatWvr8888v2uYxDMOtYHC+4sHZYzwpVSApKCjQFVdcoc6dOysyMrI033peTZs21Q8//HDBQOKpevJ3s2ll8SWd7z+3yO14j4fa67o2xWGwZffGys8r0NJZ3yo3x6nqdSP0z3HdZfdz36S0+ZtkBVUNUN04rpq4XPTrUFxy/+jJe9yOj575uT5eU/xJ2zc3rqdXBt3qOvfmkF6SpMkL1mjKwjWSpHeXb5Ldx1tP9+2g4CqVlXwgXfe8/KEOnHE5b6M60Xr0tlbyt/toT+oJjZud6LqkGOVXzslcLX1rpXIyc+TrZ1e1mFDdMaa768qaG25prIK8Aq18f5VOn3Iqqk6E7nysh+seJJK05evtWrdwo+vrDyYVB9UuD7RXw1bsN/uzymqXzl9tz3gSFRWlWrVqadeuXZKkyMhI5eXlKSMjw61Kkp6erpYtW7rGHDly5Jy5jh49qoiIiBI/t80o5V98f39/JScnq1atv34TrdWrVysnJ0ddunQ57/mcnBxt2rRJbdq0KdW885Je+8trQ8Xy5OslvzQWFd9Tj5y7pwp/X4NaDDf9OZq/+GqZzLPhiVGeB12AzWbTggULLlgEkKTjx4+revXqeuutt3TvvfcqKytL1apV09y5c9W7d29JUmpqqmrUqKGlS5eqc+fOSk5OVoMGDbRhwwbdcMMNxevcsEEtWrTQL7/8Yt4ekubNm2vz5s1lEkhatWp10fMBAQGlDiMAAJQ3Vl3IlJ2drd27d7u+TklJ0ZYtWxQSEqKQkBBNnDhRd9xxh6KiorRv3z49+eSTCgsL02233SZJcjgcGjhwoEaPHq3Q0FCFhIRozJgxiouLU8eOHSVJsbGx6tKliwYNGqQZM2ZIKr7st1u3biUOI9KfCCSDBw/W6NGjdejQITVt2lQBAe73Jbj22mtLOyUAABWcNYlk06ZNateunevr3/ef9O/fX9OnT9fWrVv1/vvvKzMzU1FRUWrXrp0+/PBDBQYGur5n8uTJ8vb2Vu/evZWbm6sOHTpo9uzZrnuQSNK8efM0fPhw19U4PXr00NSpU0u11hK3bO6//35NmTJFwcHB507yv70eNptNhYWFpVqAGWjZ4Gy0bHAmWjY406Vo2bR4aXKZzLP+8UfLZJ7yqMQVkvfee08vvviiUlJSzFwPAAAVDvee86zEgeT3QkpZ7B0BAOBvhUDiUan2kJTmemIAAFCMv56elSqQ1K9f32MoOXHixEXPAwAAnK1UgeTZZ5+Vw+Eway0AAFRINBg8K1Ugueuuu857v3oAAIC/olJJB7J/BAAAmKXUV9kAAIDS4T29ZyUOJEVFRWauAwCACos84lmJWzYAAABmKfVn2QAAgFKiROIRgQQAAJOxh8QzWjYAAMByVEgAADAZBRLPCCQAAJiNROIRgQQAAJORRzxjDwkAALAcFRIAAEzGVTaeEUgAADAbicQjWjYAAMByVEgAADAZ9RHPCCQAAJiNROIRLRsAAGA5KiQAAJiMAolnBBIAAEzGRTae0bIBAACWo0ICAIDZqJB4RCABAMBk5BHPCCQAAJiMPSSesYcEAABYjkACAAAsR8sGAACT0bLxjAoJAACwHBUSAABMRoXEMyokAADAcgQSAABgOVo2AACYjJaNZwQSAABMRh7xjJYNAACwHBUSAADMRonEIwIJAAAmYw+JZwQSAABMRh7xjD0kAADAclRIAAAwGz0bjwgkAACYjDjiGS0bAABgOSokAACYjI6NZwQSAABMRiDxjJYNAACwHIEEAABYjpYNAAAmo2XjGRUSAABgOSokAACYjAKJZwQS/C0kjPCzegkoRyYNOWT1ElCODFp3CZ6EROIRgQQAAJORRzxjDwkAALAcFRIAAEzGVTaeEUgAADAZgcQzWjYAAMByBBIAACqob7/9Vt27d1d0dLRsNpsWLlzodt4wDE2cOFHR0dHy8/NT27ZttX37drcxTqdTw4YNU1hYmAICAtSjRw8dOuR+pVpGRob69esnh8Mhh8Ohfv36KTMzs1RrJZAAAGAym61sHqWVk5OjRo0aaerUqec9//LLL+vVV1/V1KlTtXHjRkVGRurmm2/Wb7/95hozcuRILViwQB988IHWrFmj7OxsdevWTYWFha4xffv21ZYtW5SYmKjExERt2bJF/fr1K9Va2UMCAEAF1bVrV3Xt2vW85wzD0JQpUzR+/HjdfvvtkqT33ntPERERmj9/vh566CFlZWXpnXfe0Zw5c9SxY0dJ0ty5cxUTE6OVK1eqc+fOSk5OVmJiotavX6/mzZtLkmbOnKn4+Hjt2LFDV111VYnWSoUEAACT2cro4XQ6dfLkSbeH0+n8U2tKSUlRWlqaOnXq5Dpmt9vVpk0brV27VpKUlJSk/Px8tzHR0dFq2LCha8y6devkcDhcYUSSWrRoIYfD4RpTEgQSAABMVlYtm4SEBNc+jd8fCQkJf2pNaWlpkqSIiAi34xEREa5zaWlp8vX1VdWqVS86Jjw8/Jz5w8PDXWNKgpYNAACXiXHjxmnUqFFux+x2+1+a03bW5hTDMM45drazx5xvfEnmORMVEgAATFZWFRK73a6goCC3x58NJJGRkZJ0ThUjPT3dVTWJjIxUXl6eMjIyLjrmyJEj58x/9OjRc6ovF0MgAQDgb6h27dqKjIzUihUrXMfy8vK0atUqtWzZUpLUtGlT+fj4uI1JTU3Vtm3bXGPi4+OVlZWl77//3jVmw4YNysrKco0pCVo2AACYzKo7tWZnZ2v37t2ur1NSUrRlyxaFhISoZs2aGjlypCZNmqR69eqpXr16mjRpkvz9/dW3b19JksPh0MCBAzV69GiFhoYqJCREY8aMUVxcnOuqm9jYWHXp0kWDBg3SjBkzJEkPPvigunXrVuIrbCQCCQAAFdamTZvUrl0719e/7z/p37+/Zs+erbFjxyo3N1eDBw9WRkaGmjdvruXLlyswMND1PZMnT5a3t7d69+6t3NxcdejQQbNnz5aXl5drzLx58zR8+HDX1Tg9evS44L1PLsRmGIbxV15seTQv6TWrl4Byhs+RwJkmDfnV6iWgHNm27mXTn6Pv/LL5uzS/74gymac8okICAIDJeFPkGZtaAQCA5aiQAABgMgoknhFIAAAwG4nEI1o2AADAclRIAAAwGZtaPSOQAABgMvKIZ7RsAACA5aiQAABgMlo2nhFIAAAwGXnEMwIJAAAmo0LiGXtIAACA5aiQAABgMgoknhFIAAAwGS0bz2jZAAAAy1EhAQDAbFRIPCKQAABgMvKIZ7RsAACA5aiQAABgMja1ekYgAQDAZOQRz2jZAAAAy1EhAQDAZLRsPCOQAABgMvKIZwQSAABMRoXEM/aQAAAAy1EhAQDAZFRIPCOQAABgMvKIZ7RsAACA5aiQAABgMlo2nhFIAAAwGXnEM1o2AADAclRIAAAwGS0bzwgkAACYjDziGS0bAABgOSokAACYjJaNZwQSAABMRh7xjEACAIDJqJB4xh4SAABgOSokAACYjAKJZwSSy8iaRUn6ZeNeHTucKW9fb8XUi1SHu1soLLqqa4xhGFr1yUb98NXPOp3jVPUrI9T1vtYKrxHiNtfBnWn6+qMN+nXPEVXyqqTIWmHq+3g3+fjyI3E52Z98WGs/26zDe48qO/OU+ozqoquvr+M6n515Siv/u057fjqo06fyVOvqKHUd0EqhUcHnzGUYhua/9Ll2/3jgnHlQ/jxwbzt1bNNQtWuF67QzX1u27tPkacu078BR15ht614+7/e+MvVzzZq3SkFBfhryQCe1vKG+IiMcyszM0Vffbtcbby1Xds5p1/jY+tU1akhXXRMbo6KiIq34eptefn2JcnPzTH+dFQUtG8/463MZ2Z98WM1ujlN03XAVFRbp6482aN6LS/TIy3fLt7KPJGntks1av+xH9XyovUKjgrV6QZLmTlqsIa/0ld3PV1JxGJn/0me6sWcTdRnQSl5elXTkwHHZ+D/mspPnzFdEzTBd1+ZqfTT5C7dzhmHow1eXqZJXJd01pqvsfr5at/RHzZm0WIP/3x8/M79bv+wn3sZdRpo1rqP/frJW25IPydurkoY/3EVvTXlAPfv+W7mn8yVJbW79l9v3tIq/Wv968k6t+HqrJCk8LEjhYUH699TPtDfliKIiq+qZsberWliQRo2fK0mqFhakt98YpMSVP+qFVxapSoBdj4/soRee6u0aA5QF9pBcRu55oruua3O1wmuEKLJWmHo81F5Zx7KVmlL8jsgwDG1I/EmtejZV7A11FR4Tqp6PdFB+XoG2rd3lmmf53O90Q+c43dSjicJrhCg0KlgNmteVt4+XVS8Nf1K962qpfZ/mir2h7jnnTqRl6dCuI7r1/jaqXjdCYdFVdev9rZV3Ot/t50GS0vYf0/rPt6jnQ+0v1dLxFz386DtatDRJe1KOaMfuVD31/EeKjqqqBlfXcI05fiLb7dGuVQN9/8MeHTp8QpK0e+8RPfrkHK1ak6yDv57Q90l79PqMRLW9qYG8vIr/PLS5MVYFBYV6/t8Lte/AUW1LPqTn/71Andpfq5gaoZa89suRzVY2j4qMQHIZc54qLpf6VbFLkjLTTyo785TqXBvjGuPt46VasdE6uDNNkpSTdUq/7j6iAIef3p3wiV55eJZm/2uhDvySeulfAExVkF8oSfL2/SNoVqpUSV7eXjqw44//3vnOfH3yxgp1va+1qgT7X/J1omxUqVJZkpR18tR5z4dWraLWN8bq0yUbLzpPYICfsnNOq7CwSJLk6+Ol/PxCGYbhGuN0FkiSmlx7RRms/O/BVkaPiszyQJKbm6s1a9bo559/Pufc6dOn9f7771/0+51Op06ePOn2yM8rMGu55YZhGFo+9zvFXBWl8JjidynZWcW/iKo43P+oVAnyV3Zm8bmM9JOSpFWfbFSTdg3U94luiqodpjmTFul4aualewEwXVh0sBxhgfryv+uVm31ahQWFWrPoB2VnnnL9PEhS4pzvFFM/Ulc3q23havFXjR3eXUlbUrR775Hznu9xS1OdOuXUym+2XXAOR5C/Hrqvg/5v4QbXsQ1JexQaGqj77mkjb28vBQX6acTDXSQVt3OAsmJpINm5c6diY2PVunVrxcXFqW3btkpN/eOdW1ZWlu67776LzpGQkCCHw+H2WDxrhdlLt9yy2at15MBx3TH0Zo9jDRmuUt/v73KatL9G17WNVdQV1dS5300KjQrWllXJZi4Zl5iXt5d6P9pZx9My9fKgd/VC/7e07+dfdeV1NWWrVPwDsWNTivZt/1Vd7r3J4tXirxg/ppfqXxmpsc/Mv+CY27pfr8++2Ky8C7xhC/C3a9or92nPviOa/s4fv0P3pBzR+Oc+VP+7W2vT18/rm8+e1qHDJ3Ts+G+uKgo8s9lsZfKoyCzd1Pr4448rLi5OmzZtUmZmpkaNGqUbb7xR33zzjWrWrFmiOcaNG6dRo0a5Hft0+0wzlltuLJu9WjuTUtT/mdsUFFrFdfz3ykh21ikFVg1wHc85mauA/52rElx8vFqNqjpTWPWqyjqWbfbScYlF1wnXwy/20elTThUWFCkgyE9vP/WxouqES5JStv+qE0ey9OLAt92+76PJX6jm1VEa8EwvC1aN0hg3qqfa3dRA/R+ZriNHs847pkmjK1SnVrgee2reec/7+9s1Y8pAncrN04gn3lfBWUFj6fItWrp8i0KrVtGp03mSYejeu1rp19QTZf56KqqKHSXKhqWBZO3atVq5cqXCwsIUFhamxYsXa8iQIWrVqpW+/vprBQQEeJzDbrfLbre7Hauol64ahqHE2av1y6YU3ftUT1UNdy+XBocHqUqwv/ZuPaSoK6pJkgoLCrU/+bA63h1fPKZaoAKrBuj44Uy37z2RmqW6jUoWAnH5qexf/P/I8dRMHd57VO163yBJuqlnEzVpH+s2dvrYD9X53htVv8kVl3qZKKUnR/dUhzYNdd/gGfo1NeOC427vfoO2Jx/Sjt3n7hUL8LdrxpQHlJ9foGGPzb5gBUWSjmcUv2m5rVszOfMKtO77XRccC5SWpX+5c3Nz5e3tvoQ333xTlSpVUps2bTR//oXLj39Hy2Z9q61rd6nP6OJLOH/fB2D395WPr7dsNpuad7lWaxYlKTTSoZBIh9Ys+kE+vt5q2LKepOKyYXy367Tq442KqBWmyFph+vHbX3TscIbuHNnZypeHPyHvdL5OpP3xrjjj6G9K23dMflXscoQFavv63QoI8pMjtIqOHDyhxPfW6Orra6vutcXhs0qw/3k3sjpCq5wTeFG+PDWml27p1FjDH39POadOKzSkuFqanXPatelUKg4cndpfq3+/8dk5c/j72/XWaw/Ir7KvRjz7XwUE2BUQUBxeMzJzVFRU3OK9+86W2vLTfp3KdSr+hnoaPfRWTZm2TL9lnz5nTpxfBe+2lAlLA8nVV1+tTZs2KTbW/R3aG2+8IcMw1KNHD4tWVj5tWrldkvT+c4vcjvd4qL2ua3O1JKll98bKzyvQ0lnfKjfHqep1I/TPcd1d9yCRpBZdG6kgv1DL56xRbo5TETVD9c9xPRQS4bh0LwZl4vDedL13xs/D8jnfSZIatb5KvR7poOzMU1o+5ztlZ+UqsKq/rm11ldrc3syq5aIM3XVHS0nS7GkPux0f/9yHWrQ0yfV115uvk81W3HY52zVXVVejhrUkScs+fsLtXKfbEnQ4rbjqEtcgRkMeuFn+fnal7E/Xv176VEsSfyjLl1PhkUc8sxlnXst1iSUkJGj16tVaunTpec8PHjxY//nPf1RUVLqNU/OSXiuL5aEC4d0JzjRpyK9WLwHlyIXuaFuWnlr+epnM83yn4WUyT3lk6VU248aNu2AYkaRp06aVOowAAIDLT8Xc/QkAQDlCkdYzAgkAACajbeyZ5XdqBQAAoEICAIDJKJB4RiABAMBktGw8o2UDAAAsR4UEAACTUSDxjEACAIDJaNl4RssGAIAKaOLEibLZbG6PyMhI13nDMDRx4kRFR0fLz89Pbdu21fbt293mcDqdGjZsmMLCwhQQEKAePXro0KFDpqyXQAIAgMlsZfQorWuuuUapqamux9atW13nXn75Zb366quaOnWqNm7cqMjISN1888367bffXGNGjhypBQsW6IMPPtCaNWuUnZ2tbt26qbCw8E+s5uJo2QAAYLJKFrVsvL293aoivzMMQ1OmTNH48eN1++23S5Lee+89RUREaP78+XrooYeUlZWld955R3PmzFHHjh0lSXPnzlVMTIxWrlypzp3L9hPiqZAAAGCysqqQOJ1OnTx50u3hdDov+Ly7du1SdHS0ateurbvuukt79+6VJKWkpCgtLU2dOnVyjbXb7WrTpo3Wrl0rSUpKSlJ+fr7bmOjoaDVs2NA1piwRSAAAuEwkJCTI4XC4PRISEs47tnnz5nr//ff1xRdfaObMmUpLS1PLli11/PhxpaWlSZIiIiLcviciIsJ1Li0tTb6+vqpateoFx5QlWjYAAJisrK6yGTdunEaNGuV2zG63n3ds165dXf8cFxen+Ph41a1bV++9955atGjxv3W5L8wwjHOOna0kY/4MKiQAAJisrFo2drtdQUFBbo8LBZKzBQQEKC4uTrt27XLtKzm70pGenu6qmkRGRiovL08ZGRkXHFOWCCQAAPwNOJ1OJScnKyoqSrVr11ZkZKRWrFjhOp+Xl6dVq1apZcuWkqSmTZvKx8fHbUxqaqq2bdvmGlOWaNkAAGAyK26MNmbMGHXv3l01a9ZUenq6nn/+eZ08eVL9+/eXzWbTyJEjNWnSJNWrV0/16tXTpEmT5O/vr759+0qSHA6HBg4cqNGjRys0NFQhISEaM2aM4uLiXFfdlCUCCQAAJrPiqt9Dhw7p7rvv1rFjx1StWjW1aNFC69evV61atSRJY8eOVW5urgYPHqyMjAw1b95cy5cvV2BgoGuOyZMny9vbW71791Zubq46dOig2bNny8vLq8zXazMMwyjzWS02L+k1q5eAcobbNuNMk4b8avUSUI5sW/ey6c/x8jevl8k8Y9sOL5N5yiMqJAAAmIw3RZ4RSAAAMBmBxDOusgEAAJajQgIAgMl49+8ZgQQAAJPRsvGMQAIAgMnII55RRQIAAJajQgIAgMlo2XhGIAEAwGTkEc9o2QAAAMtRIQEAwGS0bDwjkAAAYDLyiGe0bAAAgOWokAAAYDJaNp4RSAAAMBl5xDNaNgAAwHJUSAAAMBktG88IJAAAmIx2hGcEEgAATEaFxDNCGwAAsBwVEgAATEaBxDMCCQAAJqNl4xktGwAAYDkqJAAAmIwCiWcEEgAATEbLxjNaNgAAwHJUSAAAMBkVEs8IJAAAmIw84hktGwAAYDkqJAAAmIyWjWcEEgAATEY7wjMCCQAAJqNC4hmhDQAAWI4KCQAAJrPJsHoJ5R6BBAAAk9Gy8YyWDQAAsJzNMAzqSBWQ0+lUQkKCxo0bJ7vdbvVyUA7wM4Ez8fOA8oZAUkGdPHlSDodDWVlZCgoKsno5KAf4mcCZ+HlAeUPLBgAAWI5AAgAALEcgAQAAliOQVFB2u10TJkxgsxpc+JnAmfh5QHnDplYAAGA5KiQAAMByBBIAAGA5AgkAALAcgQQAAFiOQFJBTZs2TbVr11blypXVtGlTrV692uolwSLffvutunfvrujoaNlsNi1cuNDqJcFCCQkJuv766xUYGKjw8HD16tVLO3bssHpZAIGkIvrwww81cuRIjR8/Xps3b1arVq3UtWtXHThwwOqlwQI5OTlq1KiRpk6davVSUA6sWrVKQ4YM0fr167VixQoVFBSoU6dOysnJsXpp+Jvjst8KqHnz5mrSpImmT5/uOhYbG6tevXopISHBwpXBajabTQsWLFCvXr2sXgrKiaNHjyo8PFyrVq1S69atrV4O/saokFQweXl5SkpKUqdOndyOd+rUSWvXrrVoVQDKq6ysLElSSEiIxSvB3x2BpII5duyYCgsLFRER4XY8IiJCaWlpFq0KQHlkGIZGjRqlm266SQ0bNrR6Ofib87Z6ATCHzWZz+9owjHOOAfh7Gzp0qH766SetWbPG6qUABJKKJiwsTF5eXudUQ9LT08+pmgD4+xo2bJgWL16sb7/9VjVq1LB6OQAtm4rG19dXTZs21YoVK9yOr1ixQi1btrRoVQDKC8MwNHToUH366af66quvVLt2bauXBEiiQlIhjRo1Sv369VOzZs0UHx+vt956SwcOHNDDDz9s9dJggezsbO3evdv1dUpKirZs2aKQkBDVrFnTwpXBCkOGDNH8+fO1aNEiBQYGuqqpDodDfn5+Fq8Of2dc9ltBTZs2TS+//LJSU1PVsGFDTZ48mUv6/qa++eYbtWvX7pzj/fv31+zZsy/9gmCpC+0lmzVrlgYMGHBpFwOcgUACAAAsxx4SAABgOQIJAACwHIEEAABYjkACAAAsRyABAACWI5AAAADLEUgAAIDlCCQAAMByBBKgApo4caKuu+4619cDBgxQr169Lvk69u3bJ5vNpi1btlzy5wZweSGQAJfQgAEDZLPZZLPZ5OPjozp16mjMmDHKyckx9Xlfe+21Et8mnhABwAp8uB5wiXXp0kWzZs1Sfn6+Vq9erQceeEA5OTmaPn2627j8/Hz5+PiUyXM6HI4ymQcAzEKFBLjE7Ha7IiMjFRMTo759++qee+7RwoULXW2Wd999V3Xq1JHdbpdhGMrKytKDDz6o8PBwBQUFqX379vrxxx/d5nzxxRcVERGhwMBADRw4UKdPn3Y7f3bLpqioSC+99JKuvPJK2e121axZUy+88IIkuT6OvnHjxrLZbGrbtq3r+2bNmqXY2FhVrlxZV199taZNm+b2PN9//70aN26sypUrq1mzZtq8eXMZ/psDUJFRIQEs5ufnp/z8fEnS7t279dFHH+mTTz6Rl5eXJOnWW29VSEiIli5dKofDoRkzZqhDhw7auXOnQkJC9NFHH2nChAl688031apVK82ZM0evv/666tSpc8HnHDdunGbOnKnJkyfrpptuUmpqqn755RdJxaHihhtu0MqVK3XNNdfI19dXkjRz5kxNmDBBU6dOVePGjbV582YNGjRIAQEB6t+/v3JyctStWze1b99ec+fOVUpKikaMGGHyvz0AFYYB4JLp37+/0bNnT9fXGzZsMEJDQ43evXsbEyZMMHx8fIz09HTX+S+//NIICgoyTp8+7TZP3bp1jRkzZhiGYRjx8fHGww8/7Ha+efPmRqNGjc77vCdPnjTsdrsxc+bM864xJSXFkGRs3rzZ7XhMTIwxf/58t2PPPfecER8fbxiGYcyYMcMICQkxcnJyXOenT59+3rkA4Gy0bIBL7LPPPlOVKlVUuXJlxcfHq3Xr1nrjjTckSbVq1VK1atVcY5OSkpSdna3Q0FBVqVLF9UhJSdGePXskScnJyYqPj3d7jrO/PlNycrKcTqc6dOhQ4jUfPXpUBw8e1MCBA93W8fzzz7uto1GjRvL39y/ROgDgTLRsgEusXbt2mj59unx8fBQdHe22cTUgIMBtbFFRkaKiovTNN9+cM09wcPCfen4/P79Sf09RUZGk4rZN8+bN3c793loyDONPrQcAJAIJcMkFBAToyiuvLNHYJk2aKC0tTd7e3rriiivOOyY2Nlbr16/Xvffe6zq2fv36C85Zr149+fn56csvv9QDDzxwzvnf94wUFha6jkVERKh69erau3ev7rnnnvPO26BBA82ZM0e5ubmu0HOxdQDAmWjZAOVYx44dFR8fr169eumLL77Qvn37tHbtWj311FPatGmTJGnEiBF699139e6772rnzp2aMGGCtm/ffsE5K1eurMcff1xjx47V+++/rz179mj9+vV65513JEnh4eHy8/NTYmKijhw5oqysLEnFN1tLSEjQa6+9pp07d2rr1q2aNWuWXn31VUlS3759ValSJQ0cOFA///yzli5dqn//+98m/xsCUFEQSIByzGazaenSpWrdurXuv/9+1a9fX3fddZf27duniIgISVKfPn30zDPP6PHHH1fTpk21f/9+PfLIIxed9+mnn9bo0aP1zDPPKDY2Vn369FF6erokydvbW6+//rpmzJih6Oho9ezZU5L0wAMP6O2339bs2bMVFxenNm3aaPbs2a7LhKtUqaIlS5bo559/VuPGjTV+/Hi99NJLJv7bAVCR2AwavwAAwGJUSAAAgOUIJAAAwHIEEgAAYDkCCQAAsByBBAAAWI5AAgAALEcgAQAAliOQAAAAyxFIAACA5QgkAADAcgQSAABguf8Pfyvu5pCbF6IAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from sklearn.metrics import confusion_matrix\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "# Confusion matrix\n", "cm = confusion_matrix(test_labels, test_preds)\n", "sns.heatmap(cm, annot=True, fmt='d', cmap=\"crest\")\n", "plt.xlabel('Predicted')\n", "plt.ylabel('True')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "jupyterbook", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.9" } }, "nbformat": 4, "nbformat_minor": 2 }