{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58e41ccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import random\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.layers import (\n",
    "    Input,\n",
    "    LSTM,\n",
    "    Dense,\n",
    "    Embedding,\n",
    "    Bidirectional,\n",
    "    Concatenate,\n",
    "    Attention,\n",
    "    Dropout,\n",
    ")\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9f22b5d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data_converted.json\", \"r\") as f:\n",
    "    data = json.load(f)\n",
    "    \n",
    "# Preprocessing function\n",
    "def preprocess_text(text):\n",
    "    \"\"\"Melakukan preprocessing teks dasar\"\"\"\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"\\s+\", \" \", text).strip()\n",
    "    return text\n",
    "\n",
    "\n",
    "# Persiapkan data untuk model\n",
    "def prepare_data(data):\n",
    "    \"\"\"Siapkan data untuk model\"\"\"\n",
    "    contexts = []\n",
    "    tokens_list = []\n",
    "    ner_list = []\n",
    "    srl_list = []\n",
    "    questions = []\n",
    "    answers = []\n",
    "    q_types = []\n",
    "\n",
    "    for item in data:\n",
    "        for qa in item[\"qas\"]:\n",
    "            contexts.append(preprocess_text(item[\"context\"]))\n",
    "            tokens_list.append(item[\"tokens\"])\n",
    "            ner_list.append(item[\"ner\"])\n",
    "            srl_list.append(item[\"srl\"])\n",
    "            questions.append(preprocess_text(qa[\"question\"]))\n",
    "            answers.append(qa[\"answer\"])\n",
    "            q_types.append(qa[\"type\"])\n",
    "\n",
    "    return contexts, tokens_list, ner_list, srl_list, questions, answers, q_types\n",
    "\n",
    "\n",
    "contexts, tokens_list, ner_list, srl_list, questions, answers, q_types = prepare_data(\n",
    "    data\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c703ec2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_vocab_size = 10000\n",
    "tokenizer = Tokenizer(num_words=max_vocab_size, oov_token=\"<OOV>\")\n",
    "tokenizer.fit_on_texts(contexts + questions + [\" \".join(item) for item in tokens_list])\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "\n",
    "# Encoding untuk NER\n",
    "ner_tokenizer = Tokenizer(oov_token=\"<OOV>\")\n",
    "ner_tokenizer.fit_on_texts([\" \".join(ner) for ner in ner_list])\n",
    "ner_vocab_size = len(ner_tokenizer.word_index) + 1\n",
    "\n",
    "# Encoding untuk SRL\n",
    "srl_tokenizer = Tokenizer(oov_token=\"<OOV>\")\n",
    "srl_tokenizer.fit_on_texts([\" \".join(srl) for srl in srl_list])\n",
    "srl_vocab_size = len(srl_tokenizer.word_index) + 1\n",
    "\n",
    "# Encoding untuk tipe pertanyaan\n",
    "q_type_tokenizer = Tokenizer()\n",
    "q_type_tokenizer.fit_on_texts(q_types)\n",
    "q_type_vocab_size = len(q_type_tokenizer.word_index) + 1\n",
    "\n",
    "\n",
    "# Konversi token, ner, srl ke sequences\n",
    "def tokens_to_sequences(tokens, ner, srl):\n",
    "    \"\"\"Konversi token, ner, dan srl ke sequences\"\"\"\n",
    "    token_seqs = [tokenizer.texts_to_sequences([\" \".join(t)])[0] for t in tokens]\n",
    "    ner_seqs = [ner_tokenizer.texts_to_sequences([\" \".join(n)])[0] for n in ner]\n",
    "    srl_seqs = [srl_tokenizer.texts_to_sequences([\" \".join(s)])[0] for s in srl]\n",
    "    return token_seqs, ner_seqs, srl_seqs\n",
    "\n",
    "\n",
    "# Menentukan panjang maksimum untuk padding\n",
    "context_seqs = tokenizer.texts_to_sequences(contexts)\n",
    "question_seqs = tokenizer.texts_to_sequences(questions)\n",
    "token_seqs, ner_seqs, srl_seqs = tokens_to_sequences(tokens_list, ner_list, srl_list)\n",
    "\n",
    "max_context_len = max([len(seq) for seq in context_seqs])\n",
    "max_question_len = max([len(seq) for seq in question_seqs])\n",
    "max_token_len = max([len(seq) for seq in token_seqs])\n",
    "\n",
    "\n",
    "# Pad sequences untuk memastikan semua input sama panjang\n",
    "def pad_all_sequences(context_seqs, question_seqs, token_seqs, ner_seqs, srl_seqs):\n",
    "    \"\"\"Padding semua sequences\"\"\"\n",
    "    context_padded = pad_sequences(context_seqs, maxlen=max_context_len, padding=\"post\")\n",
    "    question_padded = pad_sequences(\n",
    "        question_seqs, maxlen=max_question_len, padding=\"post\"\n",
    "    )\n",
    "    token_padded = pad_sequences(token_seqs, maxlen=max_token_len, padding=\"post\")\n",
    "    ner_padded = pad_sequences(ner_seqs, maxlen=max_token_len, padding=\"post\")\n",
    "    srl_padded = pad_sequences(srl_seqs, maxlen=max_token_len, padding=\"post\")\n",
    "    return context_padded, question_padded, token_padded, ner_padded, srl_padded\n",
    "\n",
    "\n",
    "# Siapkan encoder untuk jawaban\n",
    "answer_tokenizer = Tokenizer(oov_token=\"<OOV>\")\n",
    "answer_tokenizer.fit_on_texts(answers)\n",
    "answer_vocab_size = len(answer_tokenizer.word_index) + 1\n",
    "\n",
    "# Encode tipe pertanyaan - FIX - Menggunakan indeks langsung bukan sequence\n",
    "q_type_indices = []\n",
    "for q_type in q_types:\n",
    "    # Dapatkan indeks tipe pertanyaan (dikurangi 1 karena indeks dimulai dari 1)\n",
    "    q_type_idx = q_type_tokenizer.word_index.get(q_type, 0)\n",
    "    q_type_indices.append(q_type_idx)\n",
    "\n",
    "# Konversi ke numpy array\n",
    "q_type_indices = np.array(q_type_indices)\n",
    "\n",
    "# One-hot encode tipe pertanyaan\n",
    "q_type_categorical = tf.keras.utils.to_categorical(\n",
    "    q_type_indices, num_classes=q_type_vocab_size\n",
    ")\n",
    "\n",
    "# Pad sequences\n",
    "context_padded, question_padded, token_padded, ner_padded, srl_padded = (\n",
    "    pad_all_sequences(context_seqs, question_seqs, token_seqs, ner_seqs, srl_seqs)\n",
    ")\n",
    "\n",
    "# Encode jawaban\n",
    "answer_seqs = answer_tokenizer.texts_to_sequences(answers)\n",
    "max_answer_len = max([len(seq) for seq in answer_seqs])\n",
    "answer_padded = pad_sequences(answer_seqs, maxlen=max_answer_len, padding=\"post\")\n",
    "\n",
    "# Split data menjadi train dan test sets\n",
    "indices = list(range(len(context_padded)))\n",
    "train_indices, test_indices = train_test_split(indices, test_size=0.2, random_state=42)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f5e6a6b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Fungsi untuk mendapatkan subset dari data berdasarkan indices\n",
    "def get_subset(data, indices):\n",
    "    return np.array([data[i] for i in indices])\n",
    "\n",
    "\n",
    "# Train data\n",
    "train_context = get_subset(context_padded, train_indices)\n",
    "train_question = get_subset(question_padded, train_indices)\n",
    "train_token = get_subset(token_padded, train_indices)\n",
    "train_ner = get_subset(ner_padded, train_indices)\n",
    "train_srl = get_subset(srl_padded, train_indices)\n",
    "train_q_type = get_subset(q_type_categorical, train_indices)\n",
    "train_answer = get_subset(answer_padded, train_indices)\n",
    "\n",
    "# Test data\n",
    "test_context = get_subset(context_padded, test_indices)\n",
    "test_question = get_subset(question_padded, test_indices)\n",
    "test_token = get_subset(token_padded, test_indices)\n",
    "test_ner = get_subset(ner_padded, test_indices)\n",
    "test_srl = get_subset(srl_padded, test_indices)\n",
    "test_q_type = get_subset(q_type_categorical, test_indices)\n",
    "test_answer = get_subset(answer_padded, test_indices)\n",
    "\n",
    "# Hyperparameters\n",
    "embedding_dim = 100\n",
    "lstm_units = 128\n",
    "ner_embedding_dim = 50\n",
    "srl_embedding_dim = 50\n",
    "dropout_rate = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "00a25f78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_1\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"functional_1\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)        </span>┃<span style=\"font-weight: bold\"> Output Shape      </span>┃<span style=\"font-weight: bold\">    Param # </span>┃<span style=\"font-weight: bold\"> Connected to      </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ context_input       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ question_input      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">12</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ token_input         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_input           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_input           │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ text_embedding      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">100</span>)   │    <span style=\"color: #00af00; text-decoration-color: #00af00\">146,000</span> │ context_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)         │                   │            │ question_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>… │\n",
       "│                     │                   │            │ token_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_embedding       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">50</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">1,550</span> │ ner_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_embedding       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">50</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">1,150</span> │ srl_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_3     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)   │    <span style=\"color: #00af00; text-decoration-color: #00af00\">234,496</span> │ text_embedding[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_4     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">12</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)   │    <span style=\"color: #00af00; text-decoration-color: #00af00\">234,496</span> │ text_embedding[<span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ token_features      │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">200</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ text_embedding[<span style=\"color: #00af00; text-decoration-color: #00af00\">2</span>… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Concatenate</span>)       │                   │            │ ner_embedding[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ srl_embedding[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ context_attention   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ bidirectional_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Attention</span>)         │                   │            │ bidirectional_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_5     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">31</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)   │    <span style=\"color: #00af00; text-decoration-color: #00af00\">336,896</span> │ token_features[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ context_att_pool    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ context_attentio… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ question_pool       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ bidirectional_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ token_pool          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ bidirectional_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ q_type_input        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">5</span>)         │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ all_features        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">773</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ context_att_pool… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Concatenate</span>)       │                   │            │ question_pool[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│                     │                   │            │ token_pool[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>], │\n",
       "│                     │                   │            │ q_type_input[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │    <span style=\"color: #00af00; text-decoration-color: #00af00\">198,144</span> │ all_features[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)       │     <span style=\"color: #00af00; text-decoration-color: #00af00\">32,896</span> │ dropout_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ answer_output       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">748</span>)       │     <span style=\"color: #00af00; text-decoration-color: #00af00\">96,492</span> │ dropout_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)             │                   │            │                   │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape     \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m   Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to     \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ context_input       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ question_input      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m12\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ token_input         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_input           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_input           │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ text_embedding      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m100\u001b[0m)   │    \u001b[38;5;34m146,000\u001b[0m │ context_input[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│ (\u001b[38;5;33mEmbedding\u001b[0m)         │                   │            │ question_input[\u001b[38;5;34m0\u001b[0m… │\n",
       "│                     │                   │            │ token_input[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_embedding       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m50\u001b[0m)    │      \u001b[38;5;34m1,550\u001b[0m │ ner_input[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mEmbedding\u001b[0m)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_embedding       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m50\u001b[0m)    │      \u001b[38;5;34m1,150\u001b[0m │ srl_input[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mEmbedding\u001b[0m)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_3     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m256\u001b[0m)   │    \u001b[38;5;34m234,496\u001b[0m │ text_embedding[\u001b[38;5;34m0\u001b[0m… │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_4     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m12\u001b[0m, \u001b[38;5;34m256\u001b[0m)   │    \u001b[38;5;34m234,496\u001b[0m │ text_embedding[\u001b[38;5;34m1\u001b[0m… │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ token_features      │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m200\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ text_embedding[\u001b[38;5;34m2\u001b[0m… │\n",
       "│ (\u001b[38;5;33mConcatenate\u001b[0m)       │                   │            │ ner_embedding[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ srl_embedding[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ context_attention   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m256\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ bidirectional_3[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mAttention\u001b[0m)         │                   │            │ bidirectional_4[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_5     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m31\u001b[0m, \u001b[38;5;34m256\u001b[0m)   │    \u001b[38;5;34m336,896\u001b[0m │ token_features[\u001b[38;5;34m0\u001b[0m… │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ context_att_pool    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ context_attentio… │\n",
       "│ (\u001b[38;5;33mGlobalMaxPooling1…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ question_pool       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ bidirectional_4[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mGlobalMaxPooling1…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ token_pool          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ bidirectional_5[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mGlobalMaxPooling1…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ q_type_input        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m5\u001b[0m)         │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "│ (\u001b[38;5;33mInputLayer\u001b[0m)        │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ all_features        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m773\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ context_att_pool… │\n",
       "│ (\u001b[38;5;33mConcatenate\u001b[0m)       │                   │            │ question_pool[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│                     │                   │            │ token_pool[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m], │\n",
       "│                     │                   │            │ q_type_input[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_1 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │    \u001b[38;5;34m198,144\u001b[0m │ all_features[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_2 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_2 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)       │     \u001b[38;5;34m32,896\u001b[0m │ dropout_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_3 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m128\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ answer_output       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m748\u001b[0m)       │     \u001b[38;5;34m96,492\u001b[0m │ dropout_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mDense\u001b[0m)             │                   │            │                   │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,282,120</span> (4.89 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m1,282,120\u001b[0m (4.89 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">1,282,120</span> (4.89 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m1,282,120\u001b[0m (4.89 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# Function untuk membuat model\n",
    "def create_qa_model():\n",
    "    # Input layers\n",
    "    context_input = Input(shape=(max_context_len,), name=\"context_input\")\n",
    "    question_input = Input(shape=(max_question_len,), name=\"question_input\")\n",
    "    token_input = Input(shape=(max_token_len,), name=\"token_input\")\n",
    "    ner_input = Input(shape=(max_token_len,), name=\"ner_input\")\n",
    "    srl_input = Input(shape=(max_token_len,), name=\"srl_input\")\n",
    "    q_type_input = Input(shape=(q_type_vocab_size,), name=\"q_type_input\")\n",
    "\n",
    "    # Shared embedding layer for text\n",
    "    text_embedding = Embedding(vocab_size, embedding_dim, name=\"text_embedding\")\n",
    "\n",
    "    # Embedding untuk NER dan SRL\n",
    "    ner_embedding = Embedding(ner_vocab_size, ner_embedding_dim, name=\"ner_embedding\")(\n",
    "        ner_input\n",
    "    )\n",
    "    srl_embedding = Embedding(srl_vocab_size, srl_embedding_dim, name=\"srl_embedding\")(\n",
    "        srl_input\n",
    "    )\n",
    "\n",
    "    # Apply embeddings\n",
    "    context_embed = text_embedding(context_input)\n",
    "    question_embed = text_embedding(question_input)\n",
    "    token_embed = text_embedding(token_input)\n",
    "\n",
    "    # Bi-directional LSTM untuk context dan token-level features\n",
    "    context_lstm = Bidirectional(\n",
    "        LSTM(lstm_units, return_sequences=True, name=\"context_lstm\")\n",
    "    )(context_embed)\n",
    "    question_lstm = Bidirectional(\n",
    "        LSTM(lstm_units, return_sequences=True, name=\"question_lstm\")\n",
    "    )(question_embed)\n",
    "\n",
    "    # Concat token features (tokens, NER, SRL)\n",
    "    token_features = Concatenate(name=\"token_features\")(\n",
    "        [token_embed, ner_embedding, srl_embedding]\n",
    "    )\n",
    "    token_lstm = Bidirectional(\n",
    "        LSTM(lstm_units, return_sequences=True, name=\"token_lstm\")\n",
    "    )(token_features)\n",
    "\n",
    "    # Attention mechanism untuk context dengan memperhatikan question\n",
    "    context_attention = tf.keras.layers.Attention(name=\"context_attention\")(\n",
    "        [context_lstm, question_lstm]\n",
    "    )\n",
    "\n",
    "    # Pool attention outputs\n",
    "    context_att_pool = tf.keras.layers.GlobalMaxPooling1D(name=\"context_att_pool\")(\n",
    "        context_attention\n",
    "    )\n",
    "    question_pool = tf.keras.layers.GlobalMaxPooling1D(name=\"question_pool\")(\n",
    "        question_lstm\n",
    "    )\n",
    "    token_pool = tf.keras.layers.GlobalMaxPooling1D(name=\"token_pool\")(token_lstm)\n",
    "\n",
    "    # Concat all features\n",
    "    all_features = Concatenate(name=\"all_features\")(\n",
    "        [context_att_pool, question_pool, token_pool, q_type_input]\n",
    "    )\n",
    "\n",
    "    # Dense layers\n",
    "    x = Dense(256, activation=\"relu\", name=\"dense_1\")(all_features)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "    x = Dense(128, activation=\"relu\", name=\"dense_2\")(x)\n",
    "    x = Dropout(dropout_rate)(x)\n",
    "\n",
    "    # Output layer untuk jawaban\n",
    "    answer_output = Dense(\n",
    "        answer_vocab_size, activation=\"softmax\", name=\"answer_output\"\n",
    "    )(x)\n",
    "\n",
    "    # Create model\n",
    "    model = Model(\n",
    "        inputs=[\n",
    "            context_input,\n",
    "            question_input,\n",
    "            token_input,\n",
    "            ner_input,\n",
    "            srl_input,\n",
    "            q_type_input,\n",
    "        ],\n",
    "        outputs=answer_output,\n",
    "    )\n",
    "\n",
    "    # Compile model\n",
    "    model.compile(\n",
    "        optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"]\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "# Buat model\n",
    "model = create_qa_model()\n",
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "88e9f158",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "\u001b[1m140/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.0361 - loss: 6.2416\n",
      "Epoch 1: val_accuracy improved from -inf to 0.09825, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m6s\u001b[0m 21ms/step - accuracy: 0.0364 - loss: 6.2299 - val_accuracy: 0.0982 - val_loss: 5.1985\n",
      "Epoch 2/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.0967 - loss: 4.9762\n",
      "Epoch 2: val_accuracy improved from 0.09825 to 0.11579, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.0966 - loss: 4.9756 - val_accuracy: 0.1158 - val_loss: 4.8629\n",
      "Epoch 3/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.1214 - loss: 4.6133\n",
      "Epoch 3: val_accuracy improved from 0.11579 to 0.19649, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.1213 - loss: 4.6122 - val_accuracy: 0.1965 - val_loss: 4.6600\n",
      "Epoch 4/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.1587 - loss: 4.2478\n",
      "Epoch 4: val_accuracy did not improve from 0.19649\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 16ms/step - accuracy: 0.1591 - loss: 4.2473 - val_accuracy: 0.1860 - val_loss: 4.4859\n",
      "Epoch 5/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.1761 - loss: 4.1033\n",
      "Epoch 5: val_accuracy improved from 0.19649 to 0.20702, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.1763 - loss: 4.1032 - val_accuracy: 0.2070 - val_loss: 4.5443\n",
      "Epoch 6/50\n",
      "\u001b[1m140/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.2003 - loss: 3.8885\n",
      "Epoch 6: val_accuracy improved from 0.20702 to 0.21053, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 16ms/step - accuracy: 0.2004 - loss: 3.8883 - val_accuracy: 0.2105 - val_loss: 4.4775\n",
      "Epoch 7/50\n",
      "\u001b[1m140/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.2059 - loss: 3.7035\n",
      "Epoch 7: val_accuracy did not improve from 0.21053\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 16ms/step - accuracy: 0.2063 - loss: 3.7017 - val_accuracy: 0.2105 - val_loss: 4.4970\n",
      "Epoch 8/50\n",
      "\u001b[1m140/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.2673 - loss: 3.3854\n",
      "Epoch 8: val_accuracy improved from 0.21053 to 0.24561, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.2671 - loss: 3.3868 - val_accuracy: 0.2456 - val_loss: 4.4437\n",
      "Epoch 9/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.2826 - loss: 3.2903\n",
      "Epoch 9: val_accuracy improved from 0.24561 to 0.26667, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.2824 - loss: 3.2903 - val_accuracy: 0.2667 - val_loss: 4.5354\n",
      "Epoch 10/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.3219 - loss: 3.0828\n",
      "Epoch 10: val_accuracy did not improve from 0.26667\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.3218 - loss: 3.0819 - val_accuracy: 0.2667 - val_loss: 4.4609\n",
      "Epoch 11/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.3562 - loss: 2.7581\n",
      "Epoch 11: val_accuracy improved from 0.26667 to 0.27719, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.3557 - loss: 2.7612 - val_accuracy: 0.2772 - val_loss: 4.4004\n",
      "Epoch 12/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 15ms/step - accuracy: 0.3917 - loss: 2.6122\n",
      "Epoch 12: val_accuracy improved from 0.27719 to 0.29474, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.3915 - loss: 2.6129 - val_accuracy: 0.2947 - val_loss: 4.5458\n",
      "Epoch 13/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.3671 - loss: 2.6738\n",
      "Epoch 13: val_accuracy improved from 0.29474 to 0.30526, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 18ms/step - accuracy: 0.3673 - loss: 2.6714 - val_accuracy: 0.3053 - val_loss: 4.5296\n",
      "Epoch 14/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.3915 - loss: 2.4747\n",
      "Epoch 14: val_accuracy improved from 0.30526 to 0.32632, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 18ms/step - accuracy: 0.3916 - loss: 2.4742 - val_accuracy: 0.3263 - val_loss: 4.5949\n",
      "Epoch 15/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.4411 - loss: 2.2083\n",
      "Epoch 15: val_accuracy did not improve from 0.32632\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.4412 - loss: 2.2083 - val_accuracy: 0.3158 - val_loss: 4.6138\n",
      "Epoch 16/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 18ms/step - accuracy: 0.5127 - loss: 1.9661\n",
      "Epoch 16: val_accuracy improved from 0.32632 to 0.36491, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 19ms/step - accuracy: 0.5124 - loss: 1.9675 - val_accuracy: 0.3649 - val_loss: 4.7892\n",
      "Epoch 17/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.5086 - loss: 1.9077\n",
      "Epoch 17: val_accuracy did not improve from 0.36491\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.5084 - loss: 1.9080 - val_accuracy: 0.3649 - val_loss: 5.0446\n",
      "Epoch 18/50\n",
      "\u001b[1m140/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.5385 - loss: 1.7653\n",
      "Epoch 18: val_accuracy improved from 0.36491 to 0.37544, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 17ms/step - accuracy: 0.5380 - loss: 1.7665 - val_accuracy: 0.3754 - val_loss: 4.8941\n",
      "Epoch 19/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 17ms/step - accuracy: 0.5773 - loss: 1.7061\n",
      "Epoch 19: val_accuracy did not improve from 0.37544\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 18ms/step - accuracy: 0.5772 - loss: 1.7058 - val_accuracy: 0.3719 - val_loss: 5.1033\n",
      "Epoch 20/50\n",
      "\u001b[1m140/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 17ms/step - accuracy: 0.5634 - loss: 1.5120\n",
      "Epoch 20: val_accuracy improved from 0.37544 to 0.40351, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 18ms/step - accuracy: 0.5633 - loss: 1.5138 - val_accuracy: 0.4035 - val_loss: 4.9617\n",
      "Epoch 21/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 16ms/step - accuracy: 0.6151 - loss: 1.3638\n",
      "Epoch 21: val_accuracy did not improve from 0.40351\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m2s\u001b[0m 17ms/step - accuracy: 0.6148 - loss: 1.3654 - val_accuracy: 0.3895 - val_loss: 5.1762\n",
      "Epoch 22/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 21ms/step - accuracy: 0.6309 - loss: 1.3643\n",
      "Epoch 22: val_accuracy improved from 0.40351 to 0.41754, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 24ms/step - accuracy: 0.6309 - loss: 1.3640 - val_accuracy: 0.4175 - val_loss: 5.5482\n",
      "Epoch 23/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 23ms/step - accuracy: 0.6541 - loss: 1.2232\n",
      "Epoch 23: val_accuracy improved from 0.41754 to 0.42456, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 26ms/step - accuracy: 0.6539 - loss: 1.2237 - val_accuracy: 0.4246 - val_loss: 5.4584\n",
      "Epoch 24/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 0.6621 - loss: 1.0867\n",
      "Epoch 24: val_accuracy improved from 0.42456 to 0.43509, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 28ms/step - accuracy: 0.6621 - loss: 1.0872 - val_accuracy: 0.4351 - val_loss: 5.5594\n",
      "Epoch 25/50\n",
      "\u001b[1m141/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 0.6814 - loss: 1.0853\n",
      "Epoch 25: val_accuracy improved from 0.43509 to 0.48070, saving model to qa_lstm_model.h5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`. \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 28ms/step - accuracy: 0.6811 - loss: 1.0865 - val_accuracy: 0.4807 - val_loss: 5.6930\n",
      "Epoch 26/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 0.6909 - loss: 0.9925\n",
      "Epoch 26: val_accuracy did not improve from 0.48070\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 27ms/step - accuracy: 0.6908 - loss: 0.9931 - val_accuracy: 0.4175 - val_loss: 6.2907\n",
      "Epoch 27/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 24ms/step - accuracy: 0.7164 - loss: 0.9724\n",
      "Epoch 27: val_accuracy did not improve from 0.48070\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 26ms/step - accuracy: 0.7162 - loss: 0.9727 - val_accuracy: 0.4526 - val_loss: 5.9290\n",
      "Epoch 28/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 0.7475 - loss: 0.8654\n",
      "Epoch 28: val_accuracy did not improve from 0.48070\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 27ms/step - accuracy: 0.7473 - loss: 0.8657 - val_accuracy: 0.4456 - val_loss: 6.3236\n",
      "Epoch 29/50\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 0.7654 - loss: 0.7537\n",
      "Epoch 29: val_accuracy did not improve from 0.48070\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 27ms/step - accuracy: 0.7652 - loss: 0.7541 - val_accuracy: 0.4316 - val_loss: 6.2431\n",
      "Epoch 30/50\n",
      "\u001b[1m142/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m━\u001b[0m \u001b[1m0s\u001b[0m 25ms/step - accuracy: 0.7920 - loss: 0.7499\n",
      "Epoch 30: val_accuracy did not improve from 0.48070\n",
      "\u001b[1m143/143\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m4s\u001b[0m 27ms/step - accuracy: 0.7918 - loss: 0.7498 - val_accuracy: 0.4561 - val_loss: 6.8981\n",
      "Epoch 30: early stopping\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Callback untuk menyimpan model terbaik\n",
    "checkpoint = ModelCheckpoint(\n",
    "    \"qa_lstm_model.h5\", monitor=\"val_accuracy\", save_best_only=True, verbose=1\n",
    ")\n",
    "\n",
    "early_stop = EarlyStopping(monitor=\"val_accuracy\", patience=5, verbose=1)\n",
    "\n",
    "# Training\n",
    "batch_size = 8\n",
    "epochs = 50\n",
    "\n",
    "# Ubah format jawaban untuk sparse categorical crossentropy\n",
    "train_answer_labels = train_answer[:, 0]  # Ambil indeks pertama dari jawaban\n",
    "test_answer_labels = test_answer[:, 0]\n",
    "\n",
    "# Train model\n",
    "history = model.fit(\n",
    "    [train_context, train_question, train_token, train_ner, train_srl, train_q_type],\n",
    "    train_answer_labels,\n",
    "    batch_size=batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=(\n",
    "        [test_context, test_question, test_token, test_ner, test_srl, test_q_type],\n",
    "        test_answer_labels,\n",
    "    ),\n",
    "    callbacks=[checkpoint, early_stop],\n",
    ")\n",
    "\n",
    "model.save(\"qa_lstm_model_final.keras\")\n",
    "\n",
    "# Simpan tokenizer\n",
    "tokenizer_data = {\n",
    "    \"word_tokenizer\": tokenizer.to_json(),\n",
    "    \"ner_tokenizer\": ner_tokenizer.to_json(),\n",
    "    \"srl_tokenizer\": srl_tokenizer.to_json(),\n",
    "    \"answer_tokenizer\": answer_tokenizer.to_json(),\n",
    "    \"q_type_tokenizer\": q_type_tokenizer.to_json(),\n",
    "    \"max_context_len\": max_context_len,\n",
    "    \"max_question_len\": max_question_len,\n",
    "    \"max_token_len\": max_token_len,\n",
    "}\n",
    "\n",
    "with open(\"qa_tokenizers.json\", \"w\") as f:\n",
    "    json.dump(tokenizer_data, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "426ad763",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/keras/src/saving/saving_lib.py:757: UserWarning: Skipping variable loading for optimizer 'rmsprop', because it has 29 variables whereas the saved optimizer has 56 variables. \n",
      "  saveable.load_own_variables(weights_store.get(inner_path))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m36/36\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 4ms/step - accuracy: 0.4400 - loss: 7.0895\n",
      "Test Loss: 6.898097991943359\n",
      "Test Accuracy: 0.45614033937454224\n"
     ]
    }
   ],
   "source": [
    "model = load_model(\"qa_lstm_model_final.keras\")\n",
    "results = model.evaluate(\n",
    "    [test_context, test_question, test_token, test_ner, test_srl, test_q_type],\n",
    "    test_answer_labels,\n",
    "    batch_size=batch_size,\n",
    ")\n",
    "\n",
    "print(\"Test Loss:\", results[0])\n",
    "print(\"Test Accuracy:\", results[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cbe7470",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_answer(context, question, tokens, ner, srl, q_type):\n",
    "    # Preprocess\n",
    "    context_seq = tokenizer.texts_to_sequences([preprocess_text(context)])\n",
    "    question_seq = tokenizer.texts_to_sequences([preprocess_text(question)])\n",
    "\n",
    "    # Convert token, ner, srl dengan benar (memperhatikan format yang sama dengan data training)\n",
    "    token_seq = [tokenizer.texts_to_sequences([\" \".join(tokens)])[0]]\n",
    "    ner_seq = [ner_tokenizer.texts_to_sequences([\" \".join(ner)])[0]]\n",
    "    srl_seq = [srl_tokenizer.texts_to_sequences([\" \".join(srl)])[0]]\n",
    "\n",
    "    # Handle tipe pertanyaan\n",
    "    q_type_idx = q_type_tokenizer.word_index.get(q_type, 0)\n",
    "    q_type_cat = tf.keras.utils.to_categorical(\n",
    "        [q_type_idx], num_classes=q_type_vocab_size\n",
    "    )\n",
    "\n",
    "    # Pad sequences\n",
    "    context_pad = pad_sequences(context_seq, maxlen=max_context_len, padding=\"post\")\n",
    "    question_pad = pad_sequences(question_seq, maxlen=max_question_len, padding=\"post\")\n",
    "    token_pad = pad_sequences(token_seq, maxlen=max_token_len, padding=\"post\")\n",
    "    ner_pad = pad_sequences(ner_seq, maxlen=max_token_len, padding=\"post\")\n",
    "    srl_pad = pad_sequences(srl_seq, maxlen=max_token_len, padding=\"post\")\n",
    "\n",
    "    # Predict\n",
    "    prediction = model.predict(\n",
    "        [context_pad, question_pad, token_pad, ner_pad, srl_pad, q_type_cat]\n",
    "    )\n",
    "\n",
    "    # Get answer index\n",
    "    answer_idx = np.argmax(prediction[0])\n",
    "\n",
    "    # Convert to answer text\n",
    "    for word, idx in answer_tokenizer.word_index.items():\n",
    "        if idx == answer_idx:\n",
    "            return word\n",
    "\n",
    "    return \"Unknown\"\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}