{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import library\n",
    "\n",
    "# Data manipulation and visualization\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "\n",
    "# Natural language processing\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "# Deep learning\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, Embedding, LSTM, Dense, Concatenate\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Metrics for model evaluation\n",
    "from sklearn.metrics import classification_report, precision_score, recall_score, accuracy_score\n",
    "\n",
    "# Utility for serialization\n",
    "import pickle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/akeon/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/akeon/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package punkt_tab to /home/akeon/nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/akeon/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# download assets\n",
    "nltk.download(\"stopwords\")\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download(\"punkt_tab\")\n",
    "nltk.download(\"wordnet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             context  \\\n",
      "0  Albert Einstein adalah fisikawan teoretis kela...   \n",
      "1  Samudra Pasifik adalah yang terbesar dan terda...   \n",
      "2  Proklamasi Kemerdekaan Indonesia dibacakan pad...   \n",
      "3  Hukum Newton adalah tiga hukum fisika yang men...   \n",
      "4  Budi Utomo adalah organisasi pemuda yang didir...   \n",
      "\n",
      "                                            qa_pairs  \n",
      "0  [{'type': 'fill_in_the_blank', 'question': 'Si...  \n",
      "1  [{'type': 'fill_in_the_blank', 'question': 'Sa...  \n",
      "2  [{'type': 'fill_in_the_blank', 'question': 'Pr...  \n",
      "3  [{'type': 'fill_in_the_blank', 'question': 'Hu...  \n",
      "4  [{'type': 'fill_in_the_blank', 'question': 'Bu...  \n",
      "\n",
      "Total Context: 49\n",
      "Total QA Pairs: 95\n"
     ]
    }
   ],
   "source": [
    "# load dataset\n",
    "df = pd.read_json(\"dataset/training_dataset.json\")\n",
    "print(df.head())\n",
    "with open(\"dataset/training_dataset.json\", \"r\", encoding=\"utf-8\") as file:\n",
    "    dataset = json.load(file)\n",
    "    \n",
    "    \n",
    "# Menghitung total context\n",
    "total_context = len(dataset)\n",
    "\n",
    "# Menghitung total qa_pairs\n",
    "total_qa_pairs = sum(len(entry[\"qa_pairs\"]) for entry in dataset)\n",
    "\n",
    "# Menampilkan hasil\n",
    "print(f\"\\nTotal Context: {total_context}\")\n",
    "print(f\"Total QA Pairs: {total_qa_pairs}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Text Preprocessing\n",
    "stop_words = set(stopwords.words(\"indonesian\"))                                                                                                                                                                                                                                                                             \n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "normalization_dict = {\n",
    "    \"yg\": \"yang\",\n",
    "    \"gokil\": \"kocak\",\n",
    "    \"kalo\": \"kalau\",\n",
    "    \"gue\": \"saya\",\n",
    "    \"elo\": \"kamu\",\n",
    "    \"nih\": \"ini\",\n",
    "    \"trs\": \"terus\",\n",
    "    \"tdk\": \"tidak\",\n",
    "    \"gmna\": \"bagaimana\",\n",
    "    \"tp\": \"tapi\",\n",
    "    \"jd\": \"jadi\",\n",
    "    \"aja\": \"saja\",\n",
    "    \"krn\": \"karena\",\n",
    "    \"blm\": \"belum\",\n",
    "    \"dgn\": \"dengan\",\n",
    "    \"skrg\": \"sekarang\",\n",
    "    \"msh\": \"masih\",\n",
    "    \"lg\": \"lagi\",\n",
    "    \"sy\": \"saya\",\n",
    "    \"sm\": \"sama\",\n",
    "    \"bgt\": \"banget\",\n",
    "    \"dr\": \"dari\",\n",
    "    \"kpn\": \"kapan\",\n",
    "    \"hrs\": \"harus\",\n",
    "    \"cm\": \"cuma\",\n",
    "    \"sbnrnya\": \"sebenarnya\",\n",
    "    \"tdr\": \"tidur\",\n",
    "    \"tdk\": \"tidak\",\n",
    "    \"kl\": \"kalau\",\n",
    "    \"org\": \"orang\",\n",
    "    \"pke\": \"pakai\",\n",
    "    \"prnh\": \"pernah\",\n",
    "    \"brgkt\": \"berangkat\",\n",
    "    \"pdhl\": \"padahal\",\n",
    "    \"btw\": \"ngomong-ngomong\",\n",
    "    \"dmn\": \"di mana\",\n",
    "    \"bsk\": \"besok\",\n",
    "    \"td\": \"tadi\",\n",
    "    \"dlm\": \"dalam\",\n",
    "    \"utk\": \"untuk\",\n",
    "    \"spt\": \"seperti\",\n",
    "    \"gpp\": \"tidak apa-apa\",\n",
    "    \"bs\": \"bisa\",\n",
    "    \"jg\": \"juga\",\n",
    "    \"tp\": \"tapi\",\n",
    "    \"dg\": \"dengan\",\n",
    "    \"klw\": \"kalau\",\n",
    "    \"wkwk\": \"haha\",\n",
    "    \"cpt\": \"cepat\",\n",
    "    \"knp\": \"kenapa\",\n",
    "    \"jgk\": \"juga\",\n",
    "    \"plg\": \"pulang\",\n",
    "    \"brp\": \"berapa\",\n",
    "    \"bkn\": \"bukan\",\n",
    "    \"mnt\": \"minta\",\n",
    "    \"udh\": \"sudah\",\n",
    "    \"sdh\": \"sudah\",\n",
    "    \"brkt\": \"berangkat\",\n",
    "    \"btw\": \"by the way\",\n",
    "    \"tdk\": \"tidak\",\n",
    "    \"sprt\": \"seperti\",\n",
    "    \"jgn\": \"jangan\",\n",
    "    \"mlm\": \"malam\",\n",
    "    \"sblm\": \"sebelum\",\n",
    "    \"stlh\": \"setelah\",\n",
    "    \"tdr\": \"tidur\",\n",
    "    \"mlh\": \"malah\",\n",
    "    \"tmn\": \"teman\",\n",
    "}\n",
    "\n",
    "\n",
    "def text_preprocessing(text):\n",
    "    #doing lower case \n",
    "    text = text.lower()\n",
    "    \n",
    "    # remove symbol and read mark\n",
    "    text = text.translate(str.maketrans(\"\", \"\", string.punctuation))\n",
    "    \n",
    "    # remove blank space\n",
    "    text = re.sub(r\"\\s+\", \" \", text).strip()\n",
    "    \n",
    "    # word tokenize \n",
    "    tokens = word_tokenize(text)\n",
    "    \n",
    "    # normalassi kata\n",
    "    tokens = [normalization_dict[word] if word in normalization_dict else word for word in tokens] \n",
    "    \n",
    "    \n",
    "    # lemmatization\n",
    "    tokens = [lemmatizer.lemmatize(word) for word in tokens]  \n",
    "    \n",
    "    # stopword removal\n",
    "    tokens = [word for word in tokens if word not in stop_words]  \n",
    "    \n",
    "    return tokens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ Data processing complete!\n",
      "Samples: 95\n"
     ]
    }
   ],
   "source": [
    "# with open(\"dataset/training_dataset.json\", \"r\", encoding=\"utf-8\") as file:\n",
    "#     dataset = json.load(file)\n",
    "\n",
    "# === Extract Data so that each QA pair has its own context === #\n",
    "contexts = []\n",
    "questions = []\n",
    "answers = []\n",
    "question_types = []\n",
    "\n",
    "for entry in dataset:\n",
    "    processed_context = text_preprocessing(entry[\"context\"])\n",
    "    for qa in entry[\"qa_pairs\"]:\n",
    "        contexts.append(processed_context)\n",
    "        questions.append(text_preprocessing(qa[\"question\"]))\n",
    "        answers.append(text_preprocessing(qa[\"answer\"]))\n",
    "        question_types.append(qa[\"type\"])\n",
    "\n",
    "# === Initialize Tokenizer and fit on all text === #\n",
    "tokenizer = Tokenizer(oov_token=\"<OOV>\")\n",
    "tokenizer.fit_on_texts(contexts + questions + answers)\n",
    "\n",
    "# === Convert Text to Sequences === #\n",
    "context_sequences = tokenizer.texts_to_sequences(contexts)\n",
    "question_sequences = tokenizer.texts_to_sequences(questions)\n",
    "answer_sequences = tokenizer.texts_to_sequences(answers)\n",
    "\n",
    "# === Define Max Length for Padding === #\n",
    "MAX_LENGTH = 100\n",
    "context_padded = pad_sequences(context_sequences, maxlen=MAX_LENGTH, padding=\"post\", truncating=\"post\")\n",
    "question_padded = pad_sequences(question_sequences, maxlen=MAX_LENGTH, padding=\"post\", truncating=\"post\")\n",
    "answer_padded = pad_sequences(answer_sequences, maxlen=MAX_LENGTH, padding=\"post\", truncating=\"post\")\n",
    "\n",
    "# Encode Question Types\n",
    "question_type_dict = {\"fill_in_the_blank\": 0, \"true_false\": 1, \"multiple_choice\": 2}\n",
    "question_type_labels = np.array([question_type_dict[q_type] for q_type in question_types])\n",
    "\n",
    "# Save the processed data (optional)\n",
    "np.save(\"context_padded.npy\", context_padded)\n",
    "np.save(\"question_padded.npy\", question_padded)\n",
    "np.save(\"answer_padded.npy\", answer_padded)\n",
    "np.save(\"question_type_labels.npy\", question_type_labels)\n",
    "with open(\"tokenizer.pkl\", \"wb\") as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "print(\"✅ Data processing complete!\")\n",
    "print(\"Samples:\", context_padded.shape[0])  # This should now match the number of QA pairs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training samples: 76\n",
      "Testing samples: 19\n"
     ]
    }
   ],
   "source": [
    "# === Split Data into Training and Testing Sets === #\n",
    "(context_train, context_test,\n",
    " question_train, question_test,\n",
    " answer_train, answer_test,\n",
    " qtype_train, qtype_test) = train_test_split(\n",
    "    context_padded,\n",
    "    question_padded,\n",
    "    answer_padded,\n",
    "    question_type_labels,\n",
    "    test_size=0.2,\n",
    "    random_state=42\n",
    ")\n",
    "\n",
    "print(\"Training samples:\", context_train.shape[0])\n",
    "print(\"Testing samples:\", context_test.shape[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 681ms/step - answer_output_accuracy: 0.0297 - answer_output_loss: 6.2965 - loss: 13.6985 - question_output_accuracy: 0.0000e+00 - question_output_loss: 6.3022 - question_type_output_accuracy: 0.2625 - question_type_output_loss: 1.0992 - val_answer_output_accuracy: 0.2044 - val_answer_output_loss: 6.2629 - val_loss: 13.6638 - val_question_output_accuracy: 0.0069 - val_question_output_loss: 6.2961 - val_question_type_output_accuracy: 0.2500 - val_question_type_output_loss: 1.1048\n",
      "Epoch 2/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 305ms/step - answer_output_accuracy: 0.2145 - answer_output_loss: 6.2378 - loss: 13.6041 - question_output_accuracy: 0.0127 - question_output_loss: 6.2865 - question_type_output_accuracy: 0.6076 - question_type_output_loss: 1.0785 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 6.1644 - val_loss: 13.5630 - val_question_output_accuracy: 0.0100 - val_question_output_loss: 6.2887 - val_question_type_output_accuracy: 0.3750 - val_question_type_output_loss: 1.1100\n",
      "Epoch 3/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 297ms/step - answer_output_accuracy: 0.9846 - answer_output_loss: 6.0887 - loss: 13.4154 - question_output_accuracy: 0.0168 - question_output_loss: 6.2692 - question_type_output_accuracy: 0.5542 - question_type_output_loss: 1.0537 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 5.8030 - val_loss: 13.2110 - val_question_output_accuracy: 0.0094 - val_question_output_loss: 6.2794 - val_question_type_output_accuracy: 0.4375 - val_question_type_output_loss: 1.1287\n",
      "Epoch 4/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 300ms/step - answer_output_accuracy: 0.9853 - answer_output_loss: 5.4983 - loss: 12.7772 - question_output_accuracy: 0.0143 - question_output_loss: 6.2471 - question_type_output_accuracy: 0.5111 - question_type_output_loss: 1.0153 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 3.9937 - val_loss: 11.4697 - val_question_output_accuracy: 0.0050 - val_question_output_loss: 6.2620 - val_question_type_output_accuracy: 0.4375 - val_question_type_output_loss: 1.2140\n",
      "Epoch 5/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 308ms/step - answer_output_accuracy: 0.9855 - answer_output_loss: 3.4938 - loss: 10.7212 - question_output_accuracy: 0.0046 - question_output_loss: 6.1959 - question_type_output_accuracy: 0.4903 - question_type_output_loss: 1.0066 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 1.4529 - val_loss: 9.4832 - val_question_output_accuracy: 0.0037 - val_question_output_loss: 6.2063 - val_question_type_output_accuracy: 0.4375 - val_question_type_output_loss: 1.8240\n",
      "Epoch 6/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 296ms/step - answer_output_accuracy: 0.9853 - answer_output_loss: 1.4168 - loss: 8.7444 - question_output_accuracy: 0.0029 - question_output_loss: 6.0149 - question_type_output_accuracy: 0.5007 - question_type_output_loss: 1.3046 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 0.7223 - val_loss: 8.8975 - val_question_output_accuracy: 0.0019 - val_question_output_loss: 6.1260 - val_question_type_output_accuracy: 0.2500 - val_question_type_output_loss: 2.0492\n",
      "Epoch 7/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 316ms/step - answer_output_accuracy: 0.9855 - answer_output_loss: 0.7297 - loss: 7.7458 - question_output_accuracy: 0.0020 - question_output_loss: 5.7405 - question_type_output_accuracy: 0.5653 - question_type_output_loss: 1.2641 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 0.6319 - val_loss: 8.0634 - val_question_output_accuracy: 0.0019 - val_question_output_loss: 6.0919 - val_question_type_output_accuracy: 0.3125 - val_question_type_output_loss: 1.3396\n",
      "Epoch 8/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 300ms/step - answer_output_accuracy: 0.9852 - answer_output_loss: 0.6516 - loss: 7.1078 - question_output_accuracy: 0.0025 - question_output_loss: 5.5006 - question_type_output_accuracy: 0.5424 - question_type_output_loss: 0.9515 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 0.6445 - val_loss: 7.8441 - val_question_output_accuracy: 0.0031 - val_question_output_loss: 6.1414 - val_question_type_output_accuracy: 0.3125 - val_question_type_output_loss: 1.0582\n",
      "Epoch 9/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 301ms/step - answer_output_accuracy: 0.9855 - answer_output_loss: 0.5892 - loss: 6.9181 - question_output_accuracy: 0.0027 - question_output_loss: 5.2758 - question_type_output_accuracy: 0.4993 - question_type_output_loss: 1.0541 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 0.6490 - val_loss: 7.9863 - val_question_output_accuracy: 0.0031 - val_question_output_loss: 6.2535 - val_question_type_output_accuracy: 0.4375 - val_question_type_output_loss: 1.0839\n",
      "Epoch 10/10\n",
      "\u001b[1m2/2\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 296ms/step - answer_output_accuracy: 0.9854 - answer_output_loss: 0.5727 - loss: 6.6346 - question_output_accuracy: 0.0027 - question_output_loss: 5.0879 - question_type_output_accuracy: 0.5319 - question_type_output_loss: 0.9754 - val_answer_output_accuracy: 0.9844 - val_answer_output_loss: 0.6354 - val_loss: 8.3545 - val_question_output_accuracy: 0.0037 - val_question_output_loss: 6.4062 - val_question_type_output_accuracy: 0.2500 - val_question_type_output_loss: 1.3129\n",
      "✅ Model training complete and saved!\n"
     ]
    }
   ],
   "source": [
    "# === Model Hyperparameters === #\n",
    "VOCAB_SIZE = len(tokenizer.word_index) + 1\n",
    "EMBEDDING_DIM = 300\n",
    "LSTM_UNITS = 256\n",
    "BATCH_SIZE = 32\n",
    "EPOCHS = 10\n",
    "\n",
    "# === Build Model === #\n",
    "# Encoder for Context\n",
    "context_input = Input(shape=(MAX_LENGTH,), name=\"context_input\")\n",
    "context_embedding = Embedding(input_dim=VOCAB_SIZE, output_dim=EMBEDDING_DIM, mask_zero=True, name=\"context_embedding\")(context_input)\n",
    "encoder_lstm = LSTM(LSTM_UNITS, return_state=True, name=\"encoder_lstm\")\n",
    "encoder_output, state_h, state_c = encoder_lstm(context_embedding)\n",
    "\n",
    "# Decoder for Question (Teacher Forcing)\n",
    "question_decoder_input = Input(shape=(MAX_LENGTH,), name=\"question_decoder_input\")\n",
    "question_embedding = Embedding(input_dim=VOCAB_SIZE, output_dim=EMBEDDING_DIM, mask_zero=True, name=\"question_embedding\")(question_decoder_input)\n",
    "question_lstm = LSTM(LSTM_UNITS, return_sequences=True, return_state=True, name=\"question_lstm\")\n",
    "question_output, _, _ = question_lstm(question_embedding, initial_state=[state_h, state_c])\n",
    "question_dense = Dense(VOCAB_SIZE, activation=\"softmax\", name=\"question_output\")(question_output)\n",
    "\n",
    "# Decoder for Answer\n",
    "answer_lstm = LSTM(LSTM_UNITS, return_sequences=True, return_state=True, name=\"answer_lstm\")\n",
    "answer_output, _, _ = answer_lstm(context_embedding, initial_state=[state_h, state_c])\n",
    "answer_dense = Dense(VOCAB_SIZE, activation=\"softmax\", name=\"answer_output\")(answer_output)\n",
    "\n",
    "# Classification Output for Question Type\n",
    "type_dense = Dense(128, activation=\"relu\")(encoder_output)\n",
    "question_type_output = Dense(3, activation=\"softmax\", name=\"question_type_output\")(type_dense)\n",
    "\n",
    "# Construct the Model\n",
    "model = Model(\n",
    "    inputs=[context_input, question_decoder_input],\n",
    "    outputs=[question_dense, answer_dense, question_type_output],\n",
    ")\n",
    "\n",
    "# === Compile the Model === #\n",
    "model.compile(\n",
    "    optimizer=\"adam\",\n",
    "    loss={\n",
    "        \"question_output\": \"sparse_categorical_crossentropy\",\n",
    "        \"answer_output\": \"sparse_categorical_crossentropy\",\n",
    "        \"question_type_output\": \"sparse_categorical_crossentropy\",\n",
    "    },\n",
    "    metrics={\n",
    "        \"question_output\": [\"accuracy\"],\n",
    "        \"answer_output\": [\"accuracy\"],\n",
    "        \"question_type_output\": [\"accuracy\"],\n",
    "    },\n",
    ")\n",
    "\n",
    "# === Train the Model === #\n",
    "model.fit(\n",
    "    [context_train, question_train],\n",
    "    {\n",
    "        \"question_output\": question_train,\n",
    "        \"answer_output\": answer_train,\n",
    "        \"question_type_output\": qtype_train,\n",
    "    },\n",
    "    batch_size=BATCH_SIZE,\n",
    "    epochs=EPOCHS,\n",
    "    validation_split=0.2,\n",
    ")\n",
    "\n",
    "# Save the Model\n",
    "model.save(\"lstm_multi_output_model.keras\")\n",
    "print(\"✅ Model training complete and saved!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m1/1\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 264ms/step\n",
      "=== Evaluation on Test Data ===\n",
      "Classification Report for Question Type:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00         8\n",
      "           1       0.50      0.83      0.62         6\n",
      "           2       0.44      0.80      0.57         5\n",
      "\n",
      "    accuracy                           0.47        19\n",
      "   macro avg       0.31      0.54      0.40        19\n",
      "weighted avg       0.27      0.47      0.35        19\n",
      "\n",
      "Accuracy: 0.47368421052631576\n",
      "Precision: 0.27485380116959063\n",
      "Recall: 0.47368421052631576\n",
      "BLEU score for first test sample (question generation): 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "# === Evaluate on Test Set === #\n",
    "pred_question, pred_answer, pred_qtype = model.predict([context_test, question_test])\n",
    "pred_qtype_labels = np.argmax(pred_qtype, axis=1)\n",
    "\n",
    "print(\"=== Evaluation on Test Data ===\")\n",
    "print(\"Classification Report for Question Type:\")\n",
    "print(classification_report(qtype_test, pred_qtype_labels))\n",
    "print(\"Accuracy:\", accuracy_score(qtype_test, pred_qtype_labels))\n",
    "print(\"Precision:\", precision_score(qtype_test, pred_qtype_labels, average='weighted'))\n",
    "print(\"Recall:\", recall_score(qtype_test, pred_qtype_labels, average='weighted'))\n",
    "\n",
    "\n",
    "def sequence_to_text(sequence, tokenizer):\n",
    "    return [tokenizer.index_word.get(idx, \"<OOV>\") for idx in sequence if idx != 0]\n",
    "\n",
    "reference = [sequence_to_text(question_test[0], tokenizer)]\n",
    "candidate = sequence_to_text(np.argmax(pred_question[0], axis=-1), tokenizer)\n",
    "bleu_score = nltk.translate.bleu_score.sentence_bleu(reference, candidate)\n",
    "print(\"BLEU score for first test sample (question generation):\", bleu_score)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
