{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "263af9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle, tensorflow as tf, numpy as np\n",
    "from tensorflow.keras.models import Model, load_model\n",
    "from tensorflow.keras.layers import (\n",
    "    Input,\n",
    "    Embedding,\n",
    "    SpatialDropout1D,\n",
    "    Bidirectional,\n",
    "    LSTM,\n",
    "    TimeDistributed,\n",
    "    Dense,\n",
    "    \n",
    ")\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from collections import Counter\n",
    "from itertools import zip_longest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4fc87f1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total kalimat  2231\n",
      "NER Label       Count      || SRL Label       Count     \n",
      "-------------------------------------------------------\n",
      "O               11464      || O               5702      \n",
      "TIME            349        || ARGM-TMP        2089      \n",
      "PER             1951       || ARG0            2637      \n",
      "LOC             2184       || V               2461      \n",
      "DATE            1461       || ARG1            2879      \n",
      "ETH             430        || ARGM-LOC        2090      \n",
      "EVENT           304        || ARG2            547       \n",
      "MISC            466        || ARGM-MOD        78        \n",
      "ORG             317        || ARGM-MNR        200       \n",
      "                           || ARGM-NEG        19        \n",
      "                           || ARGM-DIR        41        \n",
      "                           || ARGM-CAU        146       \n",
      "                           || ARG3            37        \n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "with open(\"../dataset/ner_srl_without_bio.tsv\", encoding=\"utf-8\") as f:\n",
    "    tok, ner, srl = [], [], []\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if not line:\n",
    "            if tok:\n",
    "                data.append({\"tokens\": tok, \"labels_ner\": ner, \"labels_srl\": srl})\n",
    "                tok, ner, srl = [], [], []\n",
    "        else:\n",
    "            t, n, s = line.split(\"\\t\")\n",
    "            tok.append(t.lower())\n",
    "            ner.append(n.strip())\n",
    "            srl_label = s.strip()\n",
    "            if srl_label == \"MISC\":\n",
    "                srl_label = \"O\"\n",
    "            srl.append(srl_label)\n",
    "\n",
    "\n",
    "print(\"total kalimat \", len(data))\n",
    "# ——————————————————\n",
    "sentences = [d[\"tokens\"] for d in data]\n",
    "labels_ner = [d[\"labels_ner\"] for d in data]\n",
    "labels_srl = [d[\"labels_srl\"] for d in data]\n",
    "\n",
    "ner_counter = Counter(label for seq in labels_ner for label in seq)\n",
    "\n",
    "srl_counter = Counter(label for seq in labels_srl for label in seq)\n",
    "\n",
    "\n",
    "print(f\"{'NER Label':<15} {'Count':<10} || {'SRL Label':<15} {'Count':<10}\")\n",
    "print(\"-\" * 55)\n",
    "\n",
    "for (ner_label, ner_count), (srl_label, srl_count) in zip_longest(ner_counter.items(), srl_counter.items(), fillvalue=('', '')):\n",
    "    print(f\"{ner_label:<15} {ner_count:<10} || {srl_label:<15} {srl_count:<10}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8dda2d6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER  -> Total Labels: 18926, O Count: 11464, O Percentage: 60.57%\n",
      "SRL  -> Total Labels: 18926, O Count: 5702, O Percentage: 30.13%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def calculate_o_percentage(labels):\n",
    "    counter = Counter(label for seq in labels for label in seq)\n",
    "    total = sum(counter.values())\n",
    "    count_o = counter.get(\"O\", 0)\n",
    "    percent_o = (count_o / total) * 100 if total > 0 else 0\n",
    "    return percent_o, total, count_o\n",
    "\n",
    "# Hitung persentase 'O' untuk NER\n",
    "o_ner_percent, total_ner, o_ner_count = calculate_o_percentage(labels_ner)\n",
    "\n",
    "# Hitung persentase 'O' untuk SRL\n",
    "o_srl_percent, total_srl, o_srl_count = calculate_o_percentage(labels_srl)\n",
    "\n",
    "print(f\"NER  -> Total Labels: {total_ner}, O Count: {o_ner_count}, O Percentage: {o_ner_percent:.2f}%\")\n",
    "print(f\"SRL  -> Total Labels: {total_srl}, O Count: {o_srl_count}, O Percentage: {o_srl_percent:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "48553e6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "PAD_TOKEN = \"<PAD>\"\n",
    "words = sorted({w for s in sentences for w in s})\n",
    "\n",
    "ner_tags = sorted({t for seq in labels_ner for t in seq})\n",
    "srl_tags = sorted({t for seq in labels_srl for t in seq})\n",
    "\n",
    "ner_tags.insert(0, PAD_TOKEN)\n",
    "srl_tags.insert(0, PAD_TOKEN)\n",
    "\n",
    "word2idx = {w: i + 2 for i, w in enumerate(words)}\n",
    "word2idx[\"PAD\"] = 0\n",
    "word2idx[\"UNK\"] = 1\n",
    "\n",
    "tag2idx_ner = {t: i for i, t in enumerate(ner_tags)}\n",
    "tag2idx_srl = {t: i for i, t in enumerate(srl_tags)}\n",
    "idx2tag_ner = {i: t for t, i in tag2idx_ner.items()}\n",
    "idx2tag_srl = {i: t for t, i in tag2idx_srl.items()}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "096967e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [[word2idx.get(w, 1) for w in s] for s in sentences]\n",
    "y_ner = [[tag2idx_ner[t] for t in seq] for seq in labels_ner]\n",
    "y_srl = [[tag2idx_srl[t] for t in seq] for seq in labels_srl]\n",
    "\n",
    "maxlen = max(map(len, X))\n",
    "pad_id = tag2idx_ner[PAD_TOKEN]\n",
    "\n",
    "X = pad_sequences(X, maxlen=maxlen, padding=\"post\", value=0)\n",
    "y_ner = pad_sequences(y_ner, maxlen=maxlen, padding=\"post\", value=pad_id)\n",
    "y_srl = pad_sequences(y_srl, maxlen=maxlen, padding=\"post\", value=pad_id)\n",
    "\n",
    "mask = (y_ner != pad_id).astype(\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a26893cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = train_test_split(\n",
    "    X,\n",
    "    y_ner,\n",
    "    y_srl,\n",
    "    mask,\n",
    "    test_size=0.2,\n",
    "    random_state=42,\n",
    "    shuffle=True,\n",
    ")\n",
    "X_tr, X_te, ner_tr, ner_te, srl_tr, srl_te, m_tr, m_te = splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1b4a1c61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_1\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"functional_1\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)        </span>┃<span style=\"font-weight: bold\"> Output Shape      </span>┃<span style=\"font-weight: bold\">    Param # </span>┃<span style=\"font-weight: bold\"> Connected to      </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ tokens (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ embed (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │    <span style=\"color: #00af00; text-decoration-color: #00af00\">158,720</span> │ tokens[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_1 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ embed[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ not_equal_2         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ tokens[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">NotEqual</span>)          │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_2     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)   │     <span style=\"color: #00af00; text-decoration-color: #00af00\">66,048</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │ not_equal_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_3     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)   │     <span style=\"color: #00af00; text-decoration-color: #00af00\">98,816</span> │ bidirectional_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │ not_equal_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_2  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │ bidirectional_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_3  │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │ bidirectional_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_output          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">10</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">650</span> │ time_distributed… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_output          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">38</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">14</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">910</span> │ time_distributed… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape     \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m   Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to     \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ tokens (\u001b[38;5;33mInputLayer\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ embed (\u001b[38;5;33mEmbedding\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │    \u001b[38;5;34m158,720\u001b[0m │ tokens[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_1 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ embed[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ not_equal_2         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ tokens[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mNotEqual\u001b[0m)          │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_2     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m128\u001b[0m)   │     \u001b[38;5;34m66,048\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │ not_equal_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_3     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m128\u001b[0m)   │     \u001b[38;5;34m98,816\u001b[0m │ bidirectional_2[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │ not_equal_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_2  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │      \u001b[38;5;34m8,256\u001b[0m │ bidirectional_3[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_3  │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │      \u001b[38;5;34m8,256\u001b[0m │ bidirectional_3[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_output          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m10\u001b[0m)    │        \u001b[38;5;34m650\u001b[0m │ time_distributed… │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_output          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m38\u001b[0m, \u001b[38;5;34m14\u001b[0m)    │        \u001b[38;5;34m910\u001b[0m │ time_distributed… │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">341,656</span> (1.30 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m341,656\u001b[0m (1.30 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">341,656</span> (1.30 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m341,656\u001b[0m (1.30 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "embed_dim = 64\n",
    "lstm_units = 64\n",
    "drop_embed = 0.2\n",
    "drop_lstm = 0.2\n",
    "\n",
    "inp = Input(shape=(maxlen,), name=\"tokens\")\n",
    "emb = Embedding(len(word2idx), embed_dim, mask_zero=True, name=\"embed\")(inp)\n",
    "emb = SpatialDropout1D(drop_embed)(emb)\n",
    "\n",
    "x = Bidirectional(\n",
    "    LSTM(\n",
    "        lstm_units,\n",
    "        return_sequences=True,\n",
    "        dropout=drop_lstm,\n",
    "        recurrent_dropout=drop_lstm,\n",
    "    )\n",
    ")(emb)\n",
    "x = Bidirectional(\n",
    "    LSTM(\n",
    "        lstm_units,\n",
    "        return_sequences=True,\n",
    "        dropout=drop_lstm,\n",
    "        recurrent_dropout=drop_lstm,\n",
    "    )\n",
    ")(x)\n",
    "\n",
    "ner_head = TimeDistributed(Dense(lstm_units, activation=\"relu\"))(x)\n",
    "ner_out = TimeDistributed(\n",
    "    Dense(len(tag2idx_ner), activation=\"softmax\"), name=\"ner_output\"\n",
    ")(ner_head)\n",
    "\n",
    "srl_head = TimeDistributed(Dense(lstm_units, activation=\"relu\"))(x)\n",
    "srl_out = TimeDistributed(\n",
    "    Dense(len(tag2idx_srl), activation=\"softmax\"), name=\"srl_output\"\n",
    ")(srl_head)\n",
    "\n",
    "model = Model(inp, [ner_out, srl_out])\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(3e-4),\n",
    "    loss={\n",
    "        \"ner_output\": \"sparse_categorical_crossentropy\",\n",
    "        \"srl_output\": \"sparse_categorical_crossentropy\",\n",
    "    },\n",
    "    metrics={\n",
    "        \"ner_output\": [\"sparse_categorical_accuracy\"],\n",
    "        \"srl_output\": [\"sparse_categorical_accuracy\"],\n",
    "    },\n",
    ")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f41d6012",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 92ms/step - loss: 4.9108 - ner_output_loss: 2.2818 - ner_output_sparse_categorical_accuracy: 0.2052 - srl_output_loss: 2.6290 - srl_output_sparse_categorical_accuracy: 0.1547 - val_loss: 4.6812 - val_ner_output_loss: 2.1289 - val_ner_output_sparse_categorical_accuracy: 0.1389 - val_srl_output_loss: 2.5523 - val_srl_output_sparse_categorical_accuracy: 0.0668 - learning_rate: 3.0000e-04\n",
      "Epoch 2/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 4.3571 - ner_output_loss: 1.9204 - ner_output_sparse_categorical_accuracy: 0.1344 - srl_output_loss: 2.4365 - srl_output_sparse_categorical_accuracy: 0.0673 - val_loss: 3.5487 - val_ner_output_loss: 1.5474 - val_ner_output_sparse_categorical_accuracy: 0.1389 - val_srl_output_loss: 2.0010 - val_srl_output_sparse_categorical_accuracy: 0.0905 - learning_rate: 3.0000e-04\n",
      "Epoch 3/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 3.3666 - ner_output_loss: 1.4404 - ner_output_sparse_categorical_accuracy: 0.1317 - srl_output_loss: 1.9260 - srl_output_sparse_categorical_accuracy: 0.0868 - val_loss: 2.9809 - val_ner_output_loss: 1.3097 - val_ner_output_sparse_categorical_accuracy: 0.1389 - val_srl_output_loss: 1.6709 - val_srl_output_sparse_categorical_accuracy: 0.0932 - learning_rate: 3.0000e-04\n",
      "Epoch 4/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 2.8620 - ner_output_loss: 1.2367 - ner_output_sparse_categorical_accuracy: 0.1335 - srl_output_loss: 1.6253 - srl_output_sparse_categorical_accuracy: 0.0978 - val_loss: 2.7502 - val_ner_output_loss: 1.2033 - val_ner_output_sparse_categorical_accuracy: 0.1375 - val_srl_output_loss: 1.5466 - val_srl_output_sparse_categorical_accuracy: 0.1118 - learning_rate: 3.0000e-04\n",
      "Epoch 5/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 2.6732 - ner_output_loss: 1.1535 - ner_output_sparse_categorical_accuracy: 0.1368 - srl_output_loss: 1.5197 - srl_output_sparse_categorical_accuracy: 0.1100 - val_loss: 2.5802 - val_ner_output_loss: 1.1193 - val_ner_output_sparse_categorical_accuracy: 0.1478 - val_srl_output_loss: 1.4604 - val_srl_output_sparse_categorical_accuracy: 0.1106 - learning_rate: 3.0000e-04\n",
      "Epoch 6/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 2.5011 - ner_output_loss: 1.0769 - ner_output_sparse_categorical_accuracy: 0.1441 - srl_output_loss: 1.4243 - srl_output_sparse_categorical_accuracy: 0.1083 - val_loss: 2.4460 - val_ner_output_loss: 1.0604 - val_ner_output_sparse_categorical_accuracy: 0.1491 - val_srl_output_loss: 1.3852 - val_srl_output_sparse_categorical_accuracy: 0.1230 - learning_rate: 3.0000e-04\n",
      "Epoch 7/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 2.4115 - ner_output_loss: 1.0338 - ner_output_sparse_categorical_accuracy: 0.1498 - srl_output_loss: 1.3776 - srl_output_sparse_categorical_accuracy: 0.1191 - val_loss: 2.2803 - val_ner_output_loss: 0.9900 - val_ner_output_sparse_categorical_accuracy: 0.1625 - val_srl_output_loss: 1.2898 - val_srl_output_sparse_categorical_accuracy: 0.1340 - learning_rate: 3.0000e-04\n",
      "Epoch 8/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 2.1656 - ner_output_loss: 0.9351 - ner_output_sparse_categorical_accuracy: 0.1573 - srl_output_loss: 1.2305 - srl_output_sparse_categorical_accuracy: 0.1324 - val_loss: 2.0635 - val_ner_output_loss: 0.8914 - val_ner_output_sparse_categorical_accuracy: 0.1697 - val_srl_output_loss: 1.1716 - val_srl_output_sparse_categorical_accuracy: 0.1417 - learning_rate: 3.0000e-04\n",
      "Epoch 9/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.9027 - ner_output_loss: 0.8172 - ner_output_sparse_categorical_accuracy: 0.1659 - srl_output_loss: 1.0854 - srl_output_sparse_categorical_accuracy: 0.1431 - val_loss: 1.8595 - val_ner_output_loss: 0.7923 - val_ner_output_sparse_categorical_accuracy: 0.1767 - val_srl_output_loss: 1.0667 - val_srl_output_sparse_categorical_accuracy: 0.1528 - learning_rate: 3.0000e-04\n",
      "Epoch 10/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.7463 - ner_output_loss: 0.7403 - ner_output_sparse_categorical_accuracy: 0.1740 - srl_output_loss: 1.0059 - srl_output_sparse_categorical_accuracy: 0.1508 - val_loss: 1.7059 - val_ner_output_loss: 0.7108 - val_ner_output_sparse_categorical_accuracy: 0.1833 - val_srl_output_loss: 0.9947 - val_srl_output_sparse_categorical_accuracy: 0.1601 - learning_rate: 3.0000e-04\n",
      "Epoch 11/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 49ms/step - loss: 1.6282 - ner_output_loss: 0.6822 - ner_output_sparse_categorical_accuracy: 0.1762 - srl_output_loss: 0.9460 - srl_output_sparse_categorical_accuracy: 0.1559 - val_loss: 1.6089 - val_ner_output_loss: 0.6610 - val_ner_output_sparse_categorical_accuracy: 0.1861 - val_srl_output_loss: 0.9475 - val_srl_output_sparse_categorical_accuracy: 0.1647 - learning_rate: 3.0000e-04\n",
      "Epoch 12/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.4778 - ner_output_loss: 0.6039 - ner_output_sparse_categorical_accuracy: 0.1795 - srl_output_loss: 0.8738 - srl_output_sparse_categorical_accuracy: 0.1625 - val_loss: 1.5403 - val_ner_output_loss: 0.6256 - val_ner_output_sparse_categorical_accuracy: 0.1874 - val_srl_output_loss: 0.9142 - val_srl_output_sparse_categorical_accuracy: 0.1668 - learning_rate: 3.0000e-04\n",
      "Epoch 13/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.3586 - ner_output_loss: 0.5400 - ner_output_sparse_categorical_accuracy: 0.1808 - srl_output_loss: 0.8186 - srl_output_sparse_categorical_accuracy: 0.1644 - val_loss: 1.4752 - val_ner_output_loss: 0.5969 - val_ner_output_sparse_categorical_accuracy: 0.1884 - val_srl_output_loss: 0.8780 - val_srl_output_sparse_categorical_accuracy: 0.1687 - learning_rate: 3.0000e-04\n",
      "Epoch 14/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 1.3344 - ner_output_loss: 0.5186 - ner_output_sparse_categorical_accuracy: 0.1869 - srl_output_loss: 0.8159 - srl_output_sparse_categorical_accuracy: 0.1682 - val_loss: 1.4290 - val_ner_output_loss: 0.5756 - val_ner_output_sparse_categorical_accuracy: 0.1893 - val_srl_output_loss: 0.8531 - val_srl_output_sparse_categorical_accuracy: 0.1702 - learning_rate: 3.0000e-04\n",
      "Epoch 15/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 50ms/step - loss: 1.2810 - ner_output_loss: 0.4851 - ner_output_sparse_categorical_accuracy: 0.1849 - srl_output_loss: 0.7959 - srl_output_sparse_categorical_accuracy: 0.1654 - val_loss: 1.3892 - val_ner_output_loss: 0.5600 - val_ner_output_sparse_categorical_accuracy: 0.1917 - val_srl_output_loss: 0.8289 - val_srl_output_sparse_categorical_accuracy: 0.1711 - learning_rate: 3.0000e-04\n",
      "Epoch 16/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.2174 - ner_output_loss: 0.4766 - ner_output_sparse_categorical_accuracy: 0.1882 - srl_output_loss: 0.7407 - srl_output_sparse_categorical_accuracy: 0.1719 - val_loss: 1.3588 - val_ner_output_loss: 0.5454 - val_ner_output_sparse_categorical_accuracy: 0.1937 - val_srl_output_loss: 0.8130 - val_srl_output_sparse_categorical_accuracy: 0.1728 - learning_rate: 3.0000e-04\n",
      "Epoch 17/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.1637 - ner_output_loss: 0.4539 - ner_output_sparse_categorical_accuracy: 0.1882 - srl_output_loss: 0.7099 - srl_output_sparse_categorical_accuracy: 0.1719 - val_loss: 1.3189 - val_ner_output_loss: 0.5257 - val_ner_output_sparse_categorical_accuracy: 0.1943 - val_srl_output_loss: 0.7928 - val_srl_output_sparse_categorical_accuracy: 0.1740 - learning_rate: 3.0000e-04\n",
      "Epoch 18/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 1.0587 - ner_output_loss: 0.4037 - ner_output_sparse_categorical_accuracy: 0.1884 - srl_output_loss: 0.6550 - srl_output_sparse_categorical_accuracy: 0.1721 - val_loss: 1.2925 - val_ner_output_loss: 0.5146 - val_ner_output_sparse_categorical_accuracy: 0.1959 - val_srl_output_loss: 0.7776 - val_srl_output_sparse_categorical_accuracy: 0.1750 - learning_rate: 3.0000e-04\n",
      "Epoch 19/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 1.0582 - ner_output_loss: 0.3877 - ner_output_sparse_categorical_accuracy: 0.1900 - srl_output_loss: 0.6705 - srl_output_sparse_categorical_accuracy: 0.1720 - val_loss: 1.2710 - val_ner_output_loss: 0.5035 - val_ner_output_sparse_categorical_accuracy: 0.1973 - val_srl_output_loss: 0.7672 - val_srl_output_sparse_categorical_accuracy: 0.1756 - learning_rate: 3.0000e-04\n",
      "Epoch 20/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.0201 - ner_output_loss: 0.3707 - ner_output_sparse_categorical_accuracy: 0.1972 - srl_output_loss: 0.6494 - srl_output_sparse_categorical_accuracy: 0.1782 - val_loss: 1.2472 - val_ner_output_loss: 0.4926 - val_ner_output_sparse_categorical_accuracy: 0.1975 - val_srl_output_loss: 0.7542 - val_srl_output_sparse_categorical_accuracy: 0.1755 - learning_rate: 3.0000e-04\n",
      "Epoch 21/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 0.9662 - ner_output_loss: 0.3554 - ner_output_sparse_categorical_accuracy: 0.1917 - srl_output_loss: 0.6109 - srl_output_sparse_categorical_accuracy: 0.1750 - val_loss: 1.2226 - val_ner_output_loss: 0.4826 - val_ner_output_sparse_categorical_accuracy: 0.1975 - val_srl_output_loss: 0.7397 - val_srl_output_sparse_categorical_accuracy: 0.1759 - learning_rate: 3.0000e-04\n",
      "Epoch 22/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 49ms/step - loss: 0.9947 - ner_output_loss: 0.3667 - ner_output_sparse_categorical_accuracy: 0.1983 - srl_output_loss: 0.6281 - srl_output_sparse_categorical_accuracy: 0.1813 - val_loss: 1.2104 - val_ner_output_loss: 0.4795 - val_ner_output_sparse_categorical_accuracy: 0.1978 - val_srl_output_loss: 0.7305 - val_srl_output_sparse_categorical_accuracy: 0.1764 - learning_rate: 3.0000e-04\n",
      "Epoch 23/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 0.9693 - ner_output_loss: 0.3491 - ner_output_sparse_categorical_accuracy: 0.1953 - srl_output_loss: 0.6202 - srl_output_sparse_categorical_accuracy: 0.1768 - val_loss: 1.1952 - val_ner_output_loss: 0.4718 - val_ner_output_sparse_categorical_accuracy: 0.1984 - val_srl_output_loss: 0.7231 - val_srl_output_sparse_categorical_accuracy: 0.1771 - learning_rate: 3.0000e-04\n",
      "Epoch 24/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 50ms/step - loss: 0.9162 - ner_output_loss: 0.3255 - ner_output_sparse_categorical_accuracy: 0.1984 - srl_output_loss: 0.5907 - srl_output_sparse_categorical_accuracy: 0.1805 - val_loss: 1.1879 - val_ner_output_loss: 0.4724 - val_ner_output_sparse_categorical_accuracy: 0.1991 - val_srl_output_loss: 0.7151 - val_srl_output_sparse_categorical_accuracy: 0.1784 - learning_rate: 3.0000e-04\n",
      "Epoch 25/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.8918 - ner_output_loss: 0.3157 - ner_output_sparse_categorical_accuracy: 0.1968 - srl_output_loss: 0.5762 - srl_output_sparse_categorical_accuracy: 0.1789 - val_loss: 1.1675 - val_ner_output_loss: 0.4613 - val_ner_output_sparse_categorical_accuracy: 0.1992 - val_srl_output_loss: 0.7058 - val_srl_output_sparse_categorical_accuracy: 0.1789 - learning_rate: 3.0000e-04\n",
      "Epoch 26/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.8769 - ner_output_loss: 0.3010 - ner_output_sparse_categorical_accuracy: 0.1997 - srl_output_loss: 0.5759 - srl_output_sparse_categorical_accuracy: 0.1801 - val_loss: 1.1435 - val_ner_output_loss: 0.4481 - val_ner_output_sparse_categorical_accuracy: 0.1995 - val_srl_output_loss: 0.6950 - val_srl_output_sparse_categorical_accuracy: 0.1792 - learning_rate: 3.0000e-04\n",
      "Epoch 27/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 48ms/step - loss: 0.8090 - ner_output_loss: 0.2721 - ner_output_sparse_categorical_accuracy: 0.2008 - srl_output_loss: 0.5369 - srl_output_sparse_categorical_accuracy: 0.1816 - val_loss: 1.1363 - val_ner_output_loss: 0.4474 - val_ner_output_sparse_categorical_accuracy: 0.1998 - val_srl_output_loss: 0.6884 - val_srl_output_sparse_categorical_accuracy: 0.1801 - learning_rate: 3.0000e-04\n",
      "Epoch 28/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 0.8319 - ner_output_loss: 0.2812 - ner_output_sparse_categorical_accuracy: 0.2008 - srl_output_loss: 0.5507 - srl_output_sparse_categorical_accuracy: 0.1800 - val_loss: 1.1294 - val_ner_output_loss: 0.4376 - val_ner_output_sparse_categorical_accuracy: 0.2004 - val_srl_output_loss: 0.6914 - val_srl_output_sparse_categorical_accuracy: 0.1800 - learning_rate: 3.0000e-04\n",
      "Epoch 29/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.8019 - ner_output_loss: 0.2668 - ner_output_sparse_categorical_accuracy: 0.1997 - srl_output_loss: 0.5351 - srl_output_sparse_categorical_accuracy: 0.1802 - val_loss: 1.1265 - val_ner_output_loss: 0.4411 - val_ner_output_sparse_categorical_accuracy: 0.2008 - val_srl_output_loss: 0.6850 - val_srl_output_sparse_categorical_accuracy: 0.1803 - learning_rate: 3.0000e-04\n",
      "Epoch 30/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.7622 - ner_output_loss: 0.2489 - ner_output_sparse_categorical_accuracy: 0.2034 - srl_output_loss: 0.5133 - srl_output_sparse_categorical_accuracy: 0.1847 - val_loss: 1.1217 - val_ner_output_loss: 0.4410 - val_ner_output_sparse_categorical_accuracy: 0.2005 - val_srl_output_loss: 0.6803 - val_srl_output_sparse_categorical_accuracy: 0.1806 - learning_rate: 3.0000e-04\n",
      "Epoch 31/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 0.7801 - ner_output_loss: 0.2485 - ner_output_sparse_categorical_accuracy: 0.2046 - srl_output_loss: 0.5316 - srl_output_sparse_categorical_accuracy: 0.1826 - val_loss: 1.1110 - val_ner_output_loss: 0.4355 - val_ner_output_sparse_categorical_accuracy: 0.2002 - val_srl_output_loss: 0.6751 - val_srl_output_sparse_categorical_accuracy: 0.1818 - learning_rate: 3.0000e-04\n",
      "Epoch 32/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.7747 - ner_output_loss: 0.2493 - ner_output_sparse_categorical_accuracy: 0.2018 - srl_output_loss: 0.5253 - srl_output_sparse_categorical_accuracy: 0.1832 - val_loss: 1.1026 - val_ner_output_loss: 0.4279 - val_ner_output_sparse_categorical_accuracy: 0.2009 - val_srl_output_loss: 0.6742 - val_srl_output_sparse_categorical_accuracy: 0.1819 - learning_rate: 3.0000e-04\n",
      "Epoch 33/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.6970 - ner_output_loss: 0.2145 - ner_output_sparse_categorical_accuracy: 0.2029 - srl_output_loss: 0.4825 - srl_output_sparse_categorical_accuracy: 0.1829 - val_loss: 1.0892 - val_ner_output_loss: 0.4261 - val_ner_output_sparse_categorical_accuracy: 0.2018 - val_srl_output_loss: 0.6627 - val_srl_output_sparse_categorical_accuracy: 0.1825 - learning_rate: 3.0000e-04\n",
      "Epoch 34/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.7070 - ner_output_loss: 0.2224 - ner_output_sparse_categorical_accuracy: 0.2055 - srl_output_loss: 0.4846 - srl_output_sparse_categorical_accuracy: 0.1855 - val_loss: 1.0870 - val_ner_output_loss: 0.4197 - val_ner_output_sparse_categorical_accuracy: 0.2022 - val_srl_output_loss: 0.6669 - val_srl_output_sparse_categorical_accuracy: 0.1834 - learning_rate: 3.0000e-04\n",
      "Epoch 35/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.7292 - ner_output_loss: 0.2378 - ner_output_sparse_categorical_accuracy: 0.2068 - srl_output_loss: 0.4914 - srl_output_sparse_categorical_accuracy: 0.1879 - val_loss: 1.0733 - val_ner_output_loss: 0.4164 - val_ner_output_sparse_categorical_accuracy: 0.2020 - val_srl_output_loss: 0.6564 - val_srl_output_sparse_categorical_accuracy: 0.1837 - learning_rate: 3.0000e-04\n",
      "Epoch 36/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 0.6923 - ner_output_loss: 0.2130 - ner_output_sparse_categorical_accuracy: 0.2064 - srl_output_loss: 0.4793 - srl_output_sparse_categorical_accuracy: 0.1865 - val_loss: 1.0673 - val_ner_output_loss: 0.4181 - val_ner_output_sparse_categorical_accuracy: 0.2024 - val_srl_output_loss: 0.6487 - val_srl_output_sparse_categorical_accuracy: 0.1840 - learning_rate: 3.0000e-04\n",
      "Epoch 37/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.6649 - ner_output_loss: 0.2093 - ner_output_sparse_categorical_accuracy: 0.2042 - srl_output_loss: 0.4557 - srl_output_sparse_categorical_accuracy: 0.1867 - val_loss: 1.0735 - val_ner_output_loss: 0.4241 - val_ner_output_sparse_categorical_accuracy: 0.2030 - val_srl_output_loss: 0.6489 - val_srl_output_sparse_categorical_accuracy: 0.1836 - learning_rate: 3.0000e-04\n",
      "Epoch 38/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 0.6624 - ner_output_loss: 0.2135 - ner_output_sparse_categorical_accuracy: 0.2046 - srl_output_loss: 0.4488 - srl_output_sparse_categorical_accuracy: 0.1872 - val_loss: 1.0567 - val_ner_output_loss: 0.4134 - val_ner_output_sparse_categorical_accuracy: 0.2026 - val_srl_output_loss: 0.6428 - val_srl_output_sparse_categorical_accuracy: 0.1847 - learning_rate: 3.0000e-04\n",
      "Epoch 39/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 0.6242 - ner_output_loss: 0.1945 - ner_output_sparse_categorical_accuracy: 0.2063 - srl_output_loss: 0.4297 - srl_output_sparse_categorical_accuracy: 0.1893 - val_loss: 1.0591 - val_ner_output_loss: 0.4165 - val_ner_output_sparse_categorical_accuracy: 0.2028 - val_srl_output_loss: 0.6422 - val_srl_output_sparse_categorical_accuracy: 0.1847 - learning_rate: 3.0000e-04\n",
      "Epoch 40/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 49ms/step - loss: 0.6077 - ner_output_loss: 0.1934 - ner_output_sparse_categorical_accuracy: 0.2081 - srl_output_loss: 0.4143 - srl_output_sparse_categorical_accuracy: 0.1924 - val_loss: 1.0560 - val_ner_output_loss: 0.4162 - val_ner_output_sparse_categorical_accuracy: 0.2028 - val_srl_output_loss: 0.6394 - val_srl_output_sparse_categorical_accuracy: 0.1849 - learning_rate: 3.0000e-04\n",
      "Epoch 41/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 49ms/step - loss: 0.6324 - ner_output_loss: 0.1931 - ner_output_sparse_categorical_accuracy: 0.2076 - srl_output_loss: 0.4392 - srl_output_sparse_categorical_accuracy: 0.1898 - val_loss: 1.0587 - val_ner_output_loss: 0.4232 - val_ner_output_sparse_categorical_accuracy: 0.2037 - val_srl_output_loss: 0.6349 - val_srl_output_sparse_categorical_accuracy: 0.1863 - learning_rate: 3.0000e-04\n",
      "Epoch 42/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 49ms/step - loss: 0.5964 - ner_output_loss: 0.1760 - ner_output_sparse_categorical_accuracy: 0.2003 - srl_output_loss: 0.4204 - srl_output_sparse_categorical_accuracy: 0.1841 - val_loss: 1.0573 - val_ner_output_loss: 0.4168 - val_ner_output_sparse_categorical_accuracy: 0.2033 - val_srl_output_loss: 0.6400 - val_srl_output_sparse_categorical_accuracy: 0.1855 - learning_rate: 3.0000e-04\n",
      "Epoch 43/50\n",
      "\u001b[1m28/28\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 48ms/step - loss: 0.6176 - ner_output_loss: 0.1845 - ner_output_sparse_categorical_accuracy: 0.2140 - srl_output_loss: 0.4331 - srl_output_sparse_categorical_accuracy: 0.1949 - val_loss: 1.0580 - val_ner_output_loss: 0.4177 - val_ner_output_sparse_categorical_accuracy: 0.2037 - val_srl_output_loss: 0.6397 - val_srl_output_sparse_categorical_accuracy: 0.1866 - learning_rate: 1.5000e-04\n"
     ]
    }
   ],
   "source": [
    "callbacks = [\n",
    "    tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True),\n",
    "    tf.keras.callbacks.ReduceLROnPlateau(patience=2, factor=0.5, min_lr=1e-5),\n",
    "]\n",
    "\n",
    "history = model.fit(\n",
    "    X_tr,\n",
    "    [ner_tr, srl_tr],  # y → LIST (pos 0 = ner_output, 1 = srl_output)\n",
    "    sample_weight=[m_tr, m_tr],  # sama‑persis urutan\n",
    "    validation_data=(X_te, [ner_te, srl_te], [m_te, m_te]),\n",
    "    \n",
    "    batch_size=64,\n",
    "    epochs=50,\n",
    "    callbacks=callbacks,\n",
    "    verbose=1,\n",
    ")\n",
    "\n",
    "\n",
    "# =========================\n",
    "# 7.  Save artefacts\n",
    "# =========================\n",
    "model.save(\"lstm_ner_srl_model.keras\")\n",
    "for fname, obj in [\n",
    "    (\"word2idx.pkl\", word2idx),\n",
    "    (\"tag2idx_ner.pkl\", tag2idx_ner),\n",
    "    (\"tag2idx_srl.pkl\", tag2idx_srl),\n",
    "]:\n",
    "    with open(fname, \"wb\") as f:\n",
    "        pickle.dump(obj, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "430794b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 82712 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 72852 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_training_history(history):\n",
    "    history_data = history.history\n",
    "    epochs = range(1, len(next(iter(history_data.values()))) + 1)\n",
    "\n",
    "    # Coba deteksi metric secara dinamis agar fleksibel\n",
    "    ner_acc_key = next((k for k in history_data if \"ner_output\" in k and \"accuracy\" in k), None)\n",
    "    srl_acc_key = next((k for k in history_data if \"srl_output\" in k and \"accuracy\" in k), None)\n",
    "    val_ner_acc_key = f\"val_{ner_acc_key}\" if ner_acc_key else None\n",
    "    val_srl_acc_key = f\"val_{srl_acc_key}\" if srl_acc_key else None\n",
    "\n",
    "    # --- Plot Accuracy ---\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    if ner_acc_key:\n",
    "        plt.plot(epochs, history_data[ner_acc_key], marker=\"o\", label=\"NER Accuracy (Train)\")\n",
    "    if srl_acc_key:\n",
    "        plt.plot(epochs, history_data[srl_acc_key], marker=\"s\", label=\"SRL Accuracy (Train)\")\n",
    "    if val_ner_acc_key in history_data:\n",
    "        plt.plot(epochs, history_data[val_ner_acc_key], marker=\"o\", linestyle=\"--\", label=\"NER Accuracy (Val)\")\n",
    "    if val_srl_acc_key in history_data:\n",
    "        plt.plot(epochs, history_data[val_srl_acc_key], marker=\"s\", linestyle=\"--\", label=\"SRL Accuracy (Val)\")\n",
    "\n",
    "    plt.title(\"Accuracy per Epoch\")\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(\"Accuracy\")\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "    plt.savefig(\"accuracy_plot.png\")\n",
    "    plt.show()\n",
    "\n",
    "    # --- Plot Loss ---\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    if \"ner_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"ner_output_loss\"], marker=\"o\", label=\"NER Loss (Train)\")\n",
    "    if \"srl_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"srl_output_loss\"], marker=\"s\", label=\"SRL Loss (Train)\")\n",
    "    if \"val_ner_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"val_ner_output_loss\"], marker=\"o\", linestyle=\"--\", label=\"NER Loss (Val)\")\n",
    "    if \"val_srl_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"val_srl_output_loss\"], marker=\"s\", linestyle=\"--\", label=\"SRL Loss (Val)\")\n",
    "\n",
    "    plt.title(\"Loss per Epoch\")\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(\"Loss\")\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "    plt.savefig(\"loss_plot.png\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "plot_training_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e690a0e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59756 characters]",
      "text/plain": [
       "<Figure size 800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 84612 characters]",
      "text/plain": [
       "<Figure size 800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: DATE ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         309 (TP)          27 (FN)\n",
      "Aktual Negatif           5 (FP)        3567 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ETH ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif          88 (TP)           5 (FN)\n",
      "Aktual Negatif           3 (FP)        3812 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: EVENT ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif          16 (TP)          45 (FN)\n",
      "Aktual Negatif          19 (FP)        3828 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: LOC ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         334 (TP)          66 (FN)\n",
      "Aktual Negatif          47 (FP)        3461 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: MISC ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif          19 (TP)          85 (FN)\n",
      "Aktual Negatif          55 (FP)        3749 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: O ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif        2266 (TP)          94 (FN)\n",
      "Aktual Negatif         255 (FP)        1293 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ORG ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif          16 (TP)          59 (FN)\n",
      "Aktual Negatif          43 (FP)        3790 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: PER ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         353 (TP)          51 (FN)\n",
      "Aktual Negatif          21 (FP)        3483 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: TIME ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif          44 (TP)          31 (FN)\n",
      "Aktual Negatif          15 (FP)        3818 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARG0 ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         446 (TP)         106 (FN)\n",
      "Aktual Negatif          58 (FP)        3298 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARG1 ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         542 (TP)          95 (FN)\n",
      "Aktual Negatif         365 (FP)        2906 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARG2 ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif          22 (TP)          74 (FN)\n",
      "Aktual Negatif          34 (FP)        3778 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARG3 ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif           0 (TP)           4 (FN)\n",
      "Aktual Negatif           0 (FP)        3904 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-CAU ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif           0 (TP)          32 (FN)\n",
      "Aktual Negatif           0 (FP)        3876 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-DIR ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif           0 (TP)           5 (FN)\n",
      "Aktual Negatif           0 (FP)        3903 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-LOC ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         321 (TP)          88 (FN)\n",
      "Aktual Negatif          40 (FP)        3459 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-MNR ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif           0 (TP)          55 (FN)\n",
      "Aktual Negatif           0 (FP)        3853 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-MOD ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif           0 (TP)          18 (FN)\n",
      "Aktual Negatif           0 (FP)        3890 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-NEG ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif           0 (TP)           1 (FN)\n",
      "Aktual Negatif           0 (FP)        3907 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: ARGM-TMP ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         391 (TP)          77 (FN)\n",
      "Aktual Negatif          26 (FP)        3414 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: O ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif        1033 (TP)         102 (FN)\n",
      "Aktual Negatif         181 (FP)        2592 (TN)\n",
      "\n",
      "==== Confusion Matrix Manual untuk Kelas: V ====\n",
      "               Prediksi Positif Prediksi Negatif\n",
      "Aktual Positif         386 (TP)         110 (FN)\n",
      "Aktual Negatif          63 (FP)        3349 (TN)\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 1.  Prediksi\n",
    "# ------------------------------------------------------------------\n",
    "\n",
    "model = load_model(\"lstm_ner_srl_model.keras\")\n",
    "pred_ner_prob, pred_srl_prob = model.predict(X_te, verbose=0)\n",
    "\n",
    "pred_ner = pred_ner_prob.argmax(-1)\n",
    "pred_srl = pred_srl_prob.argmax(-1)\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 2.  Siapkan masker PAD\n",
    "# ------------------------------------------------------------------\n",
    "pad_id = tag2idx_ner[\"<PAD>\"]\n",
    "\n",
    "mask_ner = ner_te != pad_id\n",
    "mask_srl = srl_te != pad_id\n",
    "\n",
    "true_ner_flat = ner_te[mask_ner]\n",
    "pred_ner_flat = pred_ner[mask_ner]\n",
    "\n",
    "true_srl_flat = srl_te[mask_srl]\n",
    "pred_srl_flat = pred_srl[mask_srl]\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 3.  Hitung confusion matrix TANPA PAD\n",
    "# ------------------------------------------------------------------\n",
    "labels_ner_no_pad = [i for i in range(len(tag2idx_ner)) if i != pad_id]\n",
    "labels_srl_no_pad = [i for i in range(len(tag2idx_srl)) if i != pad_id]\n",
    "\n",
    "cm_ner = confusion_matrix(true_ner_flat, pred_ner_flat, labels=labels_ner_no_pad)\n",
    "cm_srl = confusion_matrix(true_srl_flat, pred_srl_flat, labels=labels_srl_no_pad)\n",
    "\n",
    "display_labels_ner = [idx2tag_ner[i] for i in labels_ner_no_pad]\n",
    "display_labels_srl = [idx2tag_srl[i] for i in labels_srl_no_pad]\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 4.  Plot NER CM (tanpa PAD)\n",
    "# ------------------------------------------------------------------\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "disp_ner = ConfusionMatrixDisplay(confusion_matrix=cm_ner, display_labels=display_labels_ner)\n",
    "disp_ner.plot(include_values=True, values_format=\"d\", cmap=plt.cm.Blues, ax=ax, colorbar=False)\n",
    "ax.set_title(\"NER Confusion Matrix\", fontsize=18)\n",
    "plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 5.  Plot SRL CM (tanpa PAD)\n",
    "# ------------------------------------------------------------------\n",
    "fig, ax = plt.subplots(figsize=(8, 8))\n",
    "disp_srl = ConfusionMatrixDisplay(confusion_matrix=cm_srl, display_labels=display_labels_srl)\n",
    "disp_srl.plot(include_values=True, values_format=\"d\", cmap=plt.cm.Blues, ax=ax, colorbar=False)\n",
    "ax.set_title(\"SRL Confusion Matrix\", fontsize=18)\n",
    "plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 6.  Manual Confusion Table untuk Thesis (NER)\n",
    "# ------------------------------------------------------------------\n",
    "def manual_confusion_table(true_labels, pred_labels, labels, label_names):\n",
    "    true_labels = np.array(true_labels)\n",
    "    pred_labels = np.array(pred_labels)\n",
    "\n",
    "    for i, label in enumerate(labels):\n",
    "        label_name = label_names[i]\n",
    "\n",
    "        tp = np.sum((true_labels == label) & (pred_labels == label))\n",
    "        fn = np.sum((true_labels == label) & (pred_labels != label))\n",
    "        fp = np.sum((true_labels != label) & (pred_labels == label))\n",
    "        tn = np.sum((true_labels != label) & (pred_labels != label))\n",
    "\n",
    "        table = pd.DataFrame({\n",
    "            'Prediksi Positif': [f'{tp} (TP)', f'{fp} (FP)'],\n",
    "            'Prediksi Negatif': [f'{fn} (FN)', f'{tn} (TN)']\n",
    "        }, index=['Aktual Positif', 'Aktual Negatif'])\n",
    "\n",
    "        print(f\"\\n==== Confusion Matrix Manual untuk Kelas: {label_name} ====\")\n",
    "        print(table.to_string())\n",
    "\n",
    "# Jalankan untuk semua kelas NER\n",
    "for label_id, label_name in zip(labels_ner_no_pad, display_labels_ner):\n",
    "    manual_confusion_table(true_ner_flat, pred_ner_flat, [label_id], [label_name])\n",
    "    \n",
    "for label_id, label_name in zip(labels_srl_no_pad, display_labels_srl):\n",
    "    manual_confusion_table(true_srl_flat, pred_srl_flat, [label_id], [label_name])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01cf3044",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a49f1dfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER TAG accuracy : 88.15%\n",
      "SRL TAG accuracy : 80.37%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 3b.  Akurasi token‑level (tanpa PAD)\n",
    "# ------------------------------------------------------------------\n",
    "acc_ner = accuracy_score(true_ner_flat, pred_ner_flat)\n",
    "acc_srl = accuracy_score(true_srl_flat, pred_srl_flat)\n",
    "\n",
    "print(f\"NER TAG accuracy : {acc_ner:.2%}\")\n",
    "print(f\"SRL TAG accuracy : {acc_srl:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9adad755",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[NER] Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        DATE       0.98      0.92      0.95       336\n",
      "         ETH       0.97      0.95      0.96        93\n",
      "       EVENT       0.46      0.26      0.33        61\n",
      "         LOC       0.88      0.83      0.86       400\n",
      "        MISC       0.26      0.18      0.21       104\n",
      "           O       0.90      0.96      0.93      2360\n",
      "         ORG       0.27      0.21      0.24        75\n",
      "         PER       0.94      0.87      0.91       404\n",
      "        TIME       0.75      0.59      0.66        75\n",
      "\n",
      "    accuracy                           0.88      3908\n",
      "   macro avg       0.71      0.64      0.67      3908\n",
      "weighted avg       0.87      0.88      0.87      3908\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(\"\\n[NER] Classification report:\")\n",
    "print(\n",
    "    classification_report(\n",
    "        true_ner_flat,\n",
    "        pred_ner_flat,\n",
    "        labels=labels_ner_no_pad,\n",
    "        target_names=display_labels_ner,\n",
    "        digits=2,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7cd28380",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[SRL] Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        ARG0       0.88      0.81      0.84       552\n",
      "        ARG1       0.60      0.85      0.70       637\n",
      "        ARG2       0.39      0.23      0.29        96\n",
      "        ARG3       0.00      0.00      0.00         4\n",
      "    ARGM-CAU       0.00      0.00      0.00        32\n",
      "    ARGM-DIR       0.00      0.00      0.00         5\n",
      "    ARGM-LOC       0.89      0.78      0.83       409\n",
      "    ARGM-MNR       0.00      0.00      0.00        55\n",
      "    ARGM-MOD       0.00      0.00      0.00        18\n",
      "    ARGM-NEG       0.00      0.00      0.00         1\n",
      "    ARGM-TMP       0.94      0.84      0.88       468\n",
      "           O       0.85      0.91      0.88      1135\n",
      "           V       0.86      0.78      0.82       496\n",
      "\n",
      "    accuracy                           0.80      3908\n",
      "   macro avg       0.42      0.40      0.40      3908\n",
      "weighted avg       0.79      0.80      0.79      3908\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n[SRL] Classification report:\")\n",
    "print(\n",
    "    classification_report(\n",
    "        true_srl_flat,\n",
    "        pred_srl_flat,\n",
    "        labels=labels_srl_no_pad,\n",
    "        target_names=display_labels_srl,\n",
    "        digits=2,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "333745fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def plot_training_history(history):\n",
    "#     epochs = range(1, len(history['loss']) + 1)\n",
    "\n",
    "#     plt.figure(figsize=(14, 6))\n",
    "\n",
    "#     # Plot Loss\n",
    "#     plt.subplot(1, 2, 1)\n",
    "#     plt.plot(epochs, history['loss'], label='Training Loss')\n",
    "#     plt.plot(epochs, history['val_loss'], label='Validation Loss')\n",
    "#     plt.title('Loss During Training')\n",
    "#     plt.xlabel('Epochs')\n",
    "#     plt.ylabel('Loss')\n",
    "#     plt.legend()\n",
    "\n",
    "#     # Plot Accuracy\n",
    "#     plt.subplot(1, 2, 2)\n",
    "#     plt.plot(epochs, history['ner_output_accuracy'], label='NER Train Acc')\n",
    "#     plt.plot(epochs, history['val_ner_output_accuracy'], label='NER Val Acc')\n",
    "#     plt.plot(epochs, history['srl_output_accuracy'], label='SRL Train Acc')\n",
    "#     plt.plot(epochs, history['val_srl_output_accuracy'], label='SRL Val Acc')\n",
    "#     plt.title('Accuracy During Training')\n",
    "#     plt.xlabel('Epochs')\n",
    "#     plt.ylabel('Accuracy')\n",
    "#     plt.legend()\n",
    "\n",
    "#     plt.tight_layout()\n",
    "#     plt.show()\n",
    "\n",
    "# plot_training_history(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "df36e200",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def token_level_accuracy(y_true, y_pred):\n",
    "#     total, correct = 0, 0\n",
    "#     for true_seq, pred_seq in zip(y_true, y_pred):\n",
    "#         for t, p in zip(true_seq, pred_seq):\n",
    "#             if t.sum() == 0:\n",
    "#                 continue\n",
    "#             total += 1\n",
    "#             if t.argmax() == p.argmax():\n",
    "#                 correct += 1\n",
    "#     return correct / total\n",
    "\n",
    "# def decode_predictions(pred, true, idx2tag):\n",
    "#     true_out, pred_out = [], []\n",
    "#     for pred_seq, true_seq in zip(pred, true):\n",
    "#         t_labels, p_labels = [], []\n",
    "#         for p_tok, t_tok in zip(pred_seq, true_seq):\n",
    "#             if t_tok.sum() == 0:\n",
    "#                 continue\n",
    "#             t_labels.append(idx2tag[t_tok.argmax()])\n",
    "#             p_labels.append(idx2tag[p_tok.argmax()])\n",
    "#         true_out.append(t_labels)\n",
    "#         pred_out.append(p_labels)\n",
    "#     return true_out, pred_out\n",
    "\n",
    "# results = model.evaluate(X_test, {\"ner_output\": y_ner_test, \"srl_output\": y_srl_test}, verbose=0)\n",
    "# for name, value in zip(model.metrics_names, results):\n",
    "#     print(f\"{name}: {value}\")\n",
    "\n",
    "# y_pred_ner, y_pred_srl = model.predict(X_test, verbose=0)\n",
    "\n",
    "# true_ner, pred_ner = decode_predictions(y_pred_ner, y_ner_test, idx2tag_ner)\n",
    "# true_srl, pred_srl = decode_predictions(y_pred_srl, y_srl_test, idx2tag_srl)\n",
    "\n",
    "# acc_ner = token_level_accuracy(y_ner_test, y_pred_ner)\n",
    "# acc_srl = token_level_accuracy(y_srl_test, y_pred_srl)\n",
    "\n",
    "# print(f\"NER Token Accuracy {acc_ner:.2%}\")\n",
    "# print(f\"SRL Token Accuracy {acc_srl:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9127cce0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"[NER] Classification Report:\")\n",
    "# print(classification_report(true_ner, pred_ner, digits=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "300897b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"SRL Classification Resport:\")\n",
    "# print(classification_report(true_srl, pred_srl, digits=2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
