{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "263af9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle, tensorflow as tf, numpy as np\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import (\n",
    "    Input,\n",
    "    Embedding,\n",
    "    SpatialDropout1D,\n",
    "    Bidirectional,\n",
    "    LSTM,\n",
    "    TimeDistributed,\n",
    "    Dense,\n",
    ")\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from collections import Counter\n",
    "from itertools import zip_longest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "4fc87f1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total kalimat  1639\n",
      "NER Label       Count      || SRL Label       Count     \n",
      "-------------------------------------------------------\n",
      "O               7261       || O               4182      \n",
      "B-TIME          189        || ARGM-TMP        1292      \n",
      "B-PER           1313       || ARG0            1965      \n",
      "B-LOC           1554       || V               1738      \n",
      "I-PER           226        || ARG1            1308      \n",
      "B-DATE          339        || ARGM-LOC        1503      \n",
      "I-DATE          647        || ARG2            293       \n",
      "B-ETH           213        || ARGM-MOD        39        \n",
      "I-ETH           217        || ARGM-MNR        37        \n",
      "B-EVENT         70         || ARGM-NEG        6         \n",
      "I-EVENT         55         || ARGM-DIR        41        \n",
      "I-LOC           32         || ARGM-CAU        21        \n",
      "B-MISC          14         ||                           \n",
      "I-MISC          3          ||                           \n",
      "I-TIME          46         ||                           \n",
      "B-ORG           21         ||                           \n",
      "I-ORG           18         ||                           \n",
      "B-QUANT         46         ||                           \n",
      "B-MAT           99         ||                           \n",
      "B-UNIT          44         ||                           \n",
      "I-UNIT          1          ||                           \n",
      "I-MAT           16         ||                           \n",
      "I-QUANT         1          ||                           \n"
     ]
    }
   ],
   "source": [
    "data = []\n",
    "with open(\"../dataset/new_ner_srl.tsv\", encoding=\"utf-8\") as f:\n",
    "# with open(\"../dataset/test_ns_dataset.tsv\", encoding=\"utf-8\") as f:\n",
    "    tok, ner, srl = [], [], []\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        if not line:\n",
    "            if tok:\n",
    "                data.append({\"tokens\": tok, \"labels_ner\": ner, \"labels_srl\": srl})\n",
    "                tok, ner, srl = [], [], []\n",
    "        else:\n",
    "            t, n, s = line.split(\"\\t\")\n",
    "            tok.append(t.lower())\n",
    "            ner.append(n.strip())\n",
    "            srl.append(s.strip())\n",
    "\n",
    "print(\"total kalimat \", len(data))\n",
    "# ——————————————————\n",
    "sentences = [d[\"tokens\"] for d in data]\n",
    "labels_ner = [d[\"labels_ner\"] for d in data]\n",
    "labels_srl = [d[\"labels_srl\"] for d in data]\n",
    "\n",
    "ner_counter = Counter(label for seq in labels_ner for label in seq)\n",
    "\n",
    "srl_counter = Counter(label for seq in labels_srl for label in seq)\n",
    "\n",
    "\n",
    "print(f\"{'NER Label':<15} {'Count':<10} || {'SRL Label':<15} {'Count':<10}\")\n",
    "print(\"-\" * 55)\n",
    "\n",
    "for (ner_label, ner_count), (srl_label, srl_count) in zip_longest(ner_counter.items(), srl_counter.items(), fillvalue=('', '')):\n",
    "    print(f\"{ner_label:<15} {ner_count:<10} || {srl_label:<15} {srl_count:<10}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "8dda2d6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER  -> Total Labels: 12425, O Count: 7261, O Percentage: 58.44%\n",
      "SRL  -> Total Labels: 12425, O Count: 4182, O Percentage: 33.66%\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def calculate_o_percentage(labels):\n",
    "    counter = Counter(label for seq in labels for label in seq)\n",
    "    total = sum(counter.values())\n",
    "    count_o = counter.get(\"O\", 0)\n",
    "    percent_o = (count_o / total) * 100 if total > 0 else 0\n",
    "    return percent_o, total, count_o\n",
    "\n",
    "# Hitung persentase 'O' untuk NER\n",
    "o_ner_percent, total_ner, o_ner_count = calculate_o_percentage(labels_ner)\n",
    "\n",
    "# Hitung persentase 'O' untuk SRL\n",
    "o_srl_percent, total_srl, o_srl_count = calculate_o_percentage(labels_srl)\n",
    "\n",
    "print(f\"NER  -> Total Labels: {total_ner}, O Count: {o_ner_count}, O Percentage: {o_ner_percent:.2f}%\")\n",
    "print(f\"SRL  -> Total Labels: {total_srl}, O Count: {o_srl_count}, O Percentage: {o_srl_percent:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "48553e6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "PAD_TOKEN = \"<PAD>\"\n",
    "words = sorted({w for s in sentences for w in s})\n",
    "\n",
    "ner_tags = sorted({t for seq in labels_ner for t in seq})\n",
    "srl_tags = sorted({t for seq in labels_srl for t in seq})\n",
    "\n",
    "ner_tags.insert(0, PAD_TOKEN)\n",
    "srl_tags.insert(0, PAD_TOKEN)\n",
    "\n",
    "word2idx = {w: i + 2 for i, w in enumerate(words)}\n",
    "word2idx[\"PAD\"] = 0\n",
    "word2idx[\"UNK\"] = 1\n",
    "\n",
    "tag2idx_ner = {t: i for i, t in enumerate(ner_tags)}\n",
    "tag2idx_srl = {t: i for i, t in enumerate(srl_tags)}\n",
    "idx2tag_ner = {i: t for t, i in tag2idx_ner.items()}\n",
    "idx2tag_srl = {i: t for t, i in tag2idx_srl.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "096967e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [[word2idx.get(w, 1) for w in s] for s in sentences]\n",
    "y_ner = [[tag2idx_ner[t] for t in seq] for seq in labels_ner]\n",
    "y_srl = [[tag2idx_srl[t] for t in seq] for seq in labels_srl]\n",
    "\n",
    "maxlen = max(map(len, X))\n",
    "pad_id = tag2idx_ner[PAD_TOKEN]\n",
    "\n",
    "X = pad_sequences(X, maxlen=maxlen, padding=\"post\", value=0)\n",
    "y_ner = pad_sequences(y_ner, maxlen=maxlen, padding=\"post\", value=pad_id)\n",
    "y_srl = pad_sequences(y_srl, maxlen=maxlen, padding=\"post\", value=pad_id)\n",
    "\n",
    "mask = (y_ner != pad_id).astype(\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "a26893cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = train_test_split(\n",
    "    X, y_ner, y_srl, mask, test_size=0.2, random_state=42, shuffle=True\n",
    ")\n",
    "X_tr, X_te, ner_tr, ner_te, srl_tr, srl_te, m_tr, m_te = splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "1b4a1c61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_7\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"functional_7\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)        </span>┃<span style=\"font-weight: bold\"> Output Shape      </span>┃<span style=\"font-weight: bold\">    Param # </span>┃<span style=\"font-weight: bold\"> Connected to      </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ tokens (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ embed (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │     <span style=\"color: #00af00; text-decoration-color: #00af00\">87,296</span> │ tokens[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_7 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ embed[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ not_equal_7         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ tokens[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">NotEqual</span>)          │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_14    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)   │     <span style=\"color: #00af00; text-decoration-color: #00af00\">66,048</span> │ spatial_dropout1… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │ not_equal_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_15    │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">128</span>)   │     <span style=\"color: #00af00; text-decoration-color: #00af00\">98,816</span> │ bidirectional_14… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │ not_equal_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_14 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │ bidirectional_15… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_15 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">64</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">8,256</span> │ bidirectional_15… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_output          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">24</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">1,560</span> │ time_distributed… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_output          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">34</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">13</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">845</span> │ time_distributed… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">TimeDistributed</span>)   │                   │            │ not_equal_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape     \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m   Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to     \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ tokens (\u001b[38;5;33mInputLayer\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ embed (\u001b[38;5;33mEmbedding\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │     \u001b[38;5;34m87,296\u001b[0m │ tokens[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_7 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ embed[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ not_equal_7         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ tokens[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mNotEqual\u001b[0m)          │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_14    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m128\u001b[0m)   │     \u001b[38;5;34m66,048\u001b[0m │ spatial_dropout1… │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │ not_equal_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_15    │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m128\u001b[0m)   │     \u001b[38;5;34m98,816\u001b[0m │ bidirectional_14… │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │ not_equal_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_14 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │      \u001b[38;5;34m8,256\u001b[0m │ bidirectional_15… │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ time_distributed_15 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m64\u001b[0m)    │      \u001b[38;5;34m8,256\u001b[0m │ bidirectional_15… │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ ner_output          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m24\u001b[0m)    │      \u001b[38;5;34m1,560\u001b[0m │ time_distributed… │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ srl_output          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m34\u001b[0m, \u001b[38;5;34m13\u001b[0m)    │        \u001b[38;5;34m845\u001b[0m │ time_distributed… │\n",
       "│ (\u001b[38;5;33mTimeDistributed\u001b[0m)   │                   │            │ not_equal_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">271,077</span> (1.03 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m271,077\u001b[0m (1.03 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">271,077</span> (1.03 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m271,077\u001b[0m (1.03 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "embed_dim = 64\n",
    "lstm_units = 64\n",
    "drop_embed = 0.5\n",
    "drop_lstm = 0.5\n",
    "\n",
    "inp = Input(shape=(maxlen,), name=\"tokens\")\n",
    "emb = Embedding(len(word2idx), embed_dim, mask_zero=True, name=\"embed\")(inp)\n",
    "emb = SpatialDropout1D(drop_embed)(emb)\n",
    "\n",
    "x = Bidirectional(\n",
    "    LSTM(\n",
    "        lstm_units,\n",
    "        return_sequences=True,\n",
    "        dropout=drop_lstm,\n",
    "        recurrent_dropout=drop_lstm,\n",
    "    )\n",
    ")(emb)\n",
    "x = Bidirectional(\n",
    "    LSTM(\n",
    "        lstm_units,\n",
    "        return_sequences=True,\n",
    "        dropout=drop_lstm,\n",
    "        recurrent_dropout=drop_lstm,\n",
    "    )\n",
    ")(x)\n",
    "\n",
    "ner_head = TimeDistributed(Dense(lstm_units, activation=\"relu\"))(x)\n",
    "ner_out = TimeDistributed(\n",
    "    Dense(len(tag2idx_ner), activation=\"softmax\"), name=\"ner_output\"\n",
    ")(ner_head)\n",
    "\n",
    "srl_head = TimeDistributed(Dense(lstm_units, activation=\"relu\"))(x)\n",
    "srl_out = TimeDistributed(\n",
    "    Dense(len(tag2idx_srl), activation=\"softmax\"), name=\"srl_output\"\n",
    ")(srl_head)\n",
    "\n",
    "model = Model(inp, [ner_out, srl_out])\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(3e-4),\n",
    "    loss={\n",
    "        \"ner_output\": \"sparse_categorical_crossentropy\",\n",
    "        \"srl_output\": \"sparse_categorical_crossentropy\",\n",
    "    },\n",
    "    metrics={\n",
    "        \"ner_output\": [\"sparse_categorical_accuracy\"],\n",
    "        \"srl_output\": [\"sparse_categorical_accuracy\"],\n",
    "    },\n",
    "    # sample_weight_mode=\"temporal\"\n",
    ")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "f41d6012",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m10s\u001b[0m 105ms/step - loss: 5.7110 - ner_output_loss: 3.1579 - ner_output_sparse_categorical_accuracy: 0.2067 - srl_output_loss: 2.5530 - srl_output_sparse_categorical_accuracy: 0.1810 - val_loss: 5.5840 - val_ner_output_loss: 3.0747 - val_ner_output_sparse_categorical_accuracy: 0.1325 - val_srl_output_loss: 2.5086 - val_srl_output_sparse_categorical_accuracy: 0.0750 - learning_rate: 3.0000e-04\n",
      "Epoch 2/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 43ms/step - loss: 5.4576 - ner_output_loss: 2.9911 - ner_output_sparse_categorical_accuracy: 0.1311 - srl_output_loss: 2.4653 - srl_output_sparse_categorical_accuracy: 0.0760 - val_loss: 4.6826 - val_ner_output_loss: 2.4715 - val_ner_output_sparse_categorical_accuracy: 0.1325 - val_srl_output_loss: 2.1957 - val_srl_output_sparse_categorical_accuracy: 0.0750 - learning_rate: 3.0000e-04\n",
      "Epoch 3/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 43ms/step - loss: 4.3203 - ner_output_loss: 2.2263 - ner_output_sparse_categorical_accuracy: 0.1270 - srl_output_loss: 2.0936 - srl_output_sparse_categorical_accuracy: 0.0751 - val_loss: 3.6358 - val_ner_output_loss: 1.8780 - val_ner_output_sparse_categorical_accuracy: 0.1325 - val_srl_output_loss: 1.7769 - val_srl_output_sparse_categorical_accuracy: 0.1032 - learning_rate: 3.0000e-04\n",
      "Epoch 4/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 43ms/step - loss: 3.4818 - ner_output_loss: 1.7297 - ner_output_sparse_categorical_accuracy: 0.1284 - srl_output_loss: 1.7518 - srl_output_sparse_categorical_accuracy: 0.1028 - val_loss: 3.0980 - val_ner_output_loss: 1.5394 - val_ner_output_sparse_categorical_accuracy: 0.1325 - val_srl_output_loss: 1.5650 - val_srl_output_sparse_categorical_accuracy: 0.1015 - learning_rate: 3.0000e-04\n",
      "Epoch 5/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 42ms/step - loss: 3.0010 - ner_output_loss: 1.4352 - ner_output_sparse_categorical_accuracy: 0.1320 - srl_output_loss: 1.5651 - srl_output_sparse_categorical_accuracy: 0.1053 - val_loss: 2.7911 - val_ner_output_loss: 1.3675 - val_ner_output_sparse_categorical_accuracy: 0.1383 - val_srl_output_loss: 1.4193 - val_srl_output_sparse_categorical_accuracy: 0.1185 - learning_rate: 3.0000e-04\n",
      "Epoch 6/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 42ms/step - loss: 2.6586 - ner_output_loss: 1.2858 - ner_output_sparse_categorical_accuracy: 0.1416 - srl_output_loss: 1.3731 - srl_output_sparse_categorical_accuracy: 0.1208 - val_loss: 2.5715 - val_ner_output_loss: 1.2498 - val_ner_output_sparse_categorical_accuracy: 0.1475 - val_srl_output_loss: 1.3092 - val_srl_output_sparse_categorical_accuracy: 0.1243 - learning_rate: 3.0000e-04\n",
      "Epoch 7/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 42ms/step - loss: 2.5278 - ner_output_loss: 1.1977 - ner_output_sparse_categorical_accuracy: 0.1510 - srl_output_loss: 1.3293 - srl_output_sparse_categorical_accuracy: 0.1260 - val_loss: 2.4185 - val_ner_output_loss: 1.1771 - val_ner_output_sparse_categorical_accuracy: 0.1486 - val_srl_output_loss: 1.2246 - val_srl_output_sparse_categorical_accuracy: 0.1286 - learning_rate: 3.0000e-04\n",
      "Epoch 8/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 43ms/step - loss: 2.3210 - ner_output_loss: 1.1242 - ner_output_sparse_categorical_accuracy: 0.1475 - srl_output_loss: 1.1972 - srl_output_sparse_categorical_accuracy: 0.1244 - val_loss: 2.3181 - val_ner_output_loss: 1.1293 - val_ner_output_sparse_categorical_accuracy: 0.1491 - val_srl_output_loss: 1.1701 - val_srl_output_sparse_categorical_accuracy: 0.1266 - learning_rate: 3.0000e-04\n",
      "Epoch 9/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 42ms/step - loss: 2.1532 - ner_output_loss: 1.0439 - ner_output_sparse_categorical_accuracy: 0.1456 - srl_output_loss: 1.1101 - srl_output_sparse_categorical_accuracy: 0.1250 - val_loss: 2.2397 - val_ner_output_loss: 1.0935 - val_ner_output_sparse_categorical_accuracy: 0.1491 - val_srl_output_loss: 1.1276 - val_srl_output_sparse_categorical_accuracy: 0.1466 - learning_rate: 3.0000e-04\n",
      "Epoch 10/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 2.1745 - ner_output_loss: 1.0467 - ner_output_sparse_categorical_accuracy: 0.1509 - srl_output_loss: 1.1280 - srl_output_sparse_categorical_accuracy: 0.1340 - val_loss: 2.1663 - val_ner_output_loss: 1.0578 - val_ner_output_sparse_categorical_accuracy: 0.1653 - val_srl_output_loss: 1.0900 - val_srl_output_sparse_categorical_accuracy: 0.1437 - learning_rate: 3.0000e-04\n",
      "Epoch 11/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 43ms/step - loss: 2.0726 - ner_output_loss: 1.0098 - ner_output_sparse_categorical_accuracy: 0.1532 - srl_output_loss: 1.0632 - srl_output_sparse_categorical_accuracy: 0.1427 - val_loss: 2.0905 - val_ner_output_loss: 1.0194 - val_ner_output_sparse_categorical_accuracy: 0.1656 - val_srl_output_loss: 1.0506 - val_srl_output_sparse_categorical_accuracy: 0.1483 - learning_rate: 3.0000e-04\n",
      "Epoch 12/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.9597 - ner_output_loss: 0.9479 - ner_output_sparse_categorical_accuracy: 0.1595 - srl_output_loss: 1.0113 - srl_output_sparse_categorical_accuracy: 0.1466 - val_loss: 2.0062 - val_ner_output_loss: 0.9765 - val_ner_output_sparse_categorical_accuracy: 0.1654 - val_srl_output_loss: 1.0097 - val_srl_output_sparse_categorical_accuracy: 0.1526 - learning_rate: 3.0000e-04\n",
      "Epoch 13/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.9171 - ner_output_loss: 0.9401 - ner_output_sparse_categorical_accuracy: 0.1655 - srl_output_loss: 0.9774 - srl_output_sparse_categorical_accuracy: 0.1519 - val_loss: 1.9191 - val_ner_output_loss: 0.9321 - val_ner_output_sparse_categorical_accuracy: 0.1659 - val_srl_output_loss: 0.9655 - val_srl_output_sparse_categorical_accuracy: 0.1561 - learning_rate: 3.0000e-04\n",
      "Epoch 14/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.8593 - ner_output_loss: 0.9060 - ner_output_sparse_categorical_accuracy: 0.1702 - srl_output_loss: 0.9529 - srl_output_sparse_categorical_accuracy: 0.1545 - val_loss: 1.8181 - val_ner_output_loss: 0.8752 - val_ner_output_sparse_categorical_accuracy: 0.1736 - val_srl_output_loss: 0.9142 - val_srl_output_sparse_categorical_accuracy: 0.1606 - learning_rate: 3.0000e-04\n",
      "Epoch 15/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 43ms/step - loss: 1.7053 - ner_output_loss: 0.8312 - ner_output_sparse_categorical_accuracy: 0.1694 - srl_output_loss: 0.8744 - srl_output_sparse_categorical_accuracy: 0.1578 - val_loss: 1.7062 - val_ner_output_loss: 0.8141 - val_ner_output_sparse_categorical_accuracy: 0.1768 - val_srl_output_loss: 0.8590 - val_srl_output_sparse_categorical_accuracy: 0.1690 - learning_rate: 3.0000e-04\n",
      "Epoch 16/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.5892 - ner_output_loss: 0.7715 - ner_output_sparse_categorical_accuracy: 0.1733 - srl_output_loss: 0.8177 - srl_output_sparse_categorical_accuracy: 0.1628 - val_loss: 1.5958 - val_ner_output_loss: 0.7575 - val_ner_output_sparse_categorical_accuracy: 0.1785 - val_srl_output_loss: 0.8016 - val_srl_output_sparse_categorical_accuracy: 0.1699 - learning_rate: 3.0000e-04\n",
      "Epoch 17/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.5047 - ner_output_loss: 0.7224 - ner_output_sparse_categorical_accuracy: 0.1762 - srl_output_loss: 0.7827 - srl_output_sparse_categorical_accuracy: 0.1656 - val_loss: 1.4978 - val_ner_output_loss: 0.7097 - val_ner_output_sparse_categorical_accuracy: 0.1848 - val_srl_output_loss: 0.7469 - val_srl_output_sparse_categorical_accuracy: 0.1730 - learning_rate: 3.0000e-04\n",
      "Epoch 18/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 44ms/step - loss: 1.4376 - ner_output_loss: 0.6999 - ner_output_sparse_categorical_accuracy: 0.1783 - srl_output_loss: 0.7377 - srl_output_sparse_categorical_accuracy: 0.1696 - val_loss: 1.4164 - val_ner_output_loss: 0.6699 - val_ner_output_sparse_categorical_accuracy: 0.1860 - val_srl_output_loss: 0.7027 - val_srl_output_sparse_categorical_accuracy: 0.1755 - learning_rate: 3.0000e-04\n",
      "Epoch 19/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.3140 - ner_output_loss: 0.6430 - ner_output_sparse_categorical_accuracy: 0.1816 - srl_output_loss: 0.6698 - srl_output_sparse_categorical_accuracy: 0.1733 - val_loss: 1.3554 - val_ner_output_loss: 0.6392 - val_ner_output_sparse_categorical_accuracy: 0.1888 - val_srl_output_loss: 0.6697 - val_srl_output_sparse_categorical_accuracy: 0.1776 - learning_rate: 3.0000e-04\n",
      "Epoch 20/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.2762 - ner_output_loss: 0.6169 - ner_output_sparse_categorical_accuracy: 0.1809 - srl_output_loss: 0.6601 - srl_output_sparse_categorical_accuracy: 0.1729 - val_loss: 1.2996 - val_ner_output_loss: 0.6101 - val_ner_output_sparse_categorical_accuracy: 0.1901 - val_srl_output_loss: 0.6401 - val_srl_output_sparse_categorical_accuracy: 0.1794 - learning_rate: 3.0000e-04\n",
      "Epoch 21/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.2447 - ner_output_loss: 0.6027 - ner_output_sparse_categorical_accuracy: 0.1849 - srl_output_loss: 0.6418 - srl_output_sparse_categorical_accuracy: 0.1760 - val_loss: 1.2591 - val_ner_output_loss: 0.5895 - val_ner_output_sparse_categorical_accuracy: 0.1897 - val_srl_output_loss: 0.6165 - val_srl_output_sparse_categorical_accuracy: 0.1797 - learning_rate: 3.0000e-04\n",
      "Epoch 22/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 1.1679 - ner_output_loss: 0.5572 - ner_output_sparse_categorical_accuracy: 0.1878 - srl_output_loss: 0.6102 - srl_output_sparse_categorical_accuracy: 0.1781 - val_loss: 1.2209 - val_ner_output_loss: 0.5655 - val_ner_output_sparse_categorical_accuracy: 0.1910 - val_srl_output_loss: 0.5995 - val_srl_output_sparse_categorical_accuracy: 0.1813 - learning_rate: 3.0000e-04\n",
      "Epoch 23/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.0996 - ner_output_loss: 0.5274 - ner_output_sparse_categorical_accuracy: 0.1873 - srl_output_loss: 0.5710 - srl_output_sparse_categorical_accuracy: 0.1794 - val_loss: 1.1831 - val_ner_output_loss: 0.5454 - val_ner_output_sparse_categorical_accuracy: 0.1924 - val_srl_output_loss: 0.5802 - val_srl_output_sparse_categorical_accuracy: 0.1821 - learning_rate: 3.0000e-04\n",
      "Epoch 24/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.1842 - ner_output_loss: 0.5630 - ner_output_sparse_categorical_accuracy: 0.1890 - srl_output_loss: 0.6211 - srl_output_sparse_categorical_accuracy: 0.1791 - val_loss: 1.1559 - val_ner_output_loss: 0.5311 - val_ner_output_sparse_categorical_accuracy: 0.1940 - val_srl_output_loss: 0.5664 - val_srl_output_sparse_categorical_accuracy: 0.1832 - learning_rate: 3.0000e-04\n",
      "Epoch 25/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 44ms/step - loss: 1.0526 - ner_output_loss: 0.5143 - ner_output_sparse_categorical_accuracy: 0.1923 - srl_output_loss: 0.5390 - srl_output_sparse_categorical_accuracy: 0.1856 - val_loss: 1.1268 - val_ner_output_loss: 0.5145 - val_ner_output_sparse_categorical_accuracy: 0.1958 - val_srl_output_loss: 0.5533 - val_srl_output_sparse_categorical_accuracy: 0.1842 - learning_rate: 3.0000e-04\n",
      "Epoch 26/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 47ms/step - loss: 1.0331 - ner_output_loss: 0.4892 - ner_output_sparse_categorical_accuracy: 0.1884 - srl_output_loss: 0.5446 - srl_output_sparse_categorical_accuracy: 0.1805 - val_loss: 1.1003 - val_ner_output_loss: 0.4994 - val_ner_output_sparse_categorical_accuracy: 0.1966 - val_srl_output_loss: 0.5411 - val_srl_output_sparse_categorical_accuracy: 0.1845 - learning_rate: 3.0000e-04\n",
      "Epoch 27/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 45ms/step - loss: 0.9729 - ner_output_loss: 0.4579 - ner_output_sparse_categorical_accuracy: 0.1902 - srl_output_loss: 0.5148 - srl_output_sparse_categorical_accuracy: 0.1837 - val_loss: 1.0818 - val_ner_output_loss: 0.4883 - val_ner_output_sparse_categorical_accuracy: 0.1967 - val_srl_output_loss: 0.5338 - val_srl_output_sparse_categorical_accuracy: 0.1851 - learning_rate: 3.0000e-04\n",
      "Epoch 28/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 44ms/step - loss: 0.9333 - ner_output_loss: 0.4367 - ner_output_sparse_categorical_accuracy: 0.1909 - srl_output_loss: 0.4970 - srl_output_sparse_categorical_accuracy: 0.1826 - val_loss: 1.0529 - val_ner_output_loss: 0.4698 - val_ner_output_sparse_categorical_accuracy: 0.2007 - val_srl_output_loss: 0.5223 - val_srl_output_sparse_categorical_accuracy: 0.1865 - learning_rate: 3.0000e-04\n",
      "Epoch 29/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 1.0398 - ner_output_loss: 0.4866 - ner_output_sparse_categorical_accuracy: 0.1948 - srl_output_loss: 0.5531 - srl_output_sparse_categorical_accuracy: 0.1833 - val_loss: 1.0380 - val_ner_output_loss: 0.4601 - val_ner_output_sparse_categorical_accuracy: 0.1998 - val_srl_output_loss: 0.5154 - val_srl_output_sparse_categorical_accuracy: 0.1870 - learning_rate: 3.0000e-04\n",
      "Epoch 30/30\n",
      "\u001b[1m21/21\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 46ms/step - loss: 0.9113 - ner_output_loss: 0.4297 - ner_output_sparse_categorical_accuracy: 0.1923 - srl_output_loss: 0.4818 - srl_output_sparse_categorical_accuracy: 0.1842 - val_loss: 1.0241 - val_ner_output_loss: 0.4523 - val_ner_output_sparse_categorical_accuracy: 0.2027 - val_srl_output_loss: 0.5074 - val_srl_output_sparse_categorical_accuracy: 0.1870 - learning_rate: 3.0000e-04\n"
     ]
    }
   ],
   "source": [
    "callbacks = [\n",
    "    tf.keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True),\n",
    "    tf.keras.callbacks.ReduceLROnPlateau(patience=2, factor=0.5, min_lr=1e-5),\n",
    "]\n",
    "\n",
    "history = model.fit(\n",
    "    X_tr,\n",
    "    [ner_tr, srl_tr],  # y → LIST (pos 0 = ner_output, 1 = srl_output)\n",
    "    sample_weight=[m_tr, m_tr],  # sama‑persis urutan\n",
    "    validation_data=(X_te, [ner_te, srl_te], [m_te, m_te]),\n",
    "    \n",
    "    batch_size=64,\n",
    "    epochs=30,\n",
    "    callbacks=callbacks,\n",
    "    verbose=1,\n",
    ")\n",
    "\n",
    "\n",
    "# =========================\n",
    "# 7.  Save artefacts\n",
    "# =========================\n",
    "model.save(\"lstm_ner_srl_model.keras\")\n",
    "for fname, obj in [\n",
    "    (\"word2idx.pkl\", word2idx),\n",
    "    (\"tag2idx_ner.pkl\", tag2idx_ner),\n",
    "    (\"tag2idx_srl.pkl\", tag2idx_srl),\n",
    "]:\n",
    "    with open(fname, \"wb\") as f:\n",
    "        pickle.dump(obj, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "430794b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 86464 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 72972 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_training_history(history):\n",
    "    history_data = history.history\n",
    "    epochs = range(1, len(next(iter(history_data.values()))) + 1)\n",
    "\n",
    "    # Coba deteksi metric secara dinamis agar fleksibel\n",
    "    ner_acc_key = next((k for k in history_data if \"ner_output\" in k and \"accuracy\" in k), None)\n",
    "    srl_acc_key = next((k for k in history_data if \"srl_output\" in k and \"accuracy\" in k), None)\n",
    "    val_ner_acc_key = f\"val_{ner_acc_key}\" if ner_acc_key else None\n",
    "    val_srl_acc_key = f\"val_{srl_acc_key}\" if srl_acc_key else None\n",
    "\n",
    "    # --- Plot Accuracy ---\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    if ner_acc_key:\n",
    "        plt.plot(epochs, history_data[ner_acc_key], marker=\"o\", label=\"NER Accuracy (Train)\")\n",
    "    if srl_acc_key:\n",
    "        plt.plot(epochs, history_data[srl_acc_key], marker=\"s\", label=\"SRL Accuracy (Train)\")\n",
    "    if val_ner_acc_key in history_data:\n",
    "        plt.plot(epochs, history_data[val_ner_acc_key], marker=\"o\", linestyle=\"--\", label=\"NER Accuracy (Val)\")\n",
    "    if val_srl_acc_key in history_data:\n",
    "        plt.plot(epochs, history_data[val_srl_acc_key], marker=\"s\", linestyle=\"--\", label=\"SRL Accuracy (Val)\")\n",
    "\n",
    "    plt.title(\"Accuracy per Epoch\")\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(\"Accuracy\")\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "    plt.savefig(\"accuracy_plot.png\")\n",
    "    plt.show()\n",
    "\n",
    "    # --- Plot Loss ---\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    if \"ner_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"ner_output_loss\"], marker=\"o\", label=\"NER Loss (Train)\")\n",
    "    if \"srl_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"srl_output_loss\"], marker=\"s\", label=\"SRL Loss (Train)\")\n",
    "    if \"val_ner_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"val_ner_output_loss\"], marker=\"o\", linestyle=\"--\", label=\"NER Loss (Val)\")\n",
    "    if \"val_srl_output_loss\" in history_data:\n",
    "        plt.plot(epochs, history_data[\"val_srl_output_loss\"], marker=\"s\", linestyle=\"--\", label=\"SRL Loss (Val)\")\n",
    "\n",
    "    plt.title(\"Loss per Epoch\")\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(\"Loss\")\n",
    "    plt.legend()\n",
    "    plt.grid(True)\n",
    "    plt.savefig(\"loss_plot.png\")\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "plot_training_history(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "e690a0e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 109356 characters]",
      "text/plain": [
       "<Figure size 1000x1000 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 87692 characters]",
      "text/plain": [
       "<Figure size 1000x1000 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 1.  Prediksi\n",
    "# ------------------------------------------------------------------\n",
    "pred_ner_prob, pred_srl_prob = model.predict(X_te, verbose=0)\n",
    "\n",
    "pred_ner = pred_ner_prob.argmax(-1)\n",
    "pred_srl = pred_srl_prob.argmax(-1)\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 2.  Siapkan masker PAD\n",
    "# ------------------------------------------------------------------\n",
    "pad_id = tag2idx_ner[\"<PAD>\"]\n",
    "\n",
    "mask_ner = ner_te != pad_id\n",
    "mask_srl = srl_te != pad_id\n",
    "\n",
    "true_ner_flat = ner_te[mask_ner]\n",
    "pred_ner_flat = pred_ner[mask_ner]\n",
    "\n",
    "true_srl_flat = srl_te[mask_srl]\n",
    "pred_srl_flat = pred_srl[mask_srl]\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 3.  Hitung confusion matrix TANPA PAD\n",
    "# ------------------------------------------------------------------\n",
    "# Buang ID PAD dari label list\n",
    "labels_ner_no_pad = [i for i in range(len(tag2idx_ner)) if i != pad_id]\n",
    "labels_srl_no_pad = [i for i in range(len(tag2idx_srl)) if i != pad_id]\n",
    "\n",
    "cm_ner = confusion_matrix(true_ner_flat, pred_ner_flat, labels=labels_ner_no_pad)\n",
    "\n",
    "cm_srl = confusion_matrix(true_srl_flat, pred_srl_flat, labels=labels_srl_no_pad)\n",
    "\n",
    "# Siapkan label display TANPA PAD\n",
    "display_labels_ner = [idx2tag_ner[i] for i in labels_ner_no_pad]\n",
    "display_labels_srl = [idx2tag_srl[i] for i in labels_srl_no_pad]\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 4.  Plot NER CM (tanpa PAD)\n",
    "# ------------------------------------------------------------------\n",
    "fig, ax = plt.subplots(figsize=(10, 10))\n",
    "disp_ner = ConfusionMatrixDisplay(\n",
    "    confusion_matrix=cm_ner, display_labels=display_labels_ner\n",
    ")\n",
    "disp_ner.plot(\n",
    "    include_values=True,  # Tampilkan angka\n",
    "    values_format=\"d\",  # Format integer\n",
    "    cmap=plt.cm.Blues,  # Biru-putih\n",
    "    ax=ax,\n",
    "    colorbar=False,\n",
    ")\n",
    "ax.set_title(\"NER Confusion Matrix\", fontsize=18)\n",
    "plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 5.  Plot SRL CM (tanpa PAD)\n",
    "# ------------------------------------------------------------------\n",
    "fig, ax = plt.subplots(figsize=(10, 10))\n",
    "disp_srl = ConfusionMatrixDisplay(\n",
    "    confusion_matrix=cm_srl, display_labels=display_labels_srl\n",
    ")\n",
    "disp_srl.plot(\n",
    "    include_values=True, values_format=\"d\", cmap=plt.cm.Blues, ax=ax, colorbar=False\n",
    ")\n",
    "ax.set_title(\"SRL Confusion Matrix\", fontsize=18)\n",
    "plt.setp(ax.get_xticklabels(), rotation=45, ha=\"right\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "a49f1dfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NER TAG accuracy : 88.84%\n",
      "SRL TAG accuracy : 81.93%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 3b.  Akurasi token‑level (tanpa PAD)\n",
    "# ------------------------------------------------------------------\n",
    "acc_ner = accuracy_score(true_ner_flat, pred_ner_flat)\n",
    "acc_srl = accuracy_score(true_srl_flat, pred_srl_flat)\n",
    "\n",
    "print(f\"NER TAG accuracy : {acc_ner:.2%}\")\n",
    "print(f\"SRL TAG accuracy : {acc_srl:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "9adad755",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[NER] Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      B-DATE       0.84      0.93      0.89        75\n",
      "       B-ETH       0.78      0.80      0.79        45\n",
      "     B-EVENT       0.00      0.00      0.00        14\n",
      "       B-LOC       0.94      0.90      0.92       304\n",
      "       B-MAT       0.00      0.00      0.00        19\n",
      "      B-MISC       0.00      0.00      0.00         6\n",
      "       B-ORG       0.00      0.00      0.00         2\n",
      "       B-PER       0.87      0.93      0.90       248\n",
      "     B-QUANT       0.00      0.00      0.00        11\n",
      "      B-TIME       0.00      0.00      0.00        49\n",
      "      B-UNIT       0.00      0.00      0.00        11\n",
      "      I-DATE       0.82      0.98      0.89       146\n",
      "       I-ETH       0.89      0.70      0.78        46\n",
      "     I-EVENT       0.00      0.00      0.00         8\n",
      "       I-LOC       0.00      0.00      0.00         8\n",
      "       I-MAT       0.00      0.00      0.00         2\n",
      "      I-MISC       0.00      0.00      0.00         1\n",
      "       I-ORG       0.00      0.00      0.00         1\n",
      "       I-PER       0.69      0.65      0.67        51\n",
      "     I-QUANT       0.00      0.00      0.00         0\n",
      "      I-TIME       0.00      0.00      0.00        19\n",
      "      I-UNIT       0.00      0.00      0.00         1\n",
      "           O       0.90      0.98      0.94      1478\n",
      "\n",
      "    accuracy                           0.89      2545\n",
      "   macro avg       0.29      0.30      0.29      2545\n",
      "weighted avg       0.84      0.89      0.86      2545\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "# (Opsional) tampilkan ringkasan metrik per‑label\n",
    "print(\"\\n[NER] Classification report:\")\n",
    "print(\n",
    "    classification_report(\n",
    "        true_ner_flat,\n",
    "        pred_ner_flat,\n",
    "        labels=labels_ner_no_pad,\n",
    "        target_names=display_labels_ner,\n",
    "        digits=2,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "7cd28380",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[SRL] Classification report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        ARG0       0.81      0.93      0.87       396\n",
      "        ARG1       0.61      0.60      0.60       288\n",
      "        ARG2       0.27      0.33      0.30        54\n",
      "    ARGM-CAU       0.00      0.00      0.00         4\n",
      "    ARGM-DIR       0.00      0.00      0.00        11\n",
      "    ARGM-LOC       0.88      0.87      0.87       296\n",
      "    ARGM-MNR       0.00      0.00      0.00         7\n",
      "    ARGM-MOD       0.00      0.00      0.00         8\n",
      "    ARGM-NEG       0.00      0.00      0.00         1\n",
      "    ARGM-TMP       0.83      0.82      0.83       302\n",
      "           O       0.89      0.89      0.89       836\n",
      "           V       0.89      0.80      0.84       342\n",
      "\n",
      "    accuracy                           0.82      2545\n",
      "   macro avg       0.43      0.44      0.43      2545\n",
      "weighted avg       0.81      0.82      0.82      2545\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
      "/mnt/disc1/code/thesis_quiz_project/lstm-quiz/myenv/lib64/python3.10/site-packages/sklearn/metrics/_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n[SRL] Classification report:\")\n",
    "print(\n",
    "    classification_report(\n",
    "        true_srl_flat,\n",
    "        pred_srl_flat,\n",
    "        labels=labels_srl_no_pad,\n",
    "        target_names=display_labels_srl,\n",
    "        digits=2,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "333745fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def plot_training_history(history):\n",
    "#     epochs = range(1, len(history['loss']) + 1)\n",
    "\n",
    "#     plt.figure(figsize=(14, 6))\n",
    "\n",
    "#     # Plot Loss\n",
    "#     plt.subplot(1, 2, 1)\n",
    "#     plt.plot(epochs, history['loss'], label='Training Loss')\n",
    "#     plt.plot(epochs, history['val_loss'], label='Validation Loss')\n",
    "#     plt.title('Loss During Training')\n",
    "#     plt.xlabel('Epochs')\n",
    "#     plt.ylabel('Loss')\n",
    "#     plt.legend()\n",
    "\n",
    "#     # Plot Accuracy\n",
    "#     plt.subplot(1, 2, 2)\n",
    "#     plt.plot(epochs, history['ner_output_accuracy'], label='NER Train Acc')\n",
    "#     plt.plot(epochs, history['val_ner_output_accuracy'], label='NER Val Acc')\n",
    "#     plt.plot(epochs, history['srl_output_accuracy'], label='SRL Train Acc')\n",
    "#     plt.plot(epochs, history['val_srl_output_accuracy'], label='SRL Val Acc')\n",
    "#     plt.title('Accuracy During Training')\n",
    "#     plt.xlabel('Epochs')\n",
    "#     plt.ylabel('Accuracy')\n",
    "#     plt.legend()\n",
    "\n",
    "#     plt.tight_layout()\n",
    "#     plt.show()\n",
    "\n",
    "# plot_training_history(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "df36e200",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def token_level_accuracy(y_true, y_pred):\n",
    "#     total, correct = 0, 0\n",
    "#     for true_seq, pred_seq in zip(y_true, y_pred):\n",
    "#         for t, p in zip(true_seq, pred_seq):\n",
    "#             if t.sum() == 0:\n",
    "#                 continue\n",
    "#             total += 1\n",
    "#             if t.argmax() == p.argmax():\n",
    "#                 correct += 1\n",
    "#     return correct / total\n",
    "\n",
    "# def decode_predictions(pred, true, idx2tag):\n",
    "#     true_out, pred_out = [], []\n",
    "#     for pred_seq, true_seq in zip(pred, true):\n",
    "#         t_labels, p_labels = [], []\n",
    "#         for p_tok, t_tok in zip(pred_seq, true_seq):\n",
    "#             if t_tok.sum() == 0:\n",
    "#                 continue\n",
    "#             t_labels.append(idx2tag[t_tok.argmax()])\n",
    "#             p_labels.append(idx2tag[p_tok.argmax()])\n",
    "#         true_out.append(t_labels)\n",
    "#         pred_out.append(p_labels)\n",
    "#     return true_out, pred_out\n",
    "\n",
    "# results = model.evaluate(X_test, {\"ner_output\": y_ner_test, \"srl_output\": y_srl_test}, verbose=0)\n",
    "# for name, value in zip(model.metrics_names, results):\n",
    "#     print(f\"{name}: {value}\")\n",
    "\n",
    "# y_pred_ner, y_pred_srl = model.predict(X_test, verbose=0)\n",
    "\n",
    "# true_ner, pred_ner = decode_predictions(y_pred_ner, y_ner_test, idx2tag_ner)\n",
    "# true_srl, pred_srl = decode_predictions(y_pred_srl, y_srl_test, idx2tag_srl)\n",
    "\n",
    "# acc_ner = token_level_accuracy(y_ner_test, y_pred_ner)\n",
    "# acc_srl = token_level_accuracy(y_srl_test, y_pred_srl)\n",
    "\n",
    "# print(f\"NER Token Accuracy {acc_ner:.2%}\")\n",
    "# print(f\"SRL Token Accuracy {acc_srl:.2%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "9127cce0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"[NER] Classification Report:\")\n",
    "# print(classification_report(true_ner, pred_ner, digits=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "300897b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"SRL Classification Resport:\")\n",
    "# print(classification_report(true_srl, pred_srl, digits=2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "myenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
