notebooks/tableqa/tableqa_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table Question Answering: WikiSQL dataset\n",
    "In this notebook, we will see how to fine-tune and evaluate a Tapas-based Table Question answering model on WikiSQL dataset using PrimeQA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "do_train=True\n",
    "model_name_or_path=\"google/tapas-base\"\n",
    "do_eval=True \n",
    "dataset_name=\"wikisql\" \n",
    "data_path_root=\"data/wikisql/\" \n",
    "output_dir=\"../../models/tableqa/wikisql_nb\"\n",
    "learning_rate=4e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from primeqa.tableqa.metrics.answer_accuracy import compute_denotation_accuracy\n",
    "from primeqa.tableqa.models.tableqa_model import TableQAModel\n",
    "from primeqa.tableqa.trainers.tableqa_trainer import TableQATrainer\n",
    "from transformers import TapasConfig\n",
    "from transformers import (\n",
    "    DataCollator,\n",
    "    HfArgumentParser,\n",
    "    TrainingArguments,\n",
    "    set_seed,default_data_collator,\n",
    ")\n",
    "from primeqa.tableqa.run_tableqa import TableQAArguments\n",
    "from primeqa.tableqa.utils.data_collator import TapasCollator\n",
    "from primeqa.tableqa.preprocessors.wikisql_preprocessor import load_data\n",
    "from primeqa.tableqa.postprocessor.wikisql import WikiSQLPostprocessor\n",
    "from primeqa.tableqa.metrics.answer_accuracy import compute_denotation_accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the TableQA specific arguments needed for TAPAS training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of TapasForQuestionAnswering were not initialized from the model checkpoint at google/tapas-base and are newly initialized: ['output_bias', 'column_output_weights', 'output_weights', 'column_output_bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tqa_args = TableQAArguments()\n",
    "tqa_args.dataset_name=dataset_name\n",
    "tqa_args.data_path_root=data_path_root\n",
    "tqa_args.use_answer_as_supervision=True\n",
    "config = TapasConfig(tqa_args)\n",
    "tableqa_model = TableQAModel(\"google/tapas-base\",config=config)\n",
    "model = tableqa_model.model\n",
    "tokenizer = tableqa_model.tokenizer\n",
    "\n",
    "post_obj = WikiSQLPostprocessor(tokenizer,tqa_args)\n",
    "train_args = TrainingArguments(\n",
    "    output_dir=output_dir,\n",
    "    overwrite_output_dir=True,\n",
    "    do_train=True,\n",
    "    do_eval=True,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the wikisql data \n",
    "Note: The call load_data also internally converts the data to TAPAS traiming format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing wikisql dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n",
      "Using custom data configuration default\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing done\n",
      "Preprocessing done\n"
     ]
    }
   ],
   "source": [
    "# only a small fraction of training, dev data has been used for demonstration purpose\n",
    "train_dataset,eval_dataset = load_data(tqa_args.data_path_root,tokenizer,100,50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use the TableQATrainer with TAPAS specific collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = TableQATrainer(model=model,\n",
    "                                args=train_args,\n",
    "                                train_dataset=train_dataset if train_args.do_train else None,\n",
    "                                eval_dataset=eval_dataset if train_args.do_eval else None,\n",
    "                                tokenizer=tableqa_model.tokenizer,\n",
    "                                data_collator=TapasCollator(),\n",
    "                                post_process_function= post_obj.postprocess_prediction,\n",
    "                                compute_metrics=compute_denotation_accuracy         \n",
    "                                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the trainer metrics for training and validation phases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dccstor/cssblr/jaydeep/pqa_env/lib/python3.7/site-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  FutureWarning,\n",
      "***** Running training *****\n",
      "  Num examples = 68\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 27\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='27' max='27' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [27/27 05:14, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Saving model checkpoint to ../../models/tableqa/wikisql_nb\n",
      "Configuration saved in ../../models/tableqa/wikisql_nb/config.json\n",
      "Model weights saved in ../../models/tableqa/wikisql_nb/pytorch_model.bin\n",
      "tokenizer config file saved in ../../models/tableqa/wikisql_nb/tokenizer_config.json\n",
      "Special tokens file saved in ../../models/tableqa/wikisql_nb/special_tokens_map.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** train metrics *****\n",
      "  epoch                    =        3.0\n",
      "  total_flos               =    49988GF\n",
      "  train_loss               =     1.0374\n",
      "  train_runtime            = 0:05:27.66\n",
      "  train_samples_per_second =      0.623\n",
      "  train_steps_per_second   =      0.082\n"
     ]
    }
   ],
   "source": [
    "if train_args.do_train:\n",
    "    train_result = trainer.train()\n",
    "    trainer.save_model()\n",
    "    metrics = train_result.metrics\n",
    "    trainer.log_metrics(\"train\", metrics)\n",
    "    trainer.save_metrics(\"train\", metrics)\n",
    "    trainer.save_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 42\n",
      "  Batch size = 8\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** eval metrics *****\n",
      "  epoch                    =    3.0\n",
      "  eval_Denotation accuracy = 0.4524\n"
     ]
    }
   ],
   "source": [
    "if train_args.do_eval:\n",
    "      metrics = trainer.evaluate()\n",
    "      trainer.log_metrics(\"eval\", metrics)\n",
    "      trainer.save_metrics(\"eval\", metrics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.13",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "7d21c3d80586c4cada912fbe8386ca6018c05632b838068c2d847893c1029454"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}