Skip to content

Commit

Permalink
minor edits in tapex inference notebook rendering
Browse files Browse the repository at this point in the history
  • Loading branch information
@jdpsen
jdpsen committed Jan 24, 2023
1 parent 1254b3e commit f94946f
Showing 1 changed file with 285 additions and 31 deletions.
316 changes: 285 additions & 31 deletions notebooks/tableqa/tapex_inference.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,6 @@
"import pandas as pd"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Loading pretrained model from huggingface\n",
"\n",
"This model was trained using PrimeQA library and uploaded to huggingface."
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -89,9 +80,10 @@
"output_type": "stream",
"text": [
"in predict for TapexModel with data: {'Actors': ['Brad Pitt', 'Leonardo Di Caprio', 'George Clooney'], 'Number of movies': ['87', '53', '69']} ,queries: ['how many movies Brad Pitt acted in', 'Name the actor who has been in 53 movies']\n",
"{\"time\":\"2023-01-23 15:31:41,610\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"loading from config at ../../primeqa/tableqa/tapex/configs/tapex_config_inference_wtq.json\"}\n",
"{\"time\":\"2023-01-23 15:31:42,074\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"WARNING\", \"message\": \"Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\"}\n",
"{\"time\":\"2023-01-23 15:31:42,076\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"Training/evaluation parameters Seq2SeqTrainingArguments(\n",
"{\"time\":\"2023-01-24 05:00:50,900\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"loading from config at ../../primeqa/tableqa/tapex/configs/tapex_config_inference_wtq.json\"}\n",
"{\"time\":\"2023-01-24 05:00:50,904\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"loading from config at ../../primeqa/tableqa/tapex/configs/tapex_config_inference_wtq.json\"}\n",
"{\"time\":\"2023-01-24 05:00:51,497\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"WARNING\", \"message\": \"Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\"}\n",
"{\"time\":\"2023-01-24 05:00:51,499\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"Training/evaluation parameters Seq2SeqTrainingArguments(\n",
"_n_gpu=0,\n",
"adafactor=False,\n",
"adam_beta1=0.9,\n",
Expand Down Expand Up @@ -148,7 +140,7 @@
"log_level=passive,\n",
"log_level_replica=passive,\n",
"log_on_each_node=True,\n",
"logging_dir=tapex_wtq/runs/Jan23_15-31-41_cccxl010,\n",
"logging_dir=tapex_wtq/runs/Jan24_05-00-50_cccxl010,\n",
"logging_first_step=False,\n",
"logging_nan_inf_filter=True,\n",
"logging_steps=10,\n",
Expand Down Expand Up @@ -292,40 +284,302 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"reading the config from ../../primeqa/tableqa/tapex/configs/tapex_config_wtq_eval.json\n"
"reading the config from ../../primeqa/tableqa/tapex/configs/tapex_config_eval_wtq.json\n",
"{\"time\":\"2023-01-24 05:01:59,356\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"loading from config at ../../primeqa/tableqa/tapex/configs/tapex_config_eval_wtq.json\"}\n",
"{\"time\":\"2023-01-24 05:01:59,357\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"loading from config at ../../primeqa/tableqa/tapex/configs/tapex_config_eval_wtq.json\"}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"PyTorch: setting up devices\n",
"The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\"time\":\"2023-01-24 05:01:59,444\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"WARNING\", \"message\": \"Process rank: -1, device: cpu, n_gpu: 0distributed training: False, 16-bits training: False\"}\n",
"{\"time\":\"2023-01-24 05:01:59,445\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"Training/evaluation parameters Seq2SeqTrainingArguments(\n",
"_n_gpu=0,\n",
"adafactor=False,\n",
"adam_beta1=0.9,\n",
"adam_beta2=0.999,\n",
"adam_epsilon=1e-08,\n",
"auto_find_batch_size=False,\n",
"bf16=False,\n",
"bf16_full_eval=False,\n",
"data_seed=None,\n",
"dataloader_drop_last=False,\n",
"dataloader_num_workers=0,\n",
"dataloader_pin_memory=True,\n",
"ddp_bucket_cap_mb=None,\n",
"ddp_find_unused_parameters=None,\n",
"ddp_timeout=1800,\n",
"debug=[],\n",
"deepspeed=None,\n",
"disable_tqdm=False,\n",
"do_eval=False,\n",
"do_predict=False,\n",
"do_train=False,\n",
"eval_accumulation_steps=None,\n",
"eval_delay=0,\n",
"eval_steps=1,\n",
"evaluation_strategy=no,\n",
"fp16=False,\n",
"fp16_backend=auto,\n",
"fp16_full_eval=False,\n",
"fp16_opt_level=O1,\n",
"fsdp=[],\n",
"fsdp_min_num_params=0,\n",
"fsdp_transformer_layer_cls_to_wrap=None,\n",
"full_determinism=False,\n",
"generation_max_length=None,\n",
"generation_num_beams=None,\n",
"gradient_accumulation_steps=8,\n",
"gradient_checkpointing=False,\n",
"greater_is_better=None,\n",
"group_by_length=False,\n",
"half_precision_backend=auto,\n",
"hub_model_id=None,\n",
"hub_private_repo=False,\n",
"hub_strategy=every_save,\n",
"hub_token=<HUB_TOKEN>,\n",
"ignore_data_skip=False,\n",
"include_inputs_for_metrics=False,\n",
"jit_mode_eval=False,\n",
"label_names=None,\n",
"label_smoothing_factor=0.1,\n",
"learning_rate=3e-05,\n",
"length_column_name=length,\n",
"load_best_model_at_end=False,\n",
"local_rank=-1,\n",
"log_level=passive,\n",
"log_level_replica=passive,\n",
"log_on_each_node=True,\n",
"logging_dir=tapex_wtq/runs/Jan24_05-01-59_cccxl010,\n",
"logging_first_step=False,\n",
"logging_nan_inf_filter=True,\n",
"logging_steps=1,\n",
"logging_strategy=steps,\n",
"lr_scheduler_type=linear,\n",
"max_grad_norm=1.0,\n",
"max_steps=20,\n",
"metric_for_best_model=None,\n",
"mp_parameters=,\n",
"no_cuda=False,\n",
"num_train_epochs=3.0,\n",
"optim=adamw_hf,\n",
"output_dir=tapex_wtq,\n",
"overwrite_output_dir=False,\n",
"past_index=-1,\n",
"per_device_eval_batch_size=4,\n",
"per_device_train_batch_size=4,\n",
"predict_with_generate=False,\n",
"prediction_loss_only=False,\n",
"push_to_hub=False,\n",
"push_to_hub_model_id=None,\n",
"push_to_hub_organization=None,\n",
"push_to_hub_token=<PUSH_TO_HUB_TOKEN>,\n",
"ray_scope=last,\n",
"remove_unused_columns=True,\n",
"report_to=[],\n",
"resume_from_checkpoint=None,\n",
"run_name=tapex_wtq,\n",
"save_on_each_node=False,\n",
"save_steps=1,\n",
"save_strategy=steps,\n",
"save_total_limit=None,\n",
"seed=42,\n",
"sharded_ddp=[],\n",
"skip_memory_metrics=True,\n",
"sortish_sampler=False,\n",
"tf32=None,\n",
"torchdynamo=None,\n",
"tpu_metrics_debug=False,\n",
"tpu_num_cores=None,\n",
"use_ipex=False,\n",
"use_legacy_prediction_loop=False,\n",
"use_mps_device=False,\n",
"warmup_ratio=0.0,\n",
"warmup_steps=1000,\n",
"weight_decay=0.01,\n",
"xpu_backend=None,\n",
")\"}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"loading configuration file config.json from cache at /u/jaydesen/.cache/huggingface/hub/models--PrimeQA--tableqa_tapex_wtq/snapshots/6684181ee6ed047224a011f4076057c16640e964/config.json\n",
"Model config BartConfig {\n",
" \"_name_or_path\": \"PrimeQA/tableqa_tapex_wtq\",\n",
" \"activation_dropout\": 0.0,\n",
" \"activation_function\": \"gelu\",\n",
" \"architectures\": [\n",
" \"BartForConditionalGeneration\"\n",
" ],\n",
" \"attention_dropout\": 0.1,\n",
" \"bos_token_id\": 0,\n",
" \"classifier_dropout\": 0.0,\n",
" \"d_model\": 1024,\n",
" \"decoder_attention_heads\": 16,\n",
" \"decoder_ffn_dim\": 4096,\n",
" \"decoder_layerdrop\": 0.0,\n",
" \"decoder_layers\": 12,\n",
" \"decoder_start_token_id\": 2,\n",
" \"dropout\": 0.1,\n",
" \"encoder_attention_heads\": 16,\n",
" \"encoder_ffn_dim\": 4096,\n",
" \"encoder_layerdrop\": 0.0,\n",
" \"encoder_layers\": 12,\n",
" \"eos_token_id\": 2,\n",
" \"forced_bos_token_id\": 0,\n",
" \"forced_eos_token_id\": 2,\n",
" \"id2label\": {\n",
" \"0\": \"LABEL_0\",\n",
" \"1\": \"LABEL_1\",\n",
" \"2\": \"LABEL_2\"\n",
" },\n",
" \"init_std\": 0.02,\n",
" \"is_encoder_decoder\": true,\n",
" \"label2id\": {\n",
" \"LABEL_0\": 0,\n",
" \"LABEL_1\": 1,\n",
" \"LABEL_2\": 2\n",
" },\n",
" \"max_length\": 1024,\n",
" \"max_position_embeddings\": 1024,\n",
" \"model_type\": \"bart\",\n",
" \"num_beams\": 4,\n",
" \"num_hidden_layers\": 12,\n",
" \"pad_token_id\": 1,\n",
" \"scale_embedding\": false,\n",
" \"torch_dtype\": \"float32\",\n",
" \"transformers_version\": \"4.24.0\",\n",
" \"use_cache\": true,\n",
" \"vocab_size\": 50265\n",
"}\n",
"\n",
"loading file vocab.json from cache at /u/jaydesen/.cache/huggingface/hub/models--PrimeQA--tableqa_tapex_wtq/snapshots/6684181ee6ed047224a011f4076057c16640e964/vocab.json\n",
"loading file merges.txt from cache at /u/jaydesen/.cache/huggingface/hub/models--PrimeQA--tableqa_tapex_wtq/snapshots/6684181ee6ed047224a011f4076057c16640e964/merges.txt\n",
"loading file added_tokens.json from cache at None\n",
"loading file special_tokens_map.json from cache at /u/jaydesen/.cache/huggingface/hub/models--PrimeQA--tableqa_tapex_wtq/snapshots/6684181ee6ed047224a011f4076057c16640e964/special_tokens_map.json\n",
"loading file tokenizer_config.json from cache at /u/jaydesen/.cache/huggingface/hub/models--PrimeQA--tableqa_tapex_wtq/snapshots/6684181ee6ed047224a011f4076057c16640e964/tokenizer_config.json\n",
"loading weights file pytorch_model.bin from cache at /u/jaydesen/.cache/huggingface/hub/models--PrimeQA--tableqa_tapex_wtq/snapshots/6684181ee6ed047224a011f4076057c16640e964/pytorch_model.bin\n",
"All model checkpoint weights were used when initializing BartForConditionalGeneration.\n",
"\n",
"All the weights of BartForConditionalGeneration were initialized from the model checkpoint at PrimeQA/tableqa_tapex_wtq.\n",
"If your task is similar to the task the model of the checkpoint was trained on, you can already use BartForConditionalGeneration for predictions without further training.\n",
"PyTorch: setting up devices\n",
"The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\"time\":\"2023-01-24 05:02:18,910\", \"name\": \"datasets.builder\", \"level\": \"WARNING\", \"message\": \"No config specified, defaulting to: wikitablequestions/random-split-1\"}\n",
"{\"time\":\"2023-01-24 05:02:19,204\", \"name\": \"datasets.builder\", \"level\": \"WARNING\", \"message\": \"Reusing dataset wikitablequestions (/u/jaydesen/.cache/huggingface/datasets/wikitablequestions/random-split-1/1.0.2/4845d83f334ed5e4a8e0420731e64d57f696feb62e7d3a47b84037864fb8317c)\"}\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5a7c82e49d3847a1ab8c5050e314e7d2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/3 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\"time\":\"2023-01-24 05:02:19,715\", \"name\": \"datasets.arrow_dataset\", \"level\": \"WARNING\", \"message\": \"Loading cached processed dataset at /u/jaydesen/.cache/huggingface/datasets/wikitablequestions/random-split-1/1.0.2/4845d83f334ed5e4a8e0420731e64d57f696feb62e7d3a47b84037864fb8317c/cache-3228295aec1898e6.arrow\"}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"max_steps is given, it will override any value given in num_train_epochs\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\"time\":\"2023-01-24 05:02:19,874\", \"name\": \"primeqa.tableqa.tapex.tapex_component\", \"level\": \"INFO\", \"message\": \"*** Evaluate ***\"}\n"
]
},
{
"ename": "NameError",
"evalue": "name 'config_json' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipykernel_78131/3701716783.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mreader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTapexReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"../../primeqa/tableqa/tapex/configs/tapex_config_wtq_eval.json\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mreader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/dccstor/irl-tableqa/jaydeep/primeqa/primeqa/tableqa/tapex/tapex_component.py\u001b[0m in \u001b[0;36meval\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 284\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 286\u001b[0;31m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"loading from config at {config_json}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 287\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_config_json\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mNameError\u001b[0m: name 'config_json' is not defined"
"name": "stderr",
"output_type": "stream",
"text": [
"***** Running Evaluation *****\n",
" Num examples = 10\n",
" Batch size = 4\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"max_eval_samples is set as: 10\n"
]
},
{
"data": {
"text/html": [
"\n",
" <div>\n",
" \n",
" <progress value='3' max='3' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" [3/3 00:45]\n",
" </div>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"***** eval metrics *****\n",
" eval_denotation_accuracy = 0.7\n",
" eval_loss = 1.7337\n",
" eval_runtime = 0:01:17.52\n",
" eval_samples = 10\n",
" eval_samples_per_second = 0.129\n",
" eval_steps_per_second = 0.039\n"
]
}
],
"source": [
"reader = TapexReader(\"../../primeqa/tableqa/tapex/configs/tapex_config_eval_wtq.json\")\n",
"reader.eval()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down

0 comments on commit f94946f

Please sign in to comment.