Merge branch 'dev' of github.com:BeyonderXX/InstructUIE into dev

.gitignore

@@ -132,4 +132,8 @@ dmypy.json
 hostfile
 wandb/
 output/
-exp.py
+exp.py
+
+.DS_Store
+
+todo

scripts/train_gptneox.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -x
+
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+export TRANSFORMERS_CACHE=/root/.cache/huggingface
+
+port=$(shuf -i25000-30000 -n1)
+
+
+# A100 * 8 on gpt-neox-20b
+# 注意更改 model_name_or_path，data_dir等值
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 deepspeed --master_port $port ../src/run_uie.py \
+   --do_train \
+   --do_predict \
+   --predict_with_generate \
+   --model_name_or_path /root/MODELS/gpt-neox-chat-base-20b/models--togethercomputer--GPT-NeoXT-Chat-Base-20B/snapshots/gpt-neox-chat-base-20b \
+   --data_dir ../workspace/IE_data_v2 \
+   --task_config_dir ../configs/multi_task_configs \
+   --instruction_file ../configs/instruction_config.json \
+   --instruction_strategy single \
+   --output_dir ../output/gpt-neox-20b-ie-single \
+   --per_device_train_batch_size 1 \
+   --per_device_eval_batch_size 16 \
+   --gradient_accumulation_steps 6 \
+   --learning_rate 5e-05 \
+   --num_train_epochs 10 \
+   --deepspeed ../configs/ds_configs/stage2.config \
+   --run_name gpt-neox-20b-mult-test-experiment \
+   --max_source_length 512 \
+   --max_target_length 50 \
+   --generation_max_length 50 \
+   --max_num_instances_per_task 10 \
+   --max_num_instances_per_eval_task 200 \
+   --add_task_name False \
+   --add_dataset_name False \
+   --num_examples 0 \
+   --instruction_strategy multiple \
+   --overwrite_output_dir \
+   --overwrite_cache \
+   --lr_scheduler_type constant \
+   --warmup_steps 0 \
+   --logging_strategy steps \
+   --logging_steps 100 \
+   --evaluation_strategy no \
+   --save_strategy steps \
+   --save_steps 2000

src/calculate_f1.py

@@ -0,0 +1,49 @@
+import json
+import os
+
+from evaluation.evaluator import *
+
+def calculate_f1(output_dir, tasks=('RE','EE','NER')):
+    EvaluatorDict = {
+        'RE':EvaluatorRE,
+        'EE':EvaluatorEvent,
+        'NER':EvaluatorNER
+    }
+    task_dict = {task: dict() for task in tasks}
+    task_path = os.path.join(output_dir, 'predict_eval_predictions.jsonl')
+    report_dir = os.path.join(output_dir, 'report')
+    with open(task_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            data = json.loads(line)
+            task_name = data['Task']
+            dataset_name = data['Dataset']
+            Evaluator = EvaluatorDict[task_name]
+            if dataset_name not in task_dict[task_name]:
+                task_dict[task_name][dataset_name] = Evaluator()
+            task_dict[task_name][dataset_name].add(data, data['Prediction'])
+
+    # export report
+    if not os.path.exists(report_dir):
+        os.mkdir(report_dir)
+
+    # export tsv
+    for task_name, eval_dict in task_dict.items():
+        print('\n'+'-'*16+task_name+'-'*16+'\n')
+        rows = []
+        scores = []
+        for dataset_name, evaluator in eval_dict.items():
+            evaluator.dump_audit_report(os.path.join(report_dir, dataset_name+'.json'))
+            rows.append((dataset_name, evaluator.get_metric()))
+            scores.append(evaluator.get_metric())
+        rows = sorted(rows, key=lambda x: x[0].lower())
+        if len(scores) == 0:
+            continue
+        rows.append(('Average', sum(scores)/len(scores)))
+        with open(os.path.join(report_dir, 'report_%s.tsv'%task_name), 'w', encoding='utf-8') as f:
+            for row in rows:
+                f.write(f'{row[0]}\t{row[1]}\n')
+                print('%48s\t%g'%row)
+
+if __name__ == '__main__':
+    root = '../output/llama-7b'
+    calculate_f1(root)