From 78d88de6a0f3ebca92191f261fc3a6e31ae4164a Mon Sep 17 00:00:00 2001 From: BM-K Date: Wed, 10 May 2023 16:48:12 +0900 Subject: [PATCH] Update dataloader.py --- model/dataloader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/model/dataloader.py b/model/dataloader.py index 034996a..bbb39b5 100644 --- a/model/dataloader.py +++ b/model/dataloader.py @@ -1,8 +1,9 @@ from fastNLP.io import JsonLoader from transformers import AutoTokenizer +from preprocess.preprocess import TOKENIED_FILE_DIR -def get_data_path(PTM, dataset): - paths = {'train': f'tokenized_files/{dataset}/train.{PTM}.jsonl', 'val': f'tokenized_files/{dataset}/val.{PTM}.jsonl'} +def get_data_path(PTM, dataset): + paths = {'train': f'{TOKENIED_FILE_DIR}/{dataset}/train.{PTM}.jsonl', 'val': f'{TOKENIED_FILE_DIR}/{dataset}/val.{PTM}.jsonl'} return paths