Skip to content

Commit

Permalink
Slight cleanup of superfluous comment and space after comma
Browse files Browse the repository at this point in the history
  • Loading branch information
MalikMAlna committed Apr 6, 2023
1 parent dc08c43 commit b3be94a
Showing 1 changed file with 1 addition and 2 deletions.
3 changes: 1 addition & 2 deletions data.py
Original file line number Diff line number Diff line change
@@ -57,7 +57,6 @@ def load_data(config, tokenizer):
dataset_path = config["dataset_path"]

if os.path.exists(dataset_path):
# check if path is a directory
if os.path.isdir(dataset_path):
files = glob.glob(os.path.join(dataset_path, "*_clean.jsonl"))
else:
@@ -68,7 +67,7 @@ def load_data(config, tokenizer):
dataset = load_dataset("json", data_files=files, split="train")

else:
dataset = load_dataset(dataset_path,split='train')
dataset = load_dataset(dataset_path, split='train')

dataset = dataset.train_test_split(test_size=.05, seed=config["seed"])

0 comments on commit b3be94a

Please sign in to comment.