From 8ca140b134ee6874901224e97cebd1a772842aa1 Mon Sep 17 00:00:00 2001 From: Andrei Vishniakov <31008759+avishniakov@users.noreply.github.com> Date: Wed, 3 Jul 2024 16:38:31 +0200 Subject: [PATCH] proper paths handling --- template/steps/finetune.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/template/steps/finetune.py b/template/steps/finetune.py index c353ffb..ce7238c 100644 --- a/template/steps/finetune.py +++ b/template/steps/finetune.py @@ -84,8 +84,8 @@ def finetune( if should_print: logger.info("Loading datasets...") tokenizer = load_tokenizer(base_model_id, use_fast=use_fast) - tokenized_train_dataset = load_from_disk(dataset_dir / "train") - tokenized_val_dataset = load_from_disk(dataset_dir / "val") + tokenized_train_dataset = load_from_disk(str((dataset_dir / "train").absolute())) + tokenized_val_dataset = load_from_disk(str((dataset_dir / "val").absolute())) if should_print: logger.info("Loading base model...") @@ -140,7 +140,6 @@ def finetune( if should_print: logger.info("Saving model...") - ft_model_dir = Path(ft_model_dir) if not use_accelerate or accelerator.is_main_process: ft_model_dir.mkdir(parents=True, exist_ok=True) if not use_accelerate: