Fix load_dataset and set modelscope==1.10.0 (modelscope#478)

* set modelscope==1.10.0

* update

* test

* update load dataset

* fix load_dataset

* fix load_dataset and update readme

README.md

@@ -106,6 +106,7 @@ print(os.getcwd())
 !pip3 install python-slugify
 !pip3 install onnxruntime==1.15.1
 !pip3 install edge-tts
+!pip3 install modelscope==1.10.0
 
 # Step4: Start the app service, click "public URL" or "local URL", upload your images to 
 # train your own model and then generate your digital twin.
@@ -140,6 +141,7 @@ pip3 install controlnet_aux==0.0.6
 pip3 install python-slugify
 pip3 install onnxruntime==1.15.1
 pip3 install edge-tts
+pip3 install modelscope==1.10.0
 
 # Step5 clone facechain from github
 GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/modelscope/facechain.git --depth 1

README_ZH.md

@@ -108,6 +108,7 @@ print(os.getcwd())
 !pip3 install python-slugify
 !pip3 install onnxruntime==1.15.1
 !pip3 install edge-tts
+!pip3 install modelscope==1.10.0
 
 # Step4: 启动服务，点击生成的URL即可访问web页面，上传照片开始训练和预测
 !python3 app.py
@@ -144,6 +145,7 @@ pip3 install controlnet_aux==0.0.6
 pip3 install python-slugify
 pip3 install onnxruntime==1.15.1
 pip3 install edge-tts
+pip3 install modelscope==1.10.0
 
 # Step5: 获取facechain源代码
 GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/modelscope/facechain.git --depth 1

app.py

@@ -740,9 +740,10 @@ def run(
         # Check Cuda Memory
         if torch.cuda.is_available():
             device = torch.device("cuda:0")
-            required_memory_bytes = 18 * (1024 ** 3) # 18GB
+            required_memory_bytes = 18 * (1024 ** 3)    # 18GB
             try:
-                tensor = torch.empty((required_memory_bytes // 4,), device = device) # create 18GB tensor to check the memory if enough
+                # create 18GB tensor to check the memory if enough
+                tensor = torch.empty((required_memory_bytes // 4,), device=device)
                 print("显存足够")
                 del tensor
             except RuntimeError as e:

facechain/train_text_to_image_lora.py

@@ -786,26 +786,28 @@ def main():
 
     # In distributed training, the load_dataset function guarantees that only one local process can concurrently
     # download the dataset.
-    if args.dataset_name is not None:
-        # Downloading and loading a dataset from the hub.
-        dataset = load_dataset(
-            args.dataset_name,
-            args.dataset_config_name,
-            cache_dir=args.cache_dir,
-            num_proc=8,
-        )
-    else:
-        # This branch will not be called
-        data_files = {}
-        if args.train_data_dir is not None:
-            data_files["train"] = os.path.join(args.train_data_dir, "**")
-        dataset = load_dataset(
-            "imagefolder",
-            data_files=data_files,
-            cache_dir=args.cache_dir,
-        )
-        # See more about loading custom images at
-        # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder
+    dataset = load_dataset("imagefolder", data_dir=args.dataset_name)
+
+    # if args.dataset_name is not None:
+    #     # Downloading and loading a dataset from the hub.
+    #     dataset = load_dataset(
+    #         args.dataset_name,
+    #         args.dataset_config_name,
+    #         cache_dir=args.cache_dir,
+    #         num_proc=8,
+    #     )
+    # else:
+    #     # This branch will not be called
+    #     data_files = {}
+    #     if args.train_data_dir is not None:
+    #         data_files["train"] = os.path.join(args.train_data_dir, "**")
+    #     dataset = load_dataset(
+    #         "imagefolder",
+    #         data_files=data_files,
+    #         cache_dir=args.cache_dir,
+    #     )
+    #     # See more about loading custom images at
+    #     # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder
 
     # Preprocessing the datasets.
     # We need to tokenize inputs and targets.

facechain/train_text_to_image_lora_sdxl.py

@@ -803,26 +803,28 @@ def main():
 
     # In distributed training, the load_dataset function guarantees that only one local process can concurrently
     # download the dataset.
-    if args.dataset_name is not None:
-        # Downloading and loading a dataset from the hub.
-        dataset = load_dataset(
-            args.dataset_name,
-            args.dataset_config_name,
-            cache_dir=args.cache_dir,
-            num_proc=8,
-        )
-    else:
-        # This branch will not be called
-        data_files = {}
-        if args.train_data_dir is not None:
-            data_files["train"] = os.path.join(args.train_data_dir, "**")
-        dataset = load_dataset(
-            "imagefolder",
-            data_files=data_files,
-            cache_dir=args.cache_dir,
-        )
-        # See more about loading custom images at
-        # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder
+    dataset = load_dataset("imagefolder", data_dir=args.dataset_name)
+
+    # if args.dataset_name is not None:
+    #     # Downloading and loading a dataset from the hub.
+    #     dataset = load_dataset(
+    #         args.dataset_name,
+    #         args.dataset_config_name,
+    #         cache_dir=args.cache_dir,
+    #         num_proc=8,
+    #     )
+    # else:
+    #     # This branch will not be called
+    #     data_files = {}
+    #     if args.train_data_dir is not None:
+    #         data_files["train"] = os.path.join(args.train_data_dir, "**")
+    #     dataset = load_dataset(
+    #         "imagefolder",
+    #         data_files=data_files,
+    #         cache_dir=args.cache_dir,
+    #     )
+    #     # See more about loading custom images at
+    #     # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder
 
     # Preprocessing the datasets.
     # We need to tokenize inputs and targets.

requirements.txt

@@ -3,7 +3,7 @@ transformers
 onnxruntime==1.15.1
 diffusers>=0.23.0
 invisible-watermark>=0.2.0
-modelscope>=1.9.2
+modelscope==1.10.0
 Pillow
 opencv-python
 torchvision