Add RealSR (PaddlePaddle#143)

* add realsr model

configs/realsr_bicubic_noise_x4_df2k.yaml

@@ -0,0 +1,131 @@
+total_iters: 60000
+output_dir: output_dir
+# tensor range for function tensor2img
+min_max:
+  (0., 1.)
+
+model:
+  name: ESRGAN
+  generator:
+    name: RRDBNet
+    in_nc: 3
+    out_nc: 3
+    nf: 64
+    nb: 23
+  discriminator:
+    name: VGGDiscriminator128
+    in_channels: 3
+    num_feat: 64
+  pixel_criterion:
+    name: L1Loss
+    loss_weight: !!float 1e-2
+  perceptual_criterion:
+    name: PerceptualLoss
+    layer_weights:
+      '34': 1.0
+    perceptual_weight: 1.0
+    style_weight: 0.0
+    norm_img: False
+  gan_criterion:
+    name: GANLoss
+    gan_mode: vanilla
+    loss_weight: !!float 5e-3
+
+dataset:
+  train:
+    name: SRDataset
+    gt_folder: data/realsr_preprocess/DF2K/generated/tdsr/HR_sub/
+    lq_folder: data/realsr_preprocess/DF2K/generated/tdsr/LR_sub/
+    num_workers: 4
+    batch_size: 16
+    scale: 4
+    preprocess:
+      - name: LoadImageFromFile
+        key: lq
+      - name: LoadImageFromFile
+        key: gt
+      - name: Transforms
+        input_keys: [lq, gt]
+        pipeline:
+          - name: SRPairedRandomCrop
+            gt_patch_size: 128
+            scale: 4
+            keys: [image, image]
+          - name: PairedRandomHorizontalFlip
+            keys: [image, image]
+          - name: PairedRandomVerticalFlip
+            keys: [image, image]
+          - name: PairedRandomTransposeHW
+            keys: [image, image]
+          - name: Transpose
+            keys: [image, image]
+          - name: Normalize
+            mean: [0., .0, 0.]
+            std: [255., 255., 255.]
+            keys: [image, image]
+          - name: SRNoise
+            noise_path: data/realsr_preprocess/DF2K/Corrupted_noise/
+            size: 32
+            keys: [image]
+  test:
+    name: SRDataset
+    gt_folder: data/DIV2K/val_set14/Set14
+    lq_folder: data/DIV2K/val_set14/Set14_bicLRx4
+    scale: 4
+    preprocess:
+      - name: LoadImageFromFile
+        key: lq
+      - name: LoadImageFromFile
+        key: gt
+      - name: Transforms
+        input_keys: [lq, gt]
+        pipeline:
+          - name: Transpose
+            keys: [image, image]
+          - name: Normalize
+            mean: [0., .0, 0.]
+            std: [255., 255., 255.]
+            keys: [image, image]
+
+lr_scheduler:
+  name: MultiStepDecay
+  learning_rate: 0.0001
+  milestones: [5000, 10000, 20000, 30000]
+  gamma: 0.5
+
+optimizer:
+  optimG:
+    name: Adam
+    net_names:
+      - generator
+    weight_decay: 0.0
+    beta1: 0.9
+    beta2: 0.999
+  optimD:
+    name: Adam
+    net_names:
+      - discriminator
+    weight_decay: 0.0
+    beta1: 0.9
+    beta2: 0.999
+
+validate:
+  interval: 5000
+  save_img: false
+
+  metrics:
+    psnr: # metric name, can be arbitrary
+      name: PSNR
+      crop_border: 4
+      test_y_channel: false
+    ssim:
+      name: SSIM
+      crop_border: 4
+      test_y_channel: false
+
+log_config:
+  interval: 100
+  visiual_interval: 500
+
+snapshot_config:
+  interval: 5000

configs/realsr_kernel_noise_x4_dped.yaml

@@ -0,0 +1,131 @@
+total_iters: 60000
+output_dir: output_dir
+# tensor range for function tensor2img
+min_max:
+  (0., 1.)
+
+model:
+  name: ESRGAN
+  generator:
+    name: RRDBNet
+    in_nc: 3
+    out_nc: 3
+    nf: 64
+    nb: 23
+  discriminator:
+    name: VGGDiscriminator128
+    in_channels: 3
+    num_feat: 64
+  pixel_criterion:
+    name: L1Loss
+    loss_weight: !!float 1e-2
+  perceptual_criterion:
+    name: PerceptualLoss
+    layer_weights:
+      '34': 1.0
+    perceptual_weight: 1.0
+    style_weight: 0.0
+    norm_img: False
+  gan_criterion:
+    name: GANLoss
+    gan_mode: vanilla
+    loss_weight: !!float 5e-3
+
+dataset:
+  train:
+    name: SRDataset
+    gt_folder: data/realsr_preprocess/DPED/generated/clean/train_tdsr/HR/
+    lq_folder: data/realsr_preprocess/DPED/generated/clean/train_tdsr/LR/
+    num_workers: 4
+    batch_size: 16
+    scale: 4
+    preprocess:
+      - name: LoadImageFromFile
+        key: lq
+      - name: LoadImageFromFile
+        key: gt
+      - name: Transforms
+        input_keys: [lq, gt]
+        pipeline:
+          - name: SRPairedRandomCrop
+            gt_patch_size: 128
+            scale: 4
+            keys: [image, image]
+          - name: PairedRandomHorizontalFlip
+            keys: [image, image]
+          - name: PairedRandomVerticalFlip
+            keys: [image, image]
+          - name: PairedRandomTransposeHW
+            keys: [image, image]
+          - name: Transpose
+            keys: [image, image]
+          - name: Normalize
+            mean: [0., .0, 0.]
+            std: [255., 255., 255.]
+            keys: [image, image]
+          - name: SRNoise
+            noise_path: data/realsr_preprocess/DPED/DPEDiphone_noise/
+            size: 32
+            keys: [image]
+  test:
+    name: SRDataset
+    gt_folder: data/DIV2K/val_set14/Set14
+    lq_folder: data/DIV2K/val_set14/Set14_bicLRx4
+    scale: 4
+    preprocess:
+      - name: LoadImageFromFile
+        key: lq
+      - name: LoadImageFromFile
+        key: gt
+      - name: Transforms
+        input_keys: [lq, gt]
+        pipeline:
+          - name: Transpose
+            keys: [image, image]
+          - name: Normalize
+            mean: [0., .0, 0.]
+            std: [255., 255., 255.]
+            keys: [image, image]
+
+lr_scheduler:
+  name: MultiStepDecay
+  learning_rate: 0.0001
+  milestones: [5000, 10000, 20000, 30000]
+  gamma: 0.5
+
+optimizer:
+  optimG:
+    name: Adam
+    net_names:
+      - generator
+    weight_decay: 0.0
+    beta1: 0.9
+    beta2: 0.999
+  optimD:
+    name: Adam
+    net_names:
+      - discriminator
+    weight_decay: 0.0
+    beta1: 0.9
+    beta2: 0.999
+
+validate:
+  interval: 5000
+  save_img: false
+
+  metrics:
+    psnr: # metric name, can be arbitrary
+      name: PSNR
+      crop_border: 4
+      test_y_channel: false
+    ssim:
+      name: SSIM
+      crop_border: 4
+      test_y_channel: false
+
+log_config:
+  interval: 100
+  visiual_interval: 500
+
+snapshot_config:
+  interval: 5000

data/realsr_preprocess/collect_noise.py

@@ -0,0 +1,84 @@
+from PIL import Image
+import numpy as np
+import os.path as osp
+import glob
+import os
+import argparse
+import yaml
+
+parser = argparse.ArgumentParser(description='create a dataset')
+parser.add_argument('--dataset',
+                    default='df2k',
+                    type=str,
+                    help='selecting different datasets')
+parser.add_argument('--artifacts',
+                    default='',
+                    type=str,
+                    help='selecting different artifacts type')
+parser.add_argument('--cleanup_factor',
+                    default=2,
+                    type=int,
+                    help='downscaling factor for image cleanup')
+parser.add_argument('--upscale_factor',
+                    default=4,
+                    type=int,
+                    choices=[4],
+                    help='super resolution upscale factor')
+opt = parser.parse_args()
+
+# define input and target directories
+with open('./preprocess/paths.yml', 'r') as stream:
+    PATHS = yaml.load(stream)
+
+
+def noise_patch(rgb_img, sp, max_var, min_mean):
+    img = rgb_img.convert('L')
+    rgb_img = np.array(rgb_img)
+    img = np.array(img)
+
+    w, h = img.shape
+    collect_patchs = []
+
+    for i in range(0, w - sp, sp):
+        for j in range(0, h - sp, sp):
+            patch = img[i:i + sp, j:j + sp]
+            var_global = np.var(patch)
+            mean_global = np.mean(patch)
+            if var_global < max_var and mean_global > min_mean:
+                rgb_patch = rgb_img[i:i + sp, j:j + sp, :]
+                collect_patchs.append(rgb_patch)
+
+    return collect_patchs
+
+
+if __name__ == '__main__':
+
+    if opt.dataset == 'df2k':
+        img_dir = PATHS[opt.dataset][opt.artifacts]['source']
+        noise_dir = PATHS['datasets']['df2k'] + '/Corrupted_noise'
+        sp = 256
+        max_var = 20
+        min_mean = 0
+    else:
+        img_dir = PATHS[opt.dataset][opt.artifacts]['hr']['train']
+        noise_dir = PATHS['datasets']['dped'] + '/DPEDiphone_noise'
+        sp = 256
+        max_var = 20
+        min_mean = 50
+
+    assert not os.path.exists(noise_dir)
+    os.mkdir(noise_dir)
+
+    img_paths = sorted(glob.glob(osp.join(img_dir, '*.png')))
+    cnt = 0
+    for path in img_paths:
+        img_name = osp.splitext(osp.basename(path))[0]
+        print('**********', img_name, '**********')
+        img = Image.open(path).convert('RGB')
+        patchs = noise_patch(img, sp, max_var, min_mean)
+        for idx, patch in enumerate(patchs):
+            save_path = osp.join(noise_dir,
+                                 '{}_{:03}.png'.format(img_name, idx))
+            cnt += 1
+            print('collect:', cnt, save_path)
+            Image.fromarray(patch).save(save_path)