Rename paddle.batch to paddle.io.batch (PaddlePaddle#290)

demo/auto_prune/train.py

@@ -116,8 +116,8 @@ def if_exist(var):
 
         fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
 
-    val_reader = paddle.batch(val_reader, batch_size=args.batch_size)
-    train_reader = paddle.batch(
+    val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size)
+    train_reader = paddle.fluid.io.batch(
         train_reader, batch_size=args.batch_size, drop_last=True)
 
     train_feeder = feeder = fluid.DataFeeder([image, label], place)

demo/auto_prune/train_finetune.py

@@ -34,12 +34,12 @@
 
 model_list = [m for m in dir(models) if "__" not in m]
 ratiolist = [
-#    [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08],
-#    [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01],
-        ]
+    #    [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08],
+    #    [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01],
+]
 
 
-def save_model(args, exe, train_prog, eval_prog,info):
+def save_model(args, exe, train_prog, eval_prog, info):
     model_path = os.path.join(args.model_save_dir, args.model, str(info))
     if not os.path.isdir(model_path):
         os.makedirs(model_path)
@@ -58,29 +58,31 @@ def piecewise_decay(args):
         regularization=fluid.regularizer.L2Decay(args.l2_decay))
     return optimizer
 
+
 def cosine_decay(args):
     step = int(math.ceil(float(args.total_images) / args.batch_size))
     learning_rate = fluid.layers.cosine_decay(
-        learning_rate=args.lr,
-        step_each_epoch=step,
-        epochs=args.num_epochs)
+        learning_rate=args.lr, step_each_epoch=step, epochs=args.num_epochs)
     optimizer = fluid.optimizer.Momentum(
         learning_rate=learning_rate,
         momentum=args.momentum_rate,
         regularization=fluid.regularizer.L2Decay(args.l2_decay))
     return optimizer
 
+
 def create_optimizer(args):
     if args.lr_strategy == "piecewise_decay":
         return piecewise_decay(args)
     elif args.lr_strategy == "cosine_decay":
         return cosine_decay(args)
 
+
 def compress(args):
-    class_dim=1000
-    image_shape="3,224,224"
+    class_dim = 1000
+    image_shape = "3,224,224"
     image_shape = [int(m) for m in image_shape.split(",")]
-    assert args.model in model_list, "{} is not in lists: {}".format(args.model, model_list)
+    assert args.model in model_list, "{} is not in lists: {}".format(
+        args.model, model_list)
     image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
     label = fluid.layers.data(name='label', shape=[1], dtype='int64')
     # model definition
@@ -98,99 +100,122 @@ def compress(args):
     exe.run(fluid.default_startup_program())
 
     if args.pretrained_model:
+
         def if_exist(var):
-            exist =  os.path.exists(os.path.join(args.pretrained_model, var.name))
-            print("exist",exist)
+            exist = os.path.exists(
+                os.path.join(args.pretrained_model, var.name))
+            print("exist", exist)
             return exist
+
         #fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
 
-    val_reader = paddle.batch(reader.val(), batch_size=args.batch_size)
-    train_reader = paddle.batch(
+    val_reader = paddle.fluid.io.batch(reader.val(), batch_size=args.batch_size)
+    train_reader = paddle.fluid.io.batch(
         reader.train(), batch_size=args.batch_size, drop_last=True)
 
     train_feeder = feeder = fluid.DataFeeder([image, label], place)
-    val_feeder = feeder = fluid.DataFeeder([image, label], place, program=val_program)
+    val_feeder = feeder = fluid.DataFeeder(
+        [image, label], place, program=val_program)
 
     def test(epoch, program):
         batch_id = 0
         acc_top1_ns = []
         acc_top5_ns = []
         for data in val_reader():
             start_time = time.time()
-            acc_top1_n, acc_top5_n = exe.run(program,
-                                           feed=train_feeder.feed(data),
-                                           fetch_list=[acc_top1.name, acc_top5.name])
+            acc_top1_n, acc_top5_n = exe.run(
+                program,
+                feed=train_feeder.feed(data),
+                fetch_list=[acc_top1.name, acc_top5.name])
             end_time = time.time()
-            print("Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".format(epoch, batch_id, np.mean(acc_top1_n), np.mean(acc_top5_n), end_time-start_time))
+            print(
+                "Eval epoch[{}] batch[{}] - acc_top1: {}; acc_top5: {}; time: {}".
+                format(epoch, batch_id,
+                       np.mean(acc_top1_n),
+                       np.mean(acc_top5_n), end_time - start_time))
             acc_top1_ns.append(np.mean(acc_top1_n))
             acc_top5_ns.append(np.mean(acc_top5_n))
             batch_id += 1
 
-        print("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(epoch, np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
-
+        print("Final eval epoch[{}] - acc_top1: {}; acc_top5: {}".format(
+            epoch,
+            np.mean(np.array(acc_top1_ns)), np.mean(np.array(acc_top5_ns))))
 
     def train(epoch, program):
 
         build_strategy = fluid.BuildStrategy()
         exec_strategy = fluid.ExecutionStrategy()
         train_program = fluid.compiler.CompiledProgram(
-                                   program).with_data_parallel(
-                                   loss_name=avg_cost.name,
-                                   build_strategy=build_strategy,
-                                   exec_strategy=exec_strategy)
+            program).with_data_parallel(
+                loss_name=avg_cost.name,
+                build_strategy=build_strategy,
+                exec_strategy=exec_strategy)
 
         batch_id = 0
         for data in train_reader():
             start_time = time.time()
-            loss_n, acc_top1_n, acc_top5_n,lr_n = exe.run(train_program,
-                                               feed=train_feeder.feed(data),
-                                               fetch_list=[avg_cost.name, acc_top1.name, acc_top5.name,"learning_rate"])
+            loss_n, acc_top1_n, acc_top5_n, lr_n = exe.run(
+                train_program,
+                feed=train_feeder.feed(data),
+                fetch_list=[
+                    avg_cost.name, acc_top1.name, acc_top5.name,
+                    "learning_rate"
+                ])
             end_time = time.time()
             loss_n = np.mean(loss_n)
             acc_top1_n = np.mean(acc_top1_n)
             acc_top5_n = np.mean(acc_top5_n)
             lr_n = np.mean(lr_n)
-            print("epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {};lrn: {}; time: {}".format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, lr_n,end_time-start_time))
+            print(
+                "epoch[{}]-batch[{}] - loss: {}; acc_top1: {}; acc_top5: {};lrn: {}; time: {}".
+                format(epoch, batch_id, loss_n, acc_top1_n, acc_top5_n, lr_n,
+                       end_time - start_time))
             batch_id += 1
 
     params = []
     for param in fluid.default_main_program().global_block().all_parameters():
         #if "_weights" in  param.name and "conv1_weights" not in param.name:
-        if "_sep_weights" in  param.name:
+        if "_sep_weights" in param.name:
             params.append(param.name)
-    print("fops before pruning: {}".format(flops(fluid.default_main_program())))
+    print("fops before pruning: {}".format(
+        flops(fluid.default_main_program())))
     pruned_program_iter = fluid.default_main_program()
     pruned_val_program_iter = val_program
     for ratios in ratiolist:
         pruner = Pruner()
-        pruned_val_program_iter = pruner.prune(pruned_val_program_iter,
-                                  fluid.global_scope(),
-                                  params=params,
-                                  ratios=ratios,
-                                  place=place,
-                                  only_graph=True)
-
-
-        pruned_program_iter = pruner.prune(pruned_program_iter,
-                                  fluid.global_scope(),
-                                  params=params,
-                                  ratios=ratios,
-                                  place=place)
+        pruned_val_program_iter = pruner.prune(
+            pruned_val_program_iter,
+            fluid.global_scope(),
+            params=params,
+            ratios=ratios,
+            place=place,
+            only_graph=True)
+
+        pruned_program_iter = pruner.prune(
+            pruned_program_iter,
+            fluid.global_scope(),
+            params=params,
+            ratios=ratios,
+            place=place)
         print("fops after pruning: {}".format(flops(pruned_program_iter)))
-
     """ do not inherit learning rate """
-    if(os.path.exists(args.pretrained_model + "/learning_rate")):
-      os.remove( args.pretrained_model + "/learning_rate")
-    if(os.path.exists(args.pretrained_model + "/@LR_DECAY_COUNTER@")):
-      os.remove( args.pretrained_model + "/@LR_DECAY_COUNTER@")
-    fluid.io.load_vars(exe, args.pretrained_model , main_program = pruned_program_iter,  predicate=if_exist)
+    if (os.path.exists(args.pretrained_model + "/learning_rate")):
+        os.remove(args.pretrained_model + "/learning_rate")
+    if (os.path.exists(args.pretrained_model + "/@LR_DECAY_COUNTER@")):
+        os.remove(args.pretrained_model + "/@LR_DECAY_COUNTER@")
+    fluid.io.load_vars(
+        exe,
+        args.pretrained_model,
+        main_program=pruned_program_iter,
+        predicate=if_exist)
 
     pruned_program = pruned_program_iter
     pruned_val_program = pruned_val_program_iter
     for i in range(args.num_epochs):
         train(i, pruned_program)
         test(i, pruned_val_program)
-        save_model(args,exe,pruned_program,pruned_val_program,i)
+        save_model(args, exe, pruned_program, pruned_val_program, i)
+
 
 def main():
     args = parser.parse_args()

demo/auto_prune/train_iterator.py

@@ -41,9 +41,10 @@
 
 model_list = [m for m in dir(models) if "__" not in m]
 ratiolist = [
-#    [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08],
-#    [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01],
-        ]
+    #    [0.06, 0.0, 0.09, 0.03, 0.09, 0.02, 0.05, 0.03, 0.0, 0.07, 0.07, 0.05, 0.08],
+    #    [0.08, 0.02, 0.03, 0.13, 0.1, 0.06, 0.03, 0.04, 0.14, 0.02, 0.03, 0.02, 0.01],
+]
+
 
 def piecewise_decay(args):
     step = int(math.ceil(float(args.total_images) / args.batch_size))
@@ -121,8 +122,8 @@ def if_exist(var):
 
 #        fluid.io.load_vars(exe, args.pretrained_model, predicate=if_exist)
 
-    val_reader = paddle.batch(val_reader, batch_size=args.batch_size)
-    train_reader = paddle.batch(
+    val_reader = paddle.fluid.io.batch(val_reader, batch_size=args.batch_size)
+    train_reader = paddle.fluid.io.batch(
         train_reader, batch_size=args.batch_size, drop_last=True)
 
     train_feeder = feeder = fluid.DataFeeder([image, label], place)
@@ -194,21 +195,26 @@ def train(epoch, program):
 
     for ratios in ratiolist:
         pruner = Pruner()
-        pruned_val_program_iter = pruner.prune(pruned_val_program_iter,
-                                  fluid.global_scope(),
-                                  params=params,
-                                  ratios=ratios,
-                                  place=place,
-                                  only_graph=True)
-
-
-        pruned_program_iter = pruner.prune(pruned_program_iter,
-                                  fluid.global_scope(),
-                                  params=params,
-                                  ratios=ratios,
-                                  place=place)
+        pruned_val_program_iter = pruner.prune(
+            pruned_val_program_iter,
+            fluid.global_scope(),
+            params=params,
+            ratios=ratios,
+            place=place,
+            only_graph=True)
+
+        pruned_program_iter = pruner.prune(
+            pruned_program_iter,
+            fluid.global_scope(),
+            params=params,
+            ratios=ratios,
+            place=place)
         print("fops after pruning: {}".format(flops(pruned_program_iter)))
-    fluid.io.load_vars(exe, args.pretrained_model , main_program = pruned_program_iter,  predicate=if_exist)
+    fluid.io.load_vars(
+        exe,
+        args.pretrained_model,
+        main_program=pruned_program_iter,
+        predicate=if_exist)
 
     pruner = AutoPruner(
         pruned_val_program_iter,
@@ -238,8 +244,6 @@ def train(epoch, program):
         pruner.reward(score)
 
 
-
-
 def main():
     args = parser.parse_args()
     print_arguments(args)

demo/distillation/distill.py

@@ -133,9 +133,9 @@ def compress(args):
     place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
     exe = fluid.Executor(place)
 
-    train_reader = paddle.batch(
+    train_reader = paddle.fluid.io.batch(
         train_reader, batch_size=args.batch_size, drop_last=True)
-    val_reader = paddle.batch(
+    val_reader = paddle.fluid.io.batch(
         val_reader, batch_size=args.batch_size, drop_last=True)
     val_program = student_program.clone(for_test=True)
 

demo/distillation/image_classification_distillation_tutorial.ipynb

@@ -165,7 +165,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "train_reader = paddle.batch(\n",
+    "train_reader = paddle.fluid.io.batch(\n",
     "    paddle.dataset.mnist.train(), batch_size=128, drop_last=True)\n",
     "train_feeder = fluid.DataFeeder(['image', 'label'], fluid.CPUPlace(), student_program)"
    ]

demo/nas/block_sa_nas_mobilenetv2.py

@@ -137,22 +137,22 @@ def search_mobilenetv2_block(config, args, image_size):
         exe.run(startup_program)
 
         if args.data == 'cifar10':
-            train_reader = paddle.batch(
+            train_reader = paddle.fluid.io.batch(
                 paddle.reader.shuffle(
                     paddle.dataset.cifar.train10(cycle=False), buf_size=1024),
                 batch_size=args.batch_size,
                 drop_last=True)
 
-            test_reader = paddle.batch(
+            test_reader = paddle.fluid.io.batch(
                 paddle.dataset.cifar.test10(cycle=False),
                 batch_size=args.batch_size,
                 drop_last=False)
         elif args.data == 'imagenet':
-            train_reader = paddle.batch(
+            train_reader = paddle.fluid.io.batch(
                 imagenet_reader.train(),
                 batch_size=args.batch_size,
                 drop_last=True)
-            test_reader = paddle.batch(
+            test_reader = paddle.fluid.io.batch(
                 imagenet_reader.val(),
                 batch_size=args.batch_size,
                 drop_last=False)

demo/nas/image_classification_nas_quick_start.ipynb

@@ -114,9 +114,9 @@
     "    if current_flops > 321208544:\n",
     "        continue\n",
     "    \n",
-    "    train_reader = paddle.batch(paddle.reader.shuffle(paddle.dataset.cifar.train10(cycle=False),                          buf_size=1024),batch_size=256)\n",
+    "    train_reader = paddle.fluid.io.batch(paddle.reader.shuffle(paddle.dataset.cifar.train10(cycle=False),                          buf_size=1024),batch_size=256)\n",
     "    train_feeder = fluid.DataFeeder(inputs, fluid.CPUPlace())\n",
-    "    test_reader = paddle.batch(paddle.dataset.cifar.test10(cycle=False),\n",
+    "    test_reader = paddle.fluid.io.batch(paddle.dataset.cifar.test10(cycle=False),\n",
     "               batch_size=256)\n",
     "    test_feeder = fluid.DataFeeder(inputs, fluid.CPUPlace())\n",
     "\n",
@@ -160,4 +160,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

demo/nas/parl_nas_mobilenetv2.py

@@ -105,22 +105,22 @@ def search_mobilenetv2(config, args, image_size, is_server=True):
         exe.run(startup_program)
 
         if args.data == 'cifar10':
-            train_reader = paddle.batch(
+            train_reader = paddle.fluid.io.batch(
                 paddle.reader.shuffle(
                     paddle.dataset.cifar.train10(cycle=False), buf_size=1024),
                 batch_size=args.batch_size,
                 drop_last=True)
 
-            test_reader = paddle.batch(
+            test_reader = paddle.fluid.io.batch(
                 paddle.dataset.cifar.test10(cycle=False),
                 batch_size=args.batch_size,
                 drop_last=False)
         elif args.data == 'imagenet':
-            train_reader = paddle.batch(
+            train_reader = paddle.fluid.io.batch(
                 imagenet_reader.train(),
                 batch_size=args.batch_size,
                 drop_last=True)
-            test_reader = paddle.batch(
+            test_reader = paddle.fluid.io.batch(
                 imagenet_reader.val(),
                 batch_size=args.batch_size,
                 drop_last=False)