Merge pull request microsoft#602 from addf400/master-update

some update
aidanmomo · Jan 18, 2022 · 8dc1160 · 8dc1160
2 parents 65f15af + f8616f4
commit 8dc1160
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 6 deletions.
diff --git a/beit/README.md b/beit/README.md
@@ -95,7 +95,7 @@ OMP_NUM_THREADS=1 python -m torch.distributed.launch --nproc_per_node=16 run_bei
         --data_path ${DATA_PATH} --output_dir ${OUTPUT_DIR} --num_mask_patches 75 \
         --model beit_base_patch16_224_8k_vocab --discrete_vae_weight_path ${TOKENIZER_PATH} \
         --batch_size 128 --lr 1.5e-3 --warmup_steps 10000 --epochs 150 \
-        --clip_grad 3.0 --drop_path 0 --layer_scale_init_value 0.1
+        --clip_grad 3.0 --drop_path 0.1 --layer_scale_init_value 0.1
 ```
 - `--num_mask_patches`: number of the input patches need be masked.
 - `--batch_size`: batch size per GPU.
@@ -126,7 +126,7 @@ OMP_NUM_THREADS=1 python -m torch.distributed.launch --nproc_per_node=16 run_bei
         --data_path ${DATA_PATH} --output_dir ${OUTPUT_DIR} --num_mask_patches 75 \
         --model beit_base_patch16_224_8k_vocab --discrete_vae_weight_path ${TOKENIZER_PATH} \
         --batch_size 128 --lr 1.5e-3 --warmup_epochs 10 --epochs 800 \
-        --clip_grad 3.0 --drop_path 0 --layer_scale_init_value 0.1 \
+        --clip_grad 3.0 --drop_path 0.1 --layer_scale_init_value 0.1 \
         --imagenet_default_mean_and_std
 ```
 

diff --git a/beit/datasets.py b/beit/datasets.py
@@ -31,6 +31,7 @@ def __init__(self, args):
         std = IMAGENET_INCEPTION_STD if not imagenet_default_mean_and_std else IMAGENET_DEFAULT_STD
 
         self.common_transform = transforms.Compose([
+            transforms.ColorJitter(0.4, 0.4, 0.4),
             transforms.RandomHorizontalFlip(p=0.5),
             RandomResizedCropAndInterpolationWithTwoPic(
                 size=args.input_size, second_size=args.second_input_size,

diff --git a/beit/run_beit_pretraining.py b/beit/run_beit_pretraining.py
@@ -58,16 +58,16 @@ def get_args():
     parser.add_argument('--second_input_size', default=112, type=int,
                         help='images input size for discrete vae')
 
-    parser.add_argument('--drop_path', type=float, default=0, metavar='PCT',
-                        help='Drop path rate (default: 0)')
+    parser.add_argument('--drop_path', type=float, default=0.1, metavar='PCT',
+                        help='Drop path rate (default: 0.1)')
 
     # Optimizer parameters
     parser.add_argument('--opt', default='adamw', type=str, metavar='OPTIMIZER',
                         help='Optimizer (default: "adamw"')
     parser.add_argument('--opt_eps', default=1e-8, type=float, metavar='EPSILON',
                         help='Optimizer Epsilon (default: 1e-8)')
-    parser.add_argument('--opt_betas', default=[0.9, 0.98], type=float, nargs='+', metavar='BETA',
-                        help='Optimizer Betas (default: 0.9, 0.98, use opt default)')
+    parser.add_argument('--opt_betas', default=[0.9, 0.999], type=float, nargs='+', metavar='BETA',
+                        help='Optimizer Betas (default: 0.9, 0.999, use opt default)')
     parser.add_argument('--clip_grad', type=float, default=None, metavar='NORM',
                         help='Clip gradient norm (default: None, no clipping)')
     parser.add_argument('--momentum', type=float, default=0.9, metavar='M',