From fee567f8115457efb05a25e33572af701dba1b3a Mon Sep 17 00:00:00 2001 From: MzeroMiko <45236610+MzeroMiko@users.noreply.github.com> Date: Thu, 18 Jan 2024 22:52:53 +0800 Subject: [PATCH] init --- README.md | 302 ++++++++------------------------------------------- modelcard.sh | 16 +-- 2 files changed, 56 insertions(+), 262 deletions(-) diff --git a/README.md b/README.md index 092355a8..4f7223cd 100644 --- a/README.md +++ b/README.md @@ -1,310 +1,104 @@ -# Swin Transformer +# VMamba: Visual State Space Models -[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/swin-transformer-v2-scaling-up-capacity-and/object-detection-on-coco)](https://paperswithcode.com/sota/object-detection-on-coco?p=swin-transformer-v2-scaling-up-capacity-and) -[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/swin-transformer-v2-scaling-up-capacity-and/instance-segmentation-on-coco)](https://paperswithcode.com/sota/instance-segmentation-on-coco?p=swin-transformer-v2-scaling-up-capacity-and) -[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/swin-transformer-v2-scaling-up-capacity-and/semantic-segmentation-on-ade20k)](https://paperswithcode.com/sota/semantic-segmentation-on-ade20k?p=swin-transformer-v2-scaling-up-capacity-and) -[![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/swin-transformer-v2-scaling-up-capacity-and/action-classification-on-kinetics-400)](https://paperswithcode.com/sota/action-classification-on-kinetics-400?p=swin-transformer-v2-scaling-up-capacity-and) -This repo is the official implementation of ["Swin Transformer: Hierarchical Vision Transformer using Shifted Windows"](https://arxiv.org/pdf/2103.14030.pdf) as well as the follow-ups. It currently includes code and models for the following tasks: +This repo is the official implementation of ["VMamba: Visual State Space Models"](?) as well as the follow-ups. It currently includes code and models for the following tasks: -> **Image Classification**: Included in this repo. See [get_started.md](get_started.md) for a quick start. +> **Image Classification** -> **Object Detection and Instance Segmentation**: See [Swin Transformer for Object Detection](https://github.com/SwinTransformer/Swin-Transformer-Object-Detection). +> **Object Detection and Instance Segmentation** -> **Semantic Segmentation**: See [Swin Transformer for Semantic Segmentation](https://github.com/SwinTransformer/Swin-Transformer-Semantic-Segmentation). - -> **Video Action Recognition**: See [Video Swin Transformer](https://github.com/SwinTransformer/Video-Swin-Transformer). - -> **Semi-Supervised Object Detection**: See [Soft Teacher](https://github.com/microsoft/SoftTeacher). - -> **SSL: Contrasitive Learning**: See [Transformer-SSL](https://github.com/SwinTransformer/Transformer-SSL). - -> **SSL: Masked Image Modeling**: See [get_started.md#simmim-support](https://github.com/microsoft/Swin-Transformer/blob/main/get_started.md#simmim-support). - -> **Mixture-of-Experts**: See [get_started](get_started.md#mixture-of-experts-support) for more instructions. - -> **Feature-Distillation**: See [Feature-Distillation](https://github.com/SwinTransformer/Feature-Distillation). +> **Semantic Segmentation** ## Updates -***12/29/2022*** - -1. **Nvidia**'s [FasterTransformer](https://github.com/NVIDIA/FasterTransformer/blob/main/docs/swin_guide.md) now supports Swin Transformer V2 inference, which have significant speed improvements on `T4 and A100 GPUs`. - -***11/30/2022*** - -1. Models and codes of **Feature Distillation** are released. Please refer to [Feature-Distillation](https://github.com/SwinTransformer/Feature-Distillation) for details, and the checkpoints (FD-EsViT-Swin-B, FD-DeiT-ViT-B, FD-DINO-ViT-B, FD-CLIP-ViT-B, FD-CLIP-ViT-L). - -***09/24/2022*** - -1. Merged [SimMIM](https://github.com/microsoft/SimMIM), which is a **Masked Image Modeling** based pre-training approach applicable to Swin and SwinV2 (and also applicable for ViT and ResNet). Please refer to [get started with SimMIM](get_started.md#simmim-support) to play with SimMIM pre-training. - -2. Released a series of Swin and SwinV2 models pre-trained using the SimMIM approach (see [MODELHUB for SimMIM](MODELHUB.md#simmim-pretrained-swin-v2-models)), with model size ranging from SwinV2-Small-50M to SwinV2-giant-1B, data size ranging from ImageNet-1K-10% to ImageNet-22K, and iterations from 125k to 500k. You may leverage these models to study the properties of MIM methods. Please look into the [data scaling](https://arxiv.org/abs/2206.04664) paper for more details. - -***07/09/2022*** - -`News`: - -1. SwinV2-G achieves `61.4 mIoU` on ADE20K semantic segmentation (+1.5 mIoU over the previous SwinV2-G model), using an additional [feature distillation (FD)](https://github.com/SwinTransformer/Feature-Distillation) approach, **setting a new recrod** on this benchmark. FD is an approach that can generally improve the fine-tuning performance of various pre-trained models, including DeiT, DINO, and CLIP. Particularly, it improves CLIP pre-trained ViT-L by +1.6% to reach `89.0%` on ImageNet-1K image classification, which is **the most accurate ViT-L model**. -2. Merged a PR from **Nvidia** that links to faster Swin Transformer inference that have significant speed improvements on `T4 and A100 GPUs`. -3. Merged a PR from **Nvidia** that enables an option to use `pure FP16 (Apex O2)` in training, while almost maintaining the accuracy. +***01/18/2024*** -***06/03/2022*** - -1. Added **Swin-MoE**, the Mixture-of-Experts variant of Swin Transformer implemented using [Tutel](https://github.com/microsoft/tutel) (an optimized Mixture-of-Experts implementation). **Swin-MoE** is introduced in the [TuTel](https://arxiv.org/abs/2206.03382) paper. - -***05/12/2022*** - -1. Pretrained models of [Swin Transformer V2](https://arxiv.org/abs/2111.09883) on ImageNet-1K and ImageNet-22K are released. -2. ImageNet-22K pretrained models for Swin-V1-Tiny and Swin-V2-Small are released. - -***03/02/2022*** - -1. Swin Transformer V2 and SimMIM got accepted by CVPR 2022. [SimMIM](https://github.com/microsoft/SimMIM) is a self-supervised pre-training approach based on masked image modeling, a key technique that works out the 3-billion-parameter Swin V2 model using `40x less labelled data` than that of previous billion-scale models based on JFT-3B. - -***02/09/2022*** - -1. Integrated into [Huggingface Spaces 🤗](https://huggingface.co/spaces) using [Gradio](https://github.com/gradio-app/gradio). Try out the Web Demo [![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/akhaliq/Swin-Transformer) +Initial commits: +1. The supported code and models for ImageNet-1K image classification, COCO object detection and ADE20K semantic segmentation are provided. -***10/12/2021*** +TODO: +1. Pretrained models on ImageNet-1K ([VMamba-T-IN1K](...), [VMamba-S-IN1K](...), [VMamba-B-IN1K](...)) are provided. -1. Swin Transformer received ICCV 2021 best paper award (Marr Prize). +## Introduction -***08/09/2021*** -1. [Soft Teacher](https://arxiv.org/pdf/2106.09018v2.pdf) will appear at ICCV2021. The code will be released at [GitHub Repo](https://github.com/microsoft/SoftTeacher). `Soft Teacher` is an end-to-end semi-supervisd object detection method, achieving a new record on the COCO test-dev: `61.3 box AP` and `53.0 mask AP`. - -***07/03/2021*** -1. Add **Swin MLP**, which is an adaption of `Swin Transformer` by replacing all multi-head self-attention (MHSA) blocks by MLP layers (more precisely it is a group linear layer). The shifted window configuration can also significantly improve the performance of vanilla MLP architectures. +**VMamba (VSSM)** is initially described in [arxiv](?), which capably serves as a +general-purpose backbone for computer vision with linear complexity, +while still preserving the advantages associated with global receptive fields and dynamic weights. -***06/25/2021*** -1. [Video Swin Transformer](https://arxiv.org/abs/2106.13230) is released at [Video-Swin-Transformer](https://github.com/SwinTransformer/Video-Swin-Transformer). -`Video Swin Transformer` achieves state-of-the-art accuracy on a broad range of video recognition benchmarks, including action recognition (`84.9` top-1 accuracy on Kinetics-400 and `86.1` top-1 accuracy on Kinetics-600 with `~20x` less pre-training data and `~3x` smaller model size) and temporal modeling (`69.6` top-1 accuracy on Something-Something v2). +VMamba achieves strong performance on COCO object detection (`49.7 box AP` and `44.0 mask AP` on COCO val 2017) and +ADE20K semantic segmentation (`50.8 mIoU` on val), with only 44M parameters for backbone. -***05/12/2021*** -1. Used as a backbone for `Self-Supervised Learning`: [Transformer-SSL](https://github.com/SwinTransformer/Transformer-SSL) + + +**VMamba is linear complexity with 2D-Selective-Scan** -Using Swin-Transformer as the backbone for self-supervised learning enables us to evaluate the transferring performance of the learnt representations on down-stream tasks, which is missing in previous works due to the use of ViT/DeiT, which has not been well tamed for down-stream tasks. +![linear complexity](srcs/figs/fig2.jpg) -***04/12/2021*** +**VMamba has global effective receptive field** -Initial commits: +![ERF](srcs/figs/fig5.jpg) -1. Pretrained models on ImageNet-1K ([Swin-T-IN1K](https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_tiny_patch4_window7_224.pth), [Swin-S-IN1K](https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_small_patch4_window7_224.pth), [Swin-B-IN1K](https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224.pth)) and ImageNet-22K ([Swin-B-IN22K](https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224_22k.pth), [Swin-L-IN22K](https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_large_patch4_window7_224_22k.pth)) are provided. -2. The supported code and models for ImageNet-1K image classification, COCO object detection and ADE20K semantic segmentation are provided. -3. The cuda kernel implementation for the [local relation layer](https://arxiv.org/pdf/1904.11491.pdf) is provided in branch [LR-Net](https://github.com/microsoft/Swin-Transformer/tree/LR-Net). +**Main Architecture of VMamba** -## Introduction +![main arch 1](srcs/figs/fig3.jpg) +![main arch 2](srcs/figs/fig4.jpg) -**Swin Transformer** (the name `Swin` stands for **S**hifted **win**dow) is initially described in [arxiv](https://arxiv.org/abs/2103.14030), which capably serves as a -general-purpose backbone for computer vision. It is basically a hierarchical Transformer whose representation is -computed with shifted windows. The shifted windowing scheme brings greater efficiency by limiting self-attention -computation to non-overlapping local windows while also allowing for cross-window connection. +## Main Results on ImageNet -Swin Transformer achieves strong performance on COCO object detection (`58.7 box AP` and `51.1 mask AP` on test-dev) and -ADE20K semantic segmentation (`53.5 mIoU` on val), surpassing previous models by a large margin. +**ImageNet-1K VMamba Models** -![teaser](figures/teaser.png) +![main](srcs/figs/tab2.jpg) -## Main Results on ImageNet with Pretrained Models -**ImageNet-1K and ImageNet-22K Pretrained Swin-V1 Models** - -| name | pretrain | resolution |acc@1 | acc@5 | #params | FLOPs | FPS| 22K model | 1K model | + -**ImageNet-22K Pretrained Swin-MoE Models** -- Please refer to [get_started](get_started.md#mixture-of-experts-support) for instructions on running Swin-MoE. -- Pretrained models for Swin-MoE can be found in [MODEL HUB](MODELHUB.md#imagenet-22k-pretrained-swin-moe-models) ## Main Results on Downstream Tasks **COCO Object Detection (2017 val)** -| Backbone | Method | pretrain | Lr Schd | box mAP | mask mAP | #params | FLOPs | +![main](srcs/figs/tab3.jpg) + + + **ADE20K Semantic Segmentation (val)** -| Backbone | Method | pretrain | Crop Size | Lr Schd | mIoU | mIoU (ms+flip) | #params | FLOPs | +![main](srcs/figs/tab4.jpg) + + -## Citing Swin Transformer +## Citing VMamba: Visual State Space Models ``` -@inproceedings{liu2021Swin, - title={Swin Transformer: Hierarchical Vision Transformer using Shifted Windows}, - author={Liu, Ze and Lin, Yutong and Cao, Yue and Hu, Han and Wei, Yixuan and Zhang, Zheng and Lin, Stephen and Guo, Baining}, - booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, - year={2021} -} -``` -## Citing Local Relation Networks (the first full-attention visual backbone) -``` -@inproceedings{hu2019local, - title={Local Relation Networks for Image Recognition}, - author={Hu, Han and Zhang, Zheng and Xie, Zhenda and Lin, Stephen}, - booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, - pages={3464--3473}, - year={2019} -} -``` -## Citing Swin Transformer V2 -``` -@inproceedings{liu2021swinv2, - title={Swin Transformer V2: Scaling Up Capacity and Resolution}, - author={Ze Liu and Han Hu and Yutong Lin and Zhuliang Yao and Zhenda Xie and Yixuan Wei and Jia Ning and Yue Cao and Zheng Zhang and Li Dong and Furu Wei and Baining Guo}, - booktitle={International Conference on Computer Vision and Pattern Recognition (CVPR)}, - year={2022} -} -``` -## Citing SimMIM (a self-supervised approach that enables SwinV2-G) -``` -@inproceedings{xie2021simmim, - title={SimMIM: A Simple Framework for Masked Image Modeling}, - author={Xie, Zhenda and Zhang, Zheng and Cao, Yue and Lin, Yutong and Bao, Jianmin and Yao, Zhuliang and Dai, Qi and Hu, Han}, - booktitle={International Conference on Computer Vision and Pattern Recognition (CVPR)}, - year={2022} -} -``` -## Citing SimMIM-data-scaling -``` -@article{xie2022data, - title={On Data Scaling in Masked Image Modeling}, - author={Xie, Zhenda and Zhang, Zheng and Cao, Yue and Lin, Yutong and Wei, Yixuan and Dai, Qi and Hu, Han}, - journal={arXiv preprint arXiv:2206.04664}, - year={2022} -} -``` -## Citing Swin-MoE -``` -@misc{hwang2022tutel, - title={Tutel: Adaptive Mixture-of-Experts at Scale}, - author={Changho Hwang and Wei Cui and Yifan Xiong and Ziyue Yang and Ze Liu and Han Hu and Zilong Wang and Rafael Salas and Jithin Jose and Prabhat Ram and Joe Chau and Peng Cheng and Fan Yang and Mao Yang and Yongqiang Xiong}, - year={2022}, - eprint={2206.03382}, - archivePrefix={arXiv} -} +@inproceedings{...} ``` ## Getting Started -- For **Image Classification**, please see [get_started.md](get_started.md) for detailed instructions. -- For **Object Detection and Instance Segmentation**, please see [Swin Transformer for Object Detection](https://github.com/SwinTransformer/Swin-Transformer-Object-Detection). -- For **Semantic Segmentation**, please see [Swin Transformer for Semantic Segmentation](https://github.com/SwinTransformer/Swin-Transformer-Semantic-Segmentation). -- For **Self-Supervised Learning**, please see [Transformer-SSL](https://github.com/SwinTransformer/Transformer-SSL). -- For **Video Recognition**, please see [Video Swin Transformer](https://github.com/SwinTransformer/Video-Swin-Transformer). +- For installation, training and testing, please see [modelcard.sh](modelcard.sh) for detailed instructions. ## Third-party Usage and Experiments -***In this pargraph, we cross link third-party repositories which use Swin and report results. You can let us know by raising an issue*** - -(`Note please report accuracy numbers and provide trained models in your new repository to facilitate others to get sense of correctness and model behavior`) +***all model files are in classification/models/vmamba, just copy that into your own project and enjoy yourself!*** -[12/29/2022] Swin Transformers (V2) inference implemented in FasterTransformer: [FasterTransformer](https://github.com/NVIDIA/FasterTransformer/blob/main/docs/swin_guide.md) - -[06/30/2022] Swin Transformers (V1) inference implemented in FasterTransformer: [FasterTransformer](https://github.com/NVIDIA/FasterTransformer/blob/main/docs/swin_guide.md) - -[05/12/2022] Swin Transformers (V1) implemented in TensorFlow with the pre-trained parameters ported into them. Find the implementation, -TensorFlow weights, code example here in [this repository](https://github.com/sayakpaul/swin-transformers-tf/). - -[04/06/2022] Swin Transformer for Audio Classification: [Hierarchical Token Semantic Audio Transformer](https://github.com/RetroCirce/HTS-Audio-Transformer). - -[12/21/2021] Swin Transformer for StyleGAN: [StyleSwin](https://github.com/microsoft/StyleSwin) - -[12/13/2021] Swin Transformer for Face Recognition: [FaceX-Zoo](https://github.com/JDAI-CV/FaceX-Zoo) - -[08/29/2021] Swin Transformer for Image Restoration: [SwinIR](https://github.com/JingyunLiang/SwinIR) - -[08/12/2021] Swin Transformer for person reID: [https://github.com/layumi/Person_reID_baseline_pytorch](https://github.com/layumi/Person_reID_baseline_pytorch) - -[06/29/2021] Swin-Transformer in PaddleClas and inference based on whl package: [https://github.com/PaddlePaddle/PaddleClas](https://github.com/PaddlePaddle/PaddleClas) - -[04/14/2021] Swin for RetinaNet in Detectron: https://github.com/xiaohu2015/SwinT_detectron2. - -[04/16/2021] Included in a famous model zoo: https://github.com/rwightman/pytorch-image-models. - -[04/20/2021] Swin-Transformer classifier inference using TorchServe: https://github.com/kamalkraj/Swin-Transformer-Serve ## Contributing -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. - -When you submit a pull request, a CLA bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -## Trademarks +Thanks to all contribute to this... (fill in later...) -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft -trademarks or logos is subject to and must follow -[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). -Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. -Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/modelcard.sh b/modelcard.sh index 495be2bd..4dbb1cb1 100644 --- a/modelcard.sh +++ b/modelcard.sh @@ -48,23 +48,23 @@ function dataset_to_memory() { } # test ok -function classification_tiny() { +function classification() { + # test ok + # ====================================================== export CODE=classification PYTHON=python log_url="logs/classification" export nnodes=1 nrank=0 nprocs=8 mport=29501 maddr="127.0.0.1" export pycmds="main.py --cfg configs/vssm/vssm_tiny_224.yaml --batch-size 64 --data-path /dataset/ImageNet2012 --output /tmp" cd ${CODE}; ${PYTHON} -m torch.distributed.launch --nnodes ${nnodes} --node_rank ${nrank} --nproc_per_node ${nprocs} --master_addr ${maddr} --master_port ${mport} --log_dir ${log_url} ${pycmds} -} -# test ok -function classification_small() { + # test ok + # ====================================================== export CODE=classification PYTHON=python log_url="logs/classification" export nnodes=1 nrank=0 nprocs=8 mport=29501 maddr="127.0.0.1" export pycmds="main.py --cfg configs/vssm/vssm_small_224.yaml --batch-size 64 --data-path /dataset/ImageNet2012 --output /tmp" cd ${CODE}; ${PYTHON} -m torch.distributed.launch --nnodes ${nnodes} --node_rank ${nrank} --nproc_per_node ${nprocs} --master_addr ${maddr} --master_port ${mport} --log_dir ${log_url} ${pycmds} -} -# test ok -function classification_base() { + # test ok + # ====================================================== export CODE=classification PYTHON=python log_url="logs/classification" export nnodes=1 nrank=0 nprocs=8 mport=29501 maddr="127.0.0.1" export pycmds="main.py --cfg configs/vssm/vssm_base_224.yaml --batch-size 64 --data-path /dataset/ImageNet2012 --output /tmp" @@ -226,6 +226,7 @@ function segmentation_test_tta() { cd $CODE; export GPUS=8 NNODES=${NNODES:-1} NODE_RANK=${NODE_RANK:-0} PORT=${PORT:-29500} MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"}; PYTHONPATH="$PWD":$PYTHONPATH $PYTHON -m torch.distributed.launch --nnodes=$NNODES --node_rank=$NODE_RANK --master_addr=$MASTER_ADDR --nproc_per_node=$GPUS --master_port=$PORT ./tools/test.py $CONFIG $CKPT --launcher pytorch --tta } +# test ok function analyze() { # test ok ===================================================== CUDA_VISIBLE_DEVICES=0 python analyze/get_erf.py > analyze/show/erf/get_erf.log 2>&1 @@ -255,4 +256,3 @@ function analyze() { -