Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Support VindLU multi-modality algorithm #2667

Merged
merged 20 commits into from
Sep 7, 2023
Merged

[Feature] Support VindLU multi-modality algorithm #2667

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
aaad5f1
[Infer] support msrvtt inferclear
cir7 Aug 2, 2023
3ffa50e
replace logging with MMLogger
cir7 Aug 2, 2023
eefd57e
implement models in OpenMMLab style
cir7 Aug 7, 2023
4df1622
support vqa training
cir7 Aug 8, 2023
a427070
support vindlu retrieval and vqa-mc inference
cir7 Aug 21, 2023
1860c61
support train msrvtt retrieval
cir7 Aug 24, 2023
7c40e45
use msrvtt-train-9k
cir7 Aug 25, 2023
27a4b28
fix mean after normalization
cir7 Aug 29, 2023
d106df2
refactor vindlu code
cir7 Aug 29, 2023
520f8be
tmp commit
cir7 Aug 30, 2023
0a026a8
simplify vindlu model code
cir7 Sep 4, 2023
7c71189
fix pre-commit format
cir7 Sep 4, 2023
c2317f1
fix pre-commit format
cir7 Sep 4, 2023
689548f
format config, add prepare dataset, add ut
cir7 Sep 5, 2023
1ddac50
update config
cir7 Sep 6, 2023
c580e22
update ci
cir7 Sep 6, 2023
0c4259e
update ckpt link
cir7 Sep 6, 2023
2770dbe
fix bug
cir7 Sep 6, 2023
fb082c0
update according to ActionDataSample
cir7 Sep 6, 2023
cac53df
remove useless comment
cir7 Sep 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
use msrvtt-train-9k
  • Loading branch information
@cir7
cir7 committed Sep 6, 2023
commit 7c40e45d76bdedadfcfa849cad248ebe03dfb9df
6 changes: 4 additions & 2 deletions configs/multimodal/vindlu/vindlu_ret_train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
_base_ = ['../../_base_/default_runtime.py']

video_root = 'data/msrvtt/msrvtt_2fps_224'
anno_file_train = 'data/msrvtt/anno_downstream/msrvtt_ret_train7k.json'
anno_file_train = 'data/msrvtt/anno_downstream/msrvtt_ret_train9k.json'
anno_file_test = 'data/msrvtt/anno_downstream/msrvtt_ret_test1k.json'
pretrained_ckpt_path = 'checkpoints/5M-pretrain.pth'
cir7 marked this conversation as resolved.
Show resolved Hide resolved

Expand Down Expand Up @@ -106,7 +106,7 @@
dataset_type = 'MSRVTT_Ret'

train_dataloader = dict(
batch_size=16,
batch_size=32,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=True),
Expand Down Expand Up @@ -177,3 +177,5 @@
auto_scale_lr = dict(enable=True, base_batch_size=128)

find_unused_parameters=True

custom_hooks = [dict(type='EmptyCacheHook', after_epoch=True)]
4 changes: 0 additions & 4 deletions mmaction/models/multimodal/vindlu_ret.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,12 +379,10 @@ def compute_score_matrix_i2t(self, img_feats, img_embeds, text_feats,
"""
use_sim = False
# compute i2t sim matrix
# print(f'mem 0 {torch.cuda.memory_allocated()// (1024*1024)}')
sim_matrix_i2t = img_feats @ text_feats.t()

score_matrix_i2t = torch.full((img_feats.size(0), text_feats.size(0)),
-100.0).to(self.device)
# print(f'mem 1 {torch.cuda.memory_allocated()// (1024*1024)}')
for i in track_on_main_process(
range(img_feats.size(0)), 'Compute I2T scores...'):
sims = sim_matrix_i2t[i]
Expand All @@ -408,8 +406,6 @@ def compute_score_matrix_i2t(self, img_feats, img_embeds, text_feats,
)
score = self.itm_head(output.last_hidden_state[:, 0, :])[:, 1]
score_matrix_i2t[i, batch_topk] = score
# print(f'mem 2 {torch.cuda.memory_allocated()// (1024*1024)}')
# print(f'mem 3 {torch.cuda.memory_allocated()// (1024*1024)}')
return score_matrix_i2t

def compute_score_matrix_t2i(self, img_feats, img_embeds, text_feats,
Expand Down