Skip to content

Commit 8b463f2

Browse files
AlexanderGroshevDanyache
AlexanderGroshev
and
Danyache
authored
Super res (ai-forever#1)
* add super res * update functions * add super res * Update download_models.sh Co-authored-by: danyache <[email protected]>
1 parent cf5cc13 commit 8b463f2

33 files changed

+4270
-46
lines changed

‎SberSwapInference.ipynb

+199-41
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
{
44
"cell_type": "code",
55
"execution_count": 1,
6-
"id": "cellular-michael",
6+
"id": "tested-scope",
77
"metadata": {},
88
"outputs": [
99
{
@@ -19,31 +19,60 @@
1919
"import cv2\n",
2020
"import torch\n",
2121
"import time\n",
22+
"import os\n",
2223
"\n",
2324
"from utils.inference.image_processing import crop_face\n",
24-
"from utils.inference.video_processing import read_video, get_final_video_frame, add_audio_from_another_video\n",
25+
"from utils.inference.video_processing import read_video, get_final_video_frame, add_audio_from_another_video, face_enhancement\n",
2526
"from utils.inference.core import model_inference\n",
2627
"\n",
2728
"from network.AEI_Net import AEI_Net\n",
2829
"from coordinate_reg.image_infer import Handler\n",
2930
"from insightface_func.face_detect_crop_single import Face_detect_crop\n",
30-
"from arcface_model.iresnet import iresnet100"
31+
"from arcface_model.iresnet import iresnet100\n",
32+
"from models.pix2pix_model import Pix2PixModel\n",
33+
"from models.config_sr import TestOptions"
3134
]
3235
},
3336
{
3437
"cell_type": "markdown",
35-
"id": "equipped-exploration",
38+
"id": "neutral-constant",
3639
"metadata": {},
3740
"source": [
3841
"### Load Models"
3942
]
4043
},
4144
{
4245
"cell_type": "code",
43-
"execution_count": null,
44-
"id": "overall-request",
46+
"execution_count": 2,
47+
"id": "elder-temperature",
4548
"metadata": {},
46-
"outputs": [],
49+
"outputs": [
50+
{
51+
"name": "stderr",
52+
"output_type": "stream",
53+
"text": [
54+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
55+
" and should_run_async(code)\n"
56+
]
57+
},
58+
{
59+
"name": "stdout",
60+
"output_type": "stream",
61+
"text": [
62+
"input mean and std: 127.5 127.5\n",
63+
"find model: ./insightface_func/models/antelope/glintr100.onnx recognition\n",
64+
"find model: ./insightface_func/models/antelope/scrfd_10g_bnkps.onnx detection\n",
65+
"set det-size: (640, 640)\n",
66+
"loading ./coordinate_reg/model/2d106det 0\n",
67+
"input mean and std: 127.5 127.5\n",
68+
"find model: ./insightface_func/models/antelope/glintr100.onnx recognition\n",
69+
"find model: ./insightface_func/models/antelope/scrfd_10g_bnkps.onnx detection\n",
70+
"set det-size: (640, 640)\n",
71+
"Network [LIPSPADEGenerator] was created. Total number of parameters: 72.2 million. To see the architecture, do print(network).\n",
72+
"Load checkpoint from path: weights/10_net_G.pth\n"
73+
]
74+
}
75+
],
4776
"source": [
4877
"app = Face_detect_crop(name='antelope', root='./insightface_func/models')\n",
4978
"app.prepare(ctx_id= 0, det_thresh=0.6, det_size=(640,640))\n",
@@ -62,23 +91,42 @@
6291
"netArc.eval()\n",
6392
"\n",
6493
"# модель, которая позволяет находить точки лица\n",
65-
"handler = Handler('./coordinate_reg/model/2d106det', 0, ctx_id=0, det_size=640)"
94+
"handler = Handler('./coordinate_reg/model/2d106det', 0, ctx_id=0, det_size=640)\n",
95+
"\n",
96+
"# модель, увеличивающая четкость лица\n",
97+
"use_sr = True\n",
98+
"if use_sr:\n",
99+
" os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
100+
" torch.backends.cudnn.benchmark = True\n",
101+
" opt = TestOptions()\n",
102+
" #opt.which_epoch ='10_7'\n",
103+
" model = Pix2PixModel(opt)\n",
104+
" model.netG.train()"
66105
]
67106
},
68107
{
69108
"cell_type": "markdown",
70-
"id": "outdoor-consensus",
109+
"id": "geographic-stream",
71110
"metadata": {},
72111
"source": [
73112
"### Set here path to source image and video for faceswap"
74113
]
75114
},
76115
{
77116
"cell_type": "code",
78-
"execution_count": null,
79-
"id": "major-employment",
117+
"execution_count": 3,
118+
"id": "familiar-prime",
80119
"metadata": {},
81-
"outputs": [],
120+
"outputs": [
121+
{
122+
"name": "stderr",
123+
"output_type": "stream",
124+
"text": [
125+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
126+
" and should_run_async(code)\n"
127+
]
128+
}
129+
],
82130
"source": [
83131
"\"\"\"\n",
84132
"choose not really long videos, coz it can take a lot of time othervise \n",
@@ -93,10 +141,26 @@
93141
},
94142
{
95143
"cell_type": "code",
96-
"execution_count": null,
97-
"id": "adult-forth",
144+
"execution_count": 4,
145+
"id": "therapeutic-regression",
98146
"metadata": {},
99-
"outputs": [],
147+
"outputs": [
148+
{
149+
"name": "stdout",
150+
"output_type": "stream",
151+
"text": [
152+
"Everything is ok!\n"
153+
]
154+
},
155+
{
156+
"name": "stderr",
157+
"output_type": "stream",
158+
"text": [
159+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
160+
" and should_run_async(code)\n"
161+
]
162+
}
163+
],
100164
"source": [
101165
"# check, if we can detect face on the source image\n",
102166
"\n",
@@ -110,10 +174,19 @@
110174
},
111175
{
112176
"cell_type": "code",
113-
"execution_count": null,
114-
"id": "under-maryland",
177+
"execution_count": 5,
178+
"id": "selected-position",
115179
"metadata": {},
116-
"outputs": [],
180+
"outputs": [
181+
{
182+
"name": "stderr",
183+
"output_type": "stream",
184+
"text": [
185+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
186+
" and should_run_async(code)\n"
187+
]
188+
}
189+
],
117190
"source": [
118191
"# read video\n",
119192
"\n",
@@ -122,28 +195,52 @@
122195
},
123196
{
124197
"cell_type": "markdown",
125-
"id": "productive-prisoner",
198+
"id": "opposite-discretion",
126199
"metadata": {},
127200
"source": [
128201
"### Model Inference"
129202
]
130203
},
131204
{
132205
"cell_type": "code",
133-
"execution_count": null,
134-
"id": "fourth-procurement",
206+
"execution_count": 6,
207+
"id": "numeric-forth",
135208
"metadata": {},
136-
"outputs": [],
209+
"outputs": [
210+
{
211+
"name": "stderr",
212+
"output_type": "stream",
213+
"text": [
214+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
215+
" and should_run_async(code)\n"
216+
]
217+
}
218+
],
137219
"source": [
138220
"START_TIME = time.time()"
139221
]
140222
},
141223
{
142224
"cell_type": "code",
143-
"execution_count": null,
144-
"id": "acceptable-liberal",
225+
"execution_count": 7,
226+
"id": "increased-malaysia",
145227
"metadata": {},
146-
"outputs": [],
228+
"outputs": [
229+
{
230+
"name": "stderr",
231+
"output_type": "stream",
232+
"text": [
233+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
234+
" and should_run_async(code)\n",
235+
"100%|██████████| 262/262 [00:02<00:00, 97.92it/s]\n",
236+
"/home/jovyan/conda/envs/faceshift_env/lib/python3.8/site-packages/torch/nn/functional.py:3000: UserWarning: The default behavior for interpolate/upsample with float scale_factor changed in 1.6.0 to align with other frameworks/libraries, and uses scale_factor directly, instead of relying on the computed output size. If you wish to keep the old behavior, please set recompute_scale_factor=True. See the documentation of nn.Upsample for details. \n",
237+
" warnings.warn(\"The default behavior for interpolate/upsample with float scale_factor changed \"\n",
238+
"262it [00:00, 5828.70it/s]\n",
239+
"100%|██████████| 5/5 [00:07<00:00, 1.43s/it]\n",
240+
"100%|██████████| 262/262 [00:00<00:00, 924228.47it/s]\n"
241+
]
242+
}
243+
],
147244
"source": [
148245
"final_frames, crop_frames, full_frames, tfm_array = model_inference(full_frames,\n",
149246
" source,\n",
@@ -155,10 +252,45 @@
155252
},
156253
{
157254
"cell_type": "code",
158-
"execution_count": null,
159-
"id": "olympic-shaft",
255+
"execution_count": 8,
256+
"id": "spoken-cinema",
160257
"metadata": {},
161-
"outputs": [],
258+
"outputs": [
259+
{
260+
"name": "stderr",
261+
"output_type": "stream",
262+
"text": [
263+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
264+
" and should_run_async(code)\n",
265+
"14it [00:07, 1.93it/s]\n"
266+
]
267+
}
268+
],
269+
"source": [
270+
"if use_sr:\n",
271+
" final_frames = face_enhancement(final_frames, model)"
272+
]
273+
},
274+
{
275+
"cell_type": "code",
276+
"execution_count": 9,
277+
"id": "heavy-charm",
278+
"metadata": {},
279+
"outputs": [
280+
{
281+
"name": "stderr",
282+
"output_type": "stream",
283+
"text": [
284+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
285+
" and should_run_async(code)\n",
286+
" 0%| | 0/262 [00:00<?, ?it/s]/home/jovyan/conda/envs/faceshift_env/lib/python3.8/site-packages/kornia/geometry/transform/imgwarp.py:184: UserWarning: The align_corners default value has been changed. By default now is set True in order to match cv2.warpAffine. In case you want to keep your previous behaviour set it to False. This warning will disappear in kornia > v0.6.\n",
287+
" warnings.warn(message)\n",
288+
"/home/jovyan/FaceShifterMulti/sber-swap/utils/inference/video_processing.py:199: UserWarning: Mixed memory format inputs detected while calling the operator. The operator will output contiguous tensor even if some of the inputs are in channels_last format. (Triggered internally at /pytorch/aten/src/ATen/native/TensorIterator.cpp:918.)\n",
289+
" final = (mask_t*swap_t + (1-mask_t)*frame).type(torch.uint8).squeeze().permute(1,2,0).cpu().detach().numpy()\n",
290+
"100%|██████████| 262/262 [00:06<00:00, 40.30it/s]\n"
291+
]
292+
}
293+
],
162294
"source": [
163295
"get_final_video_frame(final_frames,\n",
164296
" crop_frames,\n",
@@ -171,20 +303,46 @@
171303
},
172304
{
173305
"cell_type": "code",
174-
"execution_count": null,
175-
"id": "double-oxide",
306+
"execution_count": 10,
307+
"id": "breeding-assistant",
176308
"metadata": {},
177-
"outputs": [],
309+
"outputs": [
310+
{
311+
"name": "stderr",
312+
"output_type": "stream",
313+
"text": [
314+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
315+
" and should_run_async(code)\n"
316+
]
317+
}
318+
],
178319
"source": [
179320
"add_audio_from_another_video(path_to_video, OUT_VIDEO_NAME, \"audio\")"
180321
]
181322
},
182323
{
183324
"cell_type": "code",
184-
"execution_count": null,
185-
"id": "voluntary-marketing",
325+
"execution_count": 11,
326+
"id": "academic-theme",
186327
"metadata": {},
187-
"outputs": [],
328+
"outputs": [
329+
{
330+
"name": "stdout",
331+
"output_type": "stream",
332+
"text": [
333+
"Full pipeline took 26.875646114349365\n",
334+
"Video saved with path examples/results/elon2dora.mp4\n"
335+
]
336+
},
337+
{
338+
"name": "stderr",
339+
"output_type": "stream",
340+
"text": [
341+
"/home/jovyan/.local/lib/python3.8/site-packages/ipykernel/ipkernel.py:283: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
342+
" and should_run_async(code)\n"
343+
]
344+
}
345+
],
188346
"source": [
189347
"print(f'Full pipeline took {time.time() - START_TIME}')\n",
190348
"print(f\"Video saved with path {OUT_VIDEO_NAME}\")"
@@ -193,49 +351,49 @@
193351
{
194352
"cell_type": "code",
195353
"execution_count": null,
196-
"id": "american-costs",
354+
"id": "fresh-hazard",
197355
"metadata": {},
198356
"outputs": [],
199357
"source": []
200358
},
201359
{
202360
"cell_type": "code",
203361
"execution_count": null,
204-
"id": "framed-breeding",
362+
"id": "defensive-romantic",
205363
"metadata": {},
206364
"outputs": [],
207365
"source": []
208366
},
209367
{
210368
"cell_type": "code",
211369
"execution_count": null,
212-
"id": "applied-laundry",
370+
"id": "reflected-hepatitis",
213371
"metadata": {},
214372
"outputs": [],
215373
"source": []
216374
},
217375
{
218376
"cell_type": "code",
219377
"execution_count": null,
220-
"id": "detected-limitation",
378+
"id": "varying-tracker",
221379
"metadata": {},
222380
"outputs": [],
223381
"source": []
224382
},
225383
{
226384
"cell_type": "code",
227385
"execution_count": null,
228-
"id": "saved-darkness",
386+
"id": "stylish-batman",
229387
"metadata": {},
230388
"outputs": [],
231389
"source": []
232390
}
233391
],
234392
"metadata": {
235393
"kernelspec": {
236-
"display_name": "fs_env",
394+
"display_name": "fs",
237395
"language": "python",
238-
"name": "fs_env"
396+
"name": "fs"
239397
},
240398
"language_info": {
241399
"codemirror_mode": {

0 commit comments

Comments
 (0)