Next (facefusion#107)

* Allow passing the onnxruntime to install.py

* Fix CI

* Disallow none execution providers in the UI

* Use CV2 to detect fps

* Respect trim on videos with audio

* Respect trim on videos with audio (finally)

* Implement caching to speed up preview and webcam

* Define webcam UI and webcam performance

* Remove layout from components

* Add primary buttons

* Extract benchmark and webcam settings

* Introduce compact UI settings

* Caching for IO and **** prediction

* Caching for IO and **** prediction

* Introduce face analyser caching

* Fix some typing

* Improve setup for benchmark

* Clear image cache via post process

* Fix settings in UI, Simplify restore_audio() using shortest

* Select resolution and fps via webcam ui

* Introduce read_static_image() to stop caching temp images

* Use DirectShow under Windows

* Multi-threading for webcam

* Fix typing

* Refactor frame processor

* Refactor webcam processing

* Avoid warnings due capture.isOpened()

* Resume downloads (facefusion#110)

* Introduce resumable downloads

* Fix CURL commands

* Break execution_settings into pieces

* Cosmetic changes on cv2 webcam

* Update Gradio

* Move face cache to own file

* Uniform namings for threading

* Fix sorting of get_temp_frame_paths(), extend get_temp_frames_pattern()

* Minor changes from the review

* Looks stable to tme

* Update the disclaimer

* Update the disclaimer

* Update the disclaimer

README.md

@@ -24,7 +24,7 @@ Read the [installation](https://docs.facefusion.io/installation) now.
 Usage
 -----
 
-Run the program as needed.
+Run the command:
 
 ```
 python run.py [options]
@@ -64,11 +64,11 @@ python run.py [options]
 Disclaimer
 ----------
 
-This software is meant to be a productive contribution to the rapidly growing AI-generated media industry. It will help artists with tasks such as animating a custom character or using the character as a model for clothing etc.
+We acknowledge the unethical potential of FaceFusion and are resolutely dedicated to establishing safeguards against such misuse. This program has been engineered to abstain from processing inappropriate content such as nudity, graphic content and sensitive material.
 
-The developers of this software are aware of its possible unethical applications and are committed to take preventative measures against them. It has a built-in check which prevents the program from working on inappropriate media including but not limited to nudity, graphic content, sensitive material such as war footage etc. We will continue to develop this project in the positive direction while adhering to law and ethics. This project may be shut down or include watermarks on the output if requested by law.
+It is important to note that we maintain a strong stance against any type of pornographic nature and do not collaborate with any websites promoting the unauthorized use of our software.
 
-Users of this software are expected to use this software responsibly while abiding the local law. If face of a real person is being used, users are suggested to get consent from the concerned person and clearly mention that it is a deepfake when posting content online. Developers of this software will not be responsible for actions of end-users.
+Users who seek to engage in such activities will face consequences, including being banned from our community. We reserve the right to report developers on GitHub who distribute unlocked forks of our software at any time.
 
 
 Documentation

facefusion/choices.py

@@ -8,3 +8,4 @@
 face_analyser_gender : List[FaceAnalyserGender] = [ 'male', 'female' ]
 temp_frame_format : List[TempFrameFormat] = [ 'jpg', 'png' ]
 output_video_encoder : List[OutputVideoEncoder] = [ 'libx264', 'libx265', 'libvpx-vp9', 'h264_nvenc', 'hevc_nvenc' ]
+

facefusion/face_analyser.py

@@ -1,13 +1,14 @@
-import threading
 from typing import Any, Optional, List
+import threading
 import insightface
 import numpy
 
 import facefusion.globals
+from facefusion.face_cache import get_faces_cache, set_faces_cache
 from facefusion.typing import Frame, Face, FaceAnalyserDirection, FaceAnalyserAge, FaceAnalyserGender
 
 FACE_ANALYSER = None
-THREAD_LOCK = threading.Lock()
+THREAD_LOCK : threading.Lock = threading.Lock()
 
 
 def get_face_analyser() -> Any:
@@ -38,7 +39,12 @@ def get_one_face(frame : Frame, position : int = 0) -> Optional[Face]:
 
 def get_many_faces(frame : Frame) -> List[Face]:
 	try:
-		faces = get_face_analyser().get(frame)
+		faces_cache = get_faces_cache(frame)
+		if faces_cache:
+			faces = faces_cache
+		else:
+			faces = get_face_analyser().get(frame)
+			set_faces_cache(frame, faces)
 		if facefusion.globals.face_analyser_direction:
 			faces = sort_by_direction(faces, facefusion.globals.face_analyser_direction)
 		if facefusion.globals.face_analyser_age:
@@ -100,7 +106,3 @@ def filter_by_gender(faces : List[Face], gender : FaceAnalyserGender) -> List[Fa
 		if face['gender'] == 0 and gender == 'female':
 			filter_faces.append(face)
 	return filter_faces
-
-
-def get_faces_total(frame : Frame) -> int:
-	return len(get_many_faces(frame))

facefusion/face_cache.py

@@ -0,0 +1,29 @@
+from typing import Optional, List, Dict
+import hashlib
+
+from facefusion.typing import Frame, Face
+
+FACES_CACHE : Dict[str, List[Face]] = {}
+
+
+def get_faces_cache(frame : Frame) -> Optional[List[Face]]:
+	frame_hash = create_frame_hash(frame)
+	if frame_hash in FACES_CACHE:
+		return FACES_CACHE[frame_hash]
+	return None
+
+
+def set_faces_cache(frame : Frame, faces : List[Face]) -> None:
+	frame_hash = create_frame_hash(frame)
+	if frame_hash:
+		FACES_CACHE[frame_hash] = faces
+
+
+def clear_faces_cache() -> None:
+	global FACES_CACHE
+
+	FACES_CACHE = {}
+
+
+def create_frame_hash(frame : Frame) -> Optional[str]:
+	return hashlib.sha256(frame.tobytes()).hexdigest() if frame is not None else None

facefusion/globals.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from facefusion.typing import FaceRecognition, FaceAnalyserDirection, FaceAnalyserAge, FaceAnalyserGender, TempFrameFormat
+from facefusion.typing import FaceRecognition, FaceAnalyserDirection, FaceAnalyserAge, FaceAnalyserGender, TempFrameFormat, OutputVideoEncoder
 
 source_path : Optional[str] = None
 target_path : Optional[str] = None
@@ -23,7 +23,7 @@
 temp_frame_format : Optional[TempFrameFormat] = None
 temp_frame_quality : Optional[int] = None
 output_image_quality : Optional[int] = None
-output_video_encoder : Optional[str] = None
+output_video_encoder : Optional[OutputVideoEncoder] = None
 output_video_quality : Optional[int] = None
 max_memory : Optional[int] = None
 execution_providers : List[str] = []

facefusion/installer.py

@@ -1,4 +1,5 @@
 from typing import Dict, Tuple
+import argparse
 import os
 import sys
 import subprocess
@@ -8,7 +9,7 @@
 
 import inquirer
 
-from facefusion import wording
+from facefusion import metadata, wording
 
 ONNXRUNTIMES : Dict[str, Tuple[str, str]] =\
 {
@@ -22,27 +23,38 @@
 
 
 def run() -> None:
-	answers : Dict[str, str] = inquirer.prompt(
-	[
-		inquirer.List(
-			'onnxruntime_key',
-			message = wording.get('select_onnxruntime_install'),
-			choices = list(ONNXRUNTIMES.keys())
-		)
-	])
+	program = argparse.ArgumentParser(formatter_class = lambda prog: argparse.HelpFormatter(prog, max_help_position = 120))
+	program.add_argument('--onnxruntime', help = wording.get('onnxruntime_help'), dest = 'onnxruntime', choices = ONNXRUNTIMES.keys())
+	program.add_argument('-v', '--version', version = metadata.get('name') + ' ' + metadata.get('version'), action = 'version')
+	args = program.parse_args()
+
+	if args.onnxruntime:
+		answers =\
+		{
+			'onnxruntime': args.onnxruntime
+		}
+	else:
+		answers = inquirer.prompt(
+		[
+			inquirer.List(
+				'onnxruntime',
+				message = wording.get('onnxruntime_help'),
+				choices = list(ONNXRUNTIMES.keys())
+			)
+		])
 
 	if answers is not None:
-		onnxruntime_key = answers['onnxruntime_key']
-		onnxruntime_name, onnxruntime_version = ONNXRUNTIMES[onnxruntime_key]
+		onnxruntime = answers['onnxruntime']
+		onnxruntime_name, onnxruntime_version = ONNXRUNTIMES[onnxruntime]
 		python_id = 'cp' + str(sys.version_info.major) + str(sys.version_info.minor)
 		subprocess.call([ 'pip', 'uninstall', 'torch', '-y' ])
-		if onnxruntime_key == 'cuda':
+		if onnxruntime == 'cuda':
 			subprocess.call([ 'pip', 'install', '-r', 'requirements.txt', '--extra-index-url', 'https://download.pytorch.org/whl/cu118' ])
 		else:
 			subprocess.call([ 'pip', 'install', '-r', 'requirements.txt' ])
-		if onnxruntime_key != 'cpu':
+		if onnxruntime != 'cpu':
 			subprocess.call([ 'pip', 'uninstall', 'onnxruntime', onnxruntime_name, '-y' ])
-		if onnxruntime_key != 'coreml-silicon':
+		if onnxruntime != 'coreml-silicon':
 			subprocess.call([ 'pip', 'install', onnxruntime_name + '==' + onnxruntime_version ])
 		elif python_id in [ 'cp39', 'cp310', 'cp311' ]:
 			wheel_name = '-'.join([ 'onnxruntime_silicon', onnxruntime_version, python_id, python_id, 'macosx_12_0_arm64.whl' ])

facefusion/metadata.py

@@ -2,7 +2,7 @@
 {
 	'name': 'FaceFusion',
 	'description': 'Next generation face swapper and enhancer',
-	'version': '1.1.0',
+	'version': '1.2.0',
 	'license': 'MIT',
 	'author': 'Henry Ruhs',
 	'url': 'https://facefusion.io'

facefusion/predictor.py

@@ -1,4 +1,5 @@
 import threading
+from functools import lru_cache
 import numpy
 import opennsfw2
 from PIL import Image
@@ -7,7 +8,7 @@
 from facefusion.typing import Frame
 
 PREDICTOR = None
-THREAD_LOCK = threading.Lock()
+THREAD_LOCK : threading.Lock = threading.Lock()
 MAX_PROBABILITY = 0.75
 
 
@@ -34,10 +35,12 @@ def predict_frame(target_frame : Frame) -> bool:
 	return probability > MAX_PROBABILITY
 
 
-def predict_image(target_path : str) -> bool:
-	return opennsfw2.predict_image(target_path) > MAX_PROBABILITY
+@lru_cache(maxsize = None)
+def predict_image(image_path : str) -> bool:
+	return opennsfw2.predict_image(image_path) > MAX_PROBABILITY
 
 
-def predict_video(target_path : str) -> bool:
-	_, probabilities = opennsfw2.predict_video_frames(video_path = target_path, frame_interval = 100)
+@lru_cache(maxsize = None)
+def predict_video(video_path : str) -> bool:
+	_, probabilities = opennsfw2.predict_video_frames(video_path = video_path, frame_interval = 25)
 	return any(probability > MAX_PROBABILITY for probability in probabilities)

facefusion/processors/frame/core.py

@@ -57,16 +57,19 @@ def clear_frame_processors_modules() -> None:
 	FRAME_PROCESSORS_MODULES = []
 
 
-def multi_process_frame(source_path : str, temp_frame_paths : List[str], process_frames: Callable[[str, List[str], Any], None], update: Callable[[], None]) -> None:
-	with ThreadPoolExecutor(max_workers = facefusion.globals.execution_thread_count) as executor:
-		futures = []
-		queue = create_queue(temp_frame_paths)
-		queue_per_future = max(len(temp_frame_paths) // facefusion.globals.execution_thread_count * facefusion.globals.execution_queue_count, 1)
-		while not queue.empty():
-			future = executor.submit(process_frames, source_path, pick_queue(queue, queue_per_future), update)
-			futures.append(future)
-		for future in as_completed(futures):
-			future.result()
+def multi_process_frames(source_path : str, temp_frame_paths : List[str], process_frames : Callable[[str, List[str], Callable[[], None]], None]) -> None:
+	progress_bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]'
+	with tqdm(total = len(temp_frame_paths), desc = wording.get('processing'), unit = 'frame', dynamic_ncols = True, bar_format = progress_bar_format) as progress:
+		with ThreadPoolExecutor(max_workers = facefusion.globals.execution_thread_count) as executor:
+			futures = []
+			queue_temp_frame_paths : Queue[str] = create_queue(temp_frame_paths)
+			queue_per_future = max(len(temp_frame_paths) // facefusion.globals.execution_thread_count * facefusion.globals.execution_queue_count, 1)
+			while not queue_temp_frame_paths.empty():
+				payload_temp_frame_paths = pick_queue(queue_temp_frame_paths, queue_per_future)
+				future = executor.submit(process_frames, source_path, payload_temp_frame_paths, lambda: update_progress(progress))
+				futures.append(future)
+			for future_done in as_completed(futures):
+				future_done.result()
 
 
 def create_queue(temp_frame_paths : List[str]) -> Queue[str]:
@@ -84,13 +87,6 @@ def pick_queue(queue : Queue[str], queue_per_future : int) -> List[str]:
 	return queues
 
 
-def process_video(source_path : str, frame_paths : List[str], process_frames : Callable[[str, List[str], Any], None]) -> None:
-	progress_bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]'
-	total = len(frame_paths)
-	with tqdm(total = total, desc = wording.get('processing'), unit = 'frame', dynamic_ncols = True, bar_format = progress_bar_format) as progress:
-		multi_process_frame(source_path, frame_paths, process_frames, lambda: update_progress(progress))
-
-
 def update_progress(progress : Any = None) -> None:
 	process = psutil.Process(os.getpid())
 	memory_usage = process.memory_info().rss / 1024 / 1024 / 1024

facefusion/processors/frame/modules/face_enhancer.py

@@ -1,5 +1,4 @@
 from typing import Any, List, Callable
-import cv2
 import threading
 from gfpgan.utils import GFPGANer
 
@@ -9,10 +8,11 @@
 from facefusion.face_analyser import get_many_faces
 from facefusion.typing import Frame, Face, ProcessMode
 from facefusion.utilities import conditional_download, resolve_relative_path, is_image, is_video
+from facefusion.vision import read_image, read_static_image, write_image
 
 FRAME_PROCESSOR = None
-THREAD_SEMAPHORE = threading.Semaphore()
-THREAD_LOCK = threading.Lock()
+THREAD_SEMAPHORE : threading.Semaphore = threading.Semaphore()
+THREAD_LOCK : threading.Lock = threading.Lock()
 NAME = 'FACEFUSION.FRAME_PROCESSOR.FACE_ENHANCER'
 
 
@@ -54,6 +54,7 @@ def pre_process(mode : ProcessMode) -> bool:
 
 def post_process() -> None:
 	clear_frame_processor()
+	read_static_image.cache_clear()
 
 
 def enhance_face(target_face : Face, temp_frame : Frame) -> Frame:
@@ -83,20 +84,19 @@ def process_frame(source_face : Face, reference_face : Face, temp_frame : Frame)
 	return temp_frame
 
 
-def process_frames(source_path : str, temp_frame_paths : List[str], update: Callable[[], None]) -> None:
+def process_frames(source_path : str, temp_frame_paths : List[str], update_progress: Callable[[], None]) -> None:
 	for temp_frame_path in temp_frame_paths:
-		temp_frame = cv2.imread(temp_frame_path)
+		temp_frame = read_image(temp_frame_path)
 		result_frame = process_frame(None, None, temp_frame)
-		cv2.imwrite(temp_frame_path, result_frame)
-		if update:
-			update()
+		write_image(temp_frame_path, result_frame)
+		update_progress()
 
 
 def process_image(source_path : str, target_path : str, output_path : str) -> None:
-	target_frame = cv2.imread(target_path)
+	target_frame = read_static_image(target_path)
 	result_frame = process_frame(None, None, target_frame)
-	cv2.imwrite(output_path, result_frame)
+	write_image(output_path, result_frame)
 
 
 def process_video(source_path : str, temp_frame_paths : List[str]) -> None:
-	facefusion.processors.frame.core.process_video(None, temp_frame_paths, process_frames)
+	facefusion.processors.frame.core.multi_process_frames(None, temp_frame_paths, process_frames)