feat: Implement Optical Flow KPS tracking for webcam performance
Introduces Nth-frame full face detection combined with KCF bounding
box tracking and Lucas-Kanade (LK) optical flow for keypoint (KPS)
tracking on intermediate frames. This is primarily for single-face
webcam mode to improve performance while maintaining per-frame swaps.
Key Changes:
- Modified `face_swapper.py` (`process_frame`):
- Full `insightface.FaceAnalysis` runs every N frames (default 5)
or if tracking is lost.
- KCF tracker updates bounding box on intermediate frames.
- Optical flow (`cv2.calcOpticalFlowPyrLK`) tracks the 5 keypoints
from the previous frame to the current intermediate frame.
- A `Face` object is constructed with tracked bbox and KPS for
swapping on intermediate frames (detailed landmarks like
`landmark_2d_106` are None for these).
- Experimental similar logic added to `_process_live_target_v2`
for `map_faces=True` live mode (non-many_faces path).
- Robustness:
- Mouth masking and face mask creation functions in `face_swapper.py`
now handle cases where `landmark_2d_106` is `None` (e.g., by
skipping mouth mask or using bbox for face mask).
- Added division-by-zero check in `apply_color_transfer`.
- State Management:
- Introduced `reset_tracker_state()` in `face_swapper.py` to clear
all tracking-related global variables.
- `ui.py` now calls `reset_tracker_state()` at appropriate points
(webcam start, mode changes, new source image selection) to ensure
clean tracking for new sessions.
- `DETECTION_INTERVAL` in `face_swapper.py` increased to 5.
This aims to provide you with a smoother face swap experience with better FPS
by reducing the frequency of expensive full face analysis, while the
actual swap operation continues on every frame using tracked data.
pull/1298/head
parent
a01314b52c
commit
4e36622a47
File diff suppressed because it is too large
Load Diff
|
|
@ -19,6 +19,7 @@ from modules.face_analyser import (
|
|||
)
|
||||
from modules.capturer import get_video_frame, get_video_frame_total
|
||||
from modules.processors.frame.core import get_frame_processors_modules
|
||||
from modules.processors.frame.face_swapper import reset_tracker_state # Added import
|
||||
from modules.utilities import (
|
||||
is_image,
|
||||
is_video,
|
||||
|
|
@ -240,6 +241,7 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
|
|||
command=lambda: (
|
||||
setattr(modules.globals, "many_faces", many_faces_value.get()),
|
||||
save_switch_states(),
|
||||
reset_tracker_state() # Added reset call
|
||||
),
|
||||
)
|
||||
many_faces_switch.place(relx=0.6, rely=0.65)
|
||||
|
|
@ -266,7 +268,8 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
|
|||
command=lambda: (
|
||||
setattr(modules.globals, "map_faces", map_faces.get()),
|
||||
save_switch_states(),
|
||||
close_mapper_window() if not map_faces.get() else None
|
||||
close_mapper_window() if not map_faces.get() else None,
|
||||
reset_tracker_state() # Added reset call
|
||||
),
|
||||
)
|
||||
map_faces_switch.place(relx=0.1, rely=0.75)
|
||||
|
|
@ -604,9 +607,11 @@ def select_source_path() -> None:
|
|||
RECENT_DIRECTORY_SOURCE = os.path.dirname(modules.globals.source_path)
|
||||
image = render_image_preview(modules.globals.source_path, (200, 200))
|
||||
source_label.configure(image=image)
|
||||
reset_tracker_state() # Added reset call
|
||||
else:
|
||||
modules.globals.source_path = None
|
||||
source_label.configure(image=None)
|
||||
reset_tracker_state() # Added reset call even if source is cleared
|
||||
|
||||
|
||||
def swap_faces_paths() -> None:
|
||||
|
|
@ -979,6 +984,8 @@ def create_webcam_preview(camera_index: int):
|
|||
frame_count = 0
|
||||
fps = 0
|
||||
|
||||
reset_tracker_state() # Ensure tracker is reset before starting webcam loop
|
||||
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
|
|
|
|||
Loading…
Reference in New Issue