Sample video frames across the whole clip.

A 5 frame clip asked for 3 frames returned the first three, so the last frame was dropped.
This commit is contained in:
Sasha Mitchell 2026-09-29 04:36:41 +07:00
parent 7a1af71545
commit b81eb46316
No known key found for this signature in database
2 changed files with 54 additions and 2 deletions

View File

@ -0,0 +1,42 @@
import importlib.util
import sys
import types
import unittest
from pathlib import Path
def load_utils():
torch = types.ModuleType("torch")
sys.modules["torch"] = torch
tv = types.ModuleType("torchvision")
tvf = types.ModuleType("torchvision.transforms")
functional = types.ModuleType("torchvision.transforms.functional")
functional.resize = lambda *args, **kwargs: None
sys.modules["torchvision"] = tv
sys.modules["torchvision.transforms"] = tvf
sys.modules["torchvision.transforms.functional"] = functional
cv2 = types.ModuleType("cv2")
sys.modules["cv2"] = cv2
decord = types.ModuleType("decord")
decord.bridge = types.SimpleNamespace(set_bridge=lambda *args, **kwargs: None)
decord.VideoReader = object
sys.modules["decord"] = decord
path = Path(__file__).resolve().parent / "utils.py"
spec = importlib.util.spec_from_file_location("cogvideo_dataset_utils", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class FrameIndexTest(unittest.TestCase):
def test_a_short_video_keeps_the_last_frame(self):
utils = load_utils()
self.assertEqual(utils.sample_frame_indices(5, 3), [0, 2, 4])
self.assertEqual(utils.sample_frame_indices(10, 4), [0, 3, 6, 9])
self.assertEqual(utils.sample_frame_indices(5, 5), list(range(5)))
self.assertEqual(utils.sample_frame_indices(8, 1), [0])
if __name__ == "__main__":
unittest.main()

View File

@ -67,6 +67,16 @@ def load_images_from_videos(videos_path: List[Path]) -> List[Path]:
return first_frame_paths
def sample_frame_indices(num_frames: int, max_num_frames: int) -> List[int]:
if max_num_frames <= 1:
return [0]
if num_frames <= max_num_frames:
return list(range(num_frames))
span = num_frames - 1
slots = max_num_frames - 1
return [(i * span + slots // 2) // slots for i in range(max_num_frames)]
########## preprocessors ##########
@ -139,7 +149,7 @@ def preprocess_video_with_resize(
frames = torch.cat([frames, repeated_frames], dim=0)
return frames.float().permute(0, 3, 1, 2).contiguous()
else:
indices = list(range(0, video_num_frames, video_num_frames // max_num_frames))
indices = sample_frame_indices(video_num_frames, max_num_frames)
frames = video_reader.get_batch(indices)
frames = frames[:max_num_frames].float()
frames = frames.permute(0, 3, 1, 2).contiguous()
@ -182,7 +192,7 @@ def preprocess_video_with_buckets(
key=lambda bucket: video_num_frames - bucket[0],
default=1,
)[0]
frame_indices = list(range(0, video_num_frames, video_num_frames // nearest_frame_bucket))
frame_indices = sample_frame_indices(video_num_frames, nearest_frame_bucket)
frames = video_reader.get_batch(frame_indices)
frames = frames[:nearest_frame_bucket].float()
frames = frames.permute(0, 3, 1, 2).contiguous()