diff --git a/finetune/datasets/frame_indices_check.py b/finetune/datasets/frame_indices_check.py new file mode 100644 index 0000000..eee9c8b --- /dev/null +++ b/finetune/datasets/frame_indices_check.py @@ -0,0 +1,42 @@ +import importlib.util +import sys +import types +import unittest +from pathlib import Path + + +def load_utils(): + torch = types.ModuleType("torch") + sys.modules["torch"] = torch + tv = types.ModuleType("torchvision") + tvf = types.ModuleType("torchvision.transforms") + functional = types.ModuleType("torchvision.transforms.functional") + functional.resize = lambda *args, **kwargs: None + sys.modules["torchvision"] = tv + sys.modules["torchvision.transforms"] = tvf + sys.modules["torchvision.transforms.functional"] = functional + cv2 = types.ModuleType("cv2") + sys.modules["cv2"] = cv2 + decord = types.ModuleType("decord") + decord.bridge = types.SimpleNamespace(set_bridge=lambda *args, **kwargs: None) + decord.VideoReader = object + sys.modules["decord"] = decord + + path = Path(__file__).resolve().parent / "utils.py" + spec = importlib.util.spec_from_file_location("cogvideo_dataset_utils", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FrameIndexTest(unittest.TestCase): + def test_a_short_video_keeps_the_last_frame(self): + utils = load_utils() + self.assertEqual(utils.sample_frame_indices(5, 3), [0, 2, 4]) + self.assertEqual(utils.sample_frame_indices(10, 4), [0, 3, 6, 9]) + self.assertEqual(utils.sample_frame_indices(5, 5), list(range(5))) + self.assertEqual(utils.sample_frame_indices(8, 1), [0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/finetune/datasets/utils.py b/finetune/datasets/utils.py index e2667d6..c310b12 100644 --- a/finetune/datasets/utils.py +++ b/finetune/datasets/utils.py @@ -67,6 +67,16 @@ def load_images_from_videos(videos_path: List[Path]) -> List[Path]: return first_frame_paths +def sample_frame_indices(num_frames: int, max_num_frames: int) -> List[int]: + if max_num_frames <= 1: + return [0] + if num_frames <= max_num_frames: + return list(range(num_frames)) + span = num_frames - 1 + slots = max_num_frames - 1 + return [(i * span + slots // 2) // slots for i in range(max_num_frames)] + + ########## preprocessors ########## @@ -139,7 +149,7 @@ def preprocess_video_with_resize( frames = torch.cat([frames, repeated_frames], dim=0) return frames.float().permute(0, 3, 1, 2).contiguous() else: - indices = list(range(0, video_num_frames, video_num_frames // max_num_frames)) + indices = sample_frame_indices(video_num_frames, max_num_frames) frames = video_reader.get_batch(indices) frames = frames[:max_num_frames].float() frames = frames.permute(0, 3, 1, 2).contiguous() @@ -182,7 +192,7 @@ def preprocess_video_with_buckets( key=lambda bucket: video_num_frames - bucket[0], default=1, )[0] - frame_indices = list(range(0, video_num_frames, video_num_frames // nearest_frame_bucket)) + frame_indices = sample_frame_indices(video_num_frames, nearest_frame_bucket) frames = video_reader.get_batch(frame_indices) frames = frames[:nearest_frame_bucket].float() frames = frames.permute(0, 3, 1, 2).contiguous()