mirror of
https://github.com/THUDM/CogVideo.git
synced 2026-11-22 18:23:34 +08:00
Sample video frames across the whole clip.
A 5 frame clip asked for 3 frames returned the first three, so the last frame was dropped.
This commit is contained in:
parent
7a1af71545
commit
b81eb46316
42
finetune/datasets/frame_indices_check.py
Normal file
42
finetune/datasets/frame_indices_check.py
Normal file
@ -0,0 +1,42 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_utils():
|
||||
torch = types.ModuleType("torch")
|
||||
sys.modules["torch"] = torch
|
||||
tv = types.ModuleType("torchvision")
|
||||
tvf = types.ModuleType("torchvision.transforms")
|
||||
functional = types.ModuleType("torchvision.transforms.functional")
|
||||
functional.resize = lambda *args, **kwargs: None
|
||||
sys.modules["torchvision"] = tv
|
||||
sys.modules["torchvision.transforms"] = tvf
|
||||
sys.modules["torchvision.transforms.functional"] = functional
|
||||
cv2 = types.ModuleType("cv2")
|
||||
sys.modules["cv2"] = cv2
|
||||
decord = types.ModuleType("decord")
|
||||
decord.bridge = types.SimpleNamespace(set_bridge=lambda *args, **kwargs: None)
|
||||
decord.VideoReader = object
|
||||
sys.modules["decord"] = decord
|
||||
|
||||
path = Path(__file__).resolve().parent / "utils.py"
|
||||
spec = importlib.util.spec_from_file_location("cogvideo_dataset_utils", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class FrameIndexTest(unittest.TestCase):
|
||||
def test_a_short_video_keeps_the_last_frame(self):
|
||||
utils = load_utils()
|
||||
self.assertEqual(utils.sample_frame_indices(5, 3), [0, 2, 4])
|
||||
self.assertEqual(utils.sample_frame_indices(10, 4), [0, 3, 6, 9])
|
||||
self.assertEqual(utils.sample_frame_indices(5, 5), list(range(5)))
|
||||
self.assertEqual(utils.sample_frame_indices(8, 1), [0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@ -67,6 +67,16 @@ def load_images_from_videos(videos_path: List[Path]) -> List[Path]:
|
||||
return first_frame_paths
|
||||
|
||||
|
||||
def sample_frame_indices(num_frames: int, max_num_frames: int) -> List[int]:
|
||||
if max_num_frames <= 1:
|
||||
return [0]
|
||||
if num_frames <= max_num_frames:
|
||||
return list(range(num_frames))
|
||||
span = num_frames - 1
|
||||
slots = max_num_frames - 1
|
||||
return [(i * span + slots // 2) // slots for i in range(max_num_frames)]
|
||||
|
||||
|
||||
########## preprocessors ##########
|
||||
|
||||
|
||||
@ -139,7 +149,7 @@ def preprocess_video_with_resize(
|
||||
frames = torch.cat([frames, repeated_frames], dim=0)
|
||||
return frames.float().permute(0, 3, 1, 2).contiguous()
|
||||
else:
|
||||
indices = list(range(0, video_num_frames, video_num_frames // max_num_frames))
|
||||
indices = sample_frame_indices(video_num_frames, max_num_frames)
|
||||
frames = video_reader.get_batch(indices)
|
||||
frames = frames[:max_num_frames].float()
|
||||
frames = frames.permute(0, 3, 1, 2).contiguous()
|
||||
@ -182,7 +192,7 @@ def preprocess_video_with_buckets(
|
||||
key=lambda bucket: video_num_frames - bucket[0],
|
||||
default=1,
|
||||
)[0]
|
||||
frame_indices = list(range(0, video_num_frames, video_num_frames // nearest_frame_bucket))
|
||||
frame_indices = sample_frame_indices(video_num_frames, nearest_frame_bucket)
|
||||
frames = video_reader.get_batch(frame_indices)
|
||||
frames = frames[:nearest_frame_bucket].float()
|
||||
frames = frames.permute(0, 3, 1, 2).contiguous()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user