Mochi video-to-video

docs/source/en/api/pipelines/mochi.md

@@ -31,6 +31,12 @@ Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers.m
   - all
   - __call__
 
+## MochiVideoToVideoPiepeline
+
+[[autodoc]] MochiVideoToVideoPiepeline
+  - all
+  - __call__
+
 ## MochiPipelineOutput
 
 [[autodoc]] pipelines.mochi.pipeline_output.MochiPipelineOutput

src/diffusers/__init__.py

@@ -319,6 +319,7 @@
             "MarigoldDepthPipeline",
             "MarigoldNormalsPipeline",
             "MochiPipeline",
+            "MochiVideoToVideoPipeline",
             "MusicLDMPipeline",
             "PaintByExamplePipeline",
             "PIAPipeline",
@@ -789,6 +790,7 @@
             MarigoldDepthPipeline,
             MarigoldNormalsPipeline,
             MochiPipeline,
+            MochiVideoToVideoPipeline,
             MusicLDMPipeline,
             PaintByExamplePipeline,
             PIAPipeline,

src/diffusers/models/transformers/transformer_mochi.py

@@ -269,6 +269,8 @@ def __init__(
         time_embed_dim: int = 256,
         activation_fn: str = "swiglu",
         max_sequence_length: int = 256,
+        sample_height: int = 60,
+        sample_width: int = 106,
     ) -> None:
         super().__init__()
 

src/diffusers/pipelines/__init__.py

@@ -252,7 +252,7 @@
             "MarigoldNormalsPipeline",
         ]
     )
-    _import_structure["mochi"] = ["MochiPipeline"]
+    _import_structure["mochi"] = ["MochiPipeline", "MochiVideoToVideoPipeline"]
     _import_structure["musicldm"] = ["MusicLDMPipeline"]
     _import_structure["paint_by_example"] = ["PaintByExamplePipeline"]
     _import_structure["pia"] = ["PIAPipeline"]
@@ -582,7 +582,7 @@
             MarigoldDepthPipeline,
             MarigoldNormalsPipeline,
         )
-        from .mochi import MochiPipeline
+        from .mochi import MochiPipeline, MochiVideoToVideoPipeline
         from .musicldm import MusicLDMPipeline
         from .pag import (
             AnimateDiffPAGPipeline,

src/diffusers/pipelines/mochi/__init__.py

@@ -23,6 +23,7 @@
     _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
 else:
     _import_structure["pipeline_mochi"] = ["MochiPipeline"]
+    _import_structure["pipeline_mochi_video2video"] = ["MochiVideoToVideoPipeline"]
 
 if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
     try:
@@ -33,6 +34,7 @@
         from ...utils.dummy_torch_and_transformers_objects import *
     else:
         from .pipeline_mochi import MochiPipeline
+        from .pipeline_mochi_video2video import MochiVideoToVideoPipeline
 
 else:
     import sys

src/diffusers/pipelines/mochi/pipeline_mochi.py

@@ -207,8 +207,6 @@ def __init__(
         self.tokenizer_max_length = (
             self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
         )
-        self.default_height = 480
-        self.default_width = 848
 
     # Adapted from diffusers.pipelines.cogvideo.pipeline_cogvideox.CogVideoXPipeline._get_t5_prompt_embeds
     def _get_t5_prompt_embeds(
@@ -577,8 +575,8 @@ def __call__(
         if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
             callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
 
-        height = height or self.default_height
-        width = width or self.default_width
+        height = height or self.transformer.config.sample_height * self.vae_spatial_scale_factor
+        width = width or self.transformer.config.sample_width * self.vae_spatial_scale_factor
 
         # 1. Check inputs. Raise error if not correct
         self.check_inputs(