add cogvideox support for gaudi. #1600
Conversation
|
test command：python cogvideo_generate.py --model_name_or_path /mnt/disk2/libo/hf_models/CogVideoX-2b/ --output_name |
panda_gaudi.mp4 |
|
could you please help to review my change @shepark |
I sent email to you, please response it. thanks. |
|
Hi @regisss , could you please help to review this PR? Thanks! |
| @@ -0,0 +1,86 @@ | |||
| import argparse | |||
There was a problem hiding this comment.
Please remove this file and use text_to_video_generation.py script for this sample (need to switch pipeline based on model, similarly as one in text_to_image_generation.py)
| @@ -1,3 +1,4 @@ | |||
| from .pipelines.cogvideox.cogvideoX_gaudi import adapt_cogvideo_to_gaudi | |||
| @@ -0,0 +1,468 @@ | |||
| from typing import Any, Dict, Optional, Tuple, Union | |||
There was a problem hiding this comment.
This file does not belong here. Either add these definitions to your cogvideox pipeline py file, or maybe put some of the attention functions under optimum/habana/diffusers/models/* and rest inside cogvideox pipeline py file.
| from diffusers.models.autoencoders.autoencoder_kl_cogvideox import AutoencoderKLCogVideoX | ||
|
|
||
|
|
||
| class AutoencoderKLCogVideoXGaudi(AutoencoderKLCogVideoX): |
There was a problem hiding this comment.
Adapted from? Please put reference here
| from diffusers.models.autoencoders.vae import DecoderOutput | ||
|
|
||
|
|
||
| class CogVideoXCausalConv3dGaudi(nn.Module): |
There was a problem hiding this comment.
Adapted from? Please put reference here
| frames: torch.Tensor | ||
|
|
||
| # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps | ||
| def retrieve_timesteps( |
There was a problem hiding this comment.
can we import this directly from diffusers?
| logger = logging.get_logger(__name__) # pylint: disable=invalid-name | ||
|
|
||
|
|
||
| class time_box_t(): |
There was a problem hiding this comment.
Please use same timing mechanism as in other pipelines (need to deal with warmup etc.)
| htcore.mark_step() | ||
| time_box.show_time('transformer_hpu') | ||
|
|
||
| #HabanaProfile.stop() |
| ).frames[0] | ||
|
|
||
| assert video is not None | ||
| assert 49 == len(video) |
| generator=torch.Generator(device="cpu").manual_seed(42), | ||
| ).frames[0] | ||
|
|
||
| assert video is not None |
There was a problem hiding this comment.
use self.assertIsNotNone here
What does this PR do?
Fixes # (issue)
Before submitting