Spaces:
Runtime error
Runtime error
File size: 1,922 Bytes
69667cb 780a177 69667cb 780a177 69667cb c628e2d 38cc643 c628e2d 38cc643 c628e2d 69667cb c628e2d 69667cb c628e2d 69667cb a5c23d1 69667cb 780a177 c628e2d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
"""
"""
from typing import Any
from typing import Callable
from typing import ParamSpec
import spaces
import torch
from spaces.zero.torch.aoti import ZeroGPUCompiledModel
from spaces.zero.torch.aoti import ZeroGPUWeights
from fa3 import FlashFusedFluxAttnProcessor3_0
P = ParamSpec('P')
INDUCTOR_CONFIGS = {
'conv_1x1_as_mm': True,
'epilogue_fusion': False,
'coordinate_descent_tuning': True,
'coordinate_descent_check_all_directions': True,
'max_autotune': True,
'triton.cudagraphs': True,
}
def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kwargs):
blocks_A = pipeline.transformer.transformer_blocks
blocks_B = pipeline.transformer.single_transformer_blocks
@spaces.GPU(duration=1500)
def compile_transformer_block_AB():
with spaces.aoti_capture(blocks_A[0]) as call_A:
pipeline(*args, **kwargs)
with spaces.aoti_capture(blocks_B[0]) as call_B:
pipeline(*args, **kwargs)
exported_A = torch.export.export(
mod=blocks_A[0],
args=call.args,
kwargs=call.kwargs,
)
exported_B = torch.export.export(
mod=blocks_B[0],
args=call.args,
kwargs=call.kwargs,
)
return (
spaces.aoti_compile(exported_A, INDUCTOR_CONFIGS).archive_file,
spaces.aoti_compile(exported_B, INDUCTOR_CONFIGS).archive_file,
)
pipeline.transformer.fuse_qkv_projections()
pipeline.transformer.set_attn_processor(FlashFusedFluxAttnProcessor3_0())
archive_file_A, archive_file_B = compile_transformer_block_AB()
for blocks, archive_file in zip((blocks_A, blocks_B), (archive_file_A, archive_file_B)):
for block in blocks:
weights = ZeroGPUWeights(block.state_dict())
block.forward = ZeroGPUCompiledModel(archive_file, weights)
|