Spaces:
Running
Running
File size: 1,089 Bytes
3a3b216 31086ae 3a3b216 31086ae 3a3b216 31086ae 3a3b216 31086ae |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import asyncio
from typing import Awaitable, Callable, List, TypeVar
from tqdm.asyncio import tqdm as tqdm_async
from graphgen.utils.log import logger
from .loop import create_event_loop
T = TypeVar("T")
R = TypeVar("R")
def run_concurrent(
coro_fn: Callable[[T], Awaitable[R]],
items: List[T],
*,
desc: str = "processing",
unit: str = "item",
) -> List[R]:
async def _run_all():
tasks = [asyncio.create_task(coro_fn(item)) for item in items]
results = []
pbar = tqdm_async(total=len(items), desc=desc, unit=unit)
for future in asyncio.as_completed(tasks):
try:
result = await future
results.append(result)
except Exception as e:
logger.exception("Task failed: %s", e)
results.append(e)
pbar.update(1)
pbar.close()
return [res for res in results if not isinstance(res, Exception)]
loop = create_event_loop()
try:
return loop.run_until_complete(_run_all())
finally:
loop.close()
|