downloader.py 1.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859
  1. from os.path import abspath as absolute_path
  2. from time import time
  3. import aiofiles
  4. from wbb import aiohttpsession as session
  5. from wbb.core.tasks import add_task
  6. def ensure_status(status_code: int):
  7. if status_code < 200 or status_code >= 300:
  8. raise Exception(f"HttpProcessingError: {status_code}")
  9. async def download_url(
  10. url,
  11. file_path,
  12. chunk_size,
  13. ):
  14. file_path = file_path or url.split("/")[-1][:20]
  15. async with session.get(url) as response:
  16. ensure_status(response.status)
  17. async with aiofiles.open(file_path, "wb") as f:
  18. # Save content in file using aiohttp streamReader.
  19. async for chunk in response.content.iter_chunked(chunk_size):
  20. await f.write(chunk)
  21. return absolute_path(file_path)
  22. async def download(
  23. url: str,
  24. file_path: str = None,
  25. chunk_size: int = 1000000, # 1MB chunk
  26. task_id: int = int(time()),
  27. ):
  28. """
  29. :url: url where the file is located
  30. :file_path: path/to/file
  31. :chunk_size: size of a single chunk
  32. Returns:
  33. (asyncio.Task, task_id), With which you can await
  34. the task, track task progress or cancel it.
  35. """
  36. # Create a task and add it to main tasks dict
  37. # So we can cancel it using .cancelTask
  38. task, task_id = await add_task(
  39. download_url,
  40. "Downloader",
  41. url=url,
  42. file_path=file_path,
  43. chunk_size=chunk_size,
  44. )
  45. return task, task_id