Progress of Python requests post

Question

I am uploading a large file using the Python requests package, and I can\'t find any way to give data back about the progress of the upload. I have seen a number of progress

Answer 1

My upload server doesn't support Chunk-Encoded so I came up with this solution. It basically just a wrapper around python IOBase and allow tqdm.wrapattr to work seamless.

import io
import requests
from typing import Union
from tqdm import tqdm
from tqdm.utils import CallbackIOWrapper

class UploadChunksIterator(Iterable):
    """
    This is an interface between python requests and tqdm.
    Make tqdm to be accessed just like IOBase for requests lib.
    """

    def __init__(
        self, file: Union[io.BufferedReader, CallbackIOWrapper], total_size: int, chunk_size: int = 16 * 1024
    ):  # 16MiB
        self.file = file
        self.chunk_size = chunk_size
        self.total_size = total_size

    def __iter__(self):
        return self

    def __next__(self):
        data = self.file.read(self.chunk_size)
        if not data:
            raise StopIteration
        return data

    # we dont retrive len from io.BufferedReader because CallbackIOWrapper only has read() method.
    def __len__(self):
        return self.total_size

fp = "data/mydata.mp4"
s3url = "example.com"
_quiet = False

with open(fp, "rb") as f:
    total_size = os.fstat(f.fileno()).st_size
    if not _quiet:
        f = tqdm.wrapattr(f, "read", desc=hv, miniters=1, total=total_size, ascii=True)

    with f as f_iter:
        res = requests.put(
            url=s3url,
            data=UploadChunksIterator(f_iter, total_size=total_size),
        )
    res.raise_for_status()