Downloading Large Number of Files in Frontend using jszip

问题

Unable to zip all the files using jszip. JS zip is reading all 402 files as shown in snapshot from the console from around 143 requests but zipping only 143 files. I am using parallelimit to process multiple async requests simultaneously and cleanly. I am How can we get all the 403 files in the result?

private downloadUntouchedFiles = () => {


let requestObjectInfo = [];
let index = 0;
this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });


this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject).then((requests) => {
  debugger;
  if (!(!requests)) {
    if (requests.length > 0) {

      var zip = new JSZip();
      var zipFileName = "ES_Unviewed_Files";
      var promises = [];
      this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });



      const downloadSubPromises = [];
      let i =0;
          requests.forEach((req) => {
            req.Folder.Files.forEach(f => {
              f.Name = this.state.initials + '_' + this.state.userId + '_' + f.Name;
              console.log(f.Name);
              i++;
              console.log(i);
              downloadSubPromises.push((submit: any) => {
                JSZipUtils.getBinaryContent(f.ServerRelativeUrl, (err, data) => {
                  try {
                    if (err) {
                      throw err;
                    }
                    zip.file(f.Name, data, { binary: true });

                    submit(null, true);

                  } catch (err) {
                    submit(err, true);
                    this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                    this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                      Component: this._canonicalName,
                      Message: ErrorMessages.COM007,
                      UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                      Group: '',
                      Notes: err,
                      Source: Constants.EXCEPTION_UI_SOURCE,
                      ExceptionID: Guid.create().toString()
                    } as ExceptionObject).then(() => {
                    });
                  }
                });

              });
            });
            requestObjectInfo.push(req);
          });
          parallelLimit(downloadSubPromises, Constants.DOWNLOAD_BATCH_MAX_FILE_LIMIT,
            (err, results) => {
              try {
                console.log(results);
                debugger;
                zip
                  .generateInternalStream({ type: "blob" })
                  .accumulate()
                  .then((content) => {
                    saveAs(content, zipFileName + ".zip");
                  });
              }
              catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
              }});

      while (index < requestObjectInfo.length) {
        this.setState({ requestObject: requestObjectInfo[index] });
        if (this.state.requestObject.Status !== Constants.ES_DOWNLOADREQUEST_STATUS) {
          this.updateESRequestStatus(Constants.ES_DOWNLOADREQUEST_STATUS);
        }
        index++;
      }
      this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
    }
  }
});

}

In this case only 55-75MB on JS Heap is used.

回答1:

This will work

private downloadUntouchedFiles = () => {
this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });
let statusUpdatePromises = [];
this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject).then((requests) => {
  if (!(!requests)) {
    if (requests.length > 0) {
      var zip = new JSZip();
      var zipFileName = "ES_Unviewed_Files";
      const downloadSubPromises = [];
      requests.forEach((req: any) => {
        req.Folder.Files.forEach((f: any) => {
          f.Name = this.state.userId + '_' + f.Name;
          downloadSubPromises.push((submit: any) => {
            JSZipUtils.getBinaryContent(`${new Constants().BASE_URL}${encodeURIComponent(f.ServerRelativeUrl).replace('%2F', '/')}`, (err, data) => {
              try {
                if (err) {
                  submit(null, true);
                } else {
                  zip.file(f.Name, data, { binary: true });
                  submit(null, true);
                }
              } catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
                submit(null, false);
              }
            });

          });
        });
        statusUpdatePromises.push((submit: any) => {
          this.setState({ requestObject: req }, () => {
            if (this.state.requestObject.Status !== Constants.ES_DOWNLOADREQUEST_STATUS) {
              this.updateESRequestStatus(Constants.ES_DOWNLOADREQUEST_STATUS).then(res => {
                submit(true);
              });
            } else {
              submit(true);
            }
          });
        });
      });
      parallelLimit(downloadSubPromises, Constants.UPLOAD_BATCH_MAX_FILE_LIMIT,
        (err: any, results: any) => {
          parallelLimit(statusUpdatePromises, Constants.UPLOAD_BATCH_MAX_FILE_LIMIT,
            (subErr: any, subResults: any) => {
              try {
                zip
                  .generateInternalStream({ type: "blob" })
                  .accumulate()
                  .then((content) => {
                    this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                    saveAs(content, zipFileName + ".zip");
                  });
              }
              catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
              }
            });
        });
    }
  }
});

}

回答2:

You can use async and await to make the code easier to understand and avoid deep nesting:

const ZIP_FILE_NAME = "ES_Unviewed_Files.zip"

const downloadUntouchedFiles = async () => {    
    const zip = new JSZip()
    const untouched = this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject)    
    if (!untouched?.length) return

    const files = untouched.reduce((acc, { Folder: { Files } }) => 
        Files.forEach(f => acc.push({ name: `${this.state.userId}+${f.Name}`, ...f })), [])

    const downloads = files.map(({name, ServerRelativeUrl}) => 
        async () => ({ name, data: await fetchBinary(ServerRelativeUrl)}))

    const responses = await batched(downloads, Constants.DOWNLOAD_BATCH_MAX_FILE_LIMIT)

    responses.forEach(({ status, value: { name, data } }) => 
        status === 'fulfilled' && zip.file(name, data, { binary: true }))

    const content = await zip.generateInternalStream({ type: "blob" }).accumulate()
    await saveAs(content, ZIP_FILE_NAME)    
}

const fetchBinary = (file) => new Promise((resolve) => 
    JSZipUtils.getBinaryContent(url, (err, data) => err ? reject(err) : resolve(data)))

async function batched(fns, batchSize = 2) {
    const results = []    
    for(let start = 0, end = batchSize; start < fns.length; start += batchSize, end = start+batchSize) {
        const slice = fns.slice(start, end)        
        const promises = slice.map((fn) => fn())
        results.push([...await Promise.allSettled(promises)])
    }
    return results.flat()
}

来源：https://stackoverflow.com/questions/59066014/downloading-large-number-of-files-in-frontend-using-jszip