Downloading Large Number of Files in Frontend using jszip

你说的曾经没有我的故事 提交于 2020-03-24 18:25:34

问题


Unable to zip all the files using jszip. JS zip is reading all 402 files as shown in snapshot from the console from around 143 requests but zipping only 143 files. I am using parallelimit to process multiple async requests simultaneously and cleanly. I am How can we get all the 403 files in the result?

private downloadUntouchedFiles = () => {


let requestObjectInfo = [];
let index = 0;
this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });


this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject).then((requests) => {
  debugger;
  if (!(!requests)) {
    if (requests.length > 0) {

      var zip = new JSZip();
      var zipFileName = "ES_Unviewed_Files";
      var promises = [];
      this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });



      const downloadSubPromises = [];
      let i =0;
          requests.forEach((req) => {
            req.Folder.Files.forEach(f => {
              f.Name = this.state.initials + '_' + this.state.userId + '_' + f.Name;
              console.log(f.Name);
              i++;
              console.log(i);
              downloadSubPromises.push((submit: any) => {
                JSZipUtils.getBinaryContent(f.ServerRelativeUrl, (err, data) => {
                  try {
                    if (err) {
                      throw err;
                    }
                    zip.file(f.Name, data, { binary: true });

                    submit(null, true);

                  } catch (err) {
                    submit(err, true);
                    this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                    this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                      Component: this._canonicalName,
                      Message: ErrorMessages.COM007,
                      UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                      Group: '',
                      Notes: err,
                      Source: Constants.EXCEPTION_UI_SOURCE,
                      ExceptionID: Guid.create().toString()
                    } as ExceptionObject).then(() => {
                    });
                  }
                });

              });
            });
            requestObjectInfo.push(req);
          });
          parallelLimit(downloadSubPromises, Constants.DOWNLOAD_BATCH_MAX_FILE_LIMIT,
            (err, results) => {
              try {
                console.log(results);
                debugger;
                zip
                  .generateInternalStream({ type: "blob" })
                  .accumulate()
                  .then((content) => {
                    saveAs(content, zipFileName + ".zip");
                  });
              }
              catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
              }});

      while (index < requestObjectInfo.length) {
        this.setState({ requestObject: requestObjectInfo[index] });
        if (this.state.requestObject.Status !== Constants.ES_DOWNLOADREQUEST_STATUS) {
          this.updateESRequestStatus(Constants.ES_DOWNLOADREQUEST_STATUS);
        }
        index++;
      }
      this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
    }
  }
});

}

In this case only 55-75MB on JS Heap is used.


回答1:


This will work

private downloadUntouchedFiles = () => {
this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: true });
let statusUpdatePromises = [];
this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject).then((requests) => {
  if (!(!requests)) {
    if (requests.length > 0) {
      var zip = new JSZip();
      var zipFileName = "ES_Unviewed_Files";
      const downloadSubPromises = [];
      requests.forEach((req: any) => {
        req.Folder.Files.forEach((f: any) => {
          f.Name = this.state.userId + '_' + f.Name;
          downloadSubPromises.push((submit: any) => {
            JSZipUtils.getBinaryContent(`${new Constants().BASE_URL}${encodeURIComponent(f.ServerRelativeUrl).replace('%2F', '/')}`, (err, data) => {
              try {
                if (err) {
                  submit(null, true);
                } else {
                  zip.file(f.Name, data, { binary: true });
                  submit(null, true);
                }
              } catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
                submit(null, false);
              }
            });

          });
        });
        statusUpdatePromises.push((submit: any) => {
          this.setState({ requestObject: req }, () => {
            if (this.state.requestObject.Status !== Constants.ES_DOWNLOADREQUEST_STATUS) {
              this.updateESRequestStatus(Constants.ES_DOWNLOADREQUEST_STATUS).then(res => {
                submit(true);
              });
            } else {
              submit(true);
            }
          });
        });
      });
      parallelLimit(downloadSubPromises, Constants.UPLOAD_BATCH_MAX_FILE_LIMIT,
        (err: any, results: any) => {
          parallelLimit(statusUpdatePromises, Constants.UPLOAD_BATCH_MAX_FILE_LIMIT,
            (subErr: any, subResults: any) => {
              try {
                zip
                  .generateInternalStream({ type: "blob" })
                  .accumulate()
                  .then((content) => {
                    this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                    saveAs(content, zipFileName + ".zip");
                  });
              }
              catch (err) {
                this._eventEmitter.emit(Constants.LOADER_CHANGE, { show: false });
                this._loggerInstance.logException(Constants.SISCC_ES_EXCEPTIONS, {
                  Component: this._canonicalName,
                  Message: ErrorMessages.COM007,
                  UserName: !(!DataSingleton.getCurrentUser()) ? DataSingleton.getCurrentUser() : '',
                  Group: '',
                  Notes: err,
                  Source: Constants.EXCEPTION_UI_SOURCE,
                  ExceptionID: Guid.create().toString()
                } as ExceptionObject).then(() => {
                });
              }
            });
        });
    }
  }
});

}




回答2:


You can use async and await to make the code easier to understand and avoid deep nesting:

const ZIP_FILE_NAME = "ES_Unviewed_Files.zip"

const downloadUntouchedFiles = async () => {    
    const zip = new JSZip()
    const untouched = this._eligibilitySubmissionInstance.getUntouchedFiles(this.state.filterObject)    
    if (!untouched?.length) return

    const files = untouched.reduce((acc, { Folder: { Files } }) => 
        Files.forEach(f => acc.push({ name: `${this.state.userId}+${f.Name}`, ...f })), [])

    const downloads = files.map(({name, ServerRelativeUrl}) => 
        async () => ({ name, data: await fetchBinary(ServerRelativeUrl)}))

    const responses = await batched(downloads, Constants.DOWNLOAD_BATCH_MAX_FILE_LIMIT)

    responses.forEach(({ status, value: { name, data } }) => 
        status === 'fulfilled' && zip.file(name, data, { binary: true }))

    const content = await zip.generateInternalStream({ type: "blob" }).accumulate()
    await saveAs(content, ZIP_FILE_NAME)    
}

const fetchBinary = (file) => new Promise((resolve) => 
    JSZipUtils.getBinaryContent(url, (err, data) => err ? reject(err) : resolve(data)))

async function batched(fns, batchSize = 2) {
    const results = []    
    for(let start = 0, end = batchSize; start < fns.length; start += batchSize, end = start+batchSize) {
        const slice = fns.slice(start, end)        
        const promises = slice.map((fn) => fn())
        results.push([...await Promise.allSettled(promises)])
    }
    return results.flat()
}


来源:https://stackoverflow.com/questions/59066014/downloading-large-number-of-files-in-frontend-using-jszip

易学教程内所有资源均来自网络或用户发布的内容,如有违反法律规定的内容欢迎反馈
该文章没有解决你所遇到的问题?点击提问,说说你的问题,让更多的人一起探讨吧!