Axios events when downloading large files with streams

Question

I'm using axios on server side.

I want to download big files .. technically this should be used with byte-ranges

1. Does axios handle the byte-range request so that the callback function is only called when all the response is ready
2. If 1 is not true, should I handle data chunks myself ?

In the code below :

axios({
  url: params.url,
  method: 'GET',
  responseType: 'stream' // important
}).then(function (response) {

  logger.debug('__download() : done!')
  let contentType = response.headers['content-type']
  let contentLength = response.headers['content-length']
  var writer = new streams.WritableStream()

  response.data.pipe(writer)
  // ....
})

Am I supposed to wait for something like response.on('end')?

The purpose of what I'm doing is to get the size of the buffer (which I could get by writer.getBuffer())

Thanks for any hint !

Solution

I found out that to download the stream, in memory in my case, I had to wait for the event on my writer (writer.on('finished',cb) vs response.on('end', cb )) (not sure if there is something like response.on('end'))...

var stream = require('stream');
var util = require('util');
var Writable = stream.Writable;


function MemoryStream(options) {
    if (!(this instanceof MemoryStream)) {
        return new MemoryStream(options);
    }
    Writable.call(this, options); // init super
}
util.inherits(MemoryStream, Writable);

MemoryStream.prototype._write = function (chunk, enc, cb) {
    var buffer = (Buffer.isBuffer(chunk)) ?
        chunk :
        Buffer.alloc(chunk, enc);

    if (Buffer.isBuffer(this.memStore)) {
        this.memStore = Buffer.concat([this.memStore, buffer]);
    } else {
        this.memStore = buffer
    }
    cb();
};


MemoryStream.prototype.toBuffer = function () {
    return this.memStore
};




module.exports = MemoryStream

and then in my download function :

axios({
  url: params.url,
  method: 'GET',
  responseType: 'stream' // important
}).then(function (response) {

  logger.debug('__download() : done!')
  let contentType = response.headers['content-type']
  let contentLength = response.headers['content-length']
  var writer = new MemoryStream()

  response.data.pipe(writer)

    writer.on('finish', function () {

        var b = writer.toBuffer()

        let computedContentLength = b.byteLength

        if (!contentLength) { contentLength = computedContentLength }

        return callback(null, { 'foo':'bar'})

    });
})

Comments

[Torc]

Is responseType: 'stream' only valid when using Node / server-side requests? I.e. it's not possible to do through the browser with XMLHttpRequests?

[Safal Pillai]

@torc responseType blob and document are allowed in browsers.