slice large file into chunks and upload using ajax and html5 FileReader

Question

What I want to implement is:

In the front end, I use the html5 file api to read the file, and then upload the file's content to the php backend using ajax, and it's ok if the filesize is small. However,if the file is big enough, it causes chrome to crash. So I split the large file into chunks using file.slice, when all chunks are uploaded to the php, merge the chunks into a single complete one.

the code is as follows:

the front end:

<style>
#container {
     min-width:300px;
     min-height:200px;
     border:3px dashed #000;
}
</style>
<div id='container'>

</div>
<script>
function addDNDListener(obj){
    obj.addEventListener('dragover',function(e){
            e.preventDefault();
            e.stopPropagation();
    },false);
    obj.addEventListener('dragenter',function(e){
            e.preventDefault();
            e.stopPropagation();
    },false);
    obj.addEventListener('drop',function(e){
            e.preventDefault();
            e.stopPropagation();
            var ul = document.createElement("ul");
            var filelist = e.dataTransfer.files;
            for(var i=0;i<filelist.length;i++){
                    var file = filelist[i];
                    var li = document.createElement('li');
                    li.innerHTML = '<label id="'+file.name+'">'+file.name+':</label>  <progress value="0" max="100"></progress>';
                    ul.appendChild(li);
            }
            document.getElementById('container').appendChild(ul);
            for(var i=0;i<filelist.length;i++){
                    var file = filelist[i];
                    uploadFile(file);
            }
    },false);
}

function uploadFile(file){
    var loaded = 0;
    var step = 1024*1024;
    var total = file.size;
    var start = 0;
    var progress = document.getElementById(file.name).nextSibling;

    var reader = new FileReader();

    reader.onprogress = function(e){
            loaded += e.loaded;
            progress.value = (loaded/total) * 100;
    };

    reader.onload = function(e){
            var xhr = new XMLHttpRequest();
            var upload = xhr.upload;
            upload.addEventListener('load',function(){
                    if(loaded <= total){
                            blob = file.slice(loaded,loaded+step+1);
                            reader.readAsBinaryString(blob);
                    }else{
                            loaded = total;
                    }
            },false);
            xhr.open("POST", "upload.php?fileName="+file.name+"&nocache="+new Date().getTime());
            xhr.overrideMimeType("application/octet-stream");
            xhr.sendAsBinary(e.target.result);
    };
    var blob = file.slice(start,start+step+1);
    reader.readAsBinaryString(blob);
}

window.onload = function(){

    addDNDListener(document.getElementById('container'));
    if(!XMLHttpRequest.prototype.sendAsBinary){ 
              XMLHttpRequest.prototype.sendAsBinary = function(datastr) {  
                        function byteValue(x) {  
                            return x.charCodeAt(0) & 0xff;  
                        }  
                        var ords = Array.prototype.map.call(datastr, byteValue);  
                        var ui8a = new Uint8Array(ords);  
                        try{
                            this.send(ui8a);
                        }catch(e){
                            this.send(ui8a.buffer);
                        }  
              };  
    }
};
</script>

the php code:

<?php
     $filename = "upload/".$_GET['fileName'];
     //$filename = "upload/".$_GET['fileName']."_".$_GET['nocache'];
     $xmlstr = $GLOBALS['HTTP_RAW_POST_DATA'];
     if(empty($xmlstr)){
             $xmlstr = file_get_contents('php://input');
     }
     $is_ok = false;
     while(!$is_ok){
            $file = fopen($filename,"ab");

            if(flock($file,LOCK_EX)){
                    fwrite($file,$xmlstr);
                    flock($file,LOCK_UN);
                    fclose($file);
                    $is_ok = true;
            }else{
                    fclose($file);
                    sleep(3);
            }
    }

The problem is, after the chunks of the file all being uploaded to the server and merged into a new one, the total file size is smaller than the original, and the merged one is broken. Where is the problem and how to fix it?

Solution 1

• Using the readAsBinaryString fn is just bad practice
• SendAsBinary is also depricated
• Reading the chunks content is just pure dum. Slicing them is enough. xhr.send(blob.slice(0,10))
• Slicing is also unnecessary unless the server don't accept such large files (such as dropbox limited REST API)
• So if anyone trying to be smart about using worker threads, base64 or FileReader for uploading large files - don't do that, it's all unnecessary.

Only time it's okey to read/slice the file is if you are deciding to encrypt/decrypt/zip the files before sending it to the server.
But only for a limited time until all browser start supporting streams.
Then you should take a look at fetch and ReadableStream

fetch(url, {method: 'post', body: new ReadableStream({...})})

---

if you just need to forward the blob to the server, just do: xhr.send(blob_or_file) and the browser will take care of reading it (correctly) and not consume any memory. And the file can be however large the file/blob is

Solution 2

There is a small error in your js script I noticed that the reader.onprogress event is triggered more times than the xhr load event. In this case some chunks are skipped. Try to increment the loaded variable inside the load function.

function uploadFile(file){
var loaded = 0;
var step = 1024*1024;
var total = file.size;
var start = 0;
var progress = document.getElementById(file.name).nextSibling.nextSibling;

var reader = new FileReader();

reader.onload = function(e){
        var xhr = new XMLHttpRequest();
        var upload = xhr.upload;
        upload.addEventListener('load',function(){
        loaded += step;
        progress.value = (loaded/total) * 100;
                if(loaded <= total){
                        blob = file.slice(loaded,loaded+step);

                        reader.readAsBinaryString(blob);
                }else{
                        loaded = total;
                }
        },false);
        xhr.open("POST", "upload.php?fileName="+file.name+"&nocache="+new Date().getTime());
        xhr.overrideMimeType("application/octet-stream");
        xhr.sendAsBinary(e.target.result);
};
var blob = file.slice(start,step);
reader.readAsBinaryString(blob); }

Solution 3

This Problem is caused mostly by global restrictions of shared Hosts. They often control the amount of data and drop the Connection if Limit is overridden. I tried this several times and several ways. Always stucking at the same Position. target file was smaller and corrupted. Even taking a smaller file for upload than this size, so that the merged data fits in the limit, the result was OK. YOu have only one Chance: Get the max size increased, for files to be opened. Every time you open the target file and write the chunk Content into it, and the size overrides the Limit, the hoster will break the Connection. Example hoster: Strato Here the Limit is globally set to 16MB. I get max merge-result size double this size. No Chance to override it.

Comments

[TheRealChx101]

It's not dum sir. See, people who have bad internet connection will have a hard time uploading a huge file back to a server. This method allows them to resume uploads after interruptions.

[Endless]

I'm not saying slicing is dum, but reading the content of the chunks is

[Endless]

if internet connection interrupts make a request to the api, ask for how much is uploaded and continue on from there.

[Endless]

it's also possible to listen for how much you have uploaded with one of xhr's progress event, send a request to the api and tell it to append the file to an existing file and start writing from offset x

[Phantom007]

Will this make multiple http request to the backend server?