Workaround AWS Lambda & S3: reading from ReadStream doesn't work, but Buffer does

This is related to #337 but since it’s an issue with readSync correctly identifying/reading a Stream, I’ll put my problem and solution here:

On an AWS Lambda function trying to read an excel file from S3, usually you’d pick the file via ReadStream:

var file = s3.getObject({Bucket: bucket, Key: key}).createReadStream();
var workbook = XLSX.read(file);

This code throws the following:

TypeError: f.substr is not a function
at firstbyte (/var/task/node_modules/xlsx/xlsx.js:11362:41)
at Object.readSync [as read] (/var/task/node_modules/xlsx/xlsx.js:11388:14)

Line https://github.com/SheetJS/js-xlsx/blob/master/xlsx.js#L11359 is the culprit, because at this point f isn’t supposed to be a String, so reading it’s first byte as such doesn’t work. So, if reading directly from a ReadStream doesn’t work to open a file, wrapping around a Buffer is enough:

    var file = s3.getObject({Bucket: bucket, Key: key}).createReadStream();
    var buffers = [];
    file.on('data', function(data) {
        buffers.push(data);
    });
    file.on('end', function() {
        var buffer = Buffer.concat(buffers);
        var workbook = XLSX.read(buffer);
    });

[annjawn]

That didn’t quite work for me especially since I am reading a .csv file with double quoted fields and writing to another bucket a resulting XLS file. What somewhat worked for me is below.

csv looks like -

"Some, value", "another value", "1.235","22","SomeMore"
"Some, value", "another value", "1.235","22","SomeMore"

const rl = readline.createInterface({
        input: s3.getObject(ReadParams).createReadStream()
    });
    rl.on('line', function(line) {
        console.log(line);        
        aoa.push(line.slice(1, -1).split("\",\""));
    })
    .on('close', function() {
        var ws = XLSX.utils.aoa_to_sheet(aoa);
        var wb = XLSX.utils.book_new();
        XLSX.utils.book_append_sheet(wb, ws, "Products");
        var buf = XLSX.write(wb, {type:'buffer', bookType: "xlsx"});
        s3.putObject({
            Bucket: destBucket,
            Key: 'TestExcel1234.xlsx',
            Body: new Buffer(buf)
        }, function(err, data) {
            if (err) {
                console.log("Cannot put object:",err)
            } else {
                console.log("Successfully uploaded data excel");
            }
   
         });
    });

[hqh07]

That didn’t quite work for me especially since I am reading a .csv file with double quoted fields and writing to another bucket a resulting XLS file. What somewhat worked for me is below.

csv looks like -

"Some, value", "another value", "1.235","22","SomeMore"
"Some, value", "another value", "1.235","22","SomeMore"

const rl = readline.createInterface({
        input: s3.getObject(ReadParams).createReadStream()
    });
    rl.on('line', function(line) {
        console.log(line);        
        aoa.push(line.slice(1, -1).split("\",\""));
    })
    .on('close', function() {
        var ws = XLSX.utils.aoa_to_sheet(aoa);
        var wb = XLSX.utils.book_new();
        XLSX.utils.book_append_sheet(wb, ws, "Products");
        var buf = XLSX.write(wb, {type:'buffer', bookType: "xlsx"});
        s3.putObject({
            Bucket: destBucket,
            Key: 'TestExcel1234.xlsx',
            Body: new Buffer(buf)
        }, function(err, data) {
            if (err) {
                console.log("Cannot put object:",err)
            } else {
                console.log("Successfully uploaded data excel");
            }
   
         });
    });

Awesome ! Thanks sir