Hanged connections on createBlockBlobFromLocalFile

Hi Everybody,

We’re experiencing issues with uploading files to Azure Blob Storage. On a high load we’ve got “Server is busy” exceptions which lead us to use ExponentialRetryPolicyFilter. To test implementation, I’ve created a small test and found that some requests just hang. I decided to simplify test and took an example from “azure-storage” library (see: https://github.com/Azure/azure-storage-node/blob/master/examples/samples/blobuploaddownloadsample.js) which still hangs. I’ve reduced complexity of the example and came up with the following:

// 
// Copyright (c) Microsoft and contributors.  All rights reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// 
// See the License for the specific language governing permissions and
// limitations under the License.
// 

/**
* 1. Demonstrates how to upload all files from a given directory in parallel
*/

var fs = require('fs');

var azure = require('azure-storage');

var container = 'any-name-you-think-is-good';
var blob = 'updownsample';

var blobService = azure.createBlobService(
    "<YOUR_STORAGE_ACCOUNT_OR_CONNECTION_STRING>",
    "<YOUR_STORAGE_ACCESS_KEY>")
.withFilter(new azure.ExponentialRetryPolicyFilter());

function uploadSample() {
  var processArguments = process.argv;
  if (processArguments.length !== 3) {
    console.log('Incorrect number of arguments. Should be: srcPath');
    process.exit(1);
  } 

  var srcPath = processArguments[2];

  console.log('Starting blobuploaddownloadsample.');

  // Create the container
  createContainer(container, function () {
    // Demonstrates how to upload all files from a given directoy
    uploadBlobs(srcPath, container, function () {
            console.log("All is done");
    });
  });
}

function createContainer (container, callback) {
  // Create the container.
  blobService.createContainerIfNotExists(container, function (error) {
    if (error) {
      console.log(error);
    } else {
      console.log('Created the container ' + container);
      callback();
    }
  });
}

function uploadBlobs(sourceDirectoryPath, containerName, callback) {
  console.log('Entering uploadBlobs.');

  // validate directory is valid.
  if (!fs.existsSync(sourceDirectoryPath)) {
    console.log(sourceDirectoryPath + ' is an invalid directory path.');
  } else {
    // Search the directory and generate a list of files to upload.
    walk(sourceDirectoryPath, function (error, files) {
      if (error) {
        console.log(error);
      } else {
        var finished = 0;
    var errored = 0;

    console.log("Total files: %s", files.length);
    // generate and schedule an upload for each file
        files.forEach(function (file) {
          var blobName = file.replace(/^.*[\\\/]/, '');

          blobService.createBlockBlobFromLocalFile(containerName, blobName, file, function (error) {
            finished++;

            if (error) {
              console.log("%s ", ++errored, error);
            } else {
              console.log(' Blob ' + blobName + ' upload finished. %s, Errors: %s', finished, errored);

              if ((finished + errored) === files.length) {
                // Wait until all workers complete and the blobs are uploaded to the server.
                console.log('All files uploaded');
                callback();
              }
            }
          });
        });
      }
    });
  }
}

// Utility function
var walk = function (dir, done) {
  var results = [];
  fs.readdir(dir, function (err, list) {
    if (err) return done(err);
    var i = 0;
    (function next() {
      var file = list[i++];
      if (!file) return done(null, results);
      file = dir + '/' + file;
      fs.stat(file, function (err2, stat) {
        if (stat && stat.isDirectory()) {
          walk(file, function (err3, res) {
            results = results.concat(res);
            next();
          });
        } else {
          results.push(file);
          next();
        }
      });
    })();
  });
};

// run
uploadSample();

I created set of 4400 files in one folder to run the test. Tried several times, it never finished: sometimes it hangs with 8, sometimes with 2 files to upload. Additionally, in case if RetryPolicy schedules operation to be retried, these calls also never finished.

I think I’m stuck with it. It seems like there is library error which prevents requests to be executed. Could you please help?

Thanks, Kirill

[yaxia]

I setup the test and it worked well with the exact code posted (1305 files). I also checked your source files which are all in small size. So the server should response fast. Could you please add the following line before uploadSample function to enable the verbose log and share the results again? Thanks.

blobService.logger.level = azure.Logger.LogLevels.DEBUG;

BTW, the code below will put all the files into read streams at once and will make the file handle exhausted.

files.forEach(function (file) {
  var blobName = file.replace(/^.*[\\\/]/, '');

  blobService.createBlockBlobFromLocalFile(containerName, blobName, file, function (error) {
    finished++;

    if (error) {
      console.log("%s ", ++errored, error);
    } else {
      console.log(' Blob ' + blobName + ' upload finished. %s, Errors: %s', finished, errored);

      if ((finished + errored) === files.length) {
        // Wait until all workers complete and the blobs are uploaded to the server.
        console.log('All files uploaded');
        callback();
      }
   }
});

[yaxia]

Glad to hear you resolved this issue:)