Merge pull request #49 from yaxia/master

Storage Client Library - 0.4.3

.gitignore

@@ -13,4 +13,4 @@ results
 
 npm-debug.log
 node_modules
-docs
+docs

.travis.yml

@@ -1,13 +1,9 @@
 language: node_js
 node_js:
-  - "0.11"
+  - "0.12"
   - "0.10"
   - "0.8"
 
-matrix:
-  allow_failures:
-    - node_js: "0.11"
-
 install:
   - npm install -g npm@1.4.23
   - npm --version

ChangeLog.txt

@@ -1,7 +1,21 @@
 Note: This is an Azure Storage only package. The all up Azure node sdk still has the old storage bits in there. In a future release, those storage bits will be removed and an npm dependency to this storage node sdk will 
 be taken. This is a CTP v1 release and the changes described below indicate the changes from the Azure node SDK 0.9.8 available here - https://github.com/Azure/azure-sdk-for-node.
 
+2015.03 Version 0.4.3
+
+ALL
+* Fixed an issue that cannot generate the SAS tokens with the specified version of 2014-02-14.
+* Fixed an issue that setting metadata keys are converted into lowercase. The metadata keys retrieved from the service will however still be converted into lowercase by the http component of Node.js.(https://github.com/joyent/node/issues/1954)
+* Included all storage error code strings in the error constants definition.
+* Documented the client request ID option in all APIs.
+
+BLOB
+* Supported listing blob virtual directories.
+* Fixed an issue that exception is thrown when downloading a blob larger than 32MB to a stream.
+* Fixed an issue that the process exits when the free memory is low.
+
 2014.12 Version 0.4.2
+
 ALL
 * Fixed an issue that batch operation could probably wait without callback.
 * Added the readable-stream module to adapt stream operations in both node 0.8 and node 0.10.

lib/common/http/webresource.js

@@ -253,17 +253,27 @@ WebResource.prototype.addOptionalMetadataHeaders = function (metadata) {
   var self = this;
 
   if (metadata) {
-    Object.keys(metadata).forEach(function (metadataHeader) {
-      if (azureutil.IsNullOrEmptyOrUndefinedOrWhiteSpace(metadataHeader)) {
+    Object.keys(metadata).forEach(function (metadataKey) {
+      if (azureutil.IsNullOrEmptyOrUndefinedOrWhiteSpace(metadataKey)) {
         throw new Error(SR.METADATA_KEY_INVALID);
       }
 
-      var value = metadata[metadataHeader];
+      var value = metadata[metadataKey];
       if (azureutil.IsNullOrEmptyOrUndefinedOrWhiteSpace(value)) {
         throw new Error(SR.METADATA_VALUE_INVALID);
       }
-
-      self.withHeader(HeaderConstants.PREFIX_FOR_STORAGE_METADATA + metadataHeader.toLowerCase(), value);
+
+      var metadataHeaderName = HeaderConstants.PREFIX_FOR_STORAGE_METADATA + metadataKey;
+      var existingMetadataHeaderName = '';
+      var headers = self.headers ? self.headers : {};
+      if (Object.keys(headers).some(function (headerName) {
+        existingMetadataHeaderName = headerName;
+        return headerName.toString().toLowerCase() === metadataHeaderName.toLowerCase();
+      })) {        
+        self.withHeader(existingMetadataHeaderName, self.headers[existingMetadataHeaderName] + ',' + value);
+      } else {
+        self.withHeader(metadataHeaderName, value);
+      }
     });
   }
 

lib/common/services/storageserviceclient.js

@@ -215,6 +215,7 @@ StorageServiceClient.prototype.performRequestInputStream = function (webResource
 * @param {Stream}      [body.outputStream]              The outgoing request data as a stream.
 * @param {Stream}      [body.inputStream]               The ingoing response data as a stream.
 * @param {object}      [options]                        The request options.
+* @param {string}      [options.clientRequestId]        The client request ID with a 1KB character limit.
 * @param {int}         [options.timeoutIntervalInMs]    The timeout interval, in milliseconds, to use for the request.
 * @param {function}    callback                         The response callback function.
 */

lib/common/signing/sharedaccesssignature.js

@@ -45,7 +45,9 @@ SharedAccessSignature.prototype.signRequest = function (webResource, callback) {
   webResource.uri += this.sasToken;
 
   // Add the api-version
-  webResource.uri += '&' + Constants.QueryStringConstants.API_VERSION + '=' + Constants.HeaderConstants.TARGET_STORAGE_VERSION;
+  if (this.sasToken.indexOf('api-version') == -1) {
+    webResource.uri += '&' + Constants.QueryStringConstants.API_VERSION + '=' + Constants.HeaderConstants.TARGET_STORAGE_VERSION;
+  }
   callback(null);
 };
 

lib/common/signing/sharedkey.js

@@ -187,15 +187,15 @@ SharedKey.prototype.generateSignedQueryString = function (path, sharedAccessPoli
     // validate and add version
     if (azureutil.objectIsNull(sasVersion)) {
       return HeaderConstants.TARGET_STORAGE_VERSION;
-    }
-    else if(sasVersion === VersionConstants.AUGUST_2013) {
-      return VersionConstants.AUGUST_2013;
-    }
-    else if(sasVersion === VersionConstants.FEBRUARY_2012) {
-      return VersionConstants.FEBRUARY_2012;
-    }
-    else {
-      throw new Error(SR.INVALID_SAS_VERSION);
+    } else {
+      var values = _.values(VersionConstants);
+      if (values.some(function(version) {
+        return version.toLowerCase() === sasVersion.toLowerCase();
+      })) {
+        return sasVersion;
+      } else {
+        throw new Error(azureutil.stringFormat(SR.INVALID_SAS_VERSION, sasVersion, values));
+      }
     }
   };
 

lib/common/streams/batchoperation.js

@@ -28,6 +28,7 @@ var DEFAULT_OPERATION_MEMORY_USAGE = Constants.BlobConstants.DEFAULT_WRITE_BLOCK
 var DEFAULT_GLOBAL_CONCURRENCY = 5; //Default http connection limitation for nodejs
 
 var SystemTotalMemory = os.totalmem();
+var CriticalFreeMemory = 0.1 * SystemTotalMemory;
 var nodeVersion = azureutil.getNodeVersion();
 var enableReuseSocket = nodeVersion.major >= 0 && nodeVersion.minor >= 10;
 
@@ -109,6 +110,7 @@ BatchOperation.prototype.IsWorkloadHeavy = function() {
     sharedRequest = 5;
   }
   return this._activeOperation >= sharedRequest * this.concurrency ||
+    this._isLowMemory() ||
     (this._activeOperation >= this.concurrency && this._getApproximateMemoryUsage() > 0.5 * SystemTotalMemory);
 };
 
@@ -121,6 +123,13 @@ BatchOperation.prototype._getApproximateMemoryUsage = function() {
   return currentUsage + futureUsage;
 };
 
+/**
+* get the approximate free memory
+*/
+BatchOperation.prototype._isLowMemory = function() {
+  return os.freemem() < CriticalFreeMemory;
+};
+
 /**
 * Add a operation into batch operation
 */

lib/common/streams/chunkstream.js

@@ -68,13 +68,25 @@ ChunkStream.prototype.setMemoryAllocator = function(allocator) {
 * Internal stream ended
 */
 ChunkStream.prototype.end = function (chunk, encoding, cb) {
+  if (typeof chunk === 'function') {
+    cb = chunk;
+    chunk = null;
+    encoding = null;
+  } else if (typeof encoding === 'function') {
+    cb = encoding;
+    encoding = null;
+  }
+
   if (chunk) {
-    this.write(chunk, encoding, cb);
+    this.write(chunk, encoding);
   }
 
   this._streamEnded = true;
   this._flushInternalBuffer();
-
+
+  if (cb) {
+    this.once('end', cb);
+  }
   this.emit('end');
 };
 

lib/common/util/constants.js

@@ -31,7 +31,7 @@ var Constants = {
   /*
   * Specifies the value to use for UserAgent header.
   */
-  USER_AGENT_PRODUCT_VERSION: '0.4.2',
+  USER_AGENT_PRODUCT_VERSION: '0.4.3',
 
   /**
   * The number of default concurrent requests for parallel operation.
@@ -494,6 +494,17 @@ var Constants = {
       BLOB: 'b'
     },
 
+    /**
+    * List blob types.
+    *
+    * @const
+    * @enum {string}
+    */
+    ListBlobTypes: {
+      Blob: 'b',
+      Directory: 'd'
+    },
+
     /**
     * Put page write options
     *
@@ -1985,6 +1996,14 @@ var Constants = {
   },
 
   VersionConstants: {
+    /**
+    * Constant for the 2014-02-14 version.
+    *
+    * @const
+    * @type {string}
+    */
+    FEBRUARY_2014: '2014-02-14',
+
     /**
     * Constant for the 2013-08-15 version.
     *
@@ -2031,43 +2050,104 @@ var Constants = {
     INVALID_MARKER: 'InvalidMarker'
   },
 
+  /**
+  * Constants for storage error strings
+  *
+  * More details are at: http://msdn.microsoft.com/en-us/library/azure/dd179357.aspx
+  */
   StorageErrorCodeStrings: {
-    UNSUPPORTED_HTTP_VERB: 'UnsupportedHttpVerb',
-    MISSING_CONTENT_LENGTH_HEADER: 'MissingContentLengthHeader',
-    MISSING_REQUIRED_HEADER: 'MissingRequiredHeader',
-    MISSING_REQUIRED_XML_NODE: 'MissingRequiredXmlNode',
+    // Not Modified (304): The condition specified in the conditional header(s) was not met for a read operation.
+    // Precondition Failed (412): The condition specified in the conditional header(s) was not met for a write operation.
+    CONDITION_NOT_MET: 'ConditionNotMet', 
+    // Bad Request (400): A required HTTP header was not specified.
+    MISSING_REQUIRED_HEADER: 'MissingRequiredHeader', 
+    // Bad Request (400): A required XML node was not specified in the request body.
+    MISSING_REQUIRED_XML_NODE: 'MissingRequiredXmlNode', 
+    // Bad Request (400): One of the HTTP headers specified in the request is not supported.
     UNSUPPORTED_HEADER: 'UnsupportedHeader',
-    UNSUPPORTED_XML_NODE: 'UnsupportedXmlNode',
-    INVALID_HEADER_VALUE: 'InvalidHeaderValue',
+    // Bad Request (400): One of the XML nodes specified in the request body is not supported.
+    UNSUPPORTED_XML_NODE: 'UnsupportedXmlNode', 
+    // Bad Request (400): The value provided for one of the HTTP headers was not in the correct format.
+    INVALID_HEADER_VALUE: 'InvalidHeaderValue', 
+    // Bad Request (400): The value provided for one of the XML nodes in the request body was not in the correct format.
     INVALID_XML_NODE_VALUE: 'InvalidXmlNodeValue',
+    // Bad Request (400): A required query parameter was not specified for this request.
     MISSING_REQUIRED_QUERY_PARAMETER: 'MissingRequiredQueryParameter',
+    // Bad Request (400): One of the query parameters specified in the request URI is not supported.
     UNSUPPORTED_QUERY_PARAMETER: 'UnsupportedQueryParameter',
+    // Bad Request (400): An invalid value was specified for one of the query parameters in the request URI.
     INVALID_QUERY_PARAMETER_VALUE: 'InvalidQueryParameterValue',
+    // Bad Request (400): A query parameter specified in the request URI is outside the permissible range.
     OUT_OF_RANGE_QUERY_PARAMETER_VALUE: 'OutOfRangeQueryParameterValue',
+    // Bad Request (400): The url in the request could not be parsed.
+    REQUEST_URL_FAILED_TO_PARSE: 'RequestUrlFailedToParse',
+    // Bad Request (400): The requested URI does not represent any resource on the server.
     INVALID_URI: 'InvalidUri',
+    // Bad Request (400): The HTTP verb specified was not recognized by the server.
     INVALID_HTTP_VERB: 'InvalidHttpVerb',
+    // Bad Request (400): The key for one of the metadata key-value pairs is empty.
     EMPTY_METADATA_KEY: 'EmptyMetadataKey',
-    REQUEST_BODY_TOO_LARGE: 'RequestBodyTooLarge',
+    // Bad Request (400): The specified XML is not syntactically valid.
     INVALID_XML_DOCUMENT: 'InvalidXmlDocument',
-    INTERNAL_ERROR: 'InternalError',
-    AUTHENTICATION_FAILED: 'AuthenticationFailed',
+    // Bad Request (400): The MD5 value specified in the request did not match the MD5 value calculated by the server.
     MD5_MISMATCH: 'Md5Mismatch',
+    // Bad Request (400): The MD5 value specified in the request is invalid. The MD5 value must be 128 bits and Base64-encoded.
     INVALID_MD5: 'InvalidMd5',
+    // Bad Request (400): One of the request inputs is out of range.
     OUT_OF_RANGE_INPUT: 'OutOfRangeInput',
+    // Bad Request (400): The authentication information was not provided in the correct format. Verify the value of Authorization header.
+    INVALID_AUTHENTICATION_INFO: 'InvalidAuthenticationInfo',
+    // Bad Request (400): One of the request inputs is not valid.
     INVALID_INPUT: 'InvalidInput',
-    OPERATION_TIMED_OUT: 'OperationTimedOut',
-    RESOURCE_NOT_FOUND: 'ResourceNotFound',
-    RESOURCE_ALREADY_EXISTS: 'ResourceAlreadyExists',
+    // Bad Request (400): The specified metadata is invalid. It includes characters that are not permitted.
     INVALID_METADATA: 'InvalidMetadata',
+    // Bad Request (400): The specifed resource name contains invalid characters.
+    INVALID_RESOURCE_NAME: 'InvalidResourceName',
+    // Bad Request (400): The size of the specified metadata exceeds the maximum size permitted.
     METADATA_TOO_LARGE: 'MetadataTooLarge',
-    CONDITION_NOT_MET: 'ConditionNotMet',
-    UPDATE_CONDITION_NOT_SATISFIED: 'UpdateConditionNotSatisfied',
+    // Bad Request (400): Condition headers are not supported.
+    CONDITION_HEADER_NOT_SUPPORTED: 'ConditionHeadersNotSupported',
+    // Bad Request (400): Multiple condition headers are not supported.
+    MULTIPLE_CONDITION_HEADER_NOT_SUPPORTED: 'MultipleConditionHeadersNotSupported',
+    // Forbidden (403): Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.
+    AUTHENTICATION_FAILED: 'AuthenticationFailed',
+    // Forbidden (403): Read-access geo-redundant replication is not enabled for the account.
+    // Forbidden (403): Write operations to the secondary location are not allowed.
+    // Forbidden (403): The account being accessed does not have sufficient permissions to execute this operation.
+    INSUFFICIENT_ACCOUNT_PERMISSIONS: 'InsufficientAccountPermissions',
+    // Not Found (404): The specified resource does not exist.
+    RESOURCE_NOT_FOUND: 'ResourceNotFound',
+    // Forbidden (403): The specified account is disabled.
+    ACCOUNT_IS_DISABLED: 'AccountIsDisabled',
+    // Method Not Allowed (405): The resource doesn't support the specified HTTP verb.
+    UNSUPPORTED_HTTP_VERB: 'UnsupportedHttpVerb',
+    // Conflict (409): The specified account already exists. 
+    ACCOUNT_ALREADY_EXISTS: 'AccountAlreadyExists',
+    // Conflict (409): The specified account is in the process of being created.
+    ACCOUNT_BEING_CREATED: 'AccountBeingCreated',
+    // Conflict (409): The specified resource already exists.
+    RESOURCE_ALREADY_EXISTS: 'ResourceAlreadyExists',
+    // Conflict (409): The specified resource type does not match the type of the existing resource.
+    RESOURCE_TYPE_MISMATCH: 'ResourceTypeMismatch',
+    // Length Required (411): The Content-Length header was not specified.
+    MISSING_CONTENT_LENGTH_HEADER: 'MissingContentLengthHeader',
+    // Request Entity Too Large (413): The size of the request body exceeds the maximum size permitted.
+    REQUEST_BODY_TOO_LARGE: 'RequestBodyTooLarge',
+    // Requested Range Not Satisfiable (416): The range specified is invalid for the current size of the resource.
     INVALID_RANGE: 'InvalidRange',
+    // Internal Server Error (500): The server encountered an internal error. Please retry the request.
+    INTERNAL_ERROR: 'InternalError',
+    // Internal Server Error (500): The operation could not be completed within the permitted time.
+    OPERATION_TIMED_OUT: 'OperationTimedOut',
+    // Service Unavailable (503): The server is currently unable to receive requests. Please retry your request. 
+    SERVER_BUSY: 'ServerBusy',
+
+    // Legacy error code strings
+    UPDATE_CONDITION_NOT_SATISFIED: 'UpdateConditionNotSatisfied',
     CONTAINER_NOT_FOUND: 'ContainerNotFound',
     CONTAINER_ALREADY_EXISTS: 'ContainerAlreadyExists',
     CONTAINER_DISABLED: 'ContainerDisabled',
     CONTAINER_BEING_DELETED: 'ContainerBeingDeleted',
-    SERVER_BUSY: 'ServerBusy'
   },
 
   TableErrorCodeStrings: {

lib/common/util/sr.js

@@ -51,7 +51,7 @@ var SR = {
 	INVALID_POP_RECEIPT: 'Pop Receipt cannot be null or undefined for deleteMessage and updateMessage operations.',
 	INVALID_PROPERTY_RESOLVER: 'The specified property resolver returned an invalid type. %s:{_:%s,$:%s }',
 	INVALID_RANGE_FOR_MD5: 'The requested range should be less than 4MB when contentMD5 is expected from the server',
-	INVALID_SAS_VERSION: 'SAS Version invalid. Valid versions include 2012-02-12 and 2013-08-15.',
+	INVALID_SAS_VERSION: 'SAS Version ? is invalid. Valid versions include: ?.',
 	INVALID_SIGNED_IDENTIFIERS: 'Signed identifiers need to be an array.',
 	INVALID_STREAM_LENGTH: 'The length of the provided stream is invalid.',
 	INVALID_STRING_ERROR: 'Invalid string error.',