| 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217 |
1
1
1
1
1
2
2
1
1
1
12
1
2178
2178
2178
2178
2178
2178
2188
2188
2188
135
8
135
2188
2188
2188
2188
2188
2188
2188
2188
10
10
10
10
10
10
10
10
10
10
10
2178
2178
2178
2178
1
2152
2152
340
340
340
1812
1
| //
// Copyright (c) Microsoft and contributors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
//
var azureutil = require('../util/util');
var Constants = require('../util/constants');
var StorageUtilities = require('../util/storageutilities');
var extend = require('util')._extend;
/**
* Creates a new RetryPolicyFilter instance.
* @class
* The RetryPolicyFilter allows you to retry operations,
* using a custom retry policy. Users are responsible to
* define the shouldRetry method.
* To apply a filter to service operations, use `withFilter`
* and specify the filter to be used when creating a service.
* @constructor
* @param {number} [retryCount=30000] The client retry count.
* @param {number} [retryInterval=3] The client retry interval, in milliseconds.
*
* @example
* var azure = require('azure-storage');
* var retryPolicy = new azure.RetryPolicyFilter();
* retryPolicy.retryCount = 3;
* retryPolicy.retryInterval = 3000;
* retryPolicy.shouldRetry = function(statusCode, retryContext) {
*
* };
* var blobService = azure.createBlobService().withFilter(retryPolicy);
*/
function RetryPolicyFilter(retryCount, retryInterval) {
this.retryCount = retryCount ? retryCount : RetryPolicyFilter.DEFAULT_CLIENT_RETRY_COUNT;
this.retryInterval = retryInterval ? retryInterval : RetryPolicyFilter.DEFAULT_CLIENT_RETRY_INTERVAL;
}
/**
* Represents the default client retry interval, in milliseconds.
*/
RetryPolicyFilter.DEFAULT_CLIENT_RETRY_INTERVAL = 1000 * 30;
/**
* Represents the default client retry count.
*/
RetryPolicyFilter.DEFAULT_CLIENT_RETRY_COUNT = 3;
/**
* Handles an operation with a retry policy.
*
* @param {Object} requestOptions The original request options.
* @param {function} next The next filter to be handled.
* @return {undefined}
*/
RetryPolicyFilter.prototype.handle = function (requestOptions, next) {
RetryPolicyFilter._handle(this, requestOptions, next);
};
/**
* Handles an operation with a retry policy.
*
* @param {Object} requestOptions The original request options.
* @param {function} next The next filter to be handled.
* @return {undefined}
*/
RetryPolicyFilter._handle = function (self, requestOptions, next) {
var retryRequestOptions = extend({}, requestOptions);
retryRequestOptions.retryInterval = 0;
// Initialize retryContext because that will be passed to the shouldRetry method which users will implement
retryRequestOptions.retryContext = {
retryCount: 0,
error: null,
retryInterval: retryRequestOptions.retryInterval,
locationMode: retryRequestOptions.locationMode,
currentLocation: retryRequestOptions.currentLocation
};
var lastPrimaryAttempt;
var lastSecondaryAttempt;
var operation = function () {
// retry policies dont really do anything to the request options
// so move on to next
Eif (next) {
next(retryRequestOptions, function (returnObject, finalCallback, nextPostCallback) {
// Previous operation ended so update the retry data
if (returnObject.error) {
if (retryRequestOptions.retryContext.error) {
returnObject.error.innerError = retryRequestOptions.retryContext.error;
}
retryRequestOptions.retryContext.error = returnObject.error;
}
// If a request sent to the secondary location fails with 404 (Not Found), it is possible
// that the resource replication is not finished yet. So, in case of 404 only in the secondary
// location, the failure should still be retryable.
var secondaryNotFound = (retryRequestOptions.currentLocation === Constants.StorageLocation.SECONDARY) && (returnObject.response && returnObject.response.statusCode === 404);
var retryInfo = self.shouldRetry(secondaryNotFound ? 500 : (azureutil.objectIsNull(returnObject.response) ? 306 : returnObject.response.statusCode), retryRequestOptions);
retryRequestOptions.retryContext.retryCount++;
Iif (retryInfo.ignore) {
returnObject.error = null;
}
// If the custom retry logic(shouldRetry) does not return a targetLocation, calculate based on the previous location and locationMode.
Eif(azureutil.objectIsNull(retryInfo.targetLocation)) {
retryInfo.targetLocation = azureutil.getNextLocation(retryRequestOptions.currentLocation, retryRequestOptions.locationMode);
}
// If the custom retry logic(shouldRetry) does not return a retryInterval, try to set it to the value on the instance if it is available. Otherwise, the default(30000) will be used.
Iif(azureutil.objectIsNull(retryInfo.retryInterval)) {
retryInfo.retryInterval = self.retryInterval;
}
// Only in the case of success from server but client side failure like MD5 or length mismatch, returnObject.retryable has a value(we explicitly set it to false). In this case, we should not retry
// the request.
if (returnObject.error && azureutil.objectIsNull(returnObject.retryable) &&
((!azureutil.objectIsNull(returnObject.response) &&
retryInfo.retryable) || (returnObject.error.code === 'ETIMEDOUT' || returnObject.error.code === 'ESOCKETTIMEDOUT' || returnObject.error.code === 'ECONNRESET'))) {
Eif (retryRequestOptions.currentLocation === Constants.StorageLocation.PRIMARY) {
lastPrimaryAttempt = returnObject.operationEndTime;
} else {
lastSecondaryAttempt = returnObject.operationEndTime;
}
// Moreover, in case of 404 when trying the secondary location, instead of retrying on the
// secondary, further requests should be sent only to the primary location, as it most
// probably has a higher chance of succeeding there.
Iif (secondaryNotFound && (retryRequestOptions.locationMode !== StorageUtilities.LocationMode.SECONDARY_ONLY))
{
retryInfo.locationMode = StorageUtilities.LocationMode.PRIMARY_ONLY;
retryInfo.targetLocation = Constants.StorageLocation.PRIMARY;
}
// Now is the time to calculate the exact retry interval. ShouldRetry call above already
// returned back how long two requests to the same location should be apart from each other.
// However, for the reasons explained above, the time spent between the last attempt to
// the target location and current time must be subtracted from the total retry interval
// that ShouldRetry returned.
var lastAttemptTime = retryInfo.targetLocation === Constants.StorageLocation.PRIMARY ? lastPrimaryAttempt : lastSecondaryAttempt;
Eif (!azureutil.objectIsNull(lastAttemptTime)) {
var sinceLastAttempt = new Date().getTime() - lastAttemptTime.getTime();
Iif (sinceLastAttempt < 0) {
sinceLastAttempt = 0;
}
retryRequestOptions.retryInterval = retryInfo.retryInterval - sinceLastAttempt;
}
else {
retryRequestOptions.retryInterval = 0;
}
Iif(!azureutil.objectIsNull(retryInfo.locationMode)) {
retryRequestOptions.locationMode = retryInfo.locationMode;
}
retryRequestOptions.currentLocation = retryInfo.targetLocation;
operation();
} else {
Iif (nextPostCallback) {
nextPostCallback(returnObject);
} else Eif (finalCallback) {
finalCallback(returnObject);
}
}
});
}
};
operation();
};
RetryPolicyFilter._shouldAbsorbConditionalError = function (statusCode, requestOptions) {
var retryInfo = (requestOptions && requestOptions.retryContext) ? requestOptions.retryContext : {};
if (statusCode >= 300) {
Eif (requestOptions && !requestOptions.absorbConditionalErrorsOnRetry) {
retryInfo.retryable = false;
return retryInfo;
}
if (statusCode == 501 || statusCode == 505) {
retryInfo.retryable = false;
} else if (statusCode == 412) {
// When appending block with precondition failure and their was a server error before, we ignore the error.
if (retryInfo.lastServerError) {
retryInfo.ignore = true;
retryInfo.retryable = true;
} else {
retryInfo.retryable = false;
}
} else if (retryInfo.retryable && statusCode >= 500 && statusCode < 600) {
retryInfo.retryable = true;
retryInfo.lastServerError = true;
}
}
return retryInfo;
};
module.exports = RetryPolicyFilter;
|