This commit is contained in:
tatianamac
2019-11-26 14:50:43 -08:00
parent 8a55660ed0
commit 6d5445ecc5
13894 changed files with 2233957 additions and 0 deletions

9
node_modules/limiter/.travis.yml generated vendored Normal file
View File

@ -0,0 +1,9 @@
language: node_js
node_js:
- "0.12"
- "0.10"
notifications:
email:
- "jhurliman@jhurliman.org"

19
node_modules/limiter/LICENSE.txt generated vendored Normal file
View File

@ -0,0 +1,19 @@
Copyright (C) 2011 by John Hurliman
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

140
node_modules/limiter/README.md generated vendored Normal file
View File

@ -0,0 +1,140 @@
# limiter #
[![Build Status](https://travis-ci.org/jhurliman/node-rate-limiter.png)](https://travis-ci.org/jhurliman/node-rate-limiter)
[![NPM Downloads](https://img.shields.io/npm/dm/limiter.svg?style=flat)](https://www.npmjs.com/package/limiter)
Provides a generic rate limiter for node.js. Useful for API clients, web
crawling, or other tasks that need to be throttled. Two classes are exposed,
RateLimiter and TokenBucket. TokenBucket provides a lower level interface to
rate limiting with a configurable burst rate and drip rate. RateLimiter sits
on top of the token bucket and adds a restriction on the maximum number of
tokens that can be removed each interval to comply with common API
restrictions like "150 requests per hour maximum".
## Installation ##
Use NPM to install:
npm install limiter
## Usage ##
A simple example allowing 150 requests per hour:
```javascript
var RateLimiter = require('limiter').RateLimiter;
// Allow 150 requests per hour (the Twitter search limit). Also understands
// 'second', 'minute', 'day', or a number of milliseconds
var limiter = new RateLimiter(150, 'hour');
// Throttle requests
limiter.removeTokens(1, function(err, remainingRequests) {
// err will only be set if we request more than the maximum number of
// requests we set in the constructor
// remainingRequests tells us how many additional requests could be sent
// right this moment
callMyRequestSendingFunction(...);
});
```
Another example allowing one message to be sent every 250ms:
```javascript
var RateLimiter = require('limiter').RateLimiter;
var limiter = new RateLimiter(1, 250);
limiter.removeTokens(1, function() {
callMyMessageSendingFunction(...);
});
```
The default behaviour is to wait for the duration of the rate limiting
thats currently in effect before the callback is fired, but if you
pass in ```true``` as the third parameter, the callback will be fired
immediately with remainingRequests set to -1:
```javascript
var RateLimiter = require('limiter').RateLimiter;
var limiter = new RateLimiter(150, 'hour', true); // fire CB immediately
// Immediately send 429 header to client when rate limiting is in effect
limiter.removeTokens(1, function(err, remainingRequests) {
if (remainingRequests < 1) {
response.writeHead(429, {'Content-Type': 'text/plain;charset=UTF-8'});
response.end('429 Too Many Requests - your IP is being rate limited');
} else {
callMyMessageSendingFunction(...);
}
});
```
A synchronous method, tryRemoveTokens(), is available in both RateLimiter and TokenBucket. This will return immediately with a boolean value indicating if the token removal was successful.
```javascript
var RateLimiter = require('limiter').RateLimiter;
var limiter = new RateLimiter(10, 'second');
if (limiter.tryRemoveTokens(5))
console.log('Tokens removed');
else
console.log('No tokens removed');
```
To get the number of remaining tokens **outside** the `removeTokens`-callback
simply use the `getTokensRemaining`-method.
```javascript
var RateLimiter = require('limiter').RateLimiter;
var limiter = new RateLimiter(1, 250);
// returns 1 since we did not remove a token and our number of tokens per interval is 1
limiter.getTokensRemaining();
```
Using the token bucket directly to throttle at the byte level:
```javascript
var BURST_RATE = 1024 * 1024 * 150; // 150KB/sec burst rate
var FILL_RATE = 1024 * 1024 * 50; // 50KB/sec sustained rate
var TokenBucket = require('limiter').TokenBucket;
// We could also pass a parent token bucket in as the last parameter to
// create a hierarchical token bucket
var bucket = new TokenBucket(BURST_RATE, FILL_RATE, 'second', null);
bucket.removeTokens(myData.byteLength, function() {
sendMyData(myData);
});
```
## Additional Notes ##
Both the token bucket and rate limiter should be used with a message queue or
some way of preventing multiple simultaneous calls to removeTokens().
Otherwise, earlier messages may get held up for long periods of time if more
recent messages are continually draining the token bucket. This can lead to
out of order messages or the appearance of "lost" messages under heavy load.
## License ##
(The MIT License)
Copyright (c) 2013 John Hurliman. &lt;jhurliman@jhurliman.org&gt;
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
'Software'), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

22
node_modules/limiter/bower.json generated vendored Normal file
View File

@ -0,0 +1,22 @@
{
"name": "limiter",
"main": "index.js",
"version": "1.0.5",
"homepage": "https://github.com/jhurliman/node-rate-limiter",
"authors": [
"John Hurliman <jhurliman@cull.tv>"
],
"description": "A generic rate limiter for node.js. Useful for API clients, web crawling, or other tasks that need to be throttled",
"moduleType": [
"node"
],
"keywords": [
"rate",
"limiting",
"throttling"
],
"license": "MIT",
"ignore": [
"package.json"
]
}

21
node_modules/limiter/index.d.ts generated vendored Normal file
View File

@ -0,0 +1,21 @@
export type TokenBucketError = string
export type Fail<T> = (error: T) => void
export type Success<T> = (error: null, data: T) => void
export type RemoveTokensCallback = Fail<TokenBucketError> | Success<number>
type Interval = number | 'second' | 'sec' | 'minute' | 'min' | 'hour' | 'hr' | 'day'
export declare class TokenBucket {
constructor(bucketSize: number, tokensPerInterval: number, interval: Interval, parentBucket?: TokenBucket)
removeTokens(count: number, callback: RemoveTokensCallback): void
tryRemoveTokens(count: number): boolean
drip(): boolean
}
export declare class RateLimiter {
constructor(tokensPerInterval: number, interval: Interval, fireImmediately?: boolean)
removeTokens(count: number, callback: RemoveTokensCallback): void
tryRemoveTokens(count: number): boolean
}

3
node_modules/limiter/index.js generated vendored Normal file
View File

@ -0,0 +1,3 @@
exports.RateLimiter = require('./lib/rateLimiter');
exports.TokenBucket = require('./lib/tokenBucket');

13
node_modules/limiter/lib/clock.js generated vendored Normal file
View File

@ -0,0 +1,13 @@
var getMilliseconds = function() {
if (typeof process !== 'undefined' && process.hrtime) {
var hrtime = process.hrtime();
var seconds = hrtime[0];
var nanoseconds = hrtime[1];
return seconds * 1e3 + Math.floor(nanoseconds / 1e6);
}
return new Date().getTime();
}
module.exports = getMilliseconds;

137
node_modules/limiter/lib/rateLimiter.js generated vendored Normal file
View File

@ -0,0 +1,137 @@
var TokenBucket = require('./tokenBucket');
var getMilliseconds = require('./clock');
/**
* A generic rate limiter. Underneath the hood, this uses a token bucket plus
* an additional check to limit how many tokens we can remove each interval.
* @author John Hurliman <jhurliman@jhurliman.org>
*
* @param {Number} tokensPerInterval Maximum number of tokens that can be
* removed at any given moment and over the course of one interval.
* @param {String|Number} interval The interval length in milliseconds, or as
* one of the following strings: 'second', 'minute', 'hour', day'.
* @param {Boolean} fireImmediately Optional. Whether or not the callback
* will fire immediately when rate limiting is in effect (default is false).
*/
var RateLimiter = function(tokensPerInterval, interval, fireImmediately) {
this.tokenBucket = new TokenBucket(tokensPerInterval, tokensPerInterval,
interval, null);
// Fill the token bucket to start
this.tokenBucket.content = tokensPerInterval;
this.curIntervalStart = getMilliseconds();
this.tokensThisInterval = 0;
this.fireImmediately = fireImmediately;
};
RateLimiter.prototype = {
tokenBucket: null,
curIntervalStart: 0,
tokensThisInterval: 0,
fireImmediately: false,
/**
* Remove the requested number of tokens and fire the given callback. If the
* rate limiter contains enough tokens and we haven't spent too many tokens
* in this interval already, this will happen immediately. Otherwise, the
* removal and callback will happen when enough tokens become available.
* @param {Number} count The number of tokens to remove.
* @param {Function} callback(err, remainingTokens)
* @returns {Boolean} True if the callback was fired immediately, otherwise
* false.
*/
removeTokens: function(count, callback) {
// Make sure the request isn't for more than we can handle
if (count > this.tokenBucket.bucketSize) {
process.nextTick(callback.bind(null, 'Requested tokens ' + count +
' exceeds maximum tokens per interval ' + this.tokenBucket.bucketSize,
null));
return false;
}
var self = this;
var now = getMilliseconds();
// Advance the current interval and reset the current interval token count
// if needed
if (now < this.curIntervalStart
|| now - this.curIntervalStart >= this.tokenBucket.interval) {
this.curIntervalStart = now;
this.tokensThisInterval = 0;
}
// If we don't have enough tokens left in this interval, wait until the
// next interval
if (count > this.tokenBucket.tokensPerInterval - this.tokensThisInterval) {
if (this.fireImmediately) {
process.nextTick(callback.bind(null, null, -1));
} else {
var waitInterval = Math.ceil(
this.curIntervalStart + this.tokenBucket.interval - now);
setTimeout(function() {
self.tokenBucket.removeTokens(count, afterTokensRemoved);
}, waitInterval);
}
return false;
}
// Remove the requested number of tokens from the token bucket
return this.tokenBucket.removeTokens(count, afterTokensRemoved);
function afterTokensRemoved(err, tokensRemaining) {
if (err) return callback(err, null);
self.tokensThisInterval += count;
callback(null, tokensRemaining);
}
},
/**
* Attempt to remove the requested number of tokens and return immediately.
* If the bucket (and any parent buckets) contains enough tokens and we
* haven't spent too many tokens in this interval already, this will return
* true. Otherwise, false is returned.
* @param {Number} count The number of tokens to remove.
* @param {Boolean} True if the tokens were successfully removed, otherwise
* false.
*/
tryRemoveTokens: function(count) {
// Make sure the request isn't for more than we can handle
if (count > this.tokenBucket.bucketSize)
return false;
var now = getMilliseconds();
// Advance the current interval and reset the current interval token count
// if needed
if (now < this.curIntervalStart
|| now - this.curIntervalStart >= this.tokenBucket.interval) {
this.curIntervalStart = now;
this.tokensThisInterval = 0;
}
// If we don't have enough tokens left in this interval, return false
if (count > this.tokenBucket.tokensPerInterval - this.tokensThisInterval)
return false;
// Try to remove the requested number of tokens from the token bucket
var removed = this.tokenBucket.tryRemoveTokens(count);
if (removed) {
this.tokensThisInterval += count;
}
return removed;
},
/**
* Returns the number of tokens remaining in the TokenBucket.
* @returns {Number} The number of tokens remaining.
*/
getTokensRemaining: function () {
this.tokenBucket.drip();
return this.tokenBucket.content;
}
};
module.exports = RateLimiter;

167
node_modules/limiter/lib/tokenBucket.js generated vendored Normal file
View File

@ -0,0 +1,167 @@
/**
* A hierarchical token bucket for rate limiting. See
* http://en.wikipedia.org/wiki/Token_bucket for more information.
* @author John Hurliman <jhurliman@cull.tv>
*
* @param {Number} bucketSize Maximum number of tokens to hold in the bucket.
* Also known as the burst rate.
* @param {Number} tokensPerInterval Number of tokens to drip into the bucket
* over the course of one interval.
* @param {String|Number} interval The interval length in milliseconds, or as
* one of the following strings: 'second', 'minute', 'hour', day'.
* @param {TokenBucket} parentBucket Optional. A token bucket that will act as
* the parent of this bucket.
*/
var TokenBucket = function(bucketSize, tokensPerInterval, interval, parentBucket) {
this.bucketSize = bucketSize;
this.tokensPerInterval = tokensPerInterval;
if (typeof interval === 'string') {
switch (interval) {
case 'sec': case 'second':
this.interval = 1000; break;
case 'min': case 'minute':
this.interval = 1000 * 60; break;
case 'hr': case 'hour':
this.interval = 1000 * 60 * 60; break;
case 'day':
this.interval = 1000 * 60 * 60 * 24; break;
default:
throw new Error('Invaid interval ' + interval);
}
} else {
this.interval = interval;
}
this.parentBucket = parentBucket;
this.content = 0;
this.lastDrip = +new Date();
};
TokenBucket.prototype = {
bucketSize: 1,
tokensPerInterval: 1,
interval: 1000,
parentBucket: null,
content: 0,
lastDrip: 0,
/**
* Remove the requested number of tokens and fire the given callback. If the
* bucket (and any parent buckets) contains enough tokens this will happen
* immediately. Otherwise, the removal and callback will happen when enough
* tokens become available.
* @param {Number} count The number of tokens to remove.
* @param {Function} callback(err, remainingTokens)
* @returns {Boolean} True if the callback was fired immediately, otherwise
* false.
*/
removeTokens: function(count, callback) {
var self = this;
// Is this an infinite size bucket?
if (!this.bucketSize) {
process.nextTick(callback.bind(null, null, count, Number.POSITIVE_INFINITY));
return true;
}
// Make sure the bucket can hold the requested number of tokens
if (count > this.bucketSize) {
process.nextTick(callback.bind(null, 'Requested tokens ' + count + ' exceeds bucket size ' +
this.bucketSize, null));
return false;
}
// Drip new tokens into this bucket
this.drip();
// If we don't have enough tokens in this bucket, come back later
if (count > this.content)
return comeBackLater();
if (this.parentBucket) {
// Remove the requested from the parent bucket first
return this.parentBucket.removeTokens(count, function(err, remainingTokens) {
if (err) return callback(err, null);
// Check that we still have enough tokens in this bucket
if (count > self.content)
return comeBackLater();
// Tokens were removed from the parent bucket, now remove them from
// this bucket and fire the callback. Note that we look at the current
// bucket and parent bucket's remaining tokens and return the smaller
// of the two values
self.content -= count;
callback(null, Math.min(remainingTokens, self.content));
});
} else {
// Remove the requested tokens from this bucket and fire the callback
this.content -= count;
process.nextTick(callback.bind(null, null, this.content));
return true;
}
function comeBackLater() {
// How long do we need to wait to make up the difference in tokens?
var waitInterval = Math.ceil(
(count - self.content) * (self.interval / self.tokensPerInterval));
setTimeout(function() { self.removeTokens(count, callback); }, waitInterval);
return false;
}
},
/**
* Attempt to remove the requested number of tokens and return immediately.
* If the bucket (and any parent buckets) contains enough tokens this will
* return true, otherwise false is returned.
* @param {Number} count The number of tokens to remove.
* @param {Boolean} True if the tokens were successfully removed, otherwise
* false.
*/
tryRemoveTokens: function(count) {
// Is this an infinite size bucket?
if (!this.bucketSize)
return true;
// Make sure the bucket can hold the requested number of tokens
if (count > this.bucketSize)
return false;
// Drip new tokens into this bucket
this.drip();
// If we don't have enough tokens in this bucket, return false
if (count > this.content)
return false;
// Try to remove the requested tokens from the parent bucket
if (this.parentBucket && !this.parentBucket.tryRemoveTokens(count))
return false;
// Remove the requested tokens from this bucket and return
this.content -= count;
return true;
},
/**
* Add any new tokens to the bucket since the last drip.
* @returns {Boolean} True if new tokens were added, otherwise false.
*/
drip: function() {
if (!this.tokensPerInterval) {
this.content = this.bucketSize;
return;
}
var now = +new Date();
var deltaMS = Math.max(now - this.lastDrip, 0);
this.lastDrip = now;
var dripAmount = deltaMS * (this.tokensPerInterval / this.interval);
this.content = Math.min(this.content + dripAmount, this.bucketSize);
}
};
module.exports = TokenBucket;

69
node_modules/limiter/package.json generated vendored Normal file
View File

@ -0,0 +1,69 @@
{
"_args": [
[
"limiter@1.1.4",
"/Users/tatiana/selfdefined"
]
],
"_from": "limiter@1.1.4",
"_id": "limiter@1.1.4",
"_inBundle": false,
"_integrity": "sha512-XCpr5bElgDI65vVgstP8TWjv6/QKWm9GU5UG0Pr5sLQ3QLo8NVKsioe+Jed5/3vFOe3IQuqE7DKwTvKQkjTHvg==",
"_location": "/limiter",
"_phantomChildren": {},
"_requested": {
"type": "version",
"registry": true,
"raw": "limiter@1.1.4",
"name": "limiter",
"escapedName": "limiter",
"rawSpec": "1.1.4",
"saveSpec": null,
"fetchSpec": "1.1.4"
},
"_requiredBy": [
"/stream-throttle"
],
"_resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.4.tgz",
"_spec": "1.1.4",
"_where": "/Users/tatiana/selfdefined",
"author": {
"name": "John Hurliman",
"email": "jhurliman@jhurliman.org"
},
"bugs": {
"url": "http://github.com/jhurliman/node-rate-limiter/issues"
},
"dependencies": {},
"description": "A generic rate limiter for node.js. Useful for API clients, web crawling, or other tasks that need to be throttled",
"devDependencies": {
"assert": "1.3.0",
"vows": "0.8.1"
},
"directories": {
"lib": "./lib/"
},
"homepage": "https://github.com/jhurliman/node-rate-limiter#readme",
"keywords": [
"rate",
"limiting",
"throttling"
],
"licenses": [
{
"type": "MIT",
"url": "http://github.com/jhurliman/node-rate-limiter/raw/master/LICENSE.txt"
}
],
"main": "./index.js",
"name": "limiter",
"repository": {
"type": "git",
"url": "git://github.com/jhurliman/node-rate-limiter.git"
},
"scripts": {
"test": "vows --spec"
},
"types": "./index.d.ts",
"version": "1.1.4"
}

20
node_modules/limiter/test/ratelimiter-test.js generated vendored Normal file
View File

@ -0,0 +1,20 @@
var vows = require('vows');
var assert = require('assert');
var RateLimiter = require('../lib/rateLimiter');
vows.describe('RateLimiter').addBatch({
'interval validation': {
'invalid interval': function() {
assert.throws(function() { new RateLimiter(1, 'junk'); }, /interval/);
},
'valid intervals': function() {
assert.doesNotThrow(function() { new RateLimiter(1, 'sec'); });
assert.doesNotThrow(function() { new RateLimiter(1, 'second'); });
assert.doesNotThrow(function() { new RateLimiter(1, 'min'); });
assert.doesNotThrow(function() { new RateLimiter(1, 'minute'); });
assert.doesNotThrow(function() { new RateLimiter(1, 'hr'); });
assert.doesNotThrow(function() { new RateLimiter(1, 'hour'); });
assert.doesNotThrow(function() { new RateLimiter(1, 'day'); });
}
}
}).export(module);

67
node_modules/limiter/test/tokenbucket-test.js generated vendored Normal file
View File

@ -0,0 +1,67 @@
var vows = require('vows');
var assert = require('assert');
var TIMING_EPSILON = 10;
var TokenBucket = require('../lib/tokenBucket');
vows.describe('TokenBucket').addBatch({
'capacity 10, 1 per 100ms': {
topic: new TokenBucket(10, 1, 100),
'is initialized empty': function(bucket) {
assert.equal(bucket.bucketSize, 10);
assert.equal(bucket.tokensPerInterval, 1);
assert.equal(bucket.content, 0);
},
'removing 10 tokens': {
topic: function(bucket) {
this.gStart = +new Date();
bucket.removeTokens(10, this.callback);
},
'takes 1 second': function(remainingTokens) {
var duration = +new Date() - this.gStart;
var diff = Math.abs(1000 - duration);
assert.ok(diff < TIMING_EPSILON, diff+'');
assert.equal(remainingTokens, 0);
},
'and removing another 10 tokens': {
topic: function(_, bucket) {
this.gStart = +new Date();
assert.equal(bucket.content, 0);
bucket.removeTokens(10, this.callback);
},
'takes 1 second': function() {
var duration = +new Date() - this.gStart;
var diff = Math.abs(1000 - duration);
assert.ok(diff < TIMING_EPSILON, diff+'');
}
},
'and waiting 2 seconds': {
topic: function(_, bucket) {
var self = this;
setTimeout(function() {
self.gStart = +new Date();
bucket.removeTokens(10, self.callback);
}, 2000);
},
'gives us only 10 tokens': function(remainingTokens) {
var duration = +new Date() - this.gStart;
assert.ok(duration < TIMING_EPSILON, duration+'');
assert.equal(remainingTokens, 0);
},
'and removing 1 token': {
topic: function(_, _, bucket) {
this.gStart = +new Date();
bucket.removeTokens(1, this.callback);
},
'takes 100ms': function() {
var duration = +new Date() - this.gStart;
var diff = Math.abs(100 - duration);
assert.ok(diff < TIMING_EPSILON, diff+'');
}
}
}
}
},
}).export(module);