Add rate-limiting functionality #45

- Add checkRateLimit option to the API. - Extend the default server.js with environment variables CORSANYWHERE_WHITELIST (re-using originWhitelist) and CORSANYWHERE_RATELIMIT (using the new checkRateLimit option) to make it easy to enforce usage limits. - Document that Heroku doesn't want open proxies.
2026-05-13 15:09:25 +00:00 · 2016-05-31 00:52:26 +02:00
parent f07bdc4ea0
commit 6c4234f2b3
7 changed files with 389 additions and 4 deletions
--- a/README.md
+++ b/README.md
@@ -12,9 +12,6 @@ cookies. Requesting [user credentials](http://www.w3.org/TR/cors/#user-credentia
 The app can be configured to require a header for proxying a request, for example to avoid
 a direct visit from the browser.

-The package also includes a Procfile, to run the app on Heroku. More information about
-Heroku can be found at https://devcenter.heroku.com/articles/nodejs.
-
 ## Example

 ```javascript
@@ -97,6 +94,8 @@ proxy requests. The following options are supported:
 * array of strings `originWhitelist` - If set, requests whose origin is not listed are blocked.  
  If this list is empty, all origins are allowed.
  Example: `['https://good.example.com', 'http://good.example.com']`
+* function `checkRateLimit` - If set, it is called with the origin (string) of the request. If this
+  function returns a non-empty string, the request is rejected and the string is send to the client.
 * boolean `redirectSameOrigin` - If true, requests to URLs from the same origin will not be proxied but redirected.
  The primary purpose for this option is to save server resources by delegating the request to the client
  (since same-origin requests should always succeed, even without proxying).
@@ -121,6 +120,40 @@ For advanced users, the following options are also provided.
 For even more advanced usage (building upon CORS Anywhere),
 see the sample code in [test/test-examples.js](test/test-examples.js).

+### Demo server
+
+A public demo of CORS Anywhere is available at https://cors-anywhere.herokuapp.com. This server is
+only provided so that you can easily and quickly try out CORS Anywhere. To ensure that the service
+stays available to everyone, the number of requests per period is limited, except for requests from
+some explicitly whitelisted origins.
+
+If you expect lots of traffic, please host your own instance of CORS Anywhere, and make sure that
+the CORS Anywhere server only whitelists your site to prevent others from using your instance of
+CORS Anywhere as an open proxy.
+
+For instance, to run a CORS Anywhere server that accepts any request from some example.com sites on
+port 8080, use:
+```
+export PORT=8080
+export CORSANYWHERE_WHITELIST=https://example.com,http://example.com,http://example.com:8080
+node server.js
+```
+
+This application can immediately be run on Heroku, see https://devcenter.heroku.com/articles/nodejs
+for instructions. Note that their [Acceptable Use Policy](https://www.heroku.com/policy/aup) forbids
+the use of Heroku for operating an open proxy, so make sure that you either enforce a whitelist as
+shown above, or severly rate-limit the number of requests.
+
+For example, to blacklist abuse.example.com and rate-limit everything to 50 requests per 3 minutes,
+except for my.example.com and my2.example.com (which may be unlimited), use:
+
+```
+export PORT=8080
+export CORSANYWHERE_BLACKLIST=https://abuse.example.com,http://abuse.example.com
+export CORSANYWHERE_RATELIMIT='50 3 my.example.com my2.example.com'
+node server.js
+```
+

 ## License

--- a/lib/cors-anywhere.js
+++ b/lib/cors-anywhere.js
@@ -213,6 +213,7 @@ function getHandler(options, proxy) {
    maxRedirects: 5,                // Maximum number of redirects to be followed.
    originBlacklist: [],            // Requests from these origins will be blocked.
    originWhitelist: [],            // If non-empty, requests not from an origin in this list will be blocked.
+    checkRateLimit: null,           // Function that may enforce a rate-limit by returning a non-empty string.
    redirectSameOrigin: false,      // Redirect the client to the requested URL for same-origin requests.
    requireHeader: null,            // Require a header to be set?
    removeHeaders: [],              // Strip these request headers.
@@ -303,6 +304,13 @@ function getHandler(options, proxy) {
      return;
    }

+    var rateLimitMessage = corsAnywhere.checkRateLimit && corsAnywhere.checkRateLimit(origin);
+    if (rateLimitMessage) {
+      res.writeHead(429, 'Too Many Requests', cors_headers);
+      res.end('The origin "' + origin + '" has sent too many requests.\n' + rateLimitMessage);
+      return;
+    }
+
    if (corsAnywhere.redirectSameOrigin && origin && location.href[origin.length] === '/' &&
        location.href.lastIndexOf(origin, 0) === 0) {
      // Send a permanent redirect to offload the server. Badly coded clients should not waste our resources.
--- a/lib/rate-limit.js
+++ b/lib/rate-limit.js
@@ -0,0 +1,74 @@
+'use strict';
+module.exports = function createRateLimitChecker(CORSANYWHERE_RATELIMIT) {
+  // Configure rate limit. The following format is accepted for CORSANYWHERE_RATELIMIT:
+  // <max requests per period> <period in minutes> <non-ratelimited hosts>
+  // where <non-ratelimited hosts> is a space-separated list of strings or regexes (/.../) that
+  // matches the whole host (ports have to be listed explicitly if applicable).
+  // <period in minutes> cannot be zero.
+  //
+  // Examples:
+  // - Allow any origin to make one request per 5 minutes:
+  //   1 5
+  //
+  // - Allow example.com to make an unlimited number of requests, and the others 1 per 5 minutes.
+  //   1 5 example.com
+  //
+  // - Allow example.com, or any subdomain to make any number of requests and block the rest:
+  //   0 1 /(.*\.)?example\.com/
+  //
+  // - Allow example.com and www.example.com, and block the rest:
+  //   0 1 example.com www.example.com
+  var rateLimitConfig = /^(\d+) (\d+)(?:\s*$|\s+(.+)$)/.exec(CORSANYWHERE_RATELIMIT);
+  if (!rateLimitConfig) {
+    // No rate limit by default.
+    return function checkRateLimit() {};
+  }
+  var maxRequestsPerPeriod = parseInt(rateLimitConfig[1]);
+  var periodInMinutes = parseInt(rateLimitConfig[2]);
+  var unlimitedPattern = rateLimitConfig[3]; // Will become a RegExp or void.
+  if (unlimitedPattern) {
+    var unlimitedPatternParts = [];
+    unlimitedPattern.trim().split(/\s+/).forEach(function(unlimitedHost, i) {
+      var startsWithSlash = unlimitedHost.charAt(0) === '/';
+      var endsWithSlash = unlimitedHost.slice(-1) === '/';
+      if (startsWithSlash || endsWithSlash) {
+        if (unlimitedHost.length === 1 || !startsWithSlash || !endsWithSlash) {
+          throw new Error('Invalid CORSANYWHERE_RATELIMIT. Regex at index ' + i +
+              ' must start and end with a slash ("/").');
+        }
+        unlimitedHost = unlimitedHost.slice(1, -1);
+        // Throws if the pattern is invalid.
+        new RegExp(unlimitedHost);
+      } else {
+        // Just escape RegExp characters even though they cannot appear in a host name.
+        // The only actual important escape is the dot.
+        unlimitedHost = unlimitedHost.replace(/[$()*+.?[\\\]^{|}]/g, '\\$&');
+      }
+      unlimitedPatternParts.push(unlimitedHost);
+    });
+    unlimitedPattern = new RegExp('^(?:' + unlimitedPatternParts.join('|') + ')$', 'i');
+  }
+
+  var accessedHosts = Object.create(null);
+  setInterval(function() {
+    accessedHosts = Object.create(null);
+  }, periodInMinutes * 60000);
+
+  var rateLimitMessage = 'The number of requests is limited to ' + maxRequestsPerPeriod +
+    (periodInMinutes === 1 ? ' per minute' : ' per ' + periodInMinutes + ' minutes') + '. ' +
+    'Please self-host CORS Anywhere if you need more quota. ' +
+    'See https://github.com/Rob--W/cors-anywhere#demo-server';
+
+  return function checkRateLimit(origin) {
+    var host = origin.replace(/^[\w\-]+:\/\//i, '');
+    if (unlimitedPattern && unlimitedPattern.test(host)) {
+      return;
+    }
+    var count = accessedHosts[host] || 0;
+    ++count;
+    if (count > maxRequestsPerPeriod) {
+      return rateLimitMessage;
+    }
+    accessedHosts[host] = count;
+  };
+};
--- a/package.json
+++ b/package.json
@@ -28,6 +28,7 @@
    "coveralls": "^2.11.6",
    "eslint": "^2.2.0",
    "istanbul": "^0.4.2",
+    "lolex": "^1.5.0",
    "mocha": "~2.2.4",
    "nock": "~1.9.0",
    "supertest": "~0.15.0"
@@ -35,7 +36,7 @@
  "scripts": {
    "lint": "eslint .",
    "test": "mocha ./test/test*.js --reporter spec",
-    "test-coverage": "istanbul cover ./node_modules/.bin/_mocha -- test/test.js --reporter spec"
+    "test-coverage": "istanbul cover ./node_modules/.bin/_mocha -- test/test.js test/test-ratelimit.js --reporter spec"
  },
  "engines": {
    "node": ">=0.6.6",
--- a/server.js
+++ b/server.js
@@ -7,11 +7,17 @@ var port = process.env.PORT || 8080;
 // immediate abuse (e.g. denial of service). If you want to block all origins except for some,
 // use originWhitelist instead.
 var originBlacklist = (process.env.CORSANYWHERE_BLACKLIST || '').split(',');
+var originWhitelist = (process.env.CORSANYWHERE_WHITELIST || '').split(',');
+
+// Set up rate-limiting to avoid abuse of the public CORS Anywhere server.
+var checkRateLimit = require('./lib/rate-limit')(process.env.CORSANYWHERE_RATELIMIT);

 var cors_proxy = require('./lib/cors-anywhere');
 cors_proxy.createServer({
  originBlacklist: originBlacklist,
+  originWhitelist: originWhitelist,
  requireHeader: ['origin', 'x-requested-with'],
+  checkRateLimit: checkRateLimit,
  removeHeaders: [
    'cookie',
    'cookie2',
--- a/test/test-ratelimit.js
+++ b/test/test-ratelimit.js
@@ -0,0 +1,231 @@
+/* eslint-env mocha */
+
+var createRateLimitChecker = require('../lib/rate-limit');
+
+var lolex = require('lolex');
+var assert = require('assert');
+
+function assertNotLimited(rateLimitReturnValue) {
+  if (rateLimitReturnValue) {
+    assert.fail('Expected no limit, but got ' + rateLimitReturnValue);
+  }
+}
+
+function assertLimited(rateLimitReturnValue, limit, period) {
+  var msg;
+  if (period === 1) {
+    msg = 'The number of requests is limited to ' + limit + ' per minute. ';
+  } else {
+    msg = 'The number of requests is limited to ' + limit + ' per ' + period + ' minutes. ';
+  }
+  msg += 'Please self-host CORS Anywhere if you need more quota. ' +
+    'See https://github.com/Rob--W/cors-anywhere#demo-server';
+  assert.equal(rateLimitReturnValue, msg);
+}
+
+describe('Rate limit', function() {
+  var clock;
+  beforeEach(function() {
+    clock = lolex.install();
+  });
+  afterEach(function() {
+    clock.uninstall();
+  });
+  it('is unlimited by default', function() {
+    var checkRateLimit = createRateLimitChecker();
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('https://example.com'));
+    assertNotLimited(checkRateLimit('https://example.com:1234'));
+
+    checkRateLimit = createRateLimitChecker('');
+    assertNotLimited(checkRateLimit('http://example.com'));
+
+    checkRateLimit = createRateLimitChecker(' ');
+    assertNotLimited(checkRateLimit('http://example.com'));
+  });
+
+  it('zero per minute / 5 minutes', function() {
+    var checkRateLimit = createRateLimitChecker('0 1');
+    assertLimited(checkRateLimit('http://example.com'), 0, 1);
+    assertLimited(checkRateLimit('https://example.com'), 0, 1);
+
+    checkRateLimit = createRateLimitChecker('0 5');
+    assertLimited(checkRateLimit('http://example.com'), 0, 5);
+    assertLimited(checkRateLimit('https://example.com'), 0, 5);
+  });
+
+  it('one per minute', function() {
+    var checkRateLimit = createRateLimitChecker('1 1');
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertLimited(checkRateLimit('http://example.com'), 1, 1);
+    assertNotLimited(checkRateLimit('http://example.com:1234'));
+    assertLimited(checkRateLimit('http://example.com:1234'), 1, 1);
+
+    clock.tick(59000);
+    assertLimited(checkRateLimit('http://example.com'), 1, 1);
+
+    clock.tick(1000);
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertLimited(checkRateLimit('http://example.com'), 1, 1);
+    assertNotLimited(checkRateLimit('http://example.com:1234'));
+    assertLimited(checkRateLimit('http://example.com:1234'), 1, 1);
+  });
+
+  it('different domains, one per minute', function() {
+    var checkRateLimit = createRateLimitChecker('1 1');
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('http://example.net'));
+    assertNotLimited(checkRateLimit('http://wexample.net'));
+    assertNotLimited(checkRateLimit('http://xample.net'));
+    assertNotLimited(checkRateLimit('http://www.example.net'));
+    assertLimited(checkRateLimit('http://example.com'), 1, 1);
+    assertLimited(checkRateLimit('http://example.net'), 1, 1);
+    assertLimited(checkRateLimit('http://wexample.net'), 1, 1);
+    assertLimited(checkRateLimit('http://xample.net'), 1, 1);
+    assertLimited(checkRateLimit('http://www.example.net'), 1, 1);
+
+    clock.tick(60000); // 1 minute
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('http://example.net'));
+    assertNotLimited(checkRateLimit('http://wexample.net'));
+    assertNotLimited(checkRateLimit('http://xample.net'));
+    assertNotLimited(checkRateLimit('http://www.example.net'));
+  });
+
+  it('unlimited domains, string', function() {
+    var checkRateLimit = createRateLimitChecker('1 2 example.com');
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('http://example.com'));
+
+    assertNotLimited(checkRateLimit('http://wexample.com'));
+    assertNotLimited(checkRateLimit('http://xample.com'));
+    assertNotLimited(checkRateLimit('http://www.example.com'));
+    assertLimited(checkRateLimit('http://wexample.com'), 1, 2);
+    assertLimited(checkRateLimit('http://xample.com'), 1, 2);
+    assertLimited(checkRateLimit('http://www.example.com'), 1, 2);
+  });
+
+  it('unlimited domains, RegExp', function() {
+    var checkRateLimit = createRateLimitChecker('1 2 /example\\.com/');
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('http://example.com'));
+
+    assertNotLimited(checkRateLimit('http://wexample.com'));
+    assertNotLimited(checkRateLimit('http://xample.com'));
+    assertNotLimited(checkRateLimit('http://www.example.com'));
+    assertLimited(checkRateLimit('http://wexample.com'), 1, 2);
+    assertLimited(checkRateLimit('http://xample.com'), 1, 2);
+    assertLimited(checkRateLimit('http://www.example.com'), 1, 2);
+  });
+
+  it('multiple domains, string', function() {
+    var checkRateLimit = createRateLimitChecker('1 2 a b  cc ');
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://cc'));
+    assertNotLimited(checkRateLimit('http://cc'));
+    assertNotLimited(checkRateLimit('http://c'));
+    assertLimited(checkRateLimit('http://c'), 1, 2);
+  });
+
+  it('multiple domains, RegExp', function() {
+    var checkRateLimit = createRateLimitChecker('1 2 /a/ /b/  /cc/ ');
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://cc'));
+    assertNotLimited(checkRateLimit('http://cc'));
+    assertNotLimited(checkRateLimit('http://ccc'));
+    assertLimited(checkRateLimit('http://ccc'), 1, 2);
+  });
+
+  it('multiple domains, string and RegExp', function() {
+    var checkRateLimit = createRateLimitChecker('1 2 a /b/');
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://ab'));
+    assertLimited(checkRateLimit('http://ab'), 1, 2);
+  });
+
+  it('multiple domains, RegExp and string', function() {
+    var checkRateLimit = createRateLimitChecker('1 2 /a/ b');
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://a'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://b'));
+    assertNotLimited(checkRateLimit('http://ab'));
+    assertLimited(checkRateLimit('http://ab'), 1, 2);
+  });
+
+  it('wildcard subdomains', function() {
+    var checkRateLimit = createRateLimitChecker('0 1 /(.*\\.)?example\\.com/');
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('http://www.example.com'));
+    assertLimited(checkRateLimit('http://xexample.com'), 0, 1);
+    assertLimited(checkRateLimit('http://example.com.br'), 0, 1);
+  });
+
+  it('wildcard ports', function() {
+    var checkRateLimit = createRateLimitChecker('0 1 /example\\.com(:\\d{1,5})?/');
+    assertNotLimited(checkRateLimit('http://example.com'));
+    assertNotLimited(checkRateLimit('http://example.com:1234'));
+  });
+
+  it('empty host', function() {
+    var checkRateLimit = createRateLimitChecker('0 1');
+    assertLimited(checkRateLimit(''), 0, 1);
+    // Empty host actually means empty origin. But let's also test for 'http://'.
+    assertLimited(checkRateLimit('http://'), 0, 1);
+
+    checkRateLimit = createRateLimitChecker('0 1    ');
+    assertLimited(checkRateLimit(''), 0, 1);
+    assertLimited(checkRateLimit('http://'), 0, 1);
+
+    checkRateLimit = createRateLimitChecker('0 1 //');
+    assertNotLimited(checkRateLimit(''));
+    assertNotLimited(checkRateLimit('http://'));
+  });
+
+  it('null origin', function() {
+    var checkRateLimit = createRateLimitChecker('0 1');
+    assertLimited(checkRateLimit('null'), 0, 1);
+    assertLimited(checkRateLimit('http://null'), 0, 1);
+
+    checkRateLimit = createRateLimitChecker('0 1 null');
+    assertNotLimited(checkRateLimit('null'));
+    assertNotLimited(checkRateLimit('http://null'));
+
+    checkRateLimit = createRateLimitChecker('0 1 /null/');
+    assertNotLimited(checkRateLimit('null'));
+    assertNotLimited(checkRateLimit('http://null'));
+  });
+
+  it('case-insensitive', function() {
+    var checkRateLimit = createRateLimitChecker('0 1 NULL');
+    assertNotLimited(checkRateLimit('null'));
+    assertNotLimited(checkRateLimit('http://null'));
+
+    checkRateLimit = createRateLimitChecker('0 1 /NULL/');
+    assertNotLimited(checkRateLimit('null'));
+    assertNotLimited(checkRateLimit('http://null'));
+  });
+
+  it('bad input', function() {
+    assert.throws(function() {
+      createRateLimitChecker('0 1 /');
+    }, 'Invalid CORSANYWHERE_RATELIMIT. Regex at index 0 must start and end with a slash ("/").');
+
+    assert.throws(function() {
+      createRateLimitChecker('0 1 a,/');
+    }, 'Invalid CORSANYWHERE_RATELIMIT. Regex at index 1 must start and end with a slash ("/").');
+
+    assert.throws(function() {
+      createRateLimitChecker('0 1 /(/');
+    }, /Invalid regular expression/);
+  });
+});
--- a/test/test.js
+++ b/test/test.js
@@ -505,6 +505,38 @@ describe('originWhitelist', function() {
  });
 });

+describe('checkRateLimit', function() {
+  afterEach(stopServer);
+
+  it('GET /example.com without rate-limit', function(done) {
+    cors_anywhere = createServer({
+      checkRateLimit: function() {},
+    });
+    cors_anywhere_port = cors_anywhere.listen(0).address().port;
+    request(cors_anywhere)
+      .get('/example.com/')
+      .expect('Access-Control-Allow-Origin', '*')
+      .expect(200, done);
+  });
+
+  it('GET /example.com with rate-limit', function(done) {
+    cors_anywhere = createServer({
+      checkRateLimit: function(origin) {
+        // Non-empty value. Let's return the origin parameter so that we also verify that the
+        // the parameter is really the origin.
+        return '[' + origin + ']';
+      },
+    });
+    cors_anywhere_port = cors_anywhere.listen(0).address().port;
+    request(cors_anywhere)
+      .get('/example.com/')
+      .set('Origin', 'http://example.net:1234')
+      .expect('Access-Control-Allow-Origin', '*')
+      .expect(429, done,
+          'The origin "http://example.net" has sent too many requests.\n[http://example.com:1234]');
+  });
+});
+
 describe('redirectSameOrigin', function() {
  before(function() {
    cors_anywhere = createServer({