How to rate-limit facebookexternalhit Requests

Website, Application, Performance
3

ok, something like this:

addEventListener('fetch', event => {
  event.respondWith(handleRequest(event.request));
})

const RATE_LIMIT = 10; // Number of requests
const TIME_WINDOW = 60 * 1000; // Time window in milliseconds (1 minute)
const BLOCK_DURATION = 5 * 60 * 1000; // Block duration in milliseconds (5 minutes)
const USER_AGENT_BLOCKED = 'BlockedUserAgent'; // User-Agent to be rate limited

async function handleRequest(request) {
  const userAgent = request.headers.get('User-Agent') || '';
  
  if (userAgent.includes(USER_AGENT_BLOCKED)) {
    const clientIP = request.headers.get('CF-Connecting-IP');
    const now = Date.now();
    const rateLimitKey = `rate_limit:${clientIP}`;
    const blockKey = `block:${clientIP}`;

    const blockTimestamp = await getCache(blockKey);
    if (blockTimestamp && (now - blockTimestamp) < BLOCK_DURATION) {
      return new Response('Too many requests, please try again later.', { status: 429 });
    }

    const rateLimitInfo = await getCache(rateLimitKey);
    let requestCount = 0;
    let firstRequestTime = now;

    if (rateLimitInfo) {
      ({ requestCount, firstRequestTime } = JSON.parse(rateLimitInfo));
    }

    if ((now - firstRequestTime) < TIME_WINDOW) {
      requestCount++;
      if (requestCount > RATE_LIMIT) {
        await setCache(blockKey, now, BLOCK_DURATION);
        return new Response('Too many requests, please try again later.', { status: 429 });
      }
    } else {
      requestCount = 1;
      firstRequestTime = now;
    }

    await setCache(rateLimitKey, JSON.stringify({ requestCount, firstRequestTime }), TIME_WINDOW);
  }

  return fetch(request);
}

async function getCache(key) {
  const cache = caches.default;
  const response = await cache.match(key);
  return response ? response.text() : null;
}

async function setCache(key, value, expirationTtl) {
  const cache = caches.default;
  const headers = { 'Content-Type': 'text/plain', 'Cache-Control': `max-age=${expirationTtl}` };
  await cache.put(key, new Response(value, { headers }));
}
1 Like