Hi There,
WAF doesn’t identify simple crawl script, here is the script i’m running i would expect the WAF to identify this bot an block it but it doesn’t, any toughts?
const rp = require(‘request-promise’)
const $ = require(‘cheerio’)
const parseOfficeProduct = require(’./get-office-product-page’)
const url = ‘https://www.mydomain.com/product/’
const fromid = 170000
const uptoid = 500000
const waitPause = 8
let ids = ;
for (let i = fromid; i < uptoid; i++) {
ids.push({
id: i,
url: url +i
})
}
const waitFor = (ms) => new Promise(r => setTimeout(r, ms));
async function asyncForEach(array, callback) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
const start = async () => {
await asyncForEach(ids, async (el) => {
await waitFor(waitPause);
console.log(‘get’, el.url)
parseOfficeProduct(el.url, el.id)
});
console.log(‘Done’);
}
start();