Conforme discutido nos comentários, se você deseja controlar o número máximo de solicitações em andamento ao mesmo tempo, pode usar o Bluebird para fazer isso facilmente assim:
const Promise = require('bluebird');
const rp = require('request-promise');
router.get('/', (req, res) => {
let collection = db.get().collection('foo');
let collection2 = db.get().collection('test');
collection.distinct('id', (err, idArr) => { // count: 3000+
if (err) {
// handle error here, send some error response
res.status(501).send(...);
} else {
Promise.map(idArr, id => {
let url = 'https://externalapi.io/id=' + id
return rp(url).then(body => {
if (error) {
console.log(error)
} else {
let resp = JSON.parse(body);
// probably want to return a promise here too, but I'm unsure what DB you're using
collection2.insert(resp);
}
}).catch(err => {
// decide what you want to do when a single request fails here
// by providing a catch handler that does not rethrow, other requests will continue
});
// pick some concurrency value here that does not cause errors
}, {concurrency: 10}).then(() => {
// all requests are done, send final response
res.send(...);
}).catch(err => {
// your code may never get here (depends upon earlier .catch() handler)
});
}
});
});
