How to run a large array of queries in smaller batches

Thanks to @Andrey for helping figure this out! This code would be useful if you are trying to run a query for all of the items in a very large array, as doing a basic triggering loop on hundreds of items can have poor performance or return errors from either Retool or your db.

// Queries in one batch will run in parallel.
// You can tweak batch size if you see server errors related to
// parallel execution

const arr = _.range(20) //evaluates as [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
const batchSize = 5 

// Note, this is wrapped in a function so that we don't evaluate
// promises immediately, but only when we run them in a batch
const queries = arr.map((item) => () => {
  return targetQuery.trigger({
    additionalScope: {"keyName":item} //passes the value being mapped over as keyName
  });
})

async function runAllQueries(queries) {
  const batches = _.chunk(queries, batchSize);
  const results = [];
  while (batches.length) {
    const batch = batches.shift();
    // This map here actually runs the promise
    const result = await Promise.all(batch.map(fn => fn()));
    results.push(result)
  }
  return _.flatten(results);
}

return runAllQueries(queries)
1 Like