https://github.com/marcbachmann/find-in-batches
Batch find and processing method. DEPRECATED: Please use native async iterators
https://github.com/marcbachmann/find-in-batches
Last synced: 9 months ago
JSON representation
Batch find and processing method. DEPRECATED: Please use native async iterators
- Host: GitHub
- URL: https://github.com/marcbachmann/find-in-batches
- Owner: marcbachmann
- Created: 2015-06-01T23:48:32.000Z (about 11 years ago)
- Default Branch: master
- Last Pushed: 2023-12-15T05:44:23.000Z (over 2 years ago)
- Last Synced: 2024-04-14T14:40:20.924Z (about 2 years ago)
- Language: JavaScript
- Homepage:
- Size: 15.6 KB
- Stars: 3
- Watchers: 4
- Forks: 0
- Open Issues: 3
-
Metadata Files:
- Readme: README.md
Awesome Lists containing this project
README
# find-in-batches
A flow control helper method that invokes a forEach function for each element that was returned in the passed find method. This is a wrapper around [async.js](https://github.com/caolan/async) and uses simple pagination to do its magic.
## API
```
findInBatches.each([options], findMethod, forEach, callback)
findInBatches.batch([options], findMethod, forBatch, callback)
findInBatches.all([options], findMethod, callback)
```
### findInBatches.each
```js
var request = require('request')
var findInBatches = require('find-in-batches')
var options = {
offset: 0, // a default offset
batchSize: 50, // this gets passed to the findMethod as limit
maximum: Infinity, // maximum count of documents that get loaded
concurrency: Infinity // how many forEach methods should run simultaneously
}
findInBatches.each(options, findMethod, forEach, function(err){
if (err) return console.error('Failed to load the entries', err)
console.log('Loaded all the entries')
});
function findMethod (options, callback) {
// options = {offset: 0, limit: 50, page: 1} // on first search
// options = {offset: 50, limit: 50, page: 2} // on second iteration
var url = 'http://maps.gelbeseiten.de'
url += '/yp/subscriberlist_ajaxAction.yp?urlSegmentSubjectString=starbucks'
url += '&recfrom=' + options.offset + '&reccount=' + options.limit
request.get({url: url, json: true}, function(err, response, body){
// the callback must be called with either an error
// or an array as second argument
if (err) return callback(err)
if (!(body && body.data && body.data.subscribers)) return callback()
callback(null, body.data.subscribers.subscribers)
})
}
// forEach gets invoked for each element that gets returned in the findMethod.
// e.g. if the findMethod would return `callback(null, [{id: 1, name: 'one'}, {id: 2, name: 'two'}])`,
// the entry would equal `{id: 1, name: 'one'}`
function forEach (entry, callback){
entry.address = entry.address || {}
entry.address.street = entry.address.street || {}
console.log(
"Name: %s, Address: %s, %s",
entry.name,
entry.address.street.name + ' ' + entry.address.street.houseno,
entry.address.location
)
callback()
}
```