alex/restaurant-search
1
Fork 0
Code Releases Activity

Adding timeout to geocoding script to deal with API rate limiting

Browse Source
This commit is contained in:
Alex Yatskov 2014-11-17 14:57:35 +09:00
parent 08ea6d9dc2
commit 0be8b78e6a
1 changed files with 18 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
scrape/geocode.js
View File

@ -5,23 +5,27 @@ var jf = require('jsonfile');
var _ = require('underscore'); var _ = require('underscore');
function queryPosition(gc, address, cache, callback) { function queryPosition(gc, address, cache, sequence, callback) {
if (_.has(cache, address)) { if (_.has(cache, address)) {
console.log('Cache lookup success for:\n\t%s', address); console.log('Cache lookup success for:\n\t%s', address);
callback(cache[address]); callback(cache[address]);
return; return sequence;
} }
gc.geocode(address, function(err, res) { setTimeout(function() {
if (err) { gc.geocode(address, function(err, res) {
console.log('Geocode lookup fail for: \n\t%s', address); if (err) {
callback(null); console.log('Geocode lookup fail for: \n\t%s', address);
} callback(null);
else { }
console.log('Geocode lookup success for: \n\t%s', address); else {
callback(cache[address] = res[0]); console.log('Geocode lookup success for: \n\t%s', address);
} callback(cache[address] = res[0]);
}); }
});
}, sequence * 200);
return sequence + 1;
} }
@ -31,9 +35,10 @@ function main() {
var srcCount = srcData.length; var srcCount = srcData.length;
var cacheData = jf.readFileSync('cache/geo.json', {throws: false}) || {}; var cacheData = jf.readFileSync('cache/geo.json', {throws: false}) || {};
var destData = []; var destData = [];
var sequence = 0;
_.each(srcData, function(srcItem) { _.each(srcData, function(srcItem) {
queryPosition(gc, srcItem.address, cacheData, function(geo) { sequence = queryPosition(gc, srcItem.address, cacheData, sequence, function(geo) {
if (geo) { if (geo) {
var destItem = _.clone(srcItem); var destItem = _.clone(srcItem);
destItem.geo = geo; destItem.geo = geo;