alex/restaurant-search
1
Fork 0
Code Releases Activity

Updating scraper and data

Browse Source
This commit is contained in:
Alex Yatskov 2015-08-13 16:37:23 +09:00
parent 2176ebebb9
commit 982ab07ece
3 changed files with 8219 additions and 7576 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15782
data/tabelog.json
View File

File diff suppressed because it is too large Load Diff

2
scrape.go
View File

@ -23,5 +23,5 @@
package main
func main() {
scrapeTabelog("http://tabelog.com/en/kanagawa/rstLst/1/", "tabelog.json", "webcache")
scrapeTabelog("http://tabelog.com/en/kanagawa/rstLst/1/", "data/tabelog.json", "webcache")
}

11
tabelog.go
View File

@ -66,8 +66,8 @@ func makeAbsUrl(base, ref string) string {
return b.ResolveReference(r).String()
}
func dumpReviews(filename string, rc chan tabelogReview, cond *sync.Cond) {
defer cond.Signal()
func dumpReviews(filename string, rc chan tabelogReview, wg *sync.WaitGroup) {
defer wg.Done()
count := 1
var reviews []tabelogReview
@ -155,12 +155,13 @@ func scrapeTabelog(url, jsonFile, cacheDir string) {
log.Fatal(err)
}
var cond sync.Cond
var wg sync.WaitGroup
wg.Add(1)
rc := make(chan tabelogReview)
go dumpReviews(jsonFile, rc, &cond)
go dumpReviews(jsonFile, rc, &wg)
scrapeIndex(url, rc, wc)
close(rc)
cond.Wait()
wg.Wait()
}