alex/restaurant-search
1
Fork 0
Code Releases Activity

Updating scraper and data

Browse Source
This commit is contained in:
Alex Yatskov 2015-08-13 16:37:23 +09:00
parent 2176ebebb9
commit 982ab07ece
3 changed files with 8219 additions and 7576 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15782
data/tabelog.json
View File

File diff suppressed because it is too large Load Diff

2
scrape.go
View File

@ -23,5 +23,5 @@
package main package main
func main() { func main() {
scrapeTabelog("http://tabelog.com/en/kanagawa/rstLst/1/", "tabelog.json", "webcache") scrapeTabelog("http://tabelog.com/en/kanagawa/rstLst/1/", "data/tabelog.json", "webcache")
} }

11
tabelog.go
View File

@ -66,8 +66,8 @@ func makeAbsUrl(base, ref string) string {
return b.ResolveReference(r).String() return b.ResolveReference(r).String()
} }
func dumpReviews(filename string, rc chan tabelogReview, cond *sync.Cond) { func dumpReviews(filename string, rc chan tabelogReview, wg *sync.WaitGroup) {
defer cond.Signal() defer wg.Done()
count := 1 count := 1
var reviews []tabelogReview var reviews []tabelogReview
@ -155,12 +155,13 @@ func scrapeTabelog(url, jsonFile, cacheDir string) {
log.Fatal(err) log.Fatal(err)
} }
var cond sync.Cond var wg sync.WaitGroup
wg.Add(1)
rc := make(chan tabelogReview) rc := make(chan tabelogReview)
go dumpReviews(jsonFile, rc, &cond) go dumpReviews(jsonFile, rc, &wg)
scrapeIndex(url, rc, wc) scrapeIndex(url, rc, wc)
close(rc) close(rc)
cond.Wait() wg.Wait()
} }