Updating scraper and data
This commit is contained in:
parent
2176ebebb9
commit
982ab07ece
15782
data/tabelog.json
15782
data/tabelog.json
File diff suppressed because it is too large
Load Diff
@ -23,5 +23,5 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
scrapeTabelog("http://tabelog.com/en/kanagawa/rstLst/1/", "tabelog.json", "webcache")
|
scrapeTabelog("http://tabelog.com/en/kanagawa/rstLst/1/", "data/tabelog.json", "webcache")
|
||||||
}
|
}
|
||||||
|
11
tabelog.go
11
tabelog.go
@ -66,8 +66,8 @@ func makeAbsUrl(base, ref string) string {
|
|||||||
return b.ResolveReference(r).String()
|
return b.ResolveReference(r).String()
|
||||||
}
|
}
|
||||||
|
|
||||||
func dumpReviews(filename string, rc chan tabelogReview, cond *sync.Cond) {
|
func dumpReviews(filename string, rc chan tabelogReview, wg *sync.WaitGroup) {
|
||||||
defer cond.Signal()
|
defer wg.Done()
|
||||||
|
|
||||||
count := 1
|
count := 1
|
||||||
var reviews []tabelogReview
|
var reviews []tabelogReview
|
||||||
@ -155,12 +155,13 @@ func scrapeTabelog(url, jsonFile, cacheDir string) {
|
|||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cond sync.Cond
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
rc := make(chan tabelogReview)
|
rc := make(chan tabelogReview)
|
||||||
go dumpReviews(jsonFile, rc, &cond)
|
go dumpReviews(jsonFile, rc, &wg)
|
||||||
|
|
||||||
scrapeIndex(url, rc, wc)
|
scrapeIndex(url, rc, wc)
|
||||||
|
|
||||||
close(rc)
|
close(rc)
|
||||||
cond.Wait()
|
wg.Wait()
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user