From 59e7590001d399e47af6e3a688bec21dbe81e928 Mon Sep 17 00:00:00 2001 From: Alex Yatskov Date: Mon, 17 Aug 2015 18:00:29 +0900 Subject: [PATCH] Adding more URLs --- scrape.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scrape.go b/scrape.go index b713cb5..2f3a5a5 100644 --- a/scrape.go +++ b/scrape.go @@ -43,16 +43,26 @@ func main() { tasks := []scrapeTask{ {"http://tabelog.com/en/kanagawa/rstLst/1/", tabelog{}}, + {"http://www.tripadvisor.com/Restaurants-g298173-Yokohama_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, - // {"http://www.tripadvisor.com/Restaurants-g1021277-Fujisawa_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, - // {"http://www.tripadvisor.com/Restaurants-g1021279-Chigasaki_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, - // {"http://www.tripadvisor.com/Restaurants-g298172-Kawasaki_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, - // {"http://www.tripadvisor.com/Restaurants-g1066854-Shinagawa_Tokyo_Tokyo_Prefecture_Kanto.html", tripadvisor{}}, - // {"http://www.tripadvisor.com/Restaurants-g298184-Tokyo_Tokyo_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g298172-Kawasaki_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g1021282-Sagamihara_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g1021277-Fujisawa_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g303156-Kamakura_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g298174-Yokosuka_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g1021278-Odawara_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g681222-Hiratsuka_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g298169-Atsugi_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g1021286-Yamato_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g1021279-Chigasaki_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, + {"http://www.tripadvisor.com/Restaurants-g1021285-Hadano_Kanagawa_Prefecture_Kanto.html", tripadvisor{}}, } + count := 0 for _, task := range tasks { - scrape(task.url, wc, gc, task.scr) + restaraunts := scrape(task.url, wc, gc, task.scr) + count += len(restaraunts) } + log.Print(count) }