Tripadvisor transitioned to toml
This commit is contained in:
parent
4cce523103
commit
8365d6a65f
@ -115,30 +115,33 @@ func main() {
|
||||
|
||||
flag.Parse()
|
||||
|
||||
log.Printf("loading geocache %s...", *geocachePath)
|
||||
log.Printf("loading geocache from %s...", *geocachePath)
|
||||
gc, err := newGeoCache(*geocachePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer gc.save()
|
||||
|
||||
log.Printf("loading webcache %s...", *webcachePath)
|
||||
log.Printf("loading webcache from %s...", *webcachePath)
|
||||
wc, err := newWebCache(*webcachePath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("loading urls %s...", *urlsPath)
|
||||
log.Printf("loading urls from %s...", *urlsPath)
|
||||
urls, err := loadUrls(*urlsPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("loading converters %s...", *convertersPath)
|
||||
log.Printf("loading converters from %s...", *convertersPath)
|
||||
converters, err := loadConverters(*convertersPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for _, c := range converters {
|
||||
log.Printf("*\t%s", c.Name)
|
||||
}
|
||||
|
||||
log.Print("scraping reviews...")
|
||||
reviews, err := scrapeReviews(urls, converters, gc, wc)
|
||||
@ -152,12 +155,12 @@ func main() {
|
||||
log.Print("computing data semantics..")
|
||||
computeSemantics(restaurants)
|
||||
|
||||
log.Print("computing station data...")
|
||||
log.Printf("computing station data from %s...", *stationsPath)
|
||||
if err := computeStations(restaurants, *stationsPath); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Print("saving data...")
|
||||
log.Printf("saving data to %s...", *dbPath)
|
||||
if err := dumpData(*dbPath, restaurants); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
84
build/cache/geocache.json
vendored
84
build/cache/geocache.json
vendored
@ -10627,10 +10627,82 @@
|
||||
"Latitude": 35.299925,
|
||||
"Longitude": 139.480876
|
||||
},
|
||||
"〒220-0005西区南幸1-3-1 横浜モアーズ8F": {
|
||||
"Latitude": 35.4672263,
|
||||
"Longitude": 139.6221788
|
||||
},
|
||||
"〒220-0011西区高島2-13-12 崎陽軒本店B1": {
|
||||
"Latitude": 35.4622889,
|
||||
"Longitude": 139.6222899
|
||||
},
|
||||
"〒220-0012西区みなとみらい 4-6-2グランドセントラルタワー 1階": {
|
||||
"Latitude": 35.4587197,
|
||||
"Longitude": 139.6294296
|
||||
},
|
||||
"〒220-8501西区北幸1-3-23横浜ベイシェラトンホテル\u0026タワーズ 8階": {
|
||||
"Latitude": 35.4667939,
|
||||
"Longitude": 139.6201484
|
||||
},
|
||||
"〒220-8522西区みなとみらい1-1-1ヨコハマグランドインターコンチネンタルホテル31階": {
|
||||
"Latitude": 35.4576034,
|
||||
"Longitude": 139.6374556
|
||||
},
|
||||
"〒221-0835神奈川区鶴屋町2-10-7タムラビル 1Ｆ": {
|
||||
"Latitude": 35.4688603,
|
||||
"Longitude": 139.6229814
|
||||
},
|
||||
"〒225-0012青葉区あざみ野南2-14-3": {
|
||||
"Latitude": 35.5652366,
|
||||
"Longitude": 139.5523784
|
||||
},
|
||||
"〒231-0007中区弁天通6-791F": {
|
||||
"Latitude": 35.4492682,
|
||||
"Longitude": 139.6353589
|
||||
},
|
||||
"〒231-0014中区常盤町5-58-2": {
|
||||
"Latitude": 35.4472157,
|
||||
"Longitude": 139.6349617
|
||||
},
|
||||
"〒231-0021中区日本大通11 情報文化センター1F": {
|
||||
"Latitude": 35.4462405,
|
||||
"Longitude": 139.6428978
|
||||
},
|
||||
"〒231-0023中区山下町10 ホテルニューグランド本館5F": {
|
||||
"Latitude": 35.444773,
|
||||
"Longitude": 139.649593
|
||||
},
|
||||
"〒231-0023中区山下町149": {
|
||||
"Latitude": 35.4436523,
|
||||
"Longitude": 139.6459251
|
||||
},
|
||||
"〒231-0023中区山下町189": {
|
||||
"Latitude": 35.442644,
|
||||
"Longitude": 139.6468862
|
||||
},
|
||||
"〒231-0023中区山下町192": {
|
||||
"Latitude": 35.4433273,
|
||||
"Longitude": 139.6467417
|
||||
},
|
||||
"〒231-0023中区山下町77": {
|
||||
"Latitude": 35.4441466,
|
||||
"Longitude": 139.6468555
|
||||
},
|
||||
"〒231-0861中区元町1-31ラ・スピーガ元町001号室": {
|
||||
"Latitude": 35.4410507,
|
||||
"Longitude": 139.6503305
|
||||
},
|
||||
"〒240-0006保土ケ谷区星川3-23-13": {
|
||||
"Latitude": 35.4557923,
|
||||
"Longitude": 139.5825558
|
||||
},
|
||||
"ラゾーナ川崎プラザ 4F": {
|
||||
"Latitude": 35.5329594,
|
||||
"Longitude": 139.6959237
|
||||
},
|
||||
"中区石川町1-8": {
|
||||
"Latitude": 35.4387639,
|
||||
"Longitude": 139.6445226
|
||||
},
|
||||
"中原区木月2-5-7": {
|
||||
"Latitude": 35.5647315,
|
||||
"Longitude": 139.6551794
|
||||
@ -10738,5 +10810,17 @@
|
||||
"神奈川県鎌倉市大船1-24-1大船駅前ビル5F": {
|
||||
"Latitude": 35.35328,
|
||||
"Longitude": 139.5325737
|
||||
},
|
||||
"西区南幸1-6-31横浜タカシマヤ 8F": {
|
||||
"Latitude": 35.4651997,
|
||||
"Longitude": 139.6191458
|
||||
},
|
||||
"西区高島2-16-B1横浜駅東口地下街ポルタ": {
|
||||
"Latitude": 35.4622889,
|
||||
"Longitude": 139.6222899
|
||||
},
|
||||
"西区高島2-19-12スカイビル 11F": {
|
||||
"Latitude": 35.4645561,
|
||||
"Longitude": 139.6246488
|
||||
}
|
||||
}
|
||||
@ -99,7 +99,7 @@ func (l *selector) locateStrings(doc *goquery.Document) ([]string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
strs = append(strs, str)
|
||||
strs = append(strs, strings.TrimSpace(str))
|
||||
})
|
||||
|
||||
return strs, err
|
||||
@ -137,6 +137,7 @@ func (l *selector) locateFloat(doc *goquery.Document) (float64, error) {
|
||||
// converter
|
||||
//
|
||||
type converter struct {
|
||||
Name string
|
||||
Domains []string
|
||||
|
||||
Index struct {
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
name = "tripadvisor"
|
||||
domains = ["www.tripadvisor.com"]
|
||||
|
||||
[index.items]
|
||||
path = "a.property_title"
|
||||
@ -12,53 +13,53 @@ name = "tripadvisor"
|
||||
path = "h1#HEADING"
|
||||
|
||||
[item.address]
|
||||
path = "address span.format_address > span:not(.extended-address)"
|
||||
path = "address span.format_address"
|
||||
|
||||
[item.weight]
|
||||
[item.count]
|
||||
path = "h3.reviews_header"
|
||||
regex = "^(\d+)"
|
||||
regEx = "^(\\d+)"
|
||||
|
||||
[item.props]
|
||||
[item.props.service]
|
||||
accomodating = 1.0
|
||||
affordable = 0.0
|
||||
atmospheric = 0.0
|
||||
delicious: 0.0
|
||||
delicious = 0.0
|
||||
scale = 5.0
|
||||
|
||||
path = "ul.barChart div.ratingRow img:nth-child(2)"
|
||||
regex = "^([0-9]*\.?[0-9]+)"
|
||||
path = "ul.barChart > li:nth-child(1) > div:nth-child(2) img"
|
||||
regEx = "^([0-9]*\\.?[0-9]+)"
|
||||
attr = "alt"
|
||||
|
||||
[item.props.food]
|
||||
accomodating = 0.0
|
||||
affordable = 0.0
|
||||
atmospheric = 0.0
|
||||
delicious: 1.0
|
||||
delicious = 1.0
|
||||
scale = 5.0
|
||||
|
||||
path = "ul.barChart div.ratingRow img:nth-child(1)"
|
||||
regex = "^([0-9]*\.?[0-9]+)"
|
||||
path = "ul.barChart > li:nth-child(1) > div:nth-child(1) img"
|
||||
regEx = "^([0-9]*\\.?[0-9]+)"
|
||||
attr = "alt"
|
||||
|
||||
[item.props.value]
|
||||
accomodating = 0.0
|
||||
affordable = 1.0
|
||||
atmospheric = 0.0
|
||||
delicious: 0.0
|
||||
delicious = 0.0
|
||||
scale = 5.0
|
||||
|
||||
path = "ul.barChart div.ratingRow img:nth-child(3)"
|
||||
regex = "^([0-9]*\.?[0-9]+)"
|
||||
path = "ul.barChart > li:nth-child(2) > div:nth-child(1) img"
|
||||
regEx = "^([0-9]*\\.?[0-9]+)"
|
||||
attr = "alt"
|
||||
|
||||
[item.props.atmosphere]
|
||||
accomodating = 0.0
|
||||
affordable = 0.0
|
||||
atmospheric = 1.0
|
||||
delicious: 0.0
|
||||
delicious = 0.0
|
||||
scale = 5.0
|
||||
|
||||
path = "ul.barChart div.ratingRow img:nth-child(4)"
|
||||
regex = "^([0-9]*\.?[0-9]+)"
|
||||
path = "ul.barChart > li:nth-child(2) > div:nth-child(2) img"
|
||||
regEx = "^([0-9]*\\.?[0-9]+)"
|
||||
attr = "alt"
|
||||
Binary file not shown.
@ -1,16 +1 @@
|
||||
http://tabelog.com/en/kanagawa/rstLst/
|
||||
|
||||
http://www.tripadvisor.com/Restaurants-g298173-Yokohama_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g298172-Kawasaki_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g1021282-Sagamihara_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g1021277-Fujisawa_Kanagawa_Prefecture_Kanto.html
|
||||
|
||||
http://www.tripadvisor.com/Restaurants-g303156-Kamakura_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g298174-Yokosuka_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g1021278-Odawara_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g681222-Hiratsuka_Kanagawa_Prefecture_Kanto.html
|
||||
|
||||
http://www.tripadvisor.com/Restaurants-g298169-Atsugi_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g1021286-Yamato_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g1021279-Chigasaki_Kanagawa_Prefecture_Kanto.html
|
||||
http://www.tripadvisor.com/Restaurants-g1021285-Hadano_Kanagawa_Prefecture_Kanto.html
|
||||
|
||||
@ -1,120 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
|
||||
* Author: Alex Yatskov <alex@foosoft.net>
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
* this software and associated documentation files (the "Software"), to deal in
|
||||
* the Software without restriction, including without limitation the rights to
|
||||
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
* the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
* subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in all
|
||||
* copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
type tripadvisor struct {
|
||||
}
|
||||
|
||||
func (tripadvisor) define(keyword string) semantics {
|
||||
return map[string]semantics{
|
||||
"food": {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 1.0},
|
||||
"service": {Accomodating: 1.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 0.0},
|
||||
"value": {Accomodating: 0.0, Affordable: 1.0, Atmospheric: 0.0, Delicious: 0.0},
|
||||
"atmosphere": {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 1.0, Delicious: 0.0},
|
||||
}[keyword]
|
||||
}
|
||||
|
||||
func (tripadvisor) index(doc *goquery.Document) (string, []string) {
|
||||
var reviewUrls []string
|
||||
doc.Find("a.property_title").Each(func(index int, sel *goquery.Selection) {
|
||||
if href, ok := sel.Attr("href"); ok {
|
||||
reviewUrls = append(reviewUrls, href)
|
||||
}
|
||||
})
|
||||
|
||||
var nextIndexUrl string
|
||||
if href, ok := doc.Find("div.deckTools.btm a.nav.next.rndBtn.rndBtnGreen.taLnk").Attr("href"); ok {
|
||||
nextIndexUrl = href
|
||||
}
|
||||
|
||||
return nextIndexUrl, reviewUrls
|
||||
}
|
||||
|
||||
func (tripadvisor) review(doc *goquery.Document) (name, address string, features map[string]float64, weight float64, err error) {
|
||||
if name = strings.TrimSpace(doc.Find("h1#HEADING").Text()); len(name) == 0 {
|
||||
err = errors.New("invalid name")
|
||||
return
|
||||
}
|
||||
|
||||
{
|
||||
var addressParts []string
|
||||
doc.Find("address span.format_address > span:not(.extended-address)").Each(func(index int, sel *goquery.Selection) {
|
||||
addressParts = append(addressParts, strings.TrimSpace(sel.Text()))
|
||||
})
|
||||
|
||||
if len(addressParts) == 0 {
|
||||
err = errors.New("invalid address")
|
||||
return
|
||||
}
|
||||
|
||||
address = strings.Join(addressParts, " ")
|
||||
}
|
||||
|
||||
{
|
||||
ratings := doc.Find("ul.barChart div.ratingRow img.sprite-rating_s_fill")
|
||||
if ratings.Length() != 4 {
|
||||
err = errors.New("invalid ratings")
|
||||
return
|
||||
}
|
||||
|
||||
features = make(map[string]float64)
|
||||
for index, category := range []string{"food", "service", "value", "atmosphere"} {
|
||||
altText, _ := ratings.Eq(index).Attr("alt")
|
||||
valueText := strings.Split(altText, " ")[0]
|
||||
|
||||
var value float64
|
||||
if value, err = strconv.ParseFloat(valueText, 8); err != nil {
|
||||
err = fmt.Errorf("invalid rating for %s", category)
|
||||
return
|
||||
}
|
||||
|
||||
features[category] = value/2.5 - 1.0
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
weightValid := false
|
||||
if weightParts := strings.Split(doc.Find("h3.reviews_header").Text(), " "); len(weightParts) > 0 {
|
||||
if weight, err = strconv.ParseFloat(weightParts[0], 8); err == nil {
|
||||
weightValid = true
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if !weightValid {
|
||||
err = fmt.Errorf("invalid review count")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user