1

Tripadvisor transitioned to toml

This commit is contained in:
Alex Yatskov 2015-09-22 11:57:07 +09:00
parent 4cce523103
commit 8365d6a65f
7 changed files with 111 additions and 157 deletions

View File

@ -115,30 +115,33 @@ func main() {
flag.Parse() flag.Parse()
log.Printf("loading geocache %s...", *geocachePath) log.Printf("loading geocache from %s...", *geocachePath)
gc, err := newGeoCache(*geocachePath) gc, err := newGeoCache(*geocachePath)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
defer gc.save() defer gc.save()
log.Printf("loading webcache %s...", *webcachePath) log.Printf("loading webcache from %s...", *webcachePath)
wc, err := newWebCache(*webcachePath) wc, err := newWebCache(*webcachePath)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
log.Printf("loading urls %s...", *urlsPath) log.Printf("loading urls from %s...", *urlsPath)
urls, err := loadUrls(*urlsPath) urls, err := loadUrls(*urlsPath)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
log.Printf("loading converters %s...", *convertersPath) log.Printf("loading converters from %s...", *convertersPath)
converters, err := loadConverters(*convertersPath) converters, err := loadConverters(*convertersPath)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
for _, c := range converters {
log.Printf("*\t%s", c.Name)
}
log.Print("scraping reviews...") log.Print("scraping reviews...")
reviews, err := scrapeReviews(urls, converters, gc, wc) reviews, err := scrapeReviews(urls, converters, gc, wc)
@ -152,12 +155,12 @@ func main() {
log.Print("computing data semantics..") log.Print("computing data semantics..")
computeSemantics(restaurants) computeSemantics(restaurants)
log.Print("computing station data...") log.Printf("computing station data from %s...", *stationsPath)
if err := computeStations(restaurants, *stationsPath); err != nil { if err := computeStations(restaurants, *stationsPath); err != nil {
log.Fatal(err) log.Fatal(err)
} }
log.Print("saving data...") log.Printf("saving data to %s...", *dbPath)
if err := dumpData(*dbPath, restaurants); err != nil { if err := dumpData(*dbPath, restaurants); err != nil {
log.Fatal(err) log.Fatal(err)
} }

View File

@ -10627,10 +10627,82 @@
"Latitude": 35.299925, "Latitude": 35.299925,
"Longitude": 139.480876 "Longitude": 139.480876
}, },
"〒220-0005西区南幸1-3-1 横浜モアーズ8F": {
"Latitude": 35.4672263,
"Longitude": 139.6221788
},
"〒220-0011西区高島2-13-12 崎陽軒本店B1": {
"Latitude": 35.4622889,
"Longitude": 139.6222899
},
"〒220-0012西区みなとみらい 4-6-2グランドセントラルタワー 1階": {
"Latitude": 35.4587197,
"Longitude": 139.6294296
},
"〒220-8501西区北幸1-3-23横浜ベイシェラトンホテル\u0026タワーズ 8階": {
"Latitude": 35.4667939,
"Longitude": 139.6201484
},
"〒220-8522西区みなとみらい1-1-1ヨコハマグランドインターコンチネンタルホテル31階": {
"Latitude": 35.4576034,
"Longitude": 139.6374556
},
"〒221-0835神奈川区鶴屋町2-10-7タムラビル 1": {
"Latitude": 35.4688603,
"Longitude": 139.6229814
},
"〒225-0012青葉区あざみ野南2-14-3": {
"Latitude": 35.5652366,
"Longitude": 139.5523784
},
"〒231-0007中区弁天通6-791F": {
"Latitude": 35.4492682,
"Longitude": 139.6353589
},
"〒231-0014中区常盤町5-58-2": {
"Latitude": 35.4472157,
"Longitude": 139.6349617
},
"〒231-0021中区日本大通11 情報文化センター1F": {
"Latitude": 35.4462405,
"Longitude": 139.6428978
},
"〒231-0023中区山下町10 ホテルニューグランド本館5F": {
"Latitude": 35.444773,
"Longitude": 139.649593
},
"〒231-0023中区山下町149": {
"Latitude": 35.4436523,
"Longitude": 139.6459251
},
"〒231-0023中区山下町189": {
"Latitude": 35.442644,
"Longitude": 139.6468862
},
"〒231-0023中区山下町192": {
"Latitude": 35.4433273,
"Longitude": 139.6467417
},
"〒231-0023中区山下町77": {
"Latitude": 35.4441466,
"Longitude": 139.6468555
},
"〒231-0861中区元町1-31ラ・スピーガ元町001号室": {
"Latitude": 35.4410507,
"Longitude": 139.6503305
},
"〒240-0006保土ケ谷区星川3-23-13": {
"Latitude": 35.4557923,
"Longitude": 139.5825558
},
"ラゾーナ川崎プラザ 4F": { "ラゾーナ川崎プラザ 4F": {
"Latitude": 35.5329594, "Latitude": 35.5329594,
"Longitude": 139.6959237 "Longitude": 139.6959237
}, },
"中区石川町1-8": {
"Latitude": 35.4387639,
"Longitude": 139.6445226
},
"中原区木月2-5-7": { "中原区木月2-5-7": {
"Latitude": 35.5647315, "Latitude": 35.5647315,
"Longitude": 139.6551794 "Longitude": 139.6551794
@ -10738,5 +10810,17 @@
"神奈川県鎌倉市大船1-24-1大船駅前ビル5F": { "神奈川県鎌倉市大船1-24-1大船駅前ビル5F": {
"Latitude": 35.35328, "Latitude": 35.35328,
"Longitude": 139.5325737 "Longitude": 139.5325737
},
"西区南幸1-6-31横浜タカシマヤ 8F": {
"Latitude": 35.4651997,
"Longitude": 139.6191458
},
"西区高島2-16-B1横浜駅東口地下街ポルタ": {
"Latitude": 35.4622889,
"Longitude": 139.6222899
},
"西区高島2-19-12スカイビル 11F": {
"Latitude": 35.4645561,
"Longitude": 139.6246488
} }
} }

View File

@ -99,7 +99,7 @@ func (l *selector) locateStrings(doc *goquery.Document) ([]string, error) {
} }
} }
strs = append(strs, str) strs = append(strs, strings.TrimSpace(str))
}) })
return strs, err return strs, err
@ -137,6 +137,7 @@ func (l *selector) locateFloat(doc *goquery.Document) (float64, error) {
// converter // converter
// //
type converter struct { type converter struct {
Name string
Domains []string Domains []string
Index struct { Index struct {

View File

@ -1,4 +1,5 @@
name = "tripadvisor" name = "tripadvisor"
domains = ["www.tripadvisor.com"]
[index.items] [index.items]
path = "a.property_title" path = "a.property_title"
@ -12,53 +13,53 @@ name = "tripadvisor"
path = "h1#HEADING" path = "h1#HEADING"
[item.address] [item.address]
path = "address span.format_address > span:not(.extended-address)" path = "address span.format_address"
[item.weight] [item.count]
path = "h3.reviews_header" path = "h3.reviews_header"
regex = "^(\d+)" regEx = "^(\\d+)"
[item.props] [item.props]
[item.props.service] [item.props.service]
accomodating = 1.0 accomodating = 1.0
affordable = 0.0 affordable = 0.0
atmospheric = 0.0 atmospheric = 0.0
delicious: 0.0 delicious = 0.0
scale = 5.0 scale = 5.0
path = "ul.barChart div.ratingRow img:nth-child(2)" path = "ul.barChart > li:nth-child(1) > div:nth-child(2) img"
regex = "^([0-9]*\.?[0-9]+)" regEx = "^([0-9]*\\.?[0-9]+)"
attr = "alt" attr = "alt"
[item.props.food] [item.props.food]
accomodating = 0.0 accomodating = 0.0
affordable = 0.0 affordable = 0.0
atmospheric = 0.0 atmospheric = 0.0
delicious: 1.0 delicious = 1.0
scale = 5.0 scale = 5.0
path = "ul.barChart div.ratingRow img:nth-child(1)" path = "ul.barChart > li:nth-child(1) > div:nth-child(1) img"
regex = "^([0-9]*\.?[0-9]+)" regEx = "^([0-9]*\\.?[0-9]+)"
attr = "alt" attr = "alt"
[item.props.value] [item.props.value]
accomodating = 0.0 accomodating = 0.0
affordable = 1.0 affordable = 1.0
atmospheric = 0.0 atmospheric = 0.0
delicious: 0.0 delicious = 0.0
scale = 5.0 scale = 5.0
path = "ul.barChart div.ratingRow img:nth-child(3)" path = "ul.barChart > li:nth-child(2) > div:nth-child(1) img"
regex = "^([0-9]*\.?[0-9]+)" regEx = "^([0-9]*\\.?[0-9]+)"
attr = "alt" attr = "alt"
[item.props.atmosphere] [item.props.atmosphere]
accomodating = 0.0 accomodating = 0.0
affordable = 0.0 affordable = 0.0
atmospheric = 1.0 atmospheric = 1.0
delicious: 0.0 delicious = 0.0
scale = 5.0 scale = 5.0
path = "ul.barChart div.ratingRow img:nth-child(4)" path = "ul.barChart > li:nth-child(2) > div:nth-child(2) img"
regex = "^([0-9]*\.?[0-9]+)" regEx = "^([0-9]*\\.?[0-9]+)"
attr = "alt" attr = "alt"

Binary file not shown.

View File

@ -1,16 +1 @@
http://tabelog.com/en/kanagawa/rstLst/
http://www.tripadvisor.com/Restaurants-g298173-Yokohama_Kanagawa_Prefecture_Kanto.html http://www.tripadvisor.com/Restaurants-g298173-Yokohama_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g298172-Kawasaki_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g1021282-Sagamihara_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g1021277-Fujisawa_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g303156-Kamakura_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g298174-Yokosuka_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g1021278-Odawara_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g681222-Hiratsuka_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g298169-Atsugi_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g1021286-Yamato_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g1021279-Chigasaki_Kanagawa_Prefecture_Kanto.html
http://www.tripadvisor.com/Restaurants-g1021285-Hadano_Kanagawa_Prefecture_Kanto.html

View File

@ -1,120 +0,0 @@
/*
* Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
* Author: Alex Yatskov <alex@foosoft.net>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package main
import (
"errors"
"fmt"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
)
type tripadvisor struct {
}
func (tripadvisor) define(keyword string) semantics {
return map[string]semantics{
"food": {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 1.0},
"service": {Accomodating: 1.0, Affordable: 0.0, Atmospheric: 0.0, Delicious: 0.0},
"value": {Accomodating: 0.0, Affordable: 1.0, Atmospheric: 0.0, Delicious: 0.0},
"atmosphere": {Accomodating: 0.0, Affordable: 0.0, Atmospheric: 1.0, Delicious: 0.0},
}[keyword]
}
func (tripadvisor) index(doc *goquery.Document) (string, []string) {
var reviewUrls []string
doc.Find("a.property_title").Each(func(index int, sel *goquery.Selection) {
if href, ok := sel.Attr("href"); ok {
reviewUrls = append(reviewUrls, href)
}
})
var nextIndexUrl string
if href, ok := doc.Find("div.deckTools.btm a.nav.next.rndBtn.rndBtnGreen.taLnk").Attr("href"); ok {
nextIndexUrl = href
}
return nextIndexUrl, reviewUrls
}
func (tripadvisor) review(doc *goquery.Document) (name, address string, features map[string]float64, weight float64, err error) {
if name = strings.TrimSpace(doc.Find("h1#HEADING").Text()); len(name) == 0 {
err = errors.New("invalid name")
return
}
{
var addressParts []string
doc.Find("address span.format_address > span:not(.extended-address)").Each(func(index int, sel *goquery.Selection) {
addressParts = append(addressParts, strings.TrimSpace(sel.Text()))
})
if len(addressParts) == 0 {
err = errors.New("invalid address")
return
}
address = strings.Join(addressParts, " ")
}
{
ratings := doc.Find("ul.barChart div.ratingRow img.sprite-rating_s_fill")
if ratings.Length() != 4 {
err = errors.New("invalid ratings")
return
}
features = make(map[string]float64)
for index, category := range []string{"food", "service", "value", "atmosphere"} {
altText, _ := ratings.Eq(index).Attr("alt")
valueText := strings.Split(altText, " ")[0]
var value float64
if value, err = strconv.ParseFloat(valueText, 8); err != nil {
err = fmt.Errorf("invalid rating for %s", category)
return
}
features[category] = value/2.5 - 1.0
}
}
{
weightValid := false
if weightParts := strings.Split(doc.Find("h3.reviews_header").Text(), " "); len(weightParts) > 0 {
if weight, err = strconv.ParseFloat(weightParts[0], 8); err == nil {
weightValid = true
return
}
}
if !weightValid {
err = fmt.Errorf("invalid review count")
return
}
}
return
}