Adding dumping to sqlite3
This commit is contained in:
parent
6b010e7dfc
commit
51e6a08eff
130
scrape.go
130
scrape.go
@ -24,9 +24,12 @@ package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
func scrapeUrls(filename string, wc *webCache, gc *geoCache) ([]restaurant, error) {
|
||||
@ -97,13 +100,136 @@ func processData(restaurants []restaurant, stationsPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpData(dbPath string, restaraunts []restaurant) error {
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
//
|
||||
// Restaurants
|
||||
//
|
||||
|
||||
_, err = db.Exec(`
|
||||
DROP TABLE IF EXISTS reviews;
|
||||
CREATE TABLE reviews(
|
||||
name VARCHAR(100) NOT NULL,
|
||||
url VARCHAR(200) NOT NULL,
|
||||
delicious FLOAT NOT NULL,
|
||||
accommodating FLOAT NOT NULL,
|
||||
affordable FLOAT NOT NULL,
|
||||
atmospheric FLOAT NOT NULL,
|
||||
latitude FLOAT NOT NULL,
|
||||
longitude FLOAT NOT NULL,
|
||||
distanceToStn FLOAT NOT NULL,
|
||||
closestStn VARCHAR(100) NOT NULL,
|
||||
accessCount INT NOT NULL,
|
||||
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY
|
||||
) DEFAULT CHARACTER SET utf8`)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, r := range restaraunts {
|
||||
_, err = db.Exec(`
|
||||
INSERT INTO reviews(
|
||||
name,
|
||||
url,
|
||||
delicious,
|
||||
accommodating,
|
||||
affordable,
|
||||
atmospheric,
|
||||
latitude,
|
||||
longitude,
|
||||
closestStnDist,
|
||||
closestStnName,
|
||||
accessCount
|
||||
) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
r.name,
|
||||
r.url,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
r.longitude,
|
||||
r.latitude,
|
||||
r.closestStnDist,
|
||||
r.closestStnName)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Categories
|
||||
//
|
||||
|
||||
_, err = db.Exec(`
|
||||
DROP TABLE IF EXISTS categories;
|
||||
CREATE TABLE categories(
|
||||
description VARCHAR(200) NOT NULL,
|
||||
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY))`)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, category := range []string{"I prefer quiet places", "I enjoy Mexican Food", "I drive a car"} {
|
||||
if _, err := db.Exec("INSERT INTO categories(description) VALUES (?)", category); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// History
|
||||
//
|
||||
|
||||
_, err = db.Exec(`
|
||||
DROP TABLE IF EXISTS history;
|
||||
CREATE TABLE history(
|
||||
date DATETIME NOT NULL,
|
||||
reviewId INT NOT NULL,
|
||||
id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
FOREIGN KEY(reviewId) REFERENCES reviews(id))`)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//
|
||||
// HistoryGroup
|
||||
//
|
||||
|
||||
_, err = db.Exec(`
|
||||
DROP TABLE IF EXISTS historyGroups;
|
||||
CREATE TABLE historyGroups(
|
||||
categoryId INT NOT NULL,
|
||||
categoryValue FLOAT NOT NULL,
|
||||
historyId INT NOT NULL,
|
||||
FOREIGN KEY(historyId) REFERENCES history(id),
|
||||
FOREIGN KEY(categoryId) REFERENCES categories(id))`)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
restaraunts, err := scrapeData("data/urls.txt", "cache/geocache.json", "cache/webcache")
|
||||
restaurants, err := scrapeData("data/urls.txt", "cache/geocache.json", "cache/webcache")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := processData(restaraunts, "data/stations.json"); err != nil {
|
||||
if err := processData(restaurants, "data/stations.json"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := dumpData("db.sqlite3", restaurants); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user