scrawl/scrawl.go

172 lines
3.5 KiB
Go
Raw Normal View History

2016-01-08 10:36:07 +00:00
/*
* Copyright (c) 2016 Alex Yatskov <alex@foosoft.net>
* Author: Alex Yatskov <alex@foosoft.net>
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* this software and associated documentation files (the "Software"), to deal in
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so,
* subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
* FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
* COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
* IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package main
import (
"bytes"
2016-01-08 11:03:40 +00:00
"errors"
2016-01-08 10:36:07 +00:00
"flag"
2016-01-08 11:03:40 +00:00
"fmt"
2016-01-08 10:36:07 +00:00
"io"
"log"
"net/http"
"net/url"
"os"
2016-01-08 13:19:36 +00:00
"path"
2016-01-08 10:36:07 +00:00
"path/filepath"
"strings"
"github.com/PuerkitoBio/goquery"
)
func scrape(url, css, attr string) (string, error) {
doc, err := goquery.NewDocument(url)
if err != nil {
return "", err
}
sel := doc.Find(css)
2016-01-08 11:03:40 +00:00
if sel.Length() == 0 {
return "", fmt.Errorf("no selection for '%s'", css)
}
2016-01-08 10:36:07 +00:00
2016-01-08 13:19:36 +00:00
sel = sel.First()
2016-01-08 10:36:07 +00:00
var res string
if len(attr) == 0 {
res = sel.Text()
} else {
2016-01-08 11:03:40 +00:00
var exists bool
if res, exists = sel.Attr(attr); !exists {
return "", fmt.Errorf("attribute '%s' not found", attr)
}
}
res = strings.TrimSpace(res)
if len(res) == 0 {
return "", errors.New("extracted empty string")
2016-01-08 10:36:07 +00:00
}
2016-01-08 11:03:40 +00:00
return res, nil
2016-01-08 10:36:07 +00:00
}
func download(url string, w io.Writer) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
io.Copy(w, resp.Body)
return nil
}
func export(path string, r io.Reader) error {
out, err := os.Create(path)
if err != nil {
log.Fatal(err)
}
defer out.Close()
io.Copy(out, r)
return nil
}
2016-01-08 13:19:36 +00:00
func usage() {
2016-01-08 13:26:19 +00:00
fmt.Fprintf(os.Stderr, "Usage: %s [options] url selector [path]\n", path.Base(os.Args[0]))
2016-01-08 13:19:36 +00:00
fmt.Fprintf(os.Stderr, "http://foosoft.net/projects/scrawl/\n\n")
fmt.Fprintf(os.Stderr, "Parameters:\n")
flag.PrintDefaults()
}
2016-01-08 10:36:07 +00:00
func main() {
2016-01-08 11:03:40 +00:00
var (
attr = flag.String("attr", "", "attribute to query")
verbose = flag.Bool("verbose", false, "verbose output")
)
2016-01-08 13:19:36 +00:00
flag.Usage = usage
2016-01-08 11:03:40 +00:00
flag.Parse()
2016-01-08 10:36:07 +00:00
if flag.NArg() < 2 {
flag.Usage()
os.Exit(2)
}
var (
2016-01-08 11:03:40 +00:00
baseRaw = flag.Arg(0)
css = flag.Arg(1)
2016-01-08 10:36:07 +00:00
)
2016-01-08 11:03:40 +00:00
base, err := url.Parse(baseRaw)
2016-01-08 10:36:07 +00:00
if err != nil {
log.Fatal(err)
}
2016-01-08 11:03:40 +00:00
if *verbose {
log.Printf("scraping page '%s'", baseRaw)
}
resRaw, err := scrape(baseRaw, css, *attr)
2016-01-08 10:36:07 +00:00
if err != nil {
log.Fatal(err)
}
2016-01-08 11:03:40 +00:00
if *verbose {
log.Printf("extracted string '%s'", resRaw)
}
2016-01-08 10:36:07 +00:00
res, err := url.Parse(resRaw)
if err != nil {
log.Fatal(err)
}
if !res.IsAbs() {
res = res.ResolveReference(base)
}
2016-01-08 11:03:40 +00:00
if *verbose {
log.Printf("downloading file '%s'", res.String())
}
2016-01-08 10:36:07 +00:00
var buff bytes.Buffer
if err := download(res.String(), &buff); err != nil {
log.Fatal(err)
}
var path string
if flag.NArg() > 2 {
2016-01-08 11:03:40 +00:00
path = flag.Arg(2)
2016-01-08 10:36:07 +00:00
} else {
path = filepath.Base(res.Path)
}
2016-01-08 11:03:40 +00:00
if *verbose {
log.Printf("writing file '%s'", path)
}
2016-01-08 10:36:07 +00:00
if err := export(path, &buff); err != nil {
log.Fatal(err)
}
}