137 lines
2.5 KiB
Go
137 lines
2.5 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
|
|
"git.redxen.eu/caskd/goboru"
|
|
"git.redxen.eu/caskd/goboru/modules/gelbooru"
|
|
)
|
|
|
|
func main() {
|
|
var (
|
|
jobs uint
|
|
)
|
|
flag.UintVar(&jobs, "maxjobs", 10, "Maximum concurrent page fetch jobs")
|
|
flag.Parse()
|
|
|
|
var (
|
|
media []goboru.Media
|
|
err error
|
|
)
|
|
|
|
if media, err = gelbooru.Query(flag.Args(), 10); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
log.Print("Fetched ", len(media), " elements")
|
|
|
|
for _, v := range media {
|
|
var i uint64
|
|
if i, err = download(v); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if i == 0 {
|
|
log.Print("Skipped ", v.MD5)
|
|
} else {
|
|
log.Print("Downloaded ", i, " bytes")
|
|
}
|
|
}
|
|
}
|
|
|
|
func download(m goboru.Media) (i uint64, err error) {
|
|
var (
|
|
f *os.File
|
|
r io.ReadCloser
|
|
skip bool
|
|
)
|
|
|
|
f, err = os.OpenFile(m.MD5, os.O_RDWR, 0644)
|
|
if err != nil {
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
f, err = os.Create(m.MD5)
|
|
if err != nil {
|
|
err = fmt.Errorf("Failed to create file: %s", err)
|
|
return
|
|
}
|
|
} else {
|
|
err = fmt.Errorf("Failed to open existing file to check MD5 checksum: %s", err)
|
|
return
|
|
}
|
|
} else {
|
|
var md []byte
|
|
md, err = hex.DecodeString(m.MD5)
|
|
if err != nil {
|
|
err = fmt.Errorf("Failed to decode hexadecimal MD5 hash: %s", err)
|
|
return
|
|
}
|
|
h := md5.New()
|
|
|
|
_, err = io.Copy(h, f)
|
|
if err != nil {
|
|
err = fmt.Errorf("Failed to copy bytes to intermediate buffer for checksumming: %s", err)
|
|
return
|
|
}
|
|
f.Seek(0, io.SeekStart)
|
|
|
|
csum := h.Sum(nil)
|
|
if bytes.Equal(csum, md) {
|
|
skip = true
|
|
} else {
|
|
log.Printf("Hash mismatch: %x != %x [%s]", csum, md, m.Source)
|
|
}
|
|
}
|
|
|
|
if !skip {
|
|
r, err = m.Content()
|
|
if err != nil {
|
|
err = fmt.Errorf("Failed to get content reader: %s", err)
|
|
return
|
|
}
|
|
|
|
var ir int64
|
|
ir, err = io.Copy(f, r)
|
|
i = uint64(ir)
|
|
if err != nil {
|
|
err = fmt.Errorf("Failed to copy bytes to file: %s", err)
|
|
return
|
|
}
|
|
r.Close()
|
|
}
|
|
f.Close()
|
|
|
|
for _, x := range m.Tags {
|
|
x = html.UnescapeString(x) // Unescape tags
|
|
x = strings.ReplaceAll(x, "/", "_") // Replace filesystem delimiters in tags
|
|
|
|
err = os.Mkdir(x, 0750)
|
|
if err != nil {
|
|
if !os.IsExist(err) {
|
|
err = fmt.Errorf("Failed to create tag directory %s: %v", x, err)
|
|
return
|
|
}
|
|
}
|
|
|
|
src, dst := strings.Join([]string{x, m.MD5}, "/"), strings.Join([]string{"..", m.MD5}, "/")
|
|
err = os.Symlink(dst, src)
|
|
if err != nil {
|
|
if !os.IsExist(err) {
|
|
err = fmt.Errorf("Failed to create symlink in tag directory %s: %v", src, err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
err = nil
|
|
return
|
|
}
|