package main import ( "bytes" "crypto/md5" "encoding/hex" "errors" "flag" "fmt" "html" "io" "io/fs" "log" "os" "strings" "git.redxen.eu/caskd/goboru" "git.redxen.eu/caskd/goboru/modules/gelbooru" ) func main() { var ( jobs uint ) flag.UintVar(&jobs, "maxjobs", 10, "Maximum concurrent page fetch jobs") flag.Parse() var ( media []goboru.Media err error ) if media, err = gelbooru.Query(flag.Args(), 10); err != nil { log.Fatal(err) } log.Print("Fetched ", len(media), " elements") for _, v := range media { var i uint64 if i, err = download(v); err != nil { log.Fatal(err) } if i == 0 { log.Print("Skipped ", v.MD5) } else { log.Print("Downloaded ", i, " bytes") } } } func download(m goboru.Media) (i uint64, err error) { var ( f *os.File r io.ReadCloser skip bool ) f, err = os.OpenFile(m.MD5, os.O_RDWR, 0644) if err != nil { if errors.Is(err, fs.ErrNotExist) { f, err = os.Create(m.MD5) if err != nil { err = fmt.Errorf("Failed to create file: %s", err) return } } else { err = fmt.Errorf("Failed to open existing file to check MD5 checksum: %s", err) return } } else { var md []byte md, err = hex.DecodeString(m.MD5) if err != nil { err = fmt.Errorf("Failed to decode hexadecimal MD5 hash: %s", err) return } h := md5.New() _, err = io.Copy(h, f) if err != nil { err = fmt.Errorf("Failed to copy bytes to intermediate buffer for checksumming: %s", err) return } f.Seek(0, io.SeekStart) csum := h.Sum(nil) if bytes.Equal(csum, md) { skip = true } else { log.Printf("Hash mismatch: %x != %x [%s]", csum, md, m.Source) } } if !skip { r, err = m.Content() if err != nil { err = fmt.Errorf("Failed to get content reader: %s", err) return } var ir int64 ir, err = io.Copy(f, r) i = uint64(ir) if err != nil { err = fmt.Errorf("Failed to copy bytes to file: %s", err) return } r.Close() } f.Close() for _, x := range m.Tags { x = html.UnescapeString(x) // Unescape tags x = strings.ReplaceAll(x, "/", "_") // Replace filesystem delimiters in tags err = os.Mkdir(x, 0750) if err != nil { if !os.IsExist(err) { err = fmt.Errorf("Failed to create tag directory %s: %v", x, err) return } } src, dst := strings.Join([]string{x, m.MD5}, "/"), strings.Join([]string{"..", m.MD5}, "/") err = os.Symlink(dst, src) if err != nil { if !os.IsExist(err) { err = fmt.Errorf("Failed to create symlink in tag directory %s: %v", src, err) return } } } err = nil return }