Skip to content

Commit

Permalink
fix doesConflict func
Browse files Browse the repository at this point in the history 
enhance title stripping to avoid false negative on different torrent serie name
add limiter for qBittorrent api to avoid updating too fast and returning conflict false
change batch tag
  • Loading branch information
@sonalys
sonalys committed Feb 7, 2024
1 parent fe54abd commit b8d1573
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 81 deletions.
2 changes: 1 addition & 1 deletion integrations/myanimelist/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ type (

AnimeListEntry struct {
Status ListStatus `json:"status"`
Title any `json:"anime_title"`
Title string `json:"anime_title"`
TitleEng string `json:"anime_title_eng"`
AiringStatus AiringStatus `json:"anime_airing_status"`
}
Expand Down
30 changes: 29 additions & 1 deletion integrations/nyaa/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import (
"fmt"
"net/http"
"net/url"
"strings"
)

type (
SourceType int
Query string
OrQuery []string
Category string
User string
)
Expand All @@ -35,8 +37,34 @@ func (q Query) Apply(req *http.Request) {
prevQuery := query.Get("q")
if prevQuery == "" {
query.Set("q", string(q))
} else {
query.Set("q", prevQuery+" "+string(q))
}
req.URL.RawQuery = query.Encode()
}

func filterNotEmpty(entries []string) []string {
notEmpty := make([]string, 0, len(entries))
for i := range entries {
if entries[i] != "" {
notEmpty = append(notEmpty, entries[i])
}
}
return notEmpty
}

func (entries OrQuery) Apply(req *http.Request) {
query := req.URL.Query()
prevQuery := query.Get("q")

entries = filterNotEmpty(entries)
curQuery := fmt.Sprintf("(%s)", strings.Join(entries, "|"))

if prevQuery == "" {
query.Set("q", curQuery)
} else {
query.Set("q", prevQuery+" "+curQuery)
}
query.Set("q", prevQuery+" "+string(q))
req.URL.RawQuery = query.Encode()
}

Expand Down
7 changes: 5 additions & 2 deletions integrations/qbittorrent/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"time"

"github.com/rs/zerolog/log"
"github.com/sonalys/animeman/internal/roundtripper"
"golang.org/x/time/rate"
)

type (
Expand All @@ -30,8 +32,9 @@ func New(ctx context.Context, host, username, password string) *API {
username: username,
password: password,
client: &http.Client{
Timeout: 3 * time.Second,
Jar: jar,
Transport: roundtripper.NewRateLimitedTransport(http.DefaultTransport, rate.NewLimiter(rate.Every(1*time.Second), 1)),
Timeout: 3 * time.Second,
Jar: jar,
},
}
var version string
Expand Down
12 changes: 6 additions & 6 deletions integrations/qbittorrent/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ func (t TorrentURL) ApplyAddTorrent(w *multipart.Writer) {
}
}

type Tag string

func (t Tag) ApplyListTorrent(v url.Values) {
v.Add("tag", url.QueryEscape(string(t)))
}

type Tags []string

func (t Tags) ApplyAddTorrent(w *multipart.Writer) {
Expand Down Expand Up @@ -72,12 +78,6 @@ type Torrent struct {
Hash string `json:"hash"`
}

func (t Tags) ApplyListTorrent(v url.Values) {
for _, tag := range t {
v.Add("tag", url.QueryEscape(tag))
}
}

type Paused bool

func (p Paused) ApplyAddTorrent(w *multipart.Writer) {
Expand Down
101 changes: 44 additions & 57 deletions internal/discovery/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"

"github.com/rs/zerolog/log"
Expand Down Expand Up @@ -67,11 +66,18 @@ func (c *Controller) Start(ctx context.Context) error {
}
}

func buildTorrentTags(title string) qbittorrent.Tags {
parsedTitle := parser.ParseTitle(title)
tags := qbittorrent.Tags{"!" + parsedTitle.Title, buildSeasonEpisodeTag(parsedTitle.Season, parsedTitle.Episode)}
if parsedTitle.Episode == "0" && parsedTitle.IsMultiEpisode {
tags = append(tags, TagBatch)
func buildSeasonEpisodeTag(parsedTitle parser.ParsedTitle) string {
resp := fmt.Sprintf("%s S%s", parsedTitle.Title, parsedTitle.Season)
if !parsedTitle.IsMultiEpisode {
resp = resp + fmt.Sprintf("E%s", parsedTitle.Episode)
}
return resp
}

func buildTorrentTags(parsedTitle parser.ParsedTitle) qbittorrent.Tags {
tags := qbittorrent.Tags{"!" + parsedTitle.Title, buildSeasonEpisodeTag(parsedTitle)}
if parsedTitle.IsMultiEpisode {
tags = append(tags, buildBatchTag(parsedTitle))
}
return tags
}
Expand All @@ -81,9 +87,9 @@ func (c *Controller) UpdateExistingTorrentsTags(ctx context.Context) error {
if err != nil {
return fmt.Errorf("listing: %w", err)
}
for i := range torrents {
torrent := &torrents[i]
if err := c.dep.QB.AddTorrentTags(ctx, []string{torrent.Hash}, buildTorrentTags(torrent.Name)); err != nil {
for _, torrent := range torrents {
parsedTitle := parser.ParseTitle(torrent.Name)
if err := c.dep.QB.AddTorrentTags(ctx, []string{torrent.Hash}, buildTorrentTags(parsedTitle)); err != nil {
return fmt.Errorf("updating tags: %w", err)
}
}
Expand All @@ -101,18 +107,7 @@ func (c *Controller) RunDiscovery(ctx context.Context) error {
log.Info().Msgf("processing %d entries from MAL", len(entries))
var totalCount int
for _, entry := range entries {
log.Debug().Msgf("Digesting entry '%s'", entry.GetTitle())
torrents, err := c.dep.NYAA.List(ctx,
nyaa.CategoryAnimeEnglishTranslated,
nyaa.Query(parser.StripTitle(entry.GetTitle())),
nyaa.Query(fmt.Sprintf("(%s)", strings.Join(c.dep.Config.Sources, "|"))),
nyaa.Query(fmt.Sprintf("(%s)", strings.Join(c.dep.Config.Qualitites, "|"))),
)
log.Debug().Str("entry", entry.GetTitle()).Msgf("Found %d torrents", len(torrents))
if err != nil {
return fmt.Errorf("getting nyaa list: %w", err)
}
count, err := c.digestEntry(ctx, entry, torrents)
count, err := c.digestEntry(ctx, entry)
if err != nil {
if errors.Is(err, qbittorrent.ErrUnauthorized) || errors.Is(err, context.Canceled) {
return fmt.Errorf("failed to digest entry: %w", err)
Expand All @@ -127,70 +122,61 @@ func (c *Controller) RunDiscovery(ctx context.Context) error {
return nil
}

func buildSeasonEpisodeTag(season, episode string) string {
resp := ""
if season != "0" {
resp = fmt.Sprintf("S%s", season)
}
if episode != "0" {
resp = resp + fmt.Sprintf("E%s", episode)
}
return resp
func buildBatchTag(parsedTitle parser.ParsedTitle) string {
return fmt.Sprintf("%s S%s batch", parsedTitle.Title, parsedTitle.Season)
}

var TagBatch = "batch"

func (c *Controller) doesConflict(ctx context.Context, title string, parsedTitle parser.ParsedTitle) (bool, error) {
func (c *Controller) doesConflict(ctx context.Context, parsedTitle parser.ParsedTitle) (bool, error) {
// check if torrent already exists, if so we skip it.
torrentList, err := c.dep.QB.List(ctx, qbittorrent.Tags{
title, TagBatch,
})
if err != nil {
return false, fmt.Errorf("listing torrents: %w", err)
}
torrentList, _ := c.dep.QB.List(ctx, qbittorrent.Tag(buildBatchTag(parsedTitle)))
if len(torrentList) > 0 {
return true, nil
}
torrentList, err = c.dep.QB.List(ctx, qbittorrent.Tags{
title, buildSeasonEpisodeTag(parsedTitle.Season, parsedTitle.Episode),
})
if err != nil {
return false, fmt.Errorf("listing torrents: %w", err)
}
torrentList, _ = c.dep.QB.List(ctx, qbittorrent.Tag(buildSeasonEpisodeTag(parsedTitle)))
if len(torrentList) > 0 {
return true, nil
}
return false, nil
}

func (c *Controller) digestEntry(ctx context.Context, entry myanimelist.AnimeListEntry, torrents []nyaa.Entry) (count int, err error) {
func (c *Controller) digestEntry(ctx context.Context, entry myanimelist.AnimeListEntry) (count int, err error) {
log.Debug().Msgf("Digesting entry '%s'", entry.GetTitle())
torrents, err := c.dep.NYAA.List(ctx,
nyaa.CategoryAnimeEnglishTranslated,
nyaa.OrQuery{parser.StripTitle(entry.TitleEng), parser.StripTitle(entry.Title)},
nyaa.OrQuery(c.dep.Config.Sources),
nyaa.OrQuery(c.dep.Config.Qualitites),
)
log.Debug().Str("entry", entry.GetTitle()).Msgf("Found %d torrents", len(torrents))
if err != nil {
return 0, fmt.Errorf("getting nyaa list: %w", err)
}
if len(torrents) == 0 {
log.Error().Msgf("no torrents found for entry '%s'", entry.GetTitle())
return 0, nil
}
for i := range torrents {
torrent := torrents[i]
log.Debug().Str("entry", entry.GetTitle()).Msgf("Analyzing torrent '%s'", torrent.Title)
parsedTitle := parser.ParseTitle(torrent.Title)
if parsedTitle.IsMultiEpisode && entry.AiringStatus == myanimelist.AiringStatusAiring {
log.Debug().Str("entry", entry.GetTitle()).Msgf("torrent '%s' dropped: multi-episode for currently airing", torrent.Title)
log.Debug().Str("entry", entry.GetTitle()).Msgf("analyzing torrent '%s'", torrent.Title)
meta := parser.ParseTitle(torrent.Title)
if meta.IsMultiEpisode && entry.AiringStatus == myanimelist.AiringStatusAiring {
log.Debug().Msgf("torrent dropped: multi-episode for currently airing")
continue
}
doesConflict, err := c.doesConflict(ctx, entry.GetTitle(), parsedTitle)
if err != nil {
doesConflict, err := c.doesConflict(ctx, meta)
if err != nil || doesConflict {
log.Debug().Str("tags", buildSeasonEpisodeTag(meta)).Msgf("torrent is conflicting")
break
}
if doesConflict {
continue
}
var savePath qbittorrent.SavePath
if c.dep.Config.CreateShowFolder {
savePath = qbittorrent.SavePath(fmt.Sprintf("%s/%s", c.dep.Config.DownloadPath, entry.GetTitle()))
} else {
savePath = qbittorrent.SavePath(c.dep.Config.DownloadPath)
}
tags := buildTorrentTags(meta)
err = c.dep.QB.AddTorrent(ctx,
buildTorrentTags(torrent.Title),
tags,
savePath,
qbittorrent.TorrentURL{torrent.Link},
qbittorrent.Category(c.dep.Config.Category),
Expand All @@ -200,7 +186,8 @@ func (c *Controller) digestEntry(ctx context.Context, entry myanimelist.AnimeLis
}
log.Info().
Str("savePath", string(savePath)).
Msgf("torrent '%s' added", entry.GetTitle())
Strs("tag", tags).
Msgf("torrent added")
}
return count, nil
}
22 changes: 8 additions & 14 deletions internal/parser/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,6 @@ import (
"strings"
)

// StripTitle returns only the main title, trimming everything after ':'.
func StripTitle(title string) string {
title, _, found := strings.Cut(title, ":")
if found {
return title
}
return title
}

type ParsedTitle struct {
Source string
Title string
Expand Down Expand Up @@ -55,12 +46,12 @@ func matchEpisode(title string) (string, bool) {
return fmt.Sprintf("%d~%d", episode1, episode2), true
}
// Some scenarios are like Frieren Season 1
return "0", true
return "", true
}

var seasonExpr = []*regexp.Regexp{
// 2nd season.
regexp.MustCompile(`(\d+)(?:nd)|(?:rd)|(?:th)(?i:\sseason)`),
regexp.MustCompile(`(\d+)(?:(?:nd)|(?:rd)|(?:th))(?i:\sseason)`),
// 2x15.
regexp.MustCompile(`(\d+)(?:x\d+)`),
// S02E15.
Expand All @@ -78,7 +69,7 @@ func matchSeason(title string) string {
season, _ := strconv.ParseInt(matches[0][1], 10, 64)
return fmt.Sprint(season)
}
return "0"
return "1"
}

func matchEpisodeIndex(title string) int {
Expand Down Expand Up @@ -112,7 +103,7 @@ func cleanWithRegex(expr *regexp.Regexp, value string) string {
return expr.ReplaceAllString(value, "")
}

func cleanupTitle(title string) string {
func StripTitle(title string) string {
for _, expr := range titleCleanupExpr {
title = cleanWithRegex(expr, title)
}
Expand All @@ -122,13 +113,16 @@ func cleanupTitle(title string) string {
if index := matchEpisodeIndex(title); index != -1 {
title = title[:index]
}
title = strings.Split(title, ": ")[0]
title = strings.Split(title, ", ")[0]
title = strings.Split(title, "- ")[0]
title = strings.ReplaceAll(title, " ", " ")
return strings.TrimSpace(title)
}

func ParseTitle(title string) ParsedTitle {
resp := ParsedTitle{
Title: cleanupTitle(title),
Title: StripTitle(title),
}
if tags := tagsExpr.FindAllStringSubmatch(title, -1); len(tags) > 0 {
resp.Source = tags[0][1]
Expand Down

0 comments on commit b8d1573

Please sign in to comment.