feat/mal-cache-check (#3)

* feat: add secrets.json.example

* feat: add mal cache checker

* fix(build): update goreleaser skip flag

* fix(build): goreleaser args

Author: varoOP

.github/workflows/release.yml

@@ -54,7 +54,7 @@ jobs:
         with:
           distribution: goreleaser
           version: latest
-          args: release --clean --skip-validate --skip-publish --parallelism 99
+          args: release --clean --skip=validate,publish --parallelism 99
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

cmd/shinkrodb/main.go

@@ -26,6 +26,7 @@ func main() {
 	switch cmd := pflag.Arg(0); cmd {
 	case "run":
 		cfg := config.NewConfig()
+		domain.CleanCache("./mal_cache")
 		domain.GetMalIds(cfg)
 		domain.ScrapeMal()
 		domain.GetTvdbIDs()

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/gocolly/colly v1.2.0
 	github.com/spf13/pflag v1.0.5
+	golang.org/x/net v0.9.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -21,7 +22,6 @@ require (
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
-	golang.org/x/net v0.9.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect

internal/domain/cache.go

@@ -0,0 +1,119 @@
+package domain
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+var (
+	delCount   int
+	nilMatches []string
+	mutex      sync.Mutex
+)
+
+func CleanCache(rootDir string) {
+	var wg sync.WaitGroup
+	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			wg.Add(1)
+			go func(path string) {
+				defer wg.Done()
+				doc := readFile(path)
+				if shouldDelete(doc, path) {
+					mutex.Lock()
+					delCount++
+					mutex.Unlock()
+					removeFile(path)
+				}
+			}(path)
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+
+	fmt.Println("Unable to check the following html pages in cache:")
+	for _, v := range nilMatches {
+		fmt.Println(v)
+	}
+
+	fmt.Println("Total number of files deleted: ", delCount)
+}
+
+func removeFile(path string) {
+	err := os.Remove(path)
+	if err != nil {
+		log.Fatalf("Failed to delete file %s: %v", path, err)
+	}
+}
+
+func readFile(path string) string {
+	f, err := os.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer f.Close()
+
+	d, err := html.Parse(f)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var b bytes.Buffer
+	err = html.Render(&b, d)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	doc := b.String()
+	return doc
+}
+
+func shouldDelete(doc, path string) bool {
+	return checkTypeAndYear(doc, path) && !checkAnidb(doc)
+}
+
+func checkAnidb(doc string) bool {
+	return strings.Contains(doc, "external-links-anime-pc-anidb")
+
+}
+
+// returns true if the anime is from the current year and it's type is tv
+func checkTypeAndYear(doc, path string) bool {
+	exp := `[\S\s]*(?:https://myanimelist\.net/topanime\.php\?type=(\w+)|Type:</span>\n\s{0,}(\w+)\s{0,}</div>)[\S\s\v]*(?:Aired:</span>\n\s+.*(\d{4}))`
+	re := regexp.MustCompile(exp)
+	match := re.FindStringSubmatch(doc)
+
+	if match == nil {
+		nilMatches = append(nilMatches, path)
+		return false
+	}
+
+	showType := match[1]
+
+	if showType == "" {
+		showType = match[2]
+	}
+
+	currentYear := strconv.Itoa(time.Now().Year())
+	return showType == "tv" && match[3] == currentYear
+}

secrets.json.example

@@ -0,0 +1,6 @@
+## Enter your id and key then rename this file to secrets.json
+
+{
+    "mal-client-id": "",
+    "tmdb-api-key": ""
+}