Very basic GUI is done

Author: theotherp

.bowerrc

@@ -0,0 +1,4 @@
+{
+  "directory": "bower_components",
+  "json": "bower.json"
+}

bower.json

@@ -0,0 +1,40 @@
+{
+  "name": "NZB Hydra",
+  "version": "0.0.1",
+  "authors": [
+    "Theotherp"
+  ],
+  "license": "Apache-2.0",
+  "private": true,
+  "ignore": [
+    "**/.*",
+    "node_modules",
+    "static/lib",
+    "test",
+    "tests",
+    "cache",
+    "bower_components"
+  ],
+  "dependencies": {
+    "angular-animate": "~1.3.11",
+    "angular-bootstrap": "~0.12.0",
+    "angular-filter": "~0.5.2",
+    "angular-cookie": "~4.0.6",
+    "angular-growl-2": "~0.7.3",
+    "angular-route": "~1.3.11",
+    "angular-loading-bar": "~0.6.0",
+    "animate-css": "~3.2.0",
+    "filesize": "~3.0.2",
+    "bootstrap": "~3.3.2",
+    "superagent": "~0.21.0",
+    "momentjs": "~2.9.0",
+    "typicons": "~2.0.7",
+    "underscore": "~1.7.0"
+  },
+  "description": "",
+  "main": "nzbhydra.py",
+  "moduleType": [
+    "globals"
+  ],
+  "homepage": "https://github.com/theotherp/nzbhydra"
+}

gruntfile.js

@@ -97,21 +97,21 @@ module.exports = function (grunt) {
 		},	
 
 		less: {
-			crapture: {
+			nzbhydra: {
 				options: {
 					strictMath: true,
 					paths: ["nzbhydra/static/less"]
 				},
 				files: {
-      				'nzbhydra/static/css/crapture.css': 'nzbhydra/static/less/nzbhydra.less'
+      				'nzbhydra/static/css/nzbhydra.css': 'nzbhydra/static/less/nzbhydra.less'
     			}
 			},
 			bootstrap: {
 				options: {
 					strictMath: true
 				},
 				files: {
-					'nzbhydra/static/css/bootstrap.css': 'nzbhydra/static/less/bootstrap/nzbhydra.less'
+					'nzbhydra/static/css/bootstrap.css': 'nzbhydra/static/less/bootstrap/bootstrap.less'
 				}
 			}
 		},
@@ -162,13 +162,13 @@ module.exports = function (grunt) {
 
 		watch: {			
 			mystuff: {
-				files: ['nzbhydra/static/**/*.*', '!nzbhydra/static/less/**/*.less'], tasks: ['shell'], options: {
+				files: ['nzbhydra/static/**/*.*', '!nzbhydra/static/less/**/*.less'],  options: {
 					livereload: true
 				}
 			},
 
 			crapLess: {
-				files: ['nzbhydra/static/less/nzbhydra.less'], tasks: ['less:nzbhydra', 'shell']
+				files: ['nzbhydra/static/less/nzbhydra.less', 'nzbhydra/static/less/bootstrap/variables.less'], tasks: ['less:nzbhydra']
 			},
 
 			bootstrapLess: {
@@ -199,7 +199,6 @@ module.exports = function (grunt) {
 	grunt.loadNpmTasks('grunt-contrib-watch');
 	grunt.loadNpmTasks('grunt-contrib-copy');
 	grunt.loadNpmTasks('grunt-contrib-less');
-	grunt.loadNpmTasks('grunt-shell');
 
 
 	grunt.registerTask(
@@ -222,4 +221,6 @@ module.exports = function (grunt) {
 
 	grunt.registerTask('default', ['watch']);
 
+	grunt.registerTask('copy', ['bowercopy']);
+	
 };

nzbhydra/api.py

@@ -0,0 +1,121 @@
+from itertools import groupby
+
+from marshmallow import Schema, fields
+
+import config
+from config import init
+
+init("ResultProcessing.duplicateSizeThresholdInPercent", 0.1, float)
+init("ResultProcessing.duplicateAgeThreshold", 36000, int)
+
+categories = {'Console': {'code': [1000, 1010, 1020, 1030, 1040, 1050, 1060, 1070, 1080], 'pretty': 'Console'},
+              'Movie': {'code': [2000, 2010, 2020], 'pretty': 'Movie'},
+              'Movie_HD': {'code': [2040, 2050, 2060], 'pretty': 'HD'},
+              'Movie_SD': {'code': [2030], 'pretty': 'SD'},
+              'Audio': {'code': [3000, 3010, 3020, 3030, 3040], 'pretty': 'Audio'},
+              'PC': {'code': [4000, 4010, 4020, 4030, 4040, 4050, 4060, 4070], 'pretty': 'PC'},
+              'TV': {'code': [5000, 5020], 'pretty': 'TV'},
+              'TV_SD': {'code': [5030], 'pretty': 'SD'},
+              'TV_HD': {'code': [5040], 'pretty': 'HD'},
+              'XXX': {'code': [6000, 6010, 6020, 6030, 6040, 6050], 'pretty': 'XXX'},
+              'Other': {'code': [7000, 7010], 'pretty': 'Other'},
+              'Ebook': {'code': [7020], 'pretty': 'Ebook'},
+              'Comics': {'code': [7030], 'pretty': 'Comics'},
+              }
+
+
+def find_duplicates(results):
+    """
+
+    :type results: list[NzbSearchResult]
+    """
+    # TODO we might want to be able to specify more precisely what item we pick of a group of duplicates, for example by indexer priority 
+
+    # Sort and group by title. We only need to check the items in each individual group against each other because we only consider items with the same title as possible duplicates
+    sorted_results = sorted(results, key=lambda x: x.title.lower())
+    grouped_by_title = groupby(sorted_results, key=lambda x: x.title.lower())
+    grouped_by_sameness = []
+
+    for key, group in grouped_by_title:
+        seen2 = set()
+        group = list(group)
+        for i in range(len(group)):
+            if group[i] in seen2:
+                continue
+            seen2.add(group[i])
+            same_results = [group[i]]  # All elements in this list are duplicates of each other 
+            for j in range(i + 1, len(group)):
+                if group[j] in seen2:
+                    continue
+                seen2.add(group[j])
+                if test_for_duplicate(group[i], group[j]):
+                    same_results.append(group[j])
+            grouped_by_sameness.append(same_results)
+
+    return grouped_by_sameness
+
+
+def test_for_duplicate(search_result_1, search_result_2):
+    """
+
+    :type search_result_1: NzbSearchResult
+    :type search_result_2: NzbSearchResult
+    """
+
+    if search_result_1.title.lower() != search_result_2.title.lower():
+        return False
+    size_threshold = config.cfg["ResultProcessing.duplicateSizeThresholdInPercent"]
+    size_difference = search_result_1.size - search_result_2.size
+    size_average = (search_result_1.size + search_result_2.size) / 2
+    size_difference_percent = abs(size_difference / size_average) * 100
+
+    
+    # TODO: Ignore age threshold if no precise date is known or account for score (if we have sth like that...) 
+    age_threshold = config.cfg["ResultProcessing.duplicateAgeThreshold"]
+    same_size = size_difference_percent <= size_threshold
+    same_age = abs(search_result_1.epoch - search_result_2.epoch) / (1000 * 60) <= age_threshold  # epoch difference (ms) to minutes
+
+    # If all nweznab providers would provide poster/group in their infos then this would be a lot easier and more precise
+    # We could also use something to combine several values to a score, say that if a two posts have the exact size their age may differe more or combine relative and absolute size comparison
+    if same_size and same_age:
+        return True
+
+
+class NzbSearchResultSchema(Schema):
+    title = fields.String()
+    link = fields.String()
+    pubDate = fields.String()
+    epoch = fields.Integer()
+    pubdate_utc = fields.String()
+    age_days = fields.Integer()
+    age_precise = fields.Boolean()
+    provider = fields.String()
+    guid = fields.String()
+    size = fields.Integer()
+    categories = fields.String()  # wthy the fuc doesnt this work with fields.Integer(many=True)
+
+
+def process_for_internal_api(results):
+    # do what ever we need do prepare the results to be shown on our own page instead of being returned as newznab-compatible API
+    """
+
+    :type results: list[NzbSearchResult]
+    """
+    grouped_by_sameness = find_duplicates(results)
+    
+    
+    # Will be sorted by GUI later anyway but makes debugging easier
+    results = sorted(grouped_by_sameness, key=lambda x: x[0].epoch, reverse=True)
+    serialized = []
+    for g in results:
+        serialized.append(serialize_nzb_search_result(g).data)
+    
+    #We give each group of results a unique count value by which they can be identified later even if they're "taken apart"
+    for count, group in enumerate(serialized):
+        for i in group:
+            i["count"] = count
+    return {"results": serialized}
+
+
+def serialize_nzb_search_result(result):
+    return NzbSearchResultSchema(many=True).dump(result)

nzbhydra/config.py

@@ -0,0 +1,20 @@
+import profig
+
+cfg = profig.Config()
+
+
+def load(filename):
+    global cfg
+    cfg.read(filename)
+    #Manually set the source to this settings file so that when syncing the settings are written back. If we don't do this it loads the settings but doesn't write them back. Or we would need to store the
+    #settings filename and call write(filename)
+    cfg.sources = [filename]
+
+
+def init(path, value, type):
+    global cfg
+    cfg.init(path, value, type)
+    #Write this to console for now, later we can use it to collect all available/expected config data
+    print("Initializing configuration with path %s and value %s" % (path, value))
+
+        

nzbhydra/database.py

@@ -0,0 +1,93 @@
+import json
+import datetime
+import arrow
+from dateutil.tz import tzutc
+
+from peewee import *
+
+db = SqliteDatabase(None)
+
+
+class JSONField(TextField):
+    db_field = "text"
+
+    def db_value(self, value):
+        return json.dumps(value)
+
+    def python_value(self, value):
+        return json.loads(value)
+
+
+class DateTimeUTCField(DateTimeField):
+    db_field = "datetime"
+    
+    def db_value(self, value):
+        return arrow.get(value).datetime if value is not None else None
+
+    def python_value(self, value):
+        return arrow.get(value, tzinfo=tzutc())
+
+
+class Provider(Model):
+    name = CharField(unique=True)
+    module = CharField()
+    enabled = BooleanField(default=True)
+    settings = JSONField(default={})
+
+    class Meta:
+        database = db
+
+
+class ProviderSearch(Model):
+    provider = ForeignKeyField(Provider)
+    time = DateTimeField(default=datetime.datetime.utcnow())
+
+    query = CharField(null=True)
+    query_generated = BooleanField(default=False)
+    identifier_key = CharField(null=True)
+    identifier_value = CharField(null=True)
+    categories = JSONField(default=[])
+    season = IntegerField(null=True)
+    episode = IntegerField(null=True)
+
+    successful = BooleanField(default=False)
+    results = IntegerField(null=True)  # number of results returned
+
+    class Meta:
+        database = db  # This model uses the "people.db" database.
+
+
+class ProviderApiAccess(Model):
+    provider = ForeignKeyField(Provider)
+    time = DateTimeUTCField(default=datetime.datetime.utcnow())
+    type = CharField()  # search, download, comments, nfo?
+    response_successful = BooleanField(default=False)
+    response_time = IntegerField(null=True)
+    error = CharField(null=True)
+
+    class Meta:
+        database = db
+
+
+class ProviderSearchApiAccess(Model):
+    search = ForeignKeyField(ProviderSearch, related_name="api_accesses")
+    api_access = ForeignKeyField(ProviderApiAccess, related_name="api_accesses") 
+
+    class Meta:
+        database = db
+        
+        
+class ProviderStatus(Model):
+    provider = ForeignKeyField(Provider, related_name="status")
+    first_failure = DateTimeUTCField(default=datetime.datetime.utcnow(), null=True)
+    latest_failure = DateTimeUTCField(default=datetime.datetime.utcnow(), null=True)
+    disabled_until = DateTimeUTCField(default=datetime.datetime.utcnow(), null=True)
+    level = IntegerField(default=0)
+    reason = CharField(null=True)
+    disabled_permanently = BooleanField(default=False) #Set to true if an error occurred that probably won't fix itself (e.g. when the apikey is wrong) 
+    
+    def __repr__(self):
+        return "%s in status %d. First failure: %s. Latest Failure: %s. Reason: %s. Disabled until: %s" % (self.provider, self.level, self.first_failure, self.latest_failure, self.reason, self.disabled_until)
+    
+    class Meta:
+        database = db

nzbhydra/datestuff.py

@@ -0,0 +1,6 @@
+#can be mocked
+import datetime
+
+
+def now():
+    return datetime.datetime.now()