Adding mongo feed icon model, bootstrapping, and moved the views. Sti…

…ll need the importer and the canonical rep to match new models.

apps/reader/views.py

@@ -23,7 +23,7 @@
 from apps.analyzer.models import get_classifiers_for_user
 from apps.reader.models import UserSubscription, UserSubscriptionFolders, MUserStory, Feature
 from apps.reader.forms import SignupForm, LoginForm, FeatureForm
-from apps.rss_feeds.models import FeedIcon
+from apps.rss_feeds.models import MFeedIcon
 try:
     from apps.rss_feeds.models import Feed, MFeedPage, DuplicateFeed, MStory, MStarredStory, FeedLoadtime
 except:
@@ -180,17 +180,11 @@ def load_feed_favicons(request):
         user_subs = user_subs.filter(active=True)
     if feed_ids:
         user_subs = user_subs.filter(feed__in=feed_ids)
-
-    favicons = {}
 
-    for sub in user_subs:
-        try:
-            feed_icon = FeedIcon.objects.get(feed__pk=sub.feed.pk)
-            favicons[sub.feed.pk] = feed_icon.data
-        except FeedIcon.DoesNotExist:
-            continue
+    feed_ids   = [sub['feed__pk'] for sub in user_subs.values('feed__pk')]
+    feed_icons = dict([(i.feed_id, i.data) for i in MFeedIcon.objects(feed_id__in=feed_ids)])
 
-    return favicons
+    return feed_icons
 
 @ajax_login_required
 @json.json_view
@@ -256,7 +250,8 @@ def refresh_feeds(request):
         user_subs = user_subs.filter(feed__in=feed_ids)
     UNREAD_CUTOFF = datetime.datetime.utcnow() - datetime.timedelta(days=settings.DAYS_OF_UNREAD)
     favicons_fetching = [int(f) for f in request.POST.getlist('favicons_fetching') if f]
-
+    feed_icons = dict([(i.feed_id, i) for i in MFeedIcon.objects(feed_id__in=favicons_fetching)])
+
     for sub in user_subs:
         if (sub.needs_unread_recalc or 
             sub.unread_count_updated < UNREAD_CUTOFF or 
@@ -275,9 +270,10 @@ def refresh_feeds(request):
         if request.POST.get('check_fetch_status', False):
             feeds[sub.feed.pk]['not_yet_fetched'] = not sub.feed.fetched_once
         if sub.feed.pk in favicons_fetching:
-            feeds[sub.feed.pk]['favicon'] = sub.feed.icon.data
-            feeds[sub.feed.pk]['favicon_color'] = sub.feed.icon.color
-            feeds[sub.feed.pk]['favicon_fetching'] = bool(not (sub.feed.icon.not_found or sub.feed.icon.data))
+            feeds[sub.feed.pk]['favicon'] = feed_icons[sub.feed.pk].data
+            feeds[sub.feed.pk]['favicon_color'] = feed_icons[sub.feed.pk].color
+            feeds[sub.feed.pk]['favicon_fetching'] = bool(not (feed_icons[sub.feed.pk].not_found or
+                                                               feed_icons[sub.feed.pk].data))
 
     if settings.DEBUG:
         diff = datetime.datetime.utcnow()-start

apps/rss_feeds/models.py

@@ -61,15 +61,16 @@ def __unicode__(self):
         return self.feed_title
 
     def canonical(self, full=False):
+        icon = MFeedIcon.objects(feed_id=self.pk)
         feed = {
             'id': self.pk,
             'feed_title': self.feed_title,
             'feed_address': self.feed_address,
             'feed_link': self.feed_link,
             'updated': relative_timesince(self.last_update),
             'subs': self.num_subscribers,
-            'favicon_color': self.icon.color,
-            'favicon_fetching': bool(not (self.icon.not_found or self.icon.data))
+            'favicon_color': icon.color,
+            'favicon_fetching': bool(not (icon.not_found or icon.data))
         }
 
         if not self.fetched_once:
@@ -883,6 +884,28 @@ def save(self, *args, **kwargs):
             if hasattr(self, 'id'): self.delete()
 
 
+class MFeedIcon(mongo.Document):
+    feed_id   = mongo.IntField(primary_key=True)
+    color     = mongo.StringField(max_length=6)
+    data      = mongo.StringField()
+    icon_url  = mongo.StringField()
+    not_found = mongo.BooleanField(default=False)
+
+    meta = {
+        'collection'        : 'feed_icons',
+        'allow_inheritance' : False,
+    }
+
+    def save(self, *args, **kwargs):
+        if self.icon_url:
+            self.icon_url = unicode(self.icon_url)
+        try:    
+            super(MFeedIcon, self).save(*args, **kwargs)
+        except (IntegrityError, OperationError):
+            # print "Error on Icon: %s" % e
+            if hasattr(self, '_id'): self.delete()
+
+
 class MFeedPage(mongo.Document):
     feed_id = mongo.IntField(primary_key=True)
     page_data = mongo.BinaryField()

utils/bootstrap_mongo.py

@@ -1,9 +1,9 @@
 from pprint import pprint
 from django.conf import settings
-from apps.reader.models import MUserStory, UserStory
-from apps.rss_feeds.models import Feed, Story, MStory, StoryAuthor, Tag, MFeedPage, FeedPage
+from apps.reader.models import MUserStory
+from apps.rss_feeds.models import Feed, MStory, MFeedPage
+from apps.rss_feeds.models import MFeedIcon, FeedIcon
 from apps.analyzer.models import MClassifierTitle, MClassifierAuthor, MClassifierFeed, MClassifierTag
-from apps.analyzer.models import ClassifierTitle, ClassifierAuthor, ClassifierFeed, ClassifierTag
 import mongoengine, pymongo
 import sys
 from mongoengine.queryset import OperationError
@@ -138,6 +138,34 @@ def bootstrap_feedpages():
 
     print "\nMongo DB feed_pages: %s" % MFeedPage.objects().count()
 
+def bootstrap_feedicons():
+    print "Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
+    db.feed_icons.drop()
+    print "Dropped! Mongo DB feed_icons: %s" % MFeedIcon.objects().count()
+
+    print "FeedIcons: %s" % FeedIcon.objects.count()
+    pprint(db.feed_icons.index_information())
+
+    feeds = Feed.objects.all().order_by('-average_stories_per_month')
+    feed_count = feeds.count()
+    i = 0
+    for feed in feeds:
+        i += 1
+        print "%s/%s: %s" % (i, feed_count, feed,)
+        sys.stdout.flush()
+
+        if not MFeedIcon.objects(feed_id=feed.pk):
+            feed_icon = FeedIcon.objects.filter(feed=feed).values()
+            if feed_icon:
+                try:
+                    MFeedIcon(**feed_icon[0]).save()
+                except:
+                    print '\n\n!\n\n'
+                    continue
+
+
+    print "\nMongo DB feed_icons: %s" % MFeedIcon.objects().count()
+
 def compress_stories():
     count = MStory.objects().count()
     print "Mongo DB stories: %s" % count
@@ -195,4 +223,5 @@ def reindex_stories():
     # bootstrap_classifiers()
     # bootstrap_feedpages()
     # compress_stories()
-    reindex_stories()
+    # reindex_stories()
+    bootstrap_feedicons()