From 590f4b602d7f6a49e999332740fdf8b5dc9256f7 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 24 Jul 2008 18:54:17 +0000 Subject: [PATCH] updated to support newforms-admin better, fixed some minor bugs, cleaned up the help strings git-svn-id: https://django-robots.googlecode.com/svn/trunk@12 12edf5ea-513a-0410-8a8c-37067077e60f committer: leidel --HG-- extra : convert_revision : 78c5bb18484f7a1683ea17ce0acb1d6a4a092ff8 --- LICENSE.txt | 2 +- robots/__init__.py | 2 +- robots/admin.py | 17 ++++++++ robots/models.py | 97 ++++++++++++++++++++++------------------------ robots/urls.py | 5 ++- robots/views.py | 14 ++++--- 6 files changed, 76 insertions(+), 61 deletions(-) create mode 100644 robots/admin.py diff --git a/LICENSE.txt b/LICENSE.txt index 75fd133..890b67f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2007, Jannis Leidel +Copyright (c) 2008, Jannis Leidel All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/robots/__init__.py b/robots/__init__.py index ae3ae76..e97ee31 100644 --- a/robots/__init__.py +++ b/robots/__init__.py @@ -1 +1 @@ -VERSION = (0, "3.2", None) +VERSION = (0, 4, None) diff --git a/robots/admin.py b/robots/admin.py new file mode 100644 index 0000000..3f953f9 --- /dev/null +++ b/robots/admin.py @@ -0,0 +1,17 @@ +from django.contrib import admin +from django.utils.translation import ugettext_lazy as _ + +from robots.models import Url, Rule + +class RuleAdmin(admin.ModelAdmin): + fieldsets = ( + (None, {'fields': ('robot', 'sites')}), + (_('URL patterns'), {'fields': ('allowed', 'disallowed')}), + (_('Advanced options'), {'classes': ('collapse',), 'fields': ('crawl_delay',)}), + ) + list_filter = ('sites',) + list_display = ('robot', 'allowed_urls', 'disallowed_urls') + search_fields = ('robot','urls') + +admin.site.register(Url) +admin.site.register(Rule, RuleAdmin) diff --git a/robots/models.py b/robots/models.py index 0057499..5ab54b6 100644 --- a/robots/models.py +++ b/robots/models.py @@ -3,32 +3,27 @@ from django.contrib.sites.models import Site from django.utils.translation import ugettext_lazy as _ from django.utils.text import get_text_list -from django.conf import settings -# for backward incompatible changes in 0.97/trunk -try: - from django.db.models import DecimalField as FloatField -except ImportError: - from django.db.models import FloatField - -try: - from django.contrib.admin.views.doc import simplify_regex -except ImportError: - from django.contrib.admindocs.views import simplify_regex class Url(models.Model): """ Defines a URL pattern for use with a robot exclusion rule. It's case-sensitive and exact, e.g., "/admin" and "/admin/" are different URLs. """ - pattern = models.CharField(_('pattern'), max_length=255, core=True, help_text=_("Case-sensitive. A missing trailing slash does also match to files which start with the name of the pattern, e.g., '/admin' matches /admin.html too. Some major search engines allow an asterisk (*) as a wildcard and a dollar sign ($) to match the end of the URL, e.g., '/*.jpg$'.")) - + pattern = models.CharField(_('pattern'), max_length=255, core=True, + help_text=_("Case-sensitive. A missing " + "trailing slash does also match " + "to files which start with the " + "name of the pattern, e.g., " + "'/admin' matches /admin.html " + "too. Some major search engines " + "allow an asterisk (*) as a " + "wildcard and a dollar sign ($) " + "to match the end of the URL, " + "e.g., '/*.jpg$'.")) class Meta: verbose_name = _('url') verbose_name_plural = _('url') - class Admin: - pass - def __unicode__(self): return u"%s" % self.pattern @@ -44,26 +39,47 @@ class Rule(models.Model): disallows the robot identified by its user agent to access the given URLs. The Site contrib app is used to enable multiple robots.txt per instance. """ - robot = models.CharField(_('robot'), max_length=255, help_text=_("This should be a user agent string like 'Googlebot'. Enter an asterisk (*) for all user agents. For a full list look at the database of Web Robots.")) - allowed = models.ManyToManyField(Url, blank=True, validator_list=[validators.RequiredIfOtherFieldNotGiven('disallowed')], related_name="allowed", help_text=_("These are URLs which are allowed to be accessed by web robots.")) - disallowed = models.ManyToManyField(Url, blank=True, validator_list=[validators.RequiredIfOtherFieldNotGiven('allowed')], related_name="disallowed", help_text=_("These are URLs which are not allowed to be accessed by web robots.")) + robot = models.CharField(_('robot'), max_length=255, help_text=_( + "This should be a user agent string like " + "'Googlebot'. Enter an asterisk (*) for all " + "user agents. For a full list look at the " + " database of Web Robots.")) + + allowed = models.ManyToManyField(Url, blank=True, related_name="allowed", + validator_list=[validators.RequiredIfOtherFieldNotGiven('disallowed')], + help_text=_("The URLs which are allowed " + "to be accessed by bots.")) + + disallowed = models.ManyToManyField(Url, blank=True, related_name="disallowed", + validator_list=[validators.RequiredIfOtherFieldNotGiven('allowed')], + help_text=_("The URLs which are not " + "allowed to be accessed " + "by bots.")) sites = models.ManyToManyField(Site) - crawl_delay = FloatField(_('crawl delay'), blank=True, null=True, max_digits=3, decimal_places=1, help_text=("From 0.1 to 99.0. This field is supported by some search engines and defines the delay between successive crawler accesses in seconds. If the crawler rate is a problem for your server, you can set the delay up to 5 or 10 or a comfortable value for your server, but it's suggested to start with small values (0.5-1), and increase as needed to an acceptable value for your server. Larger delay values add more delay between successive crawl accesses and decrease the maximum crawl rate to your web server.")) + crawl_delay = models.DecimalField(_('crawl delay'), blank=True, null=True, + max_digits=3, decimal_places=1, help_text=( + "Between 0.1 and 99.0. This field is " + "supported by some search engines and " + "defines the delay between successive " + "crawler accesses in seconds. If the " + "crawler rate is a problem for your " + "server, you can set the delay up to 5 " + "or 10 or a comfortable value for your " + "server, but it's suggested to start " + "with small values (0.5-1), and " + "increase as needed to an acceptable " + "value for your server. Larger delay " + "values add more delay between " + "successive crawl accesses and " + "decrease the maximum crawl rate to " + "your web server.")) class Meta: verbose_name = _('rule') verbose_name_plural = _('rules') - class Admin: - fields = ( - (None, {'fields': ('robot', 'sites')}), - (_('URL patterns'), {'fields': ('allowed', 'disallowed')}), - (_('Advanced options'), {'classes': 'collapse', 'fields': ('crawl_delay',)}), - ) - list_filter = ('sites',) - list_display = ('robot', 'allowed_urls', 'disallowed_urls') - search_fields = ('robot','urls') - def __unicode__(self): return u"%s" % self.robot @@ -74,24 +90,3 @@ def allowed_urls(self): def disallowed_urls(self): return get_text_list(list(self.disallowed.all()), _('and')) disallowed_urls.short_description = _('disallowed') - -try: - from django.contrib import admin - - class UrlAdmin(admin.ModelAdmin): - pass - - class RuleAdmin(admin.ModelAdmin): - fieldsets = ( - (None, {'fields': ('robot', 'sites')}), - (_('URL patterns'), {'fields': ('allowed', 'disallowed')}), - (_('Advanced options'), {'classes': ('collapse',), 'fields': ('crawl_delay',)}), - ) - list_filter = ('sites',) - list_display = ('robot', 'allowed_urls', 'disallowed_urls') - search_fields = ('robot','urls') - - admin.site.register(Url, UrlAdmin) - admin.site.register(Rule, RuleAdmin) -except: - pass diff --git a/robots/urls.py b/robots/urls.py index 17ab433..42bb61d 100644 --- a/robots/urls.py +++ b/robots/urls.py @@ -1,5 +1,6 @@ from django.conf.urls.defaults import * +from robots.views import rules_list -urlpatterns = patterns('robots.views', - (r'^$', 'rules_list'), +urlpatterns = patterns('', + url(r'^$', rules_list, name='robots_rule_list'), ) diff --git a/robots/views.py b/robots/views.py index eb5ef25..f2ed865 100644 --- a/robots/views.py +++ b/robots/views.py @@ -1,11 +1,14 @@ from django.contrib.sites.models import Site from django.template import loader, RequestContext -from django.http import HttpResponse, Http404 +from django.http import HttpResponse from django.core.urlresolvers import reverse, NoReverseMatch from django.conf import settings from robots.models import Rule +CRAWL_DELAY = getattr(settings, 'ROBOTS_CRAWL_DELAY', False) +USE_SITEMAP = getattr(settings, 'ROBOTS_USE_SITEMAP', True) + def rules_list(request, template_name='robots/rule_list.html', mimetype='text/plain', status_code=200): """ @@ -13,7 +16,7 @@ def rules_list(request, template_name='robots/rule_list.html', status code (200 or 404), sitemap url (automatically) and crawl delay (if settings.ROBOTS_CRAWL_DELAY is given). """ - protocol = request.is_secure() and 'https' or 'http' + scheme = request.is_secure() and 'https' or 'http' current_site = Site.objects.get_current() try: sitemap_url = reverse('django.contrib.sitemaps.views.index') @@ -22,9 +25,8 @@ def rules_list(request, template_name='robots/rule_list.html', sitemap_url = reverse('django.contrib.sitemaps.views.sitemap') except NoReverseMatch: sitemap_url = None - use_sitemap = getattr(settings, 'ROBOTS_USE_SITEMAP', True) - if sitemap_url is not None and use_sitemap: - sitemap_url = "%s://%s%s" % (protocol, current_site.domain, sitemap_url) + if sitemap_url is not None and USE_SITEMAP: + sitemap_url = "%s://%s%s" % (scheme, current_site.domain, sitemap_url) rules = Rule.objects.filter(sites=current_site) if not rules.count(): status_code = 404 @@ -32,6 +34,6 @@ def rules_list(request, template_name='robots/rule_list.html', c = RequestContext(request, { 'rules': rules, 'sitemap_url': sitemap_url, - 'crawl_delay': getattr(settings, 'ROBOTS_CRAWL_DELAY', False) + 'crawl_delay': CRAWL_DELAY, }) return HttpResponse(t.render(c), status=status_code, mimetype=mimetype)