diff --git a/LICENSE.txt b/LICENSE.txt index 0e9acad..2412afc 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2008-2011, Jannis Leidel +Copyright (c) 2008-2012, Jannis Leidel All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.rst b/README.rst index 97ece94..8e9d3f6 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ This is a basic Django application to manage robots.txt files following the For installation instructions, see the file "INSTALL.txt" in this directory; for instructions on how to use this application, and on what it provides, see the file "overview.txt" in the "docs/" -directory. +directory or on ReadTheDocs: http://django-robots.readthedocs.org/ .. _robots exclusion protocol: http://en.wikipedia.org/wiki/Robots_exclusion_standard .. _Django: http://www.djangoproject.com/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..9b5950f --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-robots.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-robots.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/django-robots" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-robots" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..5755ee5 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +# +# django-robots documentation build configuration file, created by +# sphinx-quickstart on Wed Nov 21 11:54:26 2012. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.txt' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'django-robots' +copyright = u'2012, Jannis Leidel' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.9' +# The full version, including alpha/beta/rc tags. +release = '0.9' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'django-robotsdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'django-robots.tex', u'django-robots Documentation', + u'Jannis Leidel', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'django-robots', u'django-robots Documentation', + [u'Jannis Leidel'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'django-robots', u'django-robots Documentation', + u'Jannis Leidel', 'django-robots', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' diff --git a/docs/index.txt b/docs/index.txt index 6d7a934..d6a250a 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -1,16 +1,153 @@ -============= -django-robots -============= +======================================= +Robots exclusion application for Django +======================================= This is a basic Django application to manage robots.txt files following the `robots exclusion protocol`_, complementing the Django_ `Sitemap contrib app`_. -.. _robots exclusion protocol: http://www.robotstxt.org/robotstxt.html +The robots exclusion application consists of two database models which are +tied together with a m2m relationship: + +* Rules_ +* URLs_ + .. _Django: http://www.djangoproject.com/ + +Installation +============ + +Get the source from the application site at:: + + http://github.com/jezdez/django-robots/ + +To install the sitemap app, follow these steps: + +1. Follow the instructions in the INSTALL.txt file +2. Add ``'robots'`` to your INSTALLED_APPS_ setting. +3. Make sure ``'django.template.loaders.app_directories.load_template_source'`` + is in your TEMPLATE_LOADERS_ setting. It's in there by default, so + you'll only need to change this if you've changed that setting. +4. Make sure you've installed the `sites framework`_. + +.. _INSTALLED_APPS: http://docs.djangoproject.com/en/dev/ref/settings/#installed-apps +.. _TEMPLATE_LOADERS: http://docs.djangoproject.com/en/dev/ref/settings/#template-loaders +.. _sites framework: http://docs.djangoproject.com/en/dev/ref/contrib/sites/ + +Sitemaps +-------- + +By default a ``Sitemap`` statement is automatically added to the resulting +robots.txt by reverse matching the URL of the installed `Sitemap contrib app`_. +This is especially useful if you allow every robot to access your whole site, +since it then gets URLs explicitly instead of searching every link. + +To change the default behaviour to omit the inclusion of a sitemap link, +change the ``ROBOTS_USE_SITEMAP`` setting in your Django settings file to:: + + ROBOTS_USE_SITEMAP = False + +In case you want to use specific sitemap URLs instead of the one that is +automatically discovered, change the ``ROBOTS_SITEMAP_URLS`` setting to:: + + ROBOTS_SITEMAP_URLS = [ + http://www.example.com/sitemap.xml, + ] + .. _Sitemap contrib app: http://docs.djangoproject.com/en/dev/ref/contrib/sitemaps/ -Contents: +Initialization +============== + +To activate robots.txt generation on your Django site, add this line to your +URLconf_:: + + (r'^robots\.txt$', include('robots.urls')), + +This tells Django to build a robots.txt when a robot accesses ``/robots.txt``. +Then, please sync your database to create the necessary tables and create +``Rule`` objects in the admin interface or via the shell. + +.. _URLconf: http://docs.djangoproject.com/en/dev/topics/http/urls/ +.. _sync your database: http://docs.djangoproject.com/en/dev/ref/django-admin/#syncdb + +Rules +===== + +``Rule`` - defines an abstract rule which is used to respond to crawling web +robots, using the `robots exclusion protocol`_, a.k.a. robots.txt. + +You can link multiple URL pattern to allows or disallows the robot identified +by its user agent to access the given URLs. + +The crawl delay field is supported by some search engines and defines the +delay between successive crawler accesses in seconds. If the crawler rate is a +problem for your server, you can set the delay up to 5 or 10 or a comfortable +value for your server, but it's suggested to start with small values (0.5-1), +and increase as needed to an acceptable value for your server. Larger delay +values add more delay between successive crawl accesses and decrease the +maximum crawl rate to your web server. + +The `sites framework`_ is used to enable multiple robots.txt per Django instance. +If no rule exists it automatically allows every web robot access to every URL. + +Please have a look at the `database of web robots`_ for a full list of +existing web robots user agent strings. + +.. _robots exclusion protocol: http://en.wikipedia.org/wiki/Robots_exclusion_standard +.. _'sites' framework: http://www.djangoproject.com/documentation/sites/ +.. _database of web robots: http://www.robotstxt.org/db.html + +URLs +==== + +``Url`` - defines a case-sensitive and exact URL pattern which is used to +allow or disallow the access for web robots. Case-sensitive. + +A missing trailing slash does also match files which start with the name of +the given pattern, e.g., ``'/admin'`` matches ``/admin.html`` too. + +Some major search engines allow an asterisk (``*``) as a wildcard to match any +sequence of characters and a dollar sign (``$``) to match the end of the URL, +e.g., ``'/*.jpg$'`` can be used to match all jpeg files. + +Caching +======= + +You can optionally cache the generation of the ``robots.txt``. Add or change +the ``ROBOTS_CACHE_TIMEOUT`` setting with a value in seconds in your Django +settings file:: + + ROBOTS_CACHE_TIMEOUT = 60*60*24 + +This tells Django to cache the ``robots.txt`` for 24 hours (86400 seconds). +The default value is ``None`` (no caching). + +Changelog +========= + +0.9 (11/21/2012) +---------------- + +- Deprecated ``ROBOTS_SITEMAP_URL`` setting. Use ``ROBOTS_SITEMAP_URLS`` + instead. + +- Refactored ``rule_list`` view to be class based. django-robots now + requires Django >= 1.3. + +- Stop returning 404 pages if there are no Rules setup on the site. Instead + dissallow access for all robots. + +- Added an initial South migration. If you're using South you have to "fake" + the initial database migration:: + + python manage.py migrate --fake robots 0001 + +- Added initial Sphinx docs. + +Support +======= -.. toctree:: +Please leave your `questions and problems`_ on the `designated Github site`_. - overview.txt +.. _designated Github site: https://github.com/jezdez/django-robots +.. _questions and problems: https://github.com/jezdez/django-robots/issues diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..a47ad5d --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,190 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-robots.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-robots.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/docs/overview.txt b/docs/overview.txt deleted file mode 100644 index f4f056e..0000000 --- a/docs/overview.txt +++ /dev/null @@ -1,131 +0,0 @@ -======================================= -Robots exclusion application for Django -======================================= - -This is a basic Django application to manage robots.txt files following the -`robots exclusion protocol`_, complementing the Django_ `Sitemap contrib app`_. - -The robots exclusion application consists of two database models which are -tied together with a m2m relationship: - -* Rules_ -* URLs_ - -.. _Django: http://www.djangoproject.com/ - -Installation -============ - -Get the source from the application site at:: - - http://github.com/jezdez/django-robots/ - -To install the sitemap app, follow these steps: - -1. Follow the instructions in the INSTALL.txt file -2. Add ``'robots'`` to your INSTALLED_APPS_ setting. -3. Make sure ``'django.template.loaders.app_directories.load_template_source'`` - is in your TEMPLATE_LOADERS_ setting. It's in there by default, so - you'll only need to change this if you've changed that setting. -4. Make sure you've installed the `sites framework`_. - -.. _INSTALLED_APPS: http://docs.djangoproject.com/en/dev/ref/settings/#installed-apps -.. _TEMPLATE_LOADERS: http://docs.djangoproject.com/en/dev/ref/settings/#template-loaders -.. _sites framework: http://docs.djangoproject.com/en/dev/ref/contrib/sites/ - -Sitemaps --------- - -By default a ``Sitemap`` statement is automatically added to the resulting -robots.txt by reverse matching the URL of the installed `Sitemap contrib app`_. -This is especially useful if you allow every robot to access your whole site, -since it then gets URLs explicitly instead of searching every link. - -To change the default behaviour to omit the inclusion of a sitemap link, -change the ``ROBOTS_USE_SITEMAP`` setting in your Django settings file to:: - - ROBOTS_USE_SITEMAP = False - -In case you want to use specific sitemap URLs instead of the one that is -automatically discovered, change the ``ROBOTS_SITEMAP_URLS`` setting to:: - - ROBOTS_SITEMAP_URLS = [ - http://www.example.com/sitemap.xml, - ] - -.. _Sitemap contrib app: http://docs.djangoproject.com/en/dev/ref/contrib/sitemaps/ - -Initialization -============== - -To activate robots.txt generation on your Django site, add this line to your -URLconf_:: - - (r'^robots\.txt$', include('robots.urls')), - -This tells Django to build a robots.txt when a robot accesses ``/robots.txt``. -Then, please sync your database to create the necessary tables and create -``Rule`` objects in the admin interface or via the shell. - -.. _URLconf: http://docs.djangoproject.com/en/dev/topics/http/urls/ -.. _sync your database: http://docs.djangoproject.com/en/dev/ref/django-admin/#syncdb - -Rules -===== - -``Rule`` - defines an abstract rule which is used to respond to crawling web -robots, using the `robots exclusion protocol`_, a.k.a. robots.txt. - -You can link multiple URL pattern to allows or disallows the robot identified -by its user agent to access the given URLs. - -The crawl delay field is supported by some search engines and defines the -delay between successive crawler accesses in seconds. If the crawler rate is a -problem for your server, you can set the delay up to 5 or 10 or a comfortable -value for your server, but it's suggested to start with small values (0.5-1), -and increase as needed to an acceptable value for your server. Larger delay -values add more delay between successive crawl accesses and decrease the -maximum crawl rate to your web server. - -The `sites framework`_ is used to enable multiple robots.txt per Django instance. -If no rule exists it automatically allows every web robot access to every URL. - -Please have a look at the `database of web robots`_ for a full list of -existing web robots user agent strings. - -.. _robots exclusion protocol: http://en.wikipedia.org/wiki/Robots_exclusion_standard -.. _'sites' framework: http://www.djangoproject.com/documentation/sites/ -.. _database of web robots: http://www.robotstxt.org/db.html - -URLs -==== - -``Url`` - defines a case-sensitive and exact URL pattern which is used to -allow or disallow the access for web robots. Case-sensitive. - -A missing trailing slash does also match files which start with the name of -the given pattern, e.g., ``'/admin'`` matches ``/admin.html`` too. - -Some major search engines allow an asterisk (``*``) as a wildcard to match any -sequence of characters and a dollar sign (``$``) to match the end of the URL, -e.g., ``'/*.jpg$'`` can be used to match all jpeg files. - -Caching -======= - -You can optionally cache the generation of the ``robots.txt``. Add or change -the ``ROBOTS_CACHE_TIMEOUT`` setting with a value in seconds in your Django -settings file:: - - ROBOTS_CACHE_TIMEOUT = 60*60*24 - -This tells Django to cache the ``robots.txt`` for 24 hours (86400 seconds). -The default value is ``None`` (no caching). - -Support -======= - -Please leave your `questions and problems`_ on the `designated Github site`_. - -.. _designated Github site: https://github.com/jezdez/django-robots -.. _questions and problems: https://github.com/jezdez/django-robots/issues