Improve structure of htaccess.j2

[ocean90]

Looking at the htaccess.j2 file and noticed a few issues:

• The iThemes Security block should be wrapped with a <IfModule mod_rewrite.c> condition

• Options -Indexes can be removed since it's now disabled by default, see 68b9e2f

• This block should be merged into one <IfModule mod_rewrite.c> section

• The WordPress rules should be extended to support a Multisite flag.

  Currently you have to add the Multisite rules to the site config which means they get added before all the other security checks. They should be the last one, just like the default WP rules.
  The rules for Multisite have a few variables which we need to define in the config. If one of them is set the provisioner would add the multisite site rules to the .htaccess file. Note that the multisite rules are a replacement of the default rules, so both versions shouldn't be added, which is currently the case.

  Pseudo code (which can be improved once we know if this is possible):

  # BEGIN WordPress
  <IfModule mod_rewrite.c>
  {% if item.value.multisite is defined %}
   RewriteEngine On
   RewriteBase /
   RewriteRule ^index\.php$ - [L]
   # add a trailing slash to /wp-admin
   RewriteRule ^{{ item.value.multisite.subdir_match }}wp-admin$ {{ item.value.multisite.subdir_replacement_01 }}wp-admin/ [R=301,L]
  
   RewriteCond %{REQUEST_FILENAME} -f [OR]
   RewriteCond %{REQUEST_FILENAME} -d
   RewriteRule ^ - [L]
   RewriteRule ^{{ item.value.multisite.subdir_match }}(wp-(content|admin|includes).*) /{{{ item.value.multisite.subdir_replacement_12 }}} [L]
   RewriteRule ^{{ item.value.multisite.subdir_match }}(.*\.php)$ /{{ item.value.multisite.subdir_replacement_12 }} [L]
   RewriteRule . index.php [L]
  {% else %}
   RewriteEngine On
   RewriteBase /
   RewriteRule ^index\.php$ - [L]
   RewriteCond %{REQUEST_FILENAME} !-f
   RewriteCond %{REQUEST_FILENAME} !-d
   RewriteRule . /index.php [L]
  {% endif %}
  </IfModule>
  # END WordPress

[ocean90]

An example of a generated .htaccess file:

#
# --- THIS FILE IS AUTOMATICALLY PROVISIONED THROUGH ANSIBLE ---
#

RewriteEngine on
# Check root separately because `%{REQUEST_FILENAME} -d` is true.
RewriteCond %{REQUEST_URI} ^/$
RewriteRule ^ /16/ [R=302,L]

RewriteCond %{REQUEST_URI} !^/16
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteRule ^(.*)$ /16/$1 [R=302,L]

# START MU Settings - Do not modify or remove this line
# add a trailing slash to /wp-admin
RewriteRule ^([_0-9a-zA-Z-]+/)?wp-admin$ $1wp-admin/ [R=301,L]

RewriteCond %{REQUEST_FILENAME} -f [OR]
RewriteCond %{REQUEST_FILENAME} -d
RewriteRule ^ - [L]
RewriteRule ^([_0-9a-zA-Z-]+/)?(wp-(content|admin|includes).*) wp/$2 [L]
RewriteRule ^([_0-9a-zA-Z-]+/)?(.*\.php)$ wp/$2 [L]
RewriteRule . index.php [L]
# END MU Settings - Do not modify or remove this line

# BEGIN iThemes Security - Do not modify or remove this line
# iThemes Security Config Details: 2
	# Enable HackRepair.com's blacklist feature - Security > Settings > Banned Users > Default Blacklist
	# Start HackRepair.com Blacklist
	RewriteEngine on
	# Start Abuse Agent Blocking
	RewriteCond %{HTTP_USER_AGENT} "^Mozilla.*Indy" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Mozilla.*NEWT" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^$" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Maxthon$" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^SeaMonkey$" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Acunetix" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^binlar" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^BlackWidow" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Bolt 0" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^BOT for JCE" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Bot mailto\:craftbot@yahoo\.com" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^casper" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^checkprivacy" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^ChinaClaw" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^clshttp" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^cmsworldmap" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^comodo" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Custo" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Default Browser 0" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^diavol" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^DIIbot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^DISCo" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^dotbot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Download Demon" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^eCatch" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^EirGrabber" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^EmailCollector" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^EmailSiphon" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^EmailWolf" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Express WebPictures" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^extract" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^ExtractorPro" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^EyeNetIE" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^feedfinder" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^FHscan" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^FlashGet" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^flicky" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^g00g1e" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^GetRight" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^GetWeb\!" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Go\!Zilla" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Go\-Ahead\-Got\-It" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^grab" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^GrabNet" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Grafula" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^harvest" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^HMView" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^ia_archiver" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Image Stripper" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Image Sucker" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^InterGET" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Internet Ninja" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^InternetSeer\.com" [NC,OR]
  # RewriteCond %{HTTP_USER_AGENT} "^jakarta" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Java" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^JetCar" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^JOC Web Spider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^kanagawa" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^kmccrew" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^larbin" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^LeechFTP" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^libwww" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Mass Downloader" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^microsoft\.url" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^MIDown tool" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^miner" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Mister PiX" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^MSFrontPage" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Navroad" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^NearSite" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Net Vampire" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^NetAnts" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^NetSpider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^NetZIP" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^nutch" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Octopus" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Offline Explorer" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Offline Navigator" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^PageGrabber" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Papa Foto" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^pavuk" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^pcBrowser" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^PeoplePal" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^planetwork" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^psbot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^purebot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^pycurl" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^RealDownload" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^ReGet" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Rippers 0" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^sitecheck\.internetseer\.com" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^SiteSnagger" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^skygrid" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^SmartDownload" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^sucker" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^SuperBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^SuperHTTP" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Surfbot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^tAkeOut" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Teleport Pro" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Toata dragostea mea pentru diavola" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^turnit" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^vikspider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^VoidEYE" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Web Image Collector" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Web Sucker" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebAuto" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebBandit" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebCopier" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebFetch" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebGo IS" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebLeacher" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebReaper" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebSauger" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Website eXtractor" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Website Quester" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebStripper" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebWhacker" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WebZIP" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Wget" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Widow" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WPScan" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WWW\-Mechanize" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^WWWOFFLE" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Xaldon WebSpider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^Zeus" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "^zmeu" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "360Spider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "AhrefsBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "CazoodleBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "discobot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "EasouSpider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "ecxi" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "GT\:\:WWW" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "heritrix" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "HTTP\:\:Lite" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "HTTrack" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "ia_archiver" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "id\-search" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "IDBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Indy Library" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "IRLbot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "ISC Systems iRc Search 2\.1" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "LinksCrawler" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "LinksManager\.com_bot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "linkwalker" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "lwp\-trivial" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "MFC_Tear_Sample" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Microsoft URL Control" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Missigua Locator" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "MJ12bot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "panscient\.com" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "PECL\:\:HTTP" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "PHPCrawl" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "PleaseCrawl" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "SBIder" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "SearchmetricsBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "SeznamBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Snoopy" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Steeler" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "URI\:\:Fetch" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "urllib" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Web Sucker" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "webalta" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "WebCollage" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "Wells Search II" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "WEP Search" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "XoviBot" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "YisouSpider" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "zermelo" [NC,OR]
	RewriteCond %{HTTP_USER_AGENT} "ZyBorg" [NC,OR]
	# End Abuse Agent Blocking
	# Start Abuse HTTP Referrer Blocking
	RewriteCond %{HTTP_REFERER} "^https?://(?:[^/]+\.)?semalt\.com" [NC,OR]
	RewriteCond %{HTTP_REFERER} "^https?://(?:[^/]+\.)?kambasoft\.com" [NC,OR]
	RewriteCond %{HTTP_REFERER} "^https?://(?:[^/]+\.)?savetubevideo\.com" [NC]
	# End Abuse HTTP Referrer Blocking
	RewriteRule ^.* - [F,L]
	# End HackRepair.com Blacklist, http://pastebin.com/u/hackrepair

	# Protect System Files - Security > Settings > System Tweaks > System Files
	<files .htaccess>
		<IfModule mod_authz_core.c>
			Require all denied
		</IfModule>
		<IfModule !mod_authz_core.c>
			Order allow,deny
			Deny from all
		</IfModule>
	</files>
	<files readme.html>
		<IfModule mod_authz_core.c>
			Require all denied
		</IfModule>
		<IfModule !mod_authz_core.c>
			Order allow,deny
			Deny from all
		</IfModule>
	</files>
	<files readme.txt>
		<IfModule mod_authz_core.c>
			Require all denied
		</IfModule>
		<IfModule !mod_authz_core.c>
			Order allow,deny
			Deny from all
		</IfModule>
	</files>
	<files install.php>
		<IfModule mod_authz_core.c>
			Require all denied
		</IfModule>
		<IfModule !mod_authz_core.c>
			Order allow,deny
			Deny from all
		</IfModule>
	</files>
	<files wp-config.php>
		<IfModule mod_authz_core.c>
			Require all denied
		</IfModule>
		<IfModule !mod_authz_core.c>
			Order allow,deny
			Deny from all
		</IfModule>
	</files>

	# Disable Directory Browsing - Security > Settings > System Tweaks > Directory Browsing
	Options -Indexes

	<IfModule mod_rewrite.c>
		RewriteEngine On

		# Protect System Files - Security > Settings > System Tweaks > System Files
		RewriteRule ^wp-admin/includes/ - [F]
		RewriteRule !^wp-includes/ - [S=3]
		RewriteCond %{SCRIPT_FILENAME} !^(.*)wp-includes/ms-files.php
		RewriteRule ^wp-includes/[^/]+\.php$ - [F]
		RewriteRule ^wp-includes/js/tinymce/langs/.+\.php - [F]
		RewriteRule ^wp-includes/theme-compat/ - [F]

		# Disable PHP in Uploads - Security > Settings > System Tweaks > Uploads
		RewriteRule ^app/uploads/.*\.(?:php[1-6]?|pht|phtml?)$ - [NC,F]

		# Filter Request Methods - Security > Settings > System Tweaks > Request Methods
		# RewriteCond %{REQUEST_METHOD} ^(TRACE|DELETE|TRACK) [NC]
		RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK) [NC]
		RewriteRule ^.* - [F]
	</IfModule>

	<IfModule mod_rewrite.c>
		RewriteEngine On

		# Reduce Comment Spam - Security > Settings > WordPress Tweaks > Comment Spam
		RewriteCond %{REQUEST_METHOD} POST
		RewriteCond %{REQUEST_URI} /wp-comments-post\.php$
		RewriteCond %{HTTP_USER_AGENT} ^$ [OR]
		RewriteRule ^.* - [F]
	</IfModule>

	# Enable the hide backend feature - Security > Settings > Hide Login Area > Hide Backend
	RewriteRule ^(/wp/)?wplogin/?$ /wp/wp-login.php [QSA,L]
	RewriteRule ^(/wp/)?wp-register-php/?$ /wplogin?action=register [QSA,L]
# END iThemes Security - Do not modify or remove this line

# BEGIN WordPress
<IfModule mod_rewrite.c>
  RewriteEngine On
  RewriteBase /
  RewriteRule ^index\.php$ - [L]
  RewriteCond %{REQUEST_FILENAME} !-f
  RewriteCond %{REQUEST_FILENAME} !-d
  RewriteRule . /index.php [L]
</IfModule>
# END WordPress