Improve structure of htaccess.j2

[ocean90]

Looking at the htaccess.j2 file and noticed a few issues:

• The iThemes Security block should be wrapped with a <IfModule mod_rewrite.c> condition
• Options -Indexes can be removed since it's now disabled by default, see 68b9e2ff1041bd9d391cb15d97d3e4ef08ae41b0
• This block should be merged into one <IfModule mod_rewrite.c> section

• The WordPress rules should be extended to support a Multisite flag.

  Currently you have to add the Multisite rules to the site config which means they get added before all the other security checks. They should be the last one, just like the default WP rules. The rules for Multisite have a few variables which we need to define in the config. If one of them is set the provisioner would add the multisite site rules to the .htaccess file. Note that the multisite rules are a replacement of the default rules, so both versions shouldn't be added, which is currently the case.

  Pseudo code (which can be improved once we know if this is possible):

  # BEGIN WordPress
  <IfModule mod_rewrite.c>
  {% if item.value.multisite is defined %}
  RewriteEngine On
  RewriteBase /
  RewriteRule ^index\.php$ - [L]
  # add a trailing slash to /wp-admin
  RewriteRule ^{{ item.value.multisite.subdir_match }}wp-admin$ {{ item.value.multisite.subdir_replacement_01 }}wp-admin/ [R=301,L]
  
  RewriteCond %{REQUEST_FILENAME} -f [OR]
  RewriteCond %{REQUEST_FILENAME} -d
  RewriteRule ^ - [L]
  RewriteRule ^{{ item.value.multisite.subdir_match }}(wp-(content|admin|includes).*) /{{{ item.value.multisite.subdir_replacement_12 }}} [L]
  RewriteRule ^{{ item.value.multisite.subdir_match }}(.*\.php)$ /{{ item.value.multisite.subdir_replacement_12 }} [L]
  RewriteRule . index.php [L]
  {% else %}
  RewriteEngine On
  RewriteBase /
  RewriteRule ^index\.php$ - [L]
  RewriteCond %{REQUEST_FILENAME} !-f
  RewriteCond %{REQUEST_FILENAME} !-d
  RewriteRule . /index.php [L]
  {% endif %}
  </IfModule>
  # END WordPress

[ocean90]

An example of a generated .htaccess file:

#
# --- THIS FILE IS AUTOMATICALLY PROVISIONED THROUGH ANSIBLE ---
#

RewriteEngine on
# Check root separately because `%{REQUEST_FILENAME} -d` is true.
RewriteCond %{REQUEST_URI} ^/$
RewriteRule ^ /16/ [R=302,L]

RewriteCond %{REQUEST_URI} !^/16
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME} !-d
RewriteRule ^(.*)$ /16/$1 [R=302,L]

# START MU Settings - Do not modify or remove this line
# add a trailing slash to /wp-admin
RewriteRule ^([_0-9a-zA-Z-]+/)?wp-admin$ $1wp-admin/ [R=301,L]

RewriteCond %{REQUEST_FILENAME} -f [OR]
RewriteCond %{REQUEST_FILENAME} -d
RewriteRule ^ - [L]
RewriteRule ^([_0-9a-zA-Z-]+/)?(wp-(content|admin|includes).*) wp/$2 [L]
RewriteRule ^([_0-9a-zA-Z-]+/)?(.*\.php)$ wp/$2 [L]
RewriteRule . index.php [L]
# END MU Settings - Do not modify or remove this line

# BEGIN iThemes Security - Do not modify or remove this line
# iThemes Security Config Details: 2
    # Enable HackRepair.com's blacklist feature - Security > Settings > Banned Users > Default Blacklist
    # Start HackRepair.com Blacklist
    RewriteEngine on
    # Start Abuse Agent Blocking
    RewriteCond %{HTTP_USER_AGENT} "^Mozilla.*Indy" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Mozilla.*NEWT" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^$" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Maxthon$" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^SeaMonkey$" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Acunetix" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^binlar" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^BlackWidow" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Bolt 0" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^BOT for JCE" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Bot mailto\:craftbot@yahoo\.com" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^casper" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^checkprivacy" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^ChinaClaw" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^clshttp" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^cmsworldmap" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^comodo" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Custo" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Default Browser 0" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^diavol" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^DIIbot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^DISCo" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^dotbot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Download Demon" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^eCatch" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^EirGrabber" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^EmailCollector" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^EmailSiphon" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^EmailWolf" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Express WebPictures" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^extract" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^ExtractorPro" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^EyeNetIE" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^feedfinder" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^FHscan" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^FlashGet" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^flicky" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^g00g1e" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^GetRight" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^GetWeb\!" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Go\!Zilla" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Go\-Ahead\-Got\-It" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^grab" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^GrabNet" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Grafula" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^harvest" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^HMView" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^ia_archiver" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Image Stripper" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Image Sucker" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^InterGET" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Internet Ninja" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^InternetSeer\.com" [NC,OR]
  # RewriteCond %{HTTP_USER_AGENT} "^jakarta" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Java" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^JetCar" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^JOC Web Spider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^kanagawa" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^kmccrew" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^larbin" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^LeechFTP" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^libwww" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Mass Downloader" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^microsoft\.url" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^MIDown tool" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^miner" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Mister PiX" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^MSFrontPage" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Navroad" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^NearSite" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Net Vampire" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^NetAnts" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^NetSpider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^NetZIP" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^nutch" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Octopus" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Offline Explorer" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Offline Navigator" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^PageGrabber" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Papa Foto" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^pavuk" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^pcBrowser" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^PeoplePal" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^planetwork" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^psbot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^purebot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^pycurl" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^RealDownload" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^ReGet" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Rippers 0" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^sitecheck\.internetseer\.com" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^SiteSnagger" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^skygrid" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^SmartDownload" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^sucker" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^SuperBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^SuperHTTP" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Surfbot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^tAkeOut" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Teleport Pro" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Toata dragostea mea pentru diavola" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^turnit" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^vikspider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^VoidEYE" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Web Image Collector" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Web Sucker" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebAuto" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebBandit" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebCopier" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebFetch" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebGo IS" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebLeacher" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebReaper" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebSauger" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Website eXtractor" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Website Quester" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebStripper" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebWhacker" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WebZIP" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Wget" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Widow" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WPScan" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WWW\-Mechanize" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^WWWOFFLE" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Xaldon WebSpider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^Zeus" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "^zmeu" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "360Spider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "AhrefsBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "CazoodleBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "discobot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "EasouSpider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "ecxi" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "GT\:\:WWW" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "heritrix" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "HTTP\:\:Lite" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "HTTrack" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "ia_archiver" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "id\-search" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "IDBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Indy Library" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "IRLbot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "ISC Systems iRc Search 2\.1" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "LinksCrawler" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "LinksManager\.com_bot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "linkwalker" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "lwp\-trivial" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "MFC_Tear_Sample" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Microsoft URL Control" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Missigua Locator" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "MJ12bot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "panscient\.com" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "PECL\:\:HTTP" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "PHPCrawl" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "PleaseCrawl" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "SBIder" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "SearchmetricsBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "SeznamBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Snoopy" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Steeler" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "URI\:\:Fetch" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "urllib" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Web Sucker" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "webalta" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "WebCollage" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "Wells Search II" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "WEP Search" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "XoviBot" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "YisouSpider" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "zermelo" [NC,OR]
    RewriteCond %{HTTP_USER_AGENT} "ZyBorg" [NC,OR]
    # End Abuse Agent Blocking
    # Start Abuse HTTP Referrer Blocking
    RewriteCond %{HTTP_REFERER} "^https?://(?:[^/]+\.)?semalt\.com" [NC,OR]
    RewriteCond %{HTTP_REFERER} "^https?://(?:[^/]+\.)?kambasoft\.com" [NC,OR]
    RewriteCond %{HTTP_REFERER} "^https?://(?:[^/]+\.)?savetubevideo\.com" [NC]
    # End Abuse HTTP Referrer Blocking
    RewriteRule ^.* - [F,L]
    # End HackRepair.com Blacklist, http://pastebin.com/u/hackrepair

    # Protect System Files - Security > Settings > System Tweaks > System Files
    <files .htaccess>
        <IfModule mod_authz_core.c>
            Require all denied
        </IfModule>
        <IfModule !mod_authz_core.c>
            Order allow,deny
            Deny from all
        </IfModule>
    </files>
    <files readme.html>
        <IfModule mod_authz_core.c>
            Require all denied
        </IfModule>
        <IfModule !mod_authz_core.c>
            Order allow,deny
            Deny from all
        </IfModule>
    </files>
    <files readme.txt>
        <IfModule mod_authz_core.c>
            Require all denied
        </IfModule>
        <IfModule !mod_authz_core.c>
            Order allow,deny
            Deny from all
        </IfModule>
    </files>
    <files install.php>
        <IfModule mod_authz_core.c>
            Require all denied
        </IfModule>
        <IfModule !mod_authz_core.c>
            Order allow,deny
            Deny from all
        </IfModule>
    </files>
    <files wp-config.php>
        <IfModule mod_authz_core.c>
            Require all denied
        </IfModule>
        <IfModule !mod_authz_core.c>
            Order allow,deny
            Deny from all
        </IfModule>
    </files>

    # Disable Directory Browsing - Security > Settings > System Tweaks > Directory Browsing
    Options -Indexes

    <IfModule mod_rewrite.c>
        RewriteEngine On

        # Protect System Files - Security > Settings > System Tweaks > System Files
        RewriteRule ^wp-admin/includes/ - [F]
        RewriteRule !^wp-includes/ - [S=3]
        RewriteCond %{SCRIPT_FILENAME} !^(.*)wp-includes/ms-files.php
        RewriteRule ^wp-includes/[^/]+\.php$ - [F]
        RewriteRule ^wp-includes/js/tinymce/langs/.+\.php - [F]
        RewriteRule ^wp-includes/theme-compat/ - [F]

        # Disable PHP in Uploads - Security > Settings > System Tweaks > Uploads
        RewriteRule ^app/uploads/.*\.(?:php[1-6]?|pht|phtml?)$ - [NC,F]

        # Filter Request Methods - Security > Settings > System Tweaks > Request Methods
        # RewriteCond %{REQUEST_METHOD} ^(TRACE|DELETE|TRACK) [NC]
        RewriteCond %{REQUEST_METHOD} ^(TRACE|TRACK) [NC]
        RewriteRule ^.* - [F]
    </IfModule>

    <IfModule mod_rewrite.c>
        RewriteEngine On

        # Reduce Comment Spam - Security > Settings > WordPress Tweaks > Comment Spam
        RewriteCond %{REQUEST_METHOD} POST
        RewriteCond %{REQUEST_URI} /wp-comments-post\.php$
        RewriteCond %{HTTP_USER_AGENT} ^$ [OR]
        RewriteRule ^.* - [F]
    </IfModule>

    # Enable the hide backend feature - Security > Settings > Hide Login Area > Hide Backend
    RewriteRule ^(/wp/)?wplogin/?$ /wp/wp-login.php [QSA,L]
    RewriteRule ^(/wp/)?wp-register-php/?$ /wplogin?action=register [QSA,L]
# END iThemes Security - Do not modify or remove this line

# BEGIN WordPress
<IfModule mod_rewrite.c>
  RewriteEngine On
  RewriteBase /
  RewriteRule ^index\.php$ - [L]
  RewriteCond %{REQUEST_FILENAME} !-f
  RewriteCond %{REQUEST_FILENAME} !-d
  RewriteRule . /index.php [L]
</IfModule>
# END WordPress