Harden Rack::Attack and pagination against aggressive web crawlers

app/controllers/photos_controller.rb

@@ -20,6 +20,9 @@ def index
     @photos = @photos.includes(:owner)
                      .order(created_at: :desc)
                      .paginate(page: params[:page], per_page: Photo.per_page)
+
+    raise ActiveRecord::RecordNotFound if @photos.out_of_bounds?
+
     respond_with(@photos)
   end
 

config/initializers/rack_attack.rb

@@ -31,6 +31,20 @@ class Rack::Attack
   blocklist('block Semrush crawler') do |request|
     request.user_agent.to_s.downcase.include?('semrush')
   end
+
+  # Honeypot: block IPs that request disallowed route /dont-crawl-me for 7 days (1 week)
+  blocklist('fail2ban/honeypot') do |req|
+    Fail2Ban.filter("honeypot-#{req.ip}", maxretry: 1, findtime: 1.day, bantime: 7.days) do
+      req.path == '/dont-crawl-me' || req.path == '/dont-crawl-me/'
+    end
+  end
+
+  # Ban crawlers that request more than 500 pages in a day for 1 week (7 days)
+  blocklist('allow2ban/excessive_crawling') do |req|
+    Allow2Ban.filter("excessive-crawling-#{req.ip}", maxretry: 500, findtime: 1.day, bantime: 1.week) do
+      req.get? && !req.path.match?(%r{\.(css|js|png|jpg|jpeg|gif|ico|svg|woff2?|eot|ttf|otf)$})
+    end
+  end
 
   ### Custom Response Headers ###
 

config/robots.txt

@@ -0,0 +1,159 @@
+# robots.txt for based on the one for http://www.wikipedia.org/ and friends
+
+# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN
+# and ignoring 429 ratelimit responses, claims to respect robots:
+# http://mj12bot.com/
+User-agent: MJ12bot
+Disallow: /
+
+# advertising-related bots:
+User-agent: Mediapartners-Google*
+Disallow: /
+
+# Wikipedia work bots:
+User-agent: IsraBot
+Disallow:
+
+User-agent: Orthogaffe
+Disallow:
+
+# Crawlers that are kind enough to obey, but which we'd rather not have
+# unless they're feeding search engines.
+User-agent: UbiCrawler
+Disallow: /
+
+User-agent: DOC
+Disallow: /
+
+User-agent: Zao
+Disallow: /
+
+# Some bots are known to be trouble, particularly those designed to copy
+# entire sites. Please obey robots.txt.
+User-agent: sitecheck.internetseer.com
+Disallow: /
+
+User-agent: Zealbot
+Disallow: /
+
+User-agent: MSIECrawler
+Disallow: /
+
+User-agent: SiteSnagger
+Disallow: /
+
+User-agent: WebStripper
+Disallow: /
+
+User-agent: WebCopier
+Disallow: /
+
+User-agent: Fetch
+Disallow: /
+
+User-agent: Offline Explorer
+Disallow: /
+
+User-agent: Teleport
+Disallow: /
+
+User-agent: TeleportPro
+Disallow: /
+
+User-agent: WebZIP
+Disallow: /
+
+User-agent: linko
+Disallow: /
+
+User-agent: HTTrack
+Disallow: /
+
+User-agent: Microsoft.URL.Control
+Disallow: /
+
+User-agent: Xenu
+Disallow: /
+
+User-agent: larbin
+Disallow: /
+
+User-agent: libwww
+Disallow: /
+
+User-agent: ZyBORG
+Disallow: /
+
+User-agent: Download Ninja
+Disallow: /
+
+# Misbehaving: requests much too fast:
+User-agent: fast
+Disallow: /
+
+#
+# Sorry, wget in its recursive mode is a frequent problem.
+# Please read the man page and use it properly; there is a
+# --wait option you can use to set the delay between hits,
+# for instance.
+#
+User-agent: wget
+Disallow: /
+
+#
+# The 'grub' distributed client has been *very* poorly behaved.
+#
+User-agent: grub-client
+Disallow: /
+
+#
+# Doesn't follow robots.txt anyway, but...
+#
+User-agent: k2spider
+Disallow: /
+
+#
+# Hits many times per second, not acceptable
+# http://www.nameprotect.com/botinfo.html
+User-agent: NPBot
+Disallow: /
+
+# A capture bot, downloads gazillions of pages with no public benefit
+# http://www.webreaper.net/
+User-agent: WebReaper
+Disallow: /
+
+# Semrush seem to crawl everything.
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: SiteAuditBot
+Disallow: /
+
+User-agent: SemrushBot-BA
+Disallow: /
+
+User-agent: SemrushBot-SI
+Disallow: /
+
+User-agent: SemrushBot-SWA
+Disallow: /
+
+User-agent: SplitSignalBot
+Disallow: /
+
+User-agent: SemrushBot-OCOB
+Disallow: /
+
+#
+# Friendly, low-speed bots are welcome viewing pages, but not
+# dynamically-generated pages please.
+#
+# Another exception is for REST API documentation, located at
+# /api/rest_v1/?doc.
+#
+User-agent: *
+Disallow: /api/
+Disallow: /dont-crawl-me
+
+Sitemap: https://growstuff-prod.s3.us-west-2.amazonaws.com/sitemap.xml.gz

config/routes.rb

@@ -4,6 +4,7 @@
   mount Rswag::Ui::Engine => '/api-docs'
   mount Rswag::Api::Engine => '/api-docs'
   get '/robots.txt' => 'robots#robots'
+  get '/dont-crawl-me' => proc { [403, { 'Content-Type' => 'text/plain' }, ['Forbidden']] }
 
   resources :garden_types
   resources :plant_parts

public/robots.txt

@@ -154,5 +154,6 @@ Disallow: /
 #
 User-agent: *
 Disallow: /api/
+Disallow: /dont-crawl-me
 
 Sitemap: https://growstuff-prod.s3.us-west-2.amazonaws.com/sitemap.xml.gz

spec/controllers/photos_controller_spec.rb

@@ -17,6 +17,11 @@
         expect(assigns(:photos).count).to eq 1
         expect(assigns(:photos).first.id).to eq photo.id
       end
+
+      it 'returns 404 not found when page is out of bounds' do
+        get :index, params: { page: 105 }
+        expect(response).to have_http_status(:not_found)
+      end
     end
 
     describe '#index crop photos' do

spec/requests/rack_attack_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe 'Rack::Attack', type: :request do
+  include ActiveSupport::Testing::TimeHelpers
+
+  before do
+    Rack::Attack.enabled = true
+    Rack::Attack.reset!
+  end
+
+  after do
+    Rack::Attack.enabled = false
+    Rack::Attack.reset!
+  end
+
+  describe 'honeypot route /dont-crawl-me' do
+    it 'bans an IP for 7 days when hitting /dont-crawl-me' do
+      get '/dont-crawl-me', headers: { 'REMOTE_ADDR' => '1.2.3.4' }
+      expect(response).to have_http_status(:forbidden)
+
+      # Next request from same IP should be blocked
+      get '/community-gardens', headers: { 'REMOTE_ADDR' => '1.2.3.4' }
+      expect(response).to have_http_status(:forbidden)
+
+      # Requests from a different IP should be allowed
+      get '/community-gardens', headers: { 'REMOTE_ADDR' => '5.6.7.8' }
+      expect(response).not_to have_http_status(:forbidden)
+
+      # Fast forward 7 days plus 1 minute
+      travel 7.days + 1.minute do
+        get '/community-gardens', headers: { 'REMOTE_ADDR' => '1.2.3.4' }
+        expect(response).not_to have_http_status(:forbidden)
+      end
+    end
+  end
+
+  describe 'excessive crawling (>500 page requests in a day)' do
+    it 'bans an IP for 1 week after 500 requests in a day' do
+      ip = '10.0.0.1'
+
+      500.times do
+        get '/community-gardens', headers: { 'REMOTE_ADDR' => ip }
+        expect(response).not_to have_http_status(:forbidden)
+      end
+
+      # 501st request should be banned
+      get '/community-gardens', headers: { 'REMOTE_ADDR' => ip }
+      expect(response).to have_http_status(:forbidden)
+
+      # Subsequent request should remain banned
+      get '/community-gardens', headers: { 'REMOTE_ADDR' => ip }
+      expect(response).to have_http_status(:forbidden)
+
+      # Fast forward 1 week plus 1 minute
+      travel 7.days + 1.minute do
+        get '/community-gardens', headers: { 'REMOTE_ADDR' => ip }
+        expect(response).not_to have_http_status(:forbidden)
+      end
+    end
+  end
+end