diff --git a/app/controllers/photos_controller.rb b/app/controllers/photos_controller.rb index 6777420074..33372b3f5a 100644 --- a/app/controllers/photos_controller.rb +++ b/app/controllers/photos_controller.rb @@ -20,6 +20,9 @@ def index @photos = @photos.includes(:owner) .order(created_at: :desc) .paginate(page: params[:page], per_page: Photo.per_page) + + raise ActiveRecord::RecordNotFound if @photos.out_of_bounds? + respond_with(@photos) end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 46c3e2d90a..672b2c7656 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -31,6 +31,20 @@ class Rack::Attack blocklist('block Semrush crawler') do |request| request.user_agent.to_s.downcase.include?('semrush') end + + # Honeypot: block IPs that request disallowed route /dont-crawl-me for 7 days (1 week) + blocklist('fail2ban/honeypot') do |req| + Fail2Ban.filter("honeypot-#{req.ip}", maxretry: 1, findtime: 1.day, bantime: 7.days) do + req.path == '/dont-crawl-me' || req.path == '/dont-crawl-me/' + end + end + + # Ban crawlers that request more than 500 pages in a day for 1 week (7 days) + blocklist('allow2ban/excessive_crawling') do |req| + Allow2Ban.filter("excessive-crawling-#{req.ip}", maxretry: 500, findtime: 1.day, bantime: 1.week) do + req.get? && !req.path.match?(%r{\.(css|js|png|jpg|jpeg|gif|ico|svg|woff2?|eot|ttf|otf)$}) + end + end ### Custom Response Headers ### diff --git a/config/robots.txt b/config/robots.txt index e69de29bb2..8105159d1a 100644 --- a/config/robots.txt +++ b/config/robots.txt @@ -0,0 +1,159 @@ +# robots.txt for based on the one for http://www.wikipedia.org/ and friends + +# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN +# and ignoring 429 ratelimit responses, claims to respect robots: +# http://mj12bot.com/ +User-agent: MJ12bot +Disallow: / + +# advertising-related bots: +User-agent: Mediapartners-Google* +Disallow: / + +# Wikipedia work bots: +User-agent: IsraBot +Disallow: + +User-agent: Orthogaffe +Disallow: + +# Crawlers that are kind enough to obey, but which we'd rather not have +# unless they're feeding search engines. +User-agent: UbiCrawler +Disallow: / + +User-agent: DOC +Disallow: / + +User-agent: Zao +Disallow: / + +# Some bots are known to be trouble, particularly those designed to copy +# entire sites. Please obey robots.txt. +User-agent: sitecheck.internetseer.com +Disallow: / + +User-agent: Zealbot +Disallow: / + +User-agent: MSIECrawler +Disallow: / + +User-agent: SiteSnagger +Disallow: / + +User-agent: WebStripper +Disallow: / + +User-agent: WebCopier +Disallow: / + +User-agent: Fetch +Disallow: / + +User-agent: Offline Explorer +Disallow: / + +User-agent: Teleport +Disallow: / + +User-agent: TeleportPro +Disallow: / + +User-agent: WebZIP +Disallow: / + +User-agent: linko +Disallow: / + +User-agent: HTTrack +Disallow: / + +User-agent: Microsoft.URL.Control +Disallow: / + +User-agent: Xenu +Disallow: / + +User-agent: larbin +Disallow: / + +User-agent: libwww +Disallow: / + +User-agent: ZyBORG +Disallow: / + +User-agent: Download Ninja +Disallow: / + +# Misbehaving: requests much too fast: +User-agent: fast +Disallow: / + +# +# Sorry, wget in its recursive mode is a frequent problem. +# Please read the man page and use it properly; there is a +# --wait option you can use to set the delay between hits, +# for instance. +# +User-agent: wget +Disallow: / + +# +# The 'grub' distributed client has been *very* poorly behaved. +# +User-agent: grub-client +Disallow: / + +# +# Doesn't follow robots.txt anyway, but... +# +User-agent: k2spider +Disallow: / + +# +# Hits many times per second, not acceptable +# http://www.nameprotect.com/botinfo.html +User-agent: NPBot +Disallow: / + +# A capture bot, downloads gazillions of pages with no public benefit +# http://www.webreaper.net/ +User-agent: WebReaper +Disallow: / + +# Semrush seem to crawl everything. +User-agent: SemrushBot +Disallow: / + +User-agent: SiteAuditBot +Disallow: / + +User-agent: SemrushBot-BA +Disallow: / + +User-agent: SemrushBot-SI +Disallow: / + +User-agent: SemrushBot-SWA +Disallow: / + +User-agent: SplitSignalBot +Disallow: / + +User-agent: SemrushBot-OCOB +Disallow: / + +# +# Friendly, low-speed bots are welcome viewing pages, but not +# dynamically-generated pages please. +# +# Another exception is for REST API documentation, located at +# /api/rest_v1/?doc. +# +User-agent: * +Disallow: /api/ +Disallow: /dont-crawl-me + +Sitemap: https://growstuff-prod.s3.us-west-2.amazonaws.com/sitemap.xml.gz diff --git a/config/routes.rb b/config/routes.rb index 361db53108..2ee16c6060 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,6 +4,7 @@ mount Rswag::Ui::Engine => '/api-docs' mount Rswag::Api::Engine => '/api-docs' get '/robots.txt' => 'robots#robots' + get '/dont-crawl-me' => proc { [403, { 'Content-Type' => 'text/plain' }, ['Forbidden']] } resources :garden_types resources :plant_parts diff --git a/public/robots.txt b/public/robots.txt index eb0c67cc83..4db87b3e8c 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -154,5 +154,6 @@ Disallow: / # User-agent: * Disallow: /api/ +Disallow: /dont-crawl-me Sitemap: https://growstuff-prod.s3.us-west-2.amazonaws.com/sitemap.xml.gz diff --git a/spec/controllers/photos_controller_spec.rb b/spec/controllers/photos_controller_spec.rb index 0dac5e7d90..6e6454892b 100644 --- a/spec/controllers/photos_controller_spec.rb +++ b/spec/controllers/photos_controller_spec.rb @@ -17,6 +17,11 @@ expect(assigns(:photos).count).to eq 1 expect(assigns(:photos).first.id).to eq photo.id end + + it 'returns 404 not found when page is out of bounds' do + get :index, params: { page: 105 } + expect(response).to have_http_status(:not_found) + end end describe '#index crop photos' do diff --git a/spec/requests/rack_attack_spec.rb b/spec/requests/rack_attack_spec.rb new file mode 100644 index 0000000000..8ed24aa918 --- /dev/null +++ b/spec/requests/rack_attack_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe 'Rack::Attack', type: :request do + include ActiveSupport::Testing::TimeHelpers + + before do + Rack::Attack.enabled = true + Rack::Attack.reset! + end + + after do + Rack::Attack.enabled = false + Rack::Attack.reset! + end + + describe 'honeypot route /dont-crawl-me' do + it 'bans an IP for 7 days when hitting /dont-crawl-me' do + get '/dont-crawl-me', headers: { 'REMOTE_ADDR' => '1.2.3.4' } + expect(response).to have_http_status(:forbidden) + + # Next request from same IP should be blocked + get '/community-gardens', headers: { 'REMOTE_ADDR' => '1.2.3.4' } + expect(response).to have_http_status(:forbidden) + + # Requests from a different IP should be allowed + get '/community-gardens', headers: { 'REMOTE_ADDR' => '5.6.7.8' } + expect(response).not_to have_http_status(:forbidden) + + # Fast forward 7 days plus 1 minute + travel 7.days + 1.minute do + get '/community-gardens', headers: { 'REMOTE_ADDR' => '1.2.3.4' } + expect(response).not_to have_http_status(:forbidden) + end + end + end + + describe 'excessive crawling (>500 page requests in a day)' do + it 'bans an IP for 1 week after 500 requests in a day' do + ip = '10.0.0.1' + + 500.times do + get '/community-gardens', headers: { 'REMOTE_ADDR' => ip } + expect(response).not_to have_http_status(:forbidden) + end + + # 501st request should be banned + get '/community-gardens', headers: { 'REMOTE_ADDR' => ip } + expect(response).to have_http_status(:forbidden) + + # Subsequent request should remain banned + get '/community-gardens', headers: { 'REMOTE_ADDR' => ip } + expect(response).to have_http_status(:forbidden) + + # Fast forward 1 week plus 1 minute + travel 7.days + 1.minute do + get '/community-gardens', headers: { 'REMOTE_ADDR' => ip } + expect(response).not_to have_http_status(:forbidden) + end + end + end +end