diff --git a/app/controllers/edits_controller.rb b/app/controllers/edits_controller.rb index 3b2388c7f..2087d5f8f 100644 --- a/app/controllers/edits_controller.rb +++ b/app/controllers/edits_controller.rb @@ -3,6 +3,7 @@ class EditsController < SettingsController helper ProjectsHelper + before_action { response.set_header('X-Robots-Tag', 'noindex, nofollow') } before_action :session_required, :redirect_unverified_account, only: [:update] before_action :find_parent, only: %i[index show update refresh] before_action :show_permissions_alert, only: :index, unless: :parent_is_account_or_license? diff --git a/app/controllers/permissions_controller.rb b/app/controllers/permissions_controller.rb index 7ffd8dfea..cd73cdf90 100644 --- a/app/controllers/permissions_controller.rb +++ b/app/controllers/permissions_controller.rb @@ -4,6 +4,7 @@ class PermissionsController < ApplicationController helper ProjectsHelper include ProjectOrOrganizationSetter + before_action { response.set_header('X-Robots-Tag', 'noindex, nofollow') } before_action :session_required, :redirect_unverified_account, only: :update before_action :find_model before_action :require_manage_authorization, only: :update diff --git a/app/controllers/session_projects_controller.rb b/app/controllers/session_projects_controller.rb index 3401a2188..458880a27 100644 --- a/app/controllers/session_projects_controller.rb +++ b/app/controllers/session_projects_controller.rb @@ -2,6 +2,7 @@ class SessionProjectsController < ApplicationController skip_before_action :store_location + before_action { response.set_header('X-Robots-Tag', 'noindex, nofollow') } before_action :prevent_bot_access before_action :set_project, only: :create before_action :set_session_projects diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index c1d74211c..6b0eac789 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -3,6 +3,7 @@ class SessionsController < Clearance::SessionsController helper StatsdHelper + before_action { response.set_header('X-Robots-Tag', 'noindex, nofollow') } before_action :account_must_exist, only: :create before_action :captcha_verify, only: :create, if: :failed_login_thrice? before_action :reset_auth_fail_count, only: :create, if: :auth_failure_timeout? diff --git a/public/robots.txt b/public/robots.txt index 4d73a02c9..cb9323af2 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,17 +1,10 @@ User-agent: * -Disallow: /accounts/*/near Disallow: /images/ Disallow: /images -Disallow: /edits/ -Disallow: /edits -Disallow: /*/edits -Disallow: /*/edits/* Disallow: /*/commits/* Disallow: /*/contributors/*.png Disallow: /*/positions/*.png -Disallow: /p/*/permissions Disallow: /p/*/stacks/near -Disallow: /sessions/new Disallow: /p/*/contributors/* Disallow: /p/*/contributors/*/commits Disallow: /p? @@ -19,7 +12,6 @@ Disallow: /people? Disallow: /people/rankings? Disallow: /search Disallow: /p/*/factoids/* -Disallow: /session_projects Crawl-Delay: 10 User-agent: discobot @@ -43,4 +35,4 @@ Disallow: / User-agent: AhrefsBot Crawl-Delay: 10 # No faster than ever 10 seconds -Sitemap: http://www.openhub.net/sitemap_index.xml +Sitemap: https://www.openhub.net/sitemap_index.xml