diff --git a/api/management/commands/notify_throttled_clients.py b/api/management/commands/notify_throttled_clients.py index 78e3cd61..cf1c40ff 100644 --- a/api/management/commands/notify_throttled_clients.py +++ b/api/management/commands/notify_throttled_clients.py @@ -125,7 +125,7 @@ def handle(self, *args, **options) -> None: prior_notices = ThrottleNotice.objects.filter(user=user).count() will_enforce = enforce and prior_notices >= options["enforce_after"] - candidates.append((user, offender, will_enforce)) + candidates.append((user, offender, will_enforce, prior_notices)) self._print_summary(candidates, send) @@ -133,7 +133,7 @@ def handle(self, *args, **options) -> None: send_pushover(self._pushover_message(candidates, send)) if send: - for user, offender, will_enforce in candidates: + for user, offender, will_enforce, _prior_notices in candidates: if will_enforce: self._enforce_and_notify(user, offender) else: @@ -144,16 +144,21 @@ def _print_summary(self, candidates, send: bool) -> None: self.stdout.write("No candidates found.") return - for user, offender, will_enforce in candidates: + for user, offender, will_enforce, prior_notices in candidates: if will_enforce: action = self._predict_enforcement_action(user) verb = f"Enforcing ({action})" if send else f"Would enforce ({action})" else: verb = "Emailing" if send else "Would email" + prior_notices_msg = ( + f"{prior_notices} prior notice{'' if prior_notices == 1 else 's'}" + if prior_notices + else "no prior notices" + ) self.stdout.write( f"{verb} {user.username} <{user.email}>: throttled on " f"{offender.days_throttled} day(s), {offender.total_count} total, " - f"worst day {offender.worst_day_count}" + f"worst day {offender.worst_day_count}, {prior_notices_msg}" ) if not send: self.stdout.write( @@ -169,7 +174,7 @@ def _predict_enforcement_action(user: CustomUser) -> str: @staticmethod def _pushover_message(candidates, send: bool) -> str: verb = "Emailed" if send else "Found" - names = ", ".join(user.username for user, _, _ in candidates) + names = ", ".join(user.username for user, _, _, _ in candidates) return f"{verb} {len(candidates)} throttled API client(s) on Metron: {names}" @staticmethod diff --git a/api/templates/api/throttle_enforcement_email.html b/api/templates/api/throttle_enforcement_email.html index 567fc54b..13852acf 100644 --- a/api/templates/api/throttle_enforcement_email.html +++ b/api/templates/api/throttle_enforcement_email.html @@ -69,10 +69,22 @@ font-size: 14px; color: #4a4a4a; font-family: Arial, Helvetica, sans-serif"> - Most recently, your client received - {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} - across {{ days_throttled }} day{{ days_throttled|pluralize }}, - including {{ worst_day_count }} in a single day. + {% if days_throttled == 1 %} + {% if total_count == worst_day_count %} + Most recently, your client received + {{ worst_day_count }} rate-limited (429) response{{ worst_day_count|pluralize }} + in a single day. + {% else %} + Most recently, your client received + {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} + recently, including {{ worst_day_count }} in a single day. + {% endif %} + {% else %} + Most recently, your client received + {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} + across {{ days_throttled }} day{{ days_throttled|pluralize }}, + including {{ worst_day_count }} in a single day. + {% endif %}

diff --git a/api/templates/api/throttle_enforcement_email.txt b/api/templates/api/throttle_enforcement_email.txt index d8307a60..4885157c 100644 --- a/api/templates/api/throttle_enforcement_email.txt +++ b/api/templates/api/throttle_enforcement_email.txt @@ -4,9 +4,11 @@ We previously warned you that your Metron API client was repeatedly hitting our rate limits without backing off. Since that behavior has continued, we've had to take action on this account: -Most recently, your client received {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} +{% if days_throttled == 1 %}{% if total_count == worst_day_count %}Most recently, your client received {{ worst_day_count }} rate-limited (429) response{{ worst_day_count|pluralize }} +in a single day.{% else %}Most recently, your client received {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} +recently, including {{ worst_day_count }} in a single day.{% endif %}{% else %}Most recently, your client received {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} across {{ days_throttled }} day{{ days_throttled|pluralize }}, including {{ worst_day_count }} -in a single day. +in a single day.{% endif %} {% if account_disabled %}This account has been disabled. You won't be able to log in or use the API until an administrator reinstates it.{% else %}All API tokens for this account have been revoked. You'll need to generate a diff --git a/api/templates/api/throttle_notice_email.html b/api/templates/api/throttle_notice_email.html index e610f0b4..7cc40782 100644 --- a/api/templates/api/throttle_notice_email.html +++ b/api/templates/api/throttle_notice_email.html @@ -51,10 +51,18 @@ font-size: 16px; color: #4a4a4a; font-family: Arial, Helvetica, sans-serif"> - We've noticed that your Metron API client has repeatedly hit our rate - limits over the past several days. When a client keeps retrying - immediately after a 429 response instead of backing off, it adds - unnecessary load to the site for everyone. + {% if days_throttled == 1 %} + We've noticed that your Metron API client hit our rate limits hard, + with {{ worst_day_count }} rate-limited (429) responses in a single + day. When a client keeps retrying immediately after a 429 response + instead of backing off, it adds unnecessary load to the site for + everyone. + {% else %} + We've noticed that your Metron API client has repeatedly hit our rate + limits over the past several days. When a client keeps retrying + immediately after a 429 response instead of backing off, it adds + unnecessary load to the site for everyone. + {% endif %}

- Your client received - {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} - across {{ days_throttled }} day{{ days_throttled|pluralize }}, - including {{ worst_day_count }} in a single day. + {% if days_throttled == 1 %} + {% if total_count == worst_day_count %} + Your client received + {{ worst_day_count }} rate-limited (429) response{{ worst_day_count|pluralize }} + in a single day. + {% else %} + Your client received + {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} + recently, including {{ worst_day_count }} in a single day. + {% endif %} + {% else %} + Your client received + {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} + across {{ days_throttled }} day{{ days_throttled|pluralize }}, + including {{ worst_day_count }} in a single day. + {% endif %}

diff --git a/api/templates/api/throttle_notice_email.txt b/api/templates/api/throttle_notice_email.txt index b3d902b9..348417e6 100644 --- a/api/templates/api/throttle_notice_email.txt +++ b/api/templates/api/throttle_notice_email.txt @@ -1,13 +1,18 @@ {% autoescape off %}Hi {{ user.username }}, -We've noticed that your Metron API client has repeatedly hit our rate limits over +{% if days_throttled == 1 %}We've noticed that your Metron API client hit our rate limits hard, with +{{ worst_day_count }} rate-limited (429) responses in a single day. When a client keeps +retrying immediately after a 429 response instead of backing off, it adds +unnecessary load to the site for everyone.{% else %}We've noticed that your Metron API client has repeatedly hit our rate limits over the past several days. When a client keeps retrying immediately after a 429 response instead of backing off, it adds unnecessary load to the site for -everyone. +everyone.{% endif %} -Your client received {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} +{% if days_throttled == 1 %}{% if total_count == worst_day_count %}Your client received {{ worst_day_count }} rate-limited (429) response{{ worst_day_count|pluralize }} +in a single day.{% else %}Your client received {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} +recently, including {{ worst_day_count }} in a single day.{% endif %}{% else %}Your client received {{ total_count }} rate-limited (429) response{{ total_count|pluralize }} across {{ days_throttled }} day{{ days_throttled|pluralize }}, including {{ worst_day_count }} -in a single day. +in a single day.{% endif %} Every API response includes X-RateLimit-* headers showing your current limit, remaining requests, and when the window resets. A 429 response also means you've