Downcase emails on write to DDB prod

.gitignore

@@ -288,3 +288,4 @@ spans*.json
 *.out
 CLAUDE.md
 .claude/*
+/*user_emails.json

cla-backend-go/approval_list/repository.go

@@ -6,6 +6,7 @@ package approval_list
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	models2 "github.com/linuxfoundation/easycla/cla-backend-go/project/models"
 
@@ -86,7 +87,7 @@ func (repo repository) AddCclaApprovalRequest(company *models.Company, project *
 	addStringAttribute(input.Item, "project_id", project.ProjectID)
 	addStringAttribute(input.Item, "project_name", project.ProjectName)
 	addStringAttribute(input.Item, "user_id", user.UserID)
-	addStringSliceAttribute(input.Item, "user_emails", []string{requesterEmail})
+	addStringSliceAttribute(input.Item, "user_emails", []string{strings.ToLower(strings.TrimSpace(requesterEmail))})
 	addStringAttribute(input.Item, "user_name", requesterName)
 	addStringAttribute(input.Item, "user_github_id", user.GithubID)
 	addStringAttribute(input.Item, "user_github_username", user.GithubUsername)

cla-backend-go/signatures/repository.go

@@ -3342,6 +3342,7 @@ func (repo repository) UpdateApprovalList(ctx context.Context, claManager *model
 			for _, email := range params.RemoveEmailApprovalList {
 				go func(email string) {
 					defer wg.Done()
+					email = strings.ToLower(strings.TrimSpace(email))
 					var iclas []*models.IclaSignature
 					var eclas []*models.Signature
 					log.WithFields(f).Debugf("getting cla user record for email: %s ", email)

cla-backend-go/users/repository.go

@@ -136,6 +136,7 @@ func (repo repository) CreateUser(user *models.User) (*models.User, error) {
 		}
 	}
 
+	user.Emails = normalizeEmails(user.Emails)
 	if len(user.Emails) > 0 {
 		attributes["user_emails"] = &dynamodb.AttributeValue{
 			SS: utils.ArrayStringPointer(user.Emails),
@@ -386,9 +387,10 @@ func (repo repository) Save(user *models.UserUpdate) (*models.User, error) {
 	}
 
 	if user.Emails != nil {
-		log.WithFields(f).Debugf("building query - adding user_emails: %v", user.Emails)
+		normalized := normalizeEmails(user.Emails)
+		log.WithFields(f).Debugf("building query - adding user_emails: %v", normalized)
 		expressionAttributeNames["#UES"] = aws.String("user_emails")
-		expressionAttributeValues[":ues"] = &dynamodb.AttributeValue{SS: aws.StringSlice(user.Emails)}
+		expressionAttributeValues[":ues"] = &dynamodb.AttributeValue{SS: aws.StringSlice(normalized)}
 		updateExpression = updateExpression + " #UES = :ues, "
 	}
 
@@ -808,6 +810,8 @@ func (repo repository) GetUsersByEmail(userEmail string) ([]*models.User, error)
 		"userEmail":    userEmail,
 	}
 
+	userEmail = strings.ToLower(strings.TrimSpace(userEmail))
+
 	// This is the filter we want to match
 	filter := expression.Name("user_emails").Contains(userEmail)
 
@@ -817,7 +821,7 @@ func (repo repository) GetUsersByEmail(userEmail string) ([]*models.User, error)
 	// Use the nice builder to create the expression
 	expr, err := expression.NewBuilder().WithFilter(filter).WithProjection(projection).Build()
 	if err != nil {
-		log.WithFields(f).Warnf("error building expression for lf_email : %s, error: %v", userEmail, err)
+		log.WithFields(f).Warnf("error building expression for user_emails : %s, error: %v", userEmail, err)
 		return nil, err
 	}
 
@@ -883,6 +887,27 @@ func (repo repository) GetUsersByEmail(userEmail string) ([]*models.User, error)
 	return users, nil
 }
 
+// normalizeEmails lower-cases, trims and de-duplicates emails (DynamoDB string sets reject duplicates).
+func normalizeEmails(emails []string) []string {
+	if emails == nil {
+		return nil
+	}
+	seen := make(map[string]struct{}, len(emails))
+	out := make([]string, 0, len(emails))
+	for _, email := range emails {
+		email = strings.ToLower(strings.TrimSpace(email))
+		if email == "" {
+			continue
+		}
+		if _, ok := seen[email]; ok {
+			continue
+		}
+		seen[email] = struct{}{}
+		out = append(out, email)
+	}
+	return out
+}
+
 // GetUsersByLFEmail fetches the user record by email
 func (repo repository) GetUsersByLFEmail(userEmail string) ([]*models.User, error) {
 	f := logrus.Fields{

cla-backend-legacy/internal/api/handlers.go

@@ -2338,7 +2338,7 @@ func (h *Handlers) InviteCompanyAdminV2(w http.ResponseWriter, r *http.Request)
 		"project_name":         &types.AttributeValueMemberS{Value: projectName},
 		"user_github_id":       &types.AttributeValueMemberS{Value: contributorID},
 		"user_github_username": &types.AttributeValueMemberS{Value: contributorName},
-		"user_emails":          &types.AttributeValueMemberSS{Value: []string{contributorEmail}},
+		"user_emails":          &types.AttributeValueMemberSS{Value: []string{strings.ToLower(strings.TrimSpace(contributorEmail))}},
 		"request_status":       &types.AttributeValueMemberS{Value: "pending"},
 		"date_created":         &types.AttributeValueMemberS{Value: now},
 		"date_modified":        &types.AttributeValueMemberS{Value: now},
@@ -2525,7 +2525,7 @@ func (h *Handlers) RequestCompanyCclaV2(w http.ResponseWriter, r *http.Request)
 		"request_id":     &types.AttributeValueMemberS{Value: reqID},
 		"company_name":   &types.AttributeValueMemberS{Value: companyName},
 		"project_name":   &types.AttributeValueMemberS{Value: projectName},
-		"user_emails":    &types.AttributeValueMemberSS{Value: []string{userEmail}},
+		"user_emails":    &types.AttributeValueMemberSS{Value: []string{strings.ToLower(strings.TrimSpace(userEmail))}},
 		"request_status": &types.AttributeValueMemberS{Value: "pending"},
 		"date_created":   &types.AttributeValueMemberS{Value: now},
 		"date_modified":  &types.AttributeValueMemberS{Value: now},

utils/downcase_emails.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+STAGE=${STAGE:-dev}
+PROFILE="lfproduct-${STAGE}"
+REGION=us-east-1
+TABLE="cla-${STAGE}-users"
+APPLY="${APPLY:-0}"
+
+aws dynamodb scan --profile "$PROFILE" --region "$REGION" --table-name "$TABLE" --projection-expression 'user_id, user_emails' --filter-expression 'attribute_exists(user_emails)' --output json > "${STAGE}_user_emails.json"
+cat "${STAGE}_user_emails.json" | jq -c '.Items[] | select(.user_emails.SS != null) | ([.user_emails.SS[] | ascii_downcase | gsub("^\\s+|\\s+$";"") | select(length > 0)] | unique) as $n | select(($n | length > 0) and ($n != (.user_emails.SS | sort)))' \
+| while IFS= read -r item; do
+    uid=$(jq -r '.user_id.S' <<<"$item")
+    newss=$(jq -c '[.user_emails.SS[] | ascii_downcase | gsub("^\\s+|\\s+$";"") | select(length > 0)] | unique' <<<"$item")   # lower + trim + drop-empty + dedupe
+    if [ "$newss" = "[]" ]
+    then
+      echo "skip $uid (no valid emails after normalize)" >&2
+      continue
+    fi
+    echo "user $uid -> $newss"
+    if [ "$APPLY" = "1" ]
+    then
+      aws dynamodb update-item --profile "$PROFILE" --region "$REGION" --table-name "$TABLE" \
+        --key "{\"user_id\":{\"S\":\"$uid\"}}" \
+        --update-expression 'SET user_emails = :e' \
+        --expression-attribute-values "{\":e\":{\"SS\":$newss}}" && echo "ok"
+    fi
+done

utils/lookup_all_logs.sh

@@ -1,15 +1,19 @@
 #!/bin/bash
-if ( [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] )
+set -euo pipefail
+if ( [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ] )
 then
   echo "usage:"
   echo "  $0 '2 hours ago' '1 second ago' 'text'"
   echo "  if 'text' = '---' then it returns all logs"
   exit 1
 fi
-if [ -z "$STAGE" ]
+if [ -z "${STAGE:-}" ]
 then
   export STAGE=dev
 fi
+
+# Fail fast (set -euo pipefail above): any lookup helper failure (aws/jq error, exit 3/4)
+# aborts here instead of running on and ending with a successful `ls` that would hide it.
 REGION=us-east-1 DEBUG=1 DTFROM="${1}" DTTO="${2}" ./utils/search_aws_log_group.sh 'githubactivity' "${3}" > githubactivity.log
 REGION=us-east-1 DEBUG=1 DTFROM="${1}" DTTO="${2}" ./utils/search_aws_log_group.sh 'apiv1' "${3}" > v1.log
 REGION=us-east-1 DEBUG=1 DTFROM="${1}" DTTO="${2}" ./utils/search_aws_log_group.sh 'apiv2' "${3}" > v2.log

utils/search_aws_log_group.sh

@@ -7,6 +7,13 @@
 # REGION=us-east-1 STAGE=prod DEBUG=1 DTFROM='15 minutes ago' DTTO='1 second ago' ./utils/search_aws_log_group.sh 'cla-backend-prod-api-v3-lambda' 'LG:api-request-path'
 # REGION=us-east-1 STAGE=prod DEBUG=1 DTFROM='15 minutes ago' DTTO='1 second ago' ./utils/search_aws_log_group.sh 'cla-backend-prod-apiv2' 'LG:api-request-path'
 # REGION=us-east-1 STAGE=prod DEBUG=1 DTFROM='15 minutes ago' DTTO='1 second ago' ./utils/search_aws_log_group.sh 'cla-backend-prod-githubactivity' 'LG:api-request-path'
+# REGION=us-east-2 STAGE=prod DTFROM='2 hours ago'  ./utils/search_aws_log_group.sh cla-backend-go-api-v4-lambda 'SSS'
+# REGION=us-east-2 STAGE=prod DTFROM='2 hours ago' ./utils/search_aws_log_group.sh cla-backend-go-api-v4-lambda 'is sanctioned'
+# REGION=us-east-2 STAGE=prod DTFROM='2 hours ago' ./utils/search_aws_log_group.sh cla-backend-go-api-v4-lambda 'persisting sanction'
+# REGION=us-east-1 STAGE=prod DTFROM='2 hours ago'  ./utils/search_aws_log_group.sh apiv2 'SSS'
+# REGION=us-east-1 STAGE=prod DTFROM='2 hours ago' ./utils/search_aws_log_group.sh apiv2 'is sanctioned'
+# REGION=us-east-2 STAGE=dev  DTFROM='1 day ago'  ./utils/search_aws_log_group.sh cla-backend-go-api-v4-lambda 'SSS'
+# REGION=us-east-1 STAGE=dev  DTFROM='1 day ago'  ./utils/search_aws_log_group.sh apiv2 'SSS'
 
 if [ -z "$STAGE" ]
 then
@@ -77,18 +84,39 @@ DTF=$(date -u -d @$(echo "${DTFROM}/1000" | bc) "+%F %T.%6N")
 DTT=$(date -u -d @$(echo "${DTTO}/1000" | bc) "+%F %T.%6N")
 echo "Date range: ${DTF} .. ${DTT} (from ${DTFROM} to ${DTTO})"
 
+# Capture aws output to a temp file first (no pipe), so an aws failure and a jq failure
+# are reported accurately and independently. In a pipe, a jq failure can SIGPIPE aws and
+# surface as 141, misclassifying it as an aws failure.
+raw_log="$(mktemp)" || { echo "ERROR: mktemp failed — cannot capture aws output" >&2; exit 5; }
+trap 'rm -f "${raw_log}"' EXIT
+
 if [ -z "${search}" ]
 then
   if [ ! -z "${DEBUG}" ]
   then
-    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\""
+    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --output json"
   fi
-  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" | jq -r '.events | sort_by(.timestamp)'
+  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --output json > "${raw_log}"
 else
   if [ ! -z "${DEBUG}" ]
   then
-    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${search}\""
+    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern '\"${search}\"' --output json"
   fi
-  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${search}\"" | jq -r '.events | sort_by(.timestamp)'
+  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${search}\"" --output json > "${raw_log}"
+fi
+aws_rc=$?
+
+# An aws failure (expired SSO, no access, crashed CLI) is NOT "no events": report it and
+# exit non-zero instead of leaving an empty/[] result that looks like "no hits". aws's own
+# error is shown above on stderr.
+if [ "${aws_rc}" -ne 0 ]
+then
+  echo "ERROR: aws failed (rc=${aws_rc}) — output above is NOT 'no events'; logs were not retrieved. Try: aws sso login --profile \"lfproduct-${STAGE}\"" >&2
+  exit 3
+fi
+if ! jq -r '.events | sort_by(.timestamp)' < "${raw_log}"
+then
+  echo "ERROR: jq failed — logs were retrieved but could not be parsed (is jq installed?)." >&2
+  exit 4
 fi