Handle errors in lookup script

utils/lookup_all_logs.sh

@@ -1,15 +1,19 @@
 #!/bin/bash
-if ( [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] )
+set -euo pipefail
+if ( [ -z "${1:-}" ] || [ -z "${2:-}" ] || [ -z "${3:-}" ] )
 then
   echo "usage:"
   echo "  $0 '2 hours ago' '1 second ago' 'text'"
   echo "  if 'text' = '---' then it returns all logs"
   exit 1
 fi
-if [ -z "$STAGE" ]
+if [ -z "${STAGE:-}" ]
 then
   export STAGE=dev
 fi
+
+# Fail fast (set -euo pipefail above): any lookup helper failure (aws/jq error, exit 3/4)
+# aborts here instead of running on and ending with a successful `ls` that would hide it.
 REGION=us-east-1 DEBUG=1 DTFROM="${1}" DTTO="${2}" ./utils/search_aws_log_group.sh 'githubactivity' "${3}" > githubactivity.log
 REGION=us-east-1 DEBUG=1 DTFROM="${1}" DTTO="${2}" ./utils/search_aws_log_group.sh 'apiv1' "${3}" > v1.log
 REGION=us-east-1 DEBUG=1 DTFROM="${1}" DTTO="${2}" ./utils/search_aws_log_group.sh 'apiv2' "${3}" > v2.log

utils/search_aws_log_group.sh

@@ -77,18 +77,39 @@ DTF=$(date -u -d @$(echo "${DTFROM}/1000" | bc) "+%F %T.%6N")
 DTT=$(date -u -d @$(echo "${DTTO}/1000" | bc) "+%F %T.%6N")
 echo "Date range: ${DTF} .. ${DTT} (from ${DTFROM} to ${DTTO})"
 
+# Capture aws output to a temp file first (no pipe), so an aws failure and a jq failure
+# are reported accurately and independently. In a pipe, a jq failure can SIGPIPE aws and
+# surface as 141, misclassifying it as an aws failure.
+raw_log="$(mktemp)" || { echo "ERROR: mktemp failed — cannot capture aws output" >&2; exit 5; }
+trap 'rm -f "${raw_log}"' EXIT
+
 if [ -z "${search}" ]
 then
   if [ ! -z "${DEBUG}" ]
   then
-    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\""
+    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --output json"
   fi
-  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" | jq -r '.events | sort_by(.timestamp)'
+  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --output json > "${raw_log}"
 else
   if [ ! -z "${DEBUG}" ]
   then
-    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${search}\""
+    echo "aws --region \"${REGION}\" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern '\"${search}\"' --output json"
   fi
-  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${search}\"" | jq -r '.events | sort_by(.timestamp)'
+  aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${search}\"" --output json > "${raw_log}"
+fi
+aws_rc=$?
+
+# An aws failure (expired SSO, no access, crashed CLI) is NOT "no events": report it and
+# exit non-zero instead of leaving an empty/[] result that looks like "no hits". aws's own
+# error is shown above on stderr.
+if [ "${aws_rc}" -ne 0 ]
+then
+  echo "ERROR: aws failed (rc=${aws_rc}) — output above is NOT 'no events'; logs were not retrieved. Try: aws sso login --profile \"lfproduct-${STAGE}\"" >&2
+  exit 3
+fi
+if ! jq -r '.events | sort_by(.timestamp)' < "${raw_log}"
+then
+  echo "ERROR: jq failed — logs were retrieved but could not be parsed (is jq installed?)." >&2
+  exit 4
 fi