Record a failed API response as a failure

src/data_sources/base.py

@@ -104,6 +104,20 @@ def write_stats_response(self, result: Any, action: str) -> None:
         """
         try:
             if type(result) is requests.Response:
+                if not result.ok:
+                    # Without this the body of an error response is written to
+                    # tmp/ as though it were data. The report generator then
+                    # finds no usable rows, preserves the previous ones, and the
+                    # run reports success while nothing new was collected.
+                    logger.error(
+                        f"{self.source} returned HTTP {result.status_code} for "
+                        f"{self.project}/{self.package} {action}"
+                    )
+                    failed_response = get_failed_result_json(result)
+                    filename = self.prep_filename("failed", action)
+                    write_json(failed_response, filename)
+                    self.write_prep_filename_metadata(action, filename)
+                    return
                 data = result.json()
             elif type(result) is pd.Series:
                 data = result.to_dict()

tests/test_data_sources.py

@@ -104,6 +104,36 @@ def test_write_stats_response_with_requests_response(self, mock_write_json):
         self.assertEqual(call_args[0], {"count": 100})
         self.assertEqual(call_args[1], "test_file.json")
 
+    @patch("src.data_sources.base.write_json")
+    def test_write_stats_response_with_error_status(self, mock_write_json):
+        """An error response is recorded as a failure, not written as data."""
+        mock_response = requests.Response()
+        mock_response.status_code = 401
+        mock_response._content = b'{"message": "Bad credentials"}'
+
+        with patch.object(self.ds, "prep_filename", return_value="failed_file.json"):
+            with patch.object(self.ds, "write_prep_filename_metadata"):
+                self.ds.write_stats_response(mock_response, "views")
+
+        mock_write_json.assert_called_once()
+        written = mock_write_json.call_args[0][0]
+        self.assertEqual(mock_write_json.call_args[0][1], "failed_file.json")
+        self.assertNotEqual(written, {"message": "Bad credentials"})
+
+    @patch("src.data_sources.base.write_json")
+    def test_error_response_is_filed_under_failed(self, mock_write_json):
+        """The failure goes to the failed folder, not to tmp."""
+        mock_response = requests.Response()
+        mock_response.status_code = 500
+        mock_response._content = b'{"message": "boom"}'
+
+        with patch.object(self.ds, "prep_filename") as mock_prep:
+            mock_prep.return_value = "failed/x.json"
+            with patch.object(self.ds, "write_prep_filename_metadata"):
+                self.ds.write_stats_response(mock_response, "views")
+
+        self.assertEqual(mock_prep.call_args[0][0], "failed")
+
     @patch("src.data_sources.base.write_json")
     def test_write_stats_response_with_pandas_series(self, mock_write_json):
         """Test write_stats_response with pandas Series object."""