enc/dec: harden integer overflow checks in size computations (defense-in-depth)

c/dec/state.c

@@ -167,9 +167,20 @@ BROTLI_BOOL BrotliDecoderHuffmanTreeGroupInit(BrotliDecoderState* s,
      This number is discovered "unlimited" "enough" calculator; it is actually
      a wee bigger than required in several cases (especially for alphabets with
      less than 16 symbols). */
+  /* defense-in-depth: each step of the size cascade can wrap size_t under
+     hypothetical caller / metablock values that bypass the parse-time caps. */
+  if (alphabet_size_limit > SIZE_MAX - 376) return BROTLI_FALSE;
   const size_t max_table_size = alphabet_size_limit + 376;
+  if (max_table_size != 0 &&
+      ntrees > SIZE_MAX / sizeof(HuffmanCode) / max_table_size) {
+    return BROTLI_FALSE;
+  }
   const size_t code_size = sizeof(HuffmanCode) * ntrees * max_table_size;
+  if (ntrees != 0 && ntrees > SIZE_MAX / sizeof(HuffmanCode*)) {
+    return BROTLI_FALSE;
+  }
   const size_t htree_size = sizeof(HuffmanCode*) * ntrees;
+  if (code_size > SIZE_MAX - htree_size) return BROTLI_FALSE;
   /* Pointer alignment is, hopefully, wider than sizeof(HuffmanCode). */
   HuffmanCode** p = (HuffmanCode**)BROTLI_DECODER_ALLOC(s,
       code_size + htree_size);

c/enc/block_encoder_inc.h

@@ -15,6 +15,11 @@ static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
     const HistogramType* histograms, const size_t histograms_size,
     const size_t alphabet_size, HuffmanTree* tree,
     size_t* storage_ix, uint8_t* storage) {
+  /* defense-in-depth: refuse if histograms_size * histogram_length_ wraps size_t */
+  if (self->histogram_length_ != 0 &&
+      histograms_size > SIZE_MAX / self->histogram_length_) {
+    return;
+  }
   const size_t table_size = histograms_size * self->histogram_length_;
   self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size);
   self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size);

c/enc/block_splitter.c

@@ -57,6 +57,13 @@ static void CopyLiteralsToByteArray(const Command* cmds,
   size_t i;
   for (i = 0; i < num_commands; ++i) {
     size_t insert_len = cmds[i].insert_len_;
+    /* defense-in-depth: refuse insert_len that wraps size_t when added to
+       from_pos. cmds[i].insert_len_ is encoder-internal and bounded by the
+       command builder; this is a defensive narrowing for hypothetical reuse
+       with caller-controlled insert_len. */
+    if (insert_len > (size_t)0 - from_pos) {
+      return;
+    }
     if (from_pos + insert_len > mask) {
       size_t head_size = mask + 1 - from_pos;
       memcpy(literals + pos, data + from_pos, head_size);

c/enc/block_splitter_inc.h

@@ -205,6 +205,8 @@ static void FN(ClusterBlocks)(MemoryManager* m,
                               const size_t num_blocks,
                               uint8_t* block_ids,
                               BlockSplit* split) {
+  /* defense-in-depth: refuse if num_blocks + 4 * HISTOGRAMS_PER_BATCH wraps. */
+  if (num_blocks > SIZE_MAX - 4 * HISTOGRAMS_PER_BATCH) return;
   uint32_t* histogram_symbols = BROTLI_ALLOC(m, uint32_t, num_blocks);
   uint32_t* u32 =
       BROTLI_ALLOC(m, uint32_t, num_blocks + 4 * HISTOGRAMS_PER_BATCH);
@@ -451,6 +453,9 @@ static void FN(SplitByteVector)(MemoryManager* m,
     uint8_t* block_ids = BROTLI_ALLOC(m, uint8_t, length);
     size_t num_blocks = 0;
     const size_t bitmaplen = (num_histograms + 7) >> 3;
+    /* defense-in-depth: refuse if either multiplication wraps size_t. */
+    if (num_histograms != 0 && data_size > SIZE_MAX / num_histograms) return;
+    if (bitmaplen     != 0 && length    > SIZE_MAX / bitmaplen)     return;
     double* insert_cost = BROTLI_ALLOC(m, double, data_size * num_histograms);
     double* cost = BROTLI_ALLOC(m, double, num_histograms);
     uint8_t* switch_signal = BROTLI_ALLOC(m, uint8_t, length * bitmaplen);