fix(libSceVideodec2): bound cumulative AVC parameter sets before insertion

core/libs/prx/libSceVideodec2/Export.cpp

@@ -42,6 +42,7 @@ constexpr std::uint32_t HeightAlignment = 32;
 constexpr std::uint32_t CodecAvc = 1;
 constexpr std::uint32_t CodecHevc = 974921;
 constexpr std::uint32_t CodecVp9 = 2382845;
+constexpr std::size_t MaxAvcParameterSetsBytes = 2u * 1024u * 1024u;
 constexpr int Videodec2ErrorAccessUnitSize = static_cast<int>(0x811D010Du);
 constexpr int Videodec2ErrorAccessUnitPointer = static_cast<int>(0x811D010Eu);
 constexpr int Videodec2ErrorStructureSize = static_cast<int>(0x811D0101u);
@@ -587,8 +588,9 @@ int APS5_VABI sceVideodec2Decode_nid_postfix(std::uint64_t handle, const InputDa
         return 0;
     }
     const auto* unit = ToAnnexB(decoder, input->auData, static_cast<std::size_t>(input->auSize));
-    auto avc = decoder.codec == CodecAvc ? Videodec2Avc::Parse_nid_no_patch(decoder.avc, *unit) : Videodec2Avc::Metadata{};
     if (decoder.codec == CodecAvc && !CarriesAvcSlice(*unit)) {
+        if (decoder.parameterSets.size() + unit->size() > MaxAvcParameterSetsBytes) throw std::runtime_error("Videodec2: invalid access unit");
+        Videodec2Avc::Parse_nid_no_patch(decoder.avc, *unit);
         decoder.parameterSets.insert(decoder.parameterSets.end(), unit->begin(), unit->end());
         EmitDecoded(handle, decoder, frame, output);
         return 0;
@@ -612,7 +614,7 @@ int APS5_VABI sceVideodec2Decode_nid_postfix(std::uint64_t handle, const InputDa
         tagged.hevc.info.attachedData = picture.attached;
     }
     if (decoder.codec == CodecAvc) {
-        tagged.avc = std::move(avc);
+        tagged.avc = Videodec2Avc::Parse_nid_no_patch(decoder.avc, *unit);
         tagged.avc.info.ptsData = picture.pts;
         tagged.avc.info.dtsData = picture.dts;
         tagged.avc.info.attachedData = picture.attached;

core/libs/tests/GuestVideodec2.cpp

@@ -29,6 +29,7 @@ using namespace SceVideodec2;
 using namespace H264Fixture;
 using namespace HevcFixture;
 
+constexpr std::uint32_t CodecAvc = 1;
 constexpr std::uint32_t CodecHevc = 974921;
 constexpr std::uint32_t CodecVp9 = 2382845;
 
@@ -1060,6 +1061,68 @@ void testConcurrentOwnership() {
     check(sceVideodec2DeleteDecoder_nid_postfix(keeper) == 0, "delete failed");
 }
 
+void testParameterSetsBound() {
+    constexpr std::uint8_t DifferentParameters[] = {
+        0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xc0, 0x0a, 0xda, 0x11,
+        0xec, 0x04, 0x40, 0x00, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00,
+        0x0f, 0x03, 0xc4, 0x89, 0xa8, 0x00, 0x00, 0x00, 0x01, 0x68,
+        0xce, 0x0f, 0x2c, 0x80,
+    };
+    const DecoderConfigInfo config{sizeof(DecoderConfigInfo), 1, CodecAvc, 1, 0, static_cast<std::int32_t>(Width), static_cast<std::int32_t>(Height), 4, 1};
+    DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)};
+    check(sceVideodec2QueryDecoderMemoryInfo_nid_postfix(&config, &memory) == 0, "memory query failed");
+    std::uint64_t handle = 0;
+    check(sceVideodec2CreateDecoder_nid_postfix(&config, &memory, &handle) == 0, "decoder creation failed");
+    std::vector<std::uint8_t> storage;
+    const auto buffer = AlignedFrameStorage(storage, memory.maxFrameBufferSize, memory.frameBufferAlignment);
+    auto units = accessUnits(false);
+    auto first = units.front();
+    std::size_t slice = 0;
+    for (std::size_t index = 0; index + 3 < first.size(); ++index) {
+        if (first[index] != 0 || first[index + 1] != 0 || first[index + 2] != 1) continue;
+        const auto type = first[index + 3] & 0x1fu;
+        if (type >= 1 && type <= 5) {
+            slice = index > 0 && first[index - 1] == 0 ? index - 1 : index;
+            break;
+        }
+        index += 2;
+    }
+    check(slice != 0, "first access unit missing slice");
+    const std::vector<std::uint8_t> validParameters(first.begin(), first.begin() + static_cast<std::ptrdiff_t>(slice));
+    first.erase(first.begin(), first.begin() + static_cast<std::ptrdiff_t>(slice));
+    const InputData paramInput{sizeof(InputData), validParameters.data(), validParameters.size(), 999, 0, 0x9f};
+    FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size(), false};
+    OutputInfo output{sizeof(OutputInfo)};
+    check(sceVideodec2Decode_nid_postfix(handle, &paramInput, &frame, &output) == 0, "valid parameter unit failed");
+    check(!output.isValid && !frame.isAccepted, "parameter unit produced picture");
+    std::vector<std::uint8_t> largeParameters(2u * 1024u * 1024u + 1u);
+    for (std::size_t offset = 0; offset + sizeof(DifferentParameters) <= largeParameters.size(); offset += sizeof(DifferentParameters)) {
+        std::memcpy(largeParameters.data() + offset, DifferentParameters, sizeof(DifferentParameters));
+    }
+    const InputData largeInput{sizeof(InputData), largeParameters.data(), largeParameters.size(), 1000, 0, 0};
+    bool threw = false;
+    try {
+        sceVideodec2Decode_nid_postfix(handle, &largeInput, &frame, &output);
+    } catch (const std::runtime_error& error) {
+        threw = std::string(error.what()) == "Videodec2: invalid access unit";
+    }
+    check(threw, "oversized parameter unit did not throw");
+    const InputData sliceInput{sizeof(InputData), first.data(), first.size(), 1001, 0, 0xa0};
+    check(sceVideodec2Decode_nid_postfix(handle, &sliceInput, &frame, &output) == 0, "slice decode failed after rejected parameter unit");
+    if (!output.isValid) check(sceVideodec2Flush_nid_postfix(handle, &frame, &output) == 0, "flush failed");
+    check(output.isValid && frame.isAccepted, "slice failed to produce picture");
+    auto expected = streamRecord(DisplayOrderUnits[0]);
+    expected.sequenceParameterSetPresentFlag = 0;
+    expected.pictureParameterSetPresentFlag = 0;
+    expected.auDelimiterPresentFlag = 0;
+    expected.ptsData = 1001;
+    expected.dtsData = 0;
+    expected.attachedData = 0xa0;
+    check(avcFields(queryAvc(output)) == avcFields(expected), "AVC picture info differs from original staged state");
+    check(sceVideodec2Reset_nid_postfix(handle) == 0, "reset failed");
+    check(sceVideodec2DeleteDecoder_nid_postfix(handle) == 0, "delete failed");
+}
+
 }
 
 void testComputeQueues() {
@@ -1118,6 +1181,7 @@ int main() {
         testFlushAfterEveryUnit();
         testPictureOwnership();
         testConcurrentOwnership();
+        testParameterSetsBound();
         std::puts("Videodec2 tests passed");
         return 0;
     } catch (const std::exception& error) {

docs/dev/debt/libSceVideodec2.md

@@ -25,7 +25,7 @@
 
 - [sceVideodec2CreateDecoder](../../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2) - One title (PPSA02225) disassembled for this entry calls it once (arguments set up at 0x8b5095-0x8b50a6) with three pointers: a structure built on its stack, a structure inside its decoder object and the address of a field of the same object. A nonzero result makes it leave the decoder setup; on one path (a flag byte set and two 32-bit fields equal to values it computed before) it passes that field to the next Videodec2 call. The codec type it requests and the meaning of the structure fields are not shown.
 
-- [AVC parameter-set units](../../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2) - an H.264 access unit with no slice NAL (types 1 to 5), such as the sequence and picture parameter sets Twispero (PPSA99474) submits on their own before a stream's first slices, is kept and put in front of the next unit that has a slice, and `sceVideodec2Decode` returns 0 with no picture for it, since libavcodec refuses a packet without a slice. What the console returns for such a unit is not known. HEVC units of parameter sets alone are still passed to libavcodec as they are
+- [AVC parameter-set units](../../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2) - an H.264 access unit with no slice NAL (types 1 to 5), such as the sequence and picture parameter sets PPSA99474 submits on their own before a stream's first slices, is kept and put in front of the next unit that has a slice, and `sceVideodec2Decode` returns 0 with no picture for it, since libavcodec refuses a packet without a slice; cumulative pending parameter sets are bounded to 2 MiB before insertion. What the console returns for such a unit is not known. HEVC units of parameter sets alone are still passed to libavcodec as they are
 
 - [HEVC and VP9 output gaps](../../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2): console `frameFormat` remains unknown and is 0; VP9 profile 2 output has no hardware evidence. Twelve-bit pictures and formats other than yuv420p10le throw. HEVC Main10 uses low-bit 16-bit little-endian samples, with interleaved UV after `framePitchInBytes * height`, following [EVO-PLAYER-PS5](https://github.com/sainsaji/EVO-PLAYER-PS5/tree/9715f6ee9bf9b53461d6fdf9a2e11b4c48357783) `projects/evoplayer/media/src/evo_vdec_native.c`, `ro_harvest`: the hardware consumer detects ten bits when byte pitch is at least twice sample pitch and uses a low-bit factor.