From f1a9ba47bf17448df52051957481b5f06f74a55f Mon Sep 17 00:00:00 2001 From: Kieran Kunhya Date: Sat, 25 Jul 2026 22:35:24 +0100 Subject: [PATCH 1/2] uref: allow a uref to carry multiple ubufs Add a chain of additional ubufs to struct uref, linked by their own uchain (unused while a ubuf is in flight), so that a single uref can carry e.g. video, audio and subtitles at the same time. The primary uref->ubuf is entry 0 and is untouched: legacy pipes keep working unmodified and transparently pass additional entries through, since uref_free frees the chain and uref_dup/uref_fork duplicate it (a cheap reference increment). uref_dup_inner deliberately does not duplicate the chain, so derived urefs (block splice/split, field split) do not fan out the additional payloads. Per-entry metadata (flow ID, flow def) is stored in the parent uref attributes under indexed names ("sub[i].*") and is thus carried and duplicated by the existing udict handling. All entries share the dates and duration of the uref by definition, since they are part of the same uref. The new uref_sub.h API provides attach/get/detach/replace/find accessors, with lookup by flow ID. Co-Authored-By: Claude Fable 5 --- include/upipe/uref.h | 57 ++++++++++-- include/upipe/uref_sub.h | 187 +++++++++++++++++++++++++++++++++++++++ lib/upipe/Build.mk | 1 + tests/Build.mk | 4 + tests/uref_sub_test.c | 162 +++++++++++++++++++++++++++++++++ 5 files changed, 405 insertions(+), 6 deletions(-) create mode 100644 include/upipe/uref_sub.h create mode 100644 tests/uref_sub_test.c diff --git a/include/upipe/uref.h b/include/upipe/uref.h index 4397a56a7..fa119e2d1 100644 --- a/include/upipe/uref.h +++ b/include/upipe/uref.h @@ -22,6 +22,7 @@ extern "C" { #include "upipe/urefcount.h" #include "upipe/ubuf.h" #include "upipe/udict.h" +#include "upipe/ulist.h" #include #include @@ -95,6 +96,10 @@ struct uref { struct ubuf *ubuf; /** pointer to udict */ struct udict *udict; + /** list of additional ubufs, chained by their own uchain; the primary + * ubuf above is entry 0 and additional ubufs are entries 1 and above + * (see uref_sub.h) */ + struct uchain sub_ubufs; /** void flags */ uint64_t flags; @@ -156,6 +161,11 @@ static inline void uref_free(struct uref *uref) return; ubuf_free(uref->ubuf); udict_free(uref->udict); + struct uchain *uchain, *uchain_tmp; + ulist_delete_foreach(&uref->sub_ubufs, uchain, uchain_tmp) { + ulist_delete(uchain); + ubuf_free(ubuf_from_uchain(uchain)); + } uref->mgr->uref_free(uref); } @@ -167,6 +177,7 @@ static inline void uref_init(struct uref *uref) { uref->ubuf = NULL; uref->udict = NULL; + ulist_init(&uref->sub_ubufs); uref->flags = 0; uref->date_sys = UINT64_MAX; @@ -237,7 +248,8 @@ static inline struct uref *uref_sibling_alloc_control(struct uref *uref) return uref_alloc_control(uref->mgr); } -/** @internal @This duplicates a uref without duplicating the ubuf. +/** @internal @This duplicates a uref without duplicating the primary ubuf, + * nor the chain of additional ubufs. * * @param uref source structure to duplicate * @return duplicated uref or NULL in case of allocation failure @@ -250,6 +262,7 @@ static inline struct uref *uref_dup_inner(struct uref *uref) return NULL; new_uref->ubuf = NULL; + ulist_init(&new_uref->sub_ubufs); if (uref->udict != NULL) { new_uref->udict = udict_dup(uref->udict); if (unlikely(new_uref->udict == NULL)) { @@ -271,7 +284,27 @@ static inline struct uref *uref_dup_inner(struct uref *uref) return new_uref; } -/** @This duplicates a uref. +/** @internal @This duplicates the chain of additional ubufs of a uref into + * another uref. The duplicated ubufs are appended to the destination chain, + * preserving order. + * + * @param new_uref destination uref + * @param uref source uref + * @return an error code + */ +static inline int uref_sub_ubufs_dup(struct uref *new_uref, struct uref *uref) +{ + struct uchain *uchain; + ulist_foreach(&uref->sub_ubufs, uchain) { + struct ubuf *ubuf = ubuf_dup(ubuf_from_uchain(uchain)); + if (unlikely(ubuf == NULL)) + return UBASE_ERR_ALLOC; + ulist_add(&new_uref->sub_ubufs, ubuf_to_uchain(ubuf)); + } + return UBASE_ERR_NONE; +} + +/** @This duplicates a uref, including its chain of additional ubufs. * * @param uref source structure to duplicate * @return duplicated uref or NULL in case of allocation failure @@ -289,11 +322,16 @@ static inline struct uref *uref_dup(struct uref *uref) return NULL; } } + if (unlikely(!ubase_check(uref_sub_ubufs_dup(new_uref, uref)))) { + uref_free(new_uref); + return NULL; + } return new_uref; } -/** @This attaches a ubuf to a given uref. The ubuf pointer may no longer be - * used by the module afterwards. +/** @This attaches a ubuf to a given uref, as the primary ubuf (entry 0). + * The chain of additional ubufs is left untouched. The ubuf pointer may no + * longer be used by the module afterwards. * * @param uref pointer to uref structure * @param ubuf pointer to ubuf structure to attach to uref @@ -306,7 +344,8 @@ static inline void uref_attach_ubuf(struct uref *uref, struct ubuf *ubuf) uref->ubuf = ubuf; } -/** @This detaches a ubuf from a uref. The returned ubuf must be freed +/** @This detaches the primary ubuf (entry 0) from a uref. The chain of + * additional ubufs is left untouched. The returned ubuf must be freed * or re-attached at some point, otherwise it will leak. * * @param uref pointer to uref structure @@ -319,7 +358,9 @@ static inline struct ubuf *uref_detach_ubuf(struct uref *uref) return ubuf; } -/** @This duplicates a uref and attaches a new ubuf to the copy. +/** @This duplicates a uref and attaches a new primary ubuf to the copy. The + * chain of additional ubufs is also duplicated. In case of error the ubuf + * is not attached and remains owned by the caller. * * @param uref source structure to duplicate * @return duplicated uref or NULL in case of allocation failure @@ -330,6 +371,10 @@ static inline struct uref *uref_fork(struct uref *uref, struct ubuf *ubuf) if (unlikely(new_uref == NULL)) return NULL; + if (unlikely(!ubase_check(uref_sub_ubufs_dup(new_uref, uref)))) { + uref_free(new_uref); + return NULL; + } uref_attach_ubuf(new_uref, ubuf); return new_uref; } diff --git a/include/upipe/uref_sub.h b/include/upipe/uref_sub.h new file mode 100644 index 000000000..48693b754 --- /dev/null +++ b/include/upipe/uref_sub.h @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2026 Open Broadcast Systems Ltd + * + * Authors: Kieran Kunhya + * + * SPDX-License-Identifier: MIT + */ + +/** @file + * @short Upipe uref handling for multiple ubufs + * This file defines the API to attach several ubufs to a single uref, so + * that one uref may for instance carry video, audio and subtitles at the + * same time. + * + * The primary ubuf (uref->ubuf) is entry 0 and remains fully usable by + * legacy pipes, which only ever see the primary ubuf. Additional ubufs are + * entries 1 and above, chained to the uref by their own uchain (which is + * unused while a ubuf is in flight). Per-entry metadata is stored in the + * uref attributes under indexed names ("sub[i].*"), so it is transparently + * carried and duplicated by udict handling. All entries share the dates and + * duration of the uref by definition, since they are part of the same uref. + * + * @ref uref_free frees all entries, and @ref uref_dup and @ref uref_fork + * duplicate the whole chain (ubuf_dup is a cheap reference increment), so + * legacy pipes transparently pass additional entries through. Note however + * that pipes deriving urefs through internal helpers (block splice/split, + * picture field split) do not carry the chain. + * + * Please note that indices of entries following a detached entry shift down + * by one, but per-entry attributes are not renumbered; it is the + * responsibility of the caller to delete or rewrite them. + */ + +#ifndef _UPIPE_UREF_SUB_H_ +/** @hidden */ +#define _UPIPE_UREF_SUB_H_ +#ifdef __cplusplus +extern "C" { +#endif + +#include "upipe/uref.h" +#include "upipe/uref_attr.h" +#include "upipe/ubuf.h" +#include "upipe/ulist.h" + +#include +#include + +UREF_ATTR_UNSIGNED_VA(sub, flow_id, "sub[%" PRIu8"].f.id", sub ubuf flow ID, + uint8_t index, index) +UREF_ATTR_STRING_VA(sub, def, "sub[%" PRIu8"].f.def", + sub ubuf flow definition, uint8_t index, index) + +/** @This walks through the chain of additional ubufs of a uref (entries 1 + * and above). The list may not be altered during the walk. + * + * @param uref pointer to uref structure + * @param uchain iterator, to be converted with @ref ubuf_from_uchain + */ +#define uref_sub_foreach(uref, uchain) \ + ulist_foreach(&(uref)->sub_ubufs, uchain) + +/** @This returns the number of ubuf entries of a uref, including the + * primary ubuf. + * + * @param uref pointer to uref structure + * @return number of entries (0 if there is no ubuf at all) + */ +static inline uint8_t uref_sub_count(struct uref *uref) +{ + size_t depth = ulist_depth(&uref->sub_ubufs); + if (depth == 0 && uref->ubuf == NULL) + return 0; + return depth + 1; +} + +/** @This returns the ubuf at the given entry of a uref, without transferring + * ownership. Entry 0 is the primary ubuf. + * + * @param uref pointer to uref structure + * @param index entry index + * @return pointer to ubuf, or NULL if the entry does not exist + */ +static inline struct ubuf *uref_sub_get(struct uref *uref, uint8_t index) +{ + if (index == 0) + return uref->ubuf; + struct uchain *uchain = ulist_at(&uref->sub_ubufs, index - 1); + return uchain != NULL ? ubuf_from_uchain(uchain) : NULL; +} + +/** @This attaches an additional ubuf to a given uref, appended at the end of + * the chain. The ubuf pointer may no longer be used by the module + * afterwards. The primary ubuf is not modified (use @ref uref_attach_ubuf + * for entry 0). + * + * @param uref pointer to uref structure + * @param ubuf pointer to ubuf structure to attach to uref + * @return entry index given to the ubuf + */ +static inline uint8_t uref_sub_attach_ubuf(struct uref *uref, + struct ubuf *ubuf) +{ + ulist_add(&uref->sub_ubufs, ubuf_to_uchain(ubuf)); + return ulist_depth(&uref->sub_ubufs); +} + +/** @This detaches the ubuf at the given entry from a uref. The returned + * ubuf must be freed or re-attached at some point, otherwise it will leak. + * Entry 0 detaches the primary ubuf (see @ref uref_detach_ubuf). Indices of + * the following entries shift down by one, but per-entry attributes are not + * renumbered. + * + * @param uref pointer to uref structure + * @param index entry index + * @return pointer to detached ubuf, or NULL if the entry does not exist + */ +static inline struct ubuf *uref_sub_detach_ubuf(struct uref *uref, + uint8_t index) +{ + if (index == 0) + return uref_detach_ubuf(uref); + struct uchain *uchain = ulist_at(&uref->sub_ubufs, index - 1); + if (uchain == NULL) + return NULL; + ulist_delete(uchain); + return ubuf_from_uchain(uchain); +} + +/** @This replaces the ubuf at the given entry of a uref, freeing the + * previous one. The ubuf pointer may no longer be used by the module + * afterwards, even in case of error. Per-entry attributes are left + * untouched. + * + * @param uref pointer to uref structure + * @param ubuf pointer to ubuf structure replacing the entry + * @param index entry index + * @return an error code + */ +static inline int uref_sub_replace_ubuf(struct uref *uref, struct ubuf *ubuf, + uint8_t index) +{ + if (index == 0) { + uref_attach_ubuf(uref, ubuf); + return UBASE_ERR_NONE; + } + struct uchain *uchain = ulist_at(&uref->sub_ubufs, index - 1); + if (uchain == NULL) { + ubuf_free(ubuf); + return UBASE_ERR_INVALID; + } + struct uchain *prev = uchain->prev; + ulist_delete(uchain); + ubuf_free(ubuf_from_uchain(uchain)); + ulist_insert(prev, prev->next, ubuf_to_uchain(ubuf)); + return UBASE_ERR_NONE; +} + +/** @This finds the entry carrying the given flow ID (as set with @ref + * uref_sub_set_flow_id, including on entry 0 for the primary ubuf) and + * returns its ubuf without transferring ownership. + * + * @param uref pointer to uref structure + * @param flow_id flow ID to look for + * @param index_p filled in with the entry index, may be NULL + * @return pointer to ubuf, or NULL if no entry carries this flow ID + */ +static inline struct ubuf *uref_sub_find_flow_id(struct uref *uref, + uint64_t flow_id, + uint8_t *index_p) +{ + uint8_t count = uref_sub_count(uref); + for (uint8_t i = 0; i < count; i++) { + uint64_t v; + if (ubase_check(uref_sub_get_flow_id(uref, &v, i)) && v == flow_id) { + if (index_p != NULL) + *index_p = i; + return uref_sub_get(uref, i); + } + } + return NULL; +} + +#ifdef __cplusplus +} +#endif +#endif diff --git a/lib/upipe/Build.mk b/lib/upipe/Build.mk index 42fc81298..2e94ad529 100644 --- a/lib/upipe/Build.mk +++ b/lib/upipe/Build.mk @@ -112,6 +112,7 @@ libupipe-includes = \ uref_sound_flow.h \ uref_sound_flow_formats.h \ uref_std.h \ + uref_sub.h \ uref_uri.h \ uref_void.h \ uref_void_flow.h \ diff --git a/tests/Build.mk b/tests/Build.mk index 783091174..292cb56ca 100644 --- a/tests/Build.mk +++ b/tests/Build.mk @@ -637,6 +637,10 @@ tests += uref_std_test uref_std_test-src = uref_std_test.c uref_std_test-libs = libupipe +tests += uref_sub_test +uref_sub_test-src = uref_sub_test.c +uref_sub_test-libs = libupipe + tests += uref_uri_test.sh uref_uri_test.sh-deps = uref_uri_test diff --git a/tests/uref_sub_test.c b/tests/uref_sub_test.c new file mode 100644 index 000000000..51fbaf495 --- /dev/null +++ b/tests/uref_sub_test.c @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2026 Open Broadcast Systems Ltd + * + * Authors: Kieran Kunhya + * + * SPDX-License-Identifier: MIT + */ + +/** @file + * @short unit tests for urefs carrying multiple ubufs + */ + +#undef NDEBUG + +#include "upipe/umem.h" +#include "upipe/umem_alloc.h" +#include "upipe/udict.h" +#include "upipe/udict_inline.h" +#include "upipe/ubuf.h" +#include "upipe/ubuf_block.h" +#include "upipe/ubuf_block_mem.h" +#include "upipe/uref.h" +#include "upipe/uref_std.h" +#include "upipe/uref_sub.h" + +#include +#include +#include + +#define UDICT_POOL_DEPTH 5 +#define UREF_POOL_DEPTH 5 +#define UBUF_POOL_DEPTH 5 + +/** @This allocates a block ubuf filled with the given octet. */ +static struct ubuf *build_block(struct ubuf_mgr *mgr, uint8_t fill, int size) +{ + struct ubuf *ubuf = ubuf_block_alloc(mgr, size); + assert(ubuf != NULL); + int wsize = -1; + uint8_t *w; + ubase_assert(ubuf_block_write(ubuf, 0, &wsize, &w)); + assert(wsize == size); + memset(w, fill, wsize); + ubase_assert(ubuf_block_unmap(ubuf, 0)); + return ubuf; +} + +/** @This checks that a block ubuf is filled with the given octet. */ +static void check_block(struct ubuf *ubuf, uint8_t fill, int size) +{ + assert(ubuf != NULL); + int rsize = -1; + const uint8_t *r; + ubase_assert(ubuf_block_read(ubuf, 0, &rsize, &r)); + assert(rsize == size); + for (int i = 0; i < rsize; i++) + assert(r[i] == fill); + ubase_assert(ubuf_block_unmap(ubuf, 0)); +} + +int main(int argc, char **argv) +{ + struct umem_mgr *umem_mgr = umem_alloc_mgr_alloc(); + assert(umem_mgr != NULL); + struct udict_mgr *udict_mgr = udict_inline_mgr_alloc(UDICT_POOL_DEPTH, + umem_mgr, -1, -1); + assert(udict_mgr != NULL); + struct uref_mgr *mgr = uref_std_mgr_alloc(UREF_POOL_DEPTH, udict_mgr, 0); + assert(mgr != NULL); + struct ubuf_mgr *ubuf_mgr = ubuf_block_mem_mgr_alloc(UBUF_POOL_DEPTH, + UBUF_POOL_DEPTH, + umem_mgr, 0, 0, 0, 0); + assert(ubuf_mgr != NULL); + + /* attach primary + two additional ubufs */ + struct uref *uref = uref_alloc(mgr); + assert(uref != NULL); + assert(uref_sub_count(uref) == 0); + assert(uref_sub_get(uref, 0) == NULL); + assert(uref_sub_get(uref, 1) == NULL); + + uref_attach_ubuf(uref, build_block(ubuf_mgr, 'V', 64)); + assert(uref_sub_count(uref) == 1); + assert(uref_sub_get(uref, 0) == uref->ubuf); + + assert(uref_sub_attach_ubuf(uref, build_block(ubuf_mgr, 'A', 32)) == 1); + assert(uref_sub_attach_ubuf(uref, build_block(ubuf_mgr, 'S', 16)) == 2); + assert(uref_sub_count(uref) == 3); + check_block(uref_sub_get(uref, 0), 'V', 64); + check_block(uref_sub_get(uref, 1), 'A', 32); + check_block(uref_sub_get(uref, 2), 'S', 16); + assert(uref_sub_get(uref, 3) == NULL); + + /* per-entry attributes */ + ubase_assert(uref_sub_set_flow_id(uref, 0x100, 0)); + ubase_assert(uref_sub_set_flow_id(uref, 0x101, 1)); + ubase_assert(uref_sub_set_flow_id(uref, 0x102, 2)); + ubase_assert(uref_sub_set_def(uref, "sound.s32.", 1)); + + uint8_t index = 0xff; + struct ubuf *found = uref_sub_find_flow_id(uref, 0x101, &index); + assert(index == 1); + check_block(found, 'A', 32); + assert(uref_sub_find_flow_id(uref, 0xdead, NULL) == NULL); + + /* uref_dup duplicates the whole chain and its attributes */ + struct uref *dup = uref_dup(uref); + assert(dup != NULL); + assert(uref_sub_count(dup) == 3); + assert(uref_sub_get(dup, 1) != uref_sub_get(uref, 1)); + check_block(uref_sub_get(dup, 0), 'V', 64); + check_block(uref_sub_get(dup, 1), 'A', 32); + check_block(uref_sub_get(dup, 2), 'S', 16); + uint64_t v; + ubase_assert(uref_sub_get_flow_id(dup, &v, 2)); + assert(v == 0x102); + const char *def; + ubase_assert(uref_sub_get_def(dup, &def, 1)); + assert(!strcmp(def, "sound.s32.")); + uref_free(dup); + + /* uref_fork replaces the primary ubuf and keeps the chain */ + struct uref *fork = uref_fork(uref, build_block(ubuf_mgr, 'W', 8)); + assert(fork != NULL); + assert(uref_sub_count(fork) == 3); + check_block(uref_sub_get(fork, 0), 'W', 8); + check_block(uref_sub_find_flow_id(fork, 0x101, NULL), 'A', 32); + uref_free(fork); + + /* replace an entry in place */ + ubase_assert(uref_sub_replace_ubuf(uref, build_block(ubuf_mgr, 'B', 32), + 1)); + check_block(uref_sub_get(uref, 1), 'B', 32); + assert(uref_sub_count(uref) == 3); + ubase_nassert(uref_sub_replace_ubuf(uref, build_block(ubuf_mgr, 'X', 8), + 42)); + + /* detach an additional entry: following entries shift down */ + struct ubuf *detached = uref_sub_detach_ubuf(uref, 1); + check_block(detached, 'B', 32); + ubuf_free(detached); + assert(uref_sub_count(uref) == 2); + check_block(uref_sub_get(uref, 1), 'S', 16); + assert(uref_sub_detach_ubuf(uref, 2) == NULL); + + /* legacy detach only touches the primary ubuf */ + detached = uref_detach_ubuf(uref); + check_block(detached, 'V', 64); + ubuf_free(detached); + assert(uref_sub_get(uref, 0) == NULL); + check_block(uref_sub_get(uref, 1), 'S', 16); + + /* uref_free releases the remaining chain (checked by refcounts on + * manager release below) */ + uref_free(uref); + + ubuf_mgr_release(ubuf_mgr); + uref_mgr_release(mgr); + udict_mgr_release(udict_mgr); + umem_mgr_release(umem_mgr); + return 0; +} From 1298a17623e608e7b343756c7fd7a6b22d4de9db Mon Sep 17 00:00:00 2001 From: Kieran Kunhya Date: Sat, 25 Jul 2026 23:09:11 +0100 Subject: [PATCH 2/2] uref_sub: add generic attribute handling for entries Add helpers to move attributes between the plain namespace of a standalone uref and the per-entry "sub[i]." namespace of a carrier uref, built generically on udict_iterate so they cover any attribute type, including ones defined in the future: - uref_sub_merge() consumes a standalone uref and appends its ubuf as a new entry, copying its plain attributes (with shorthands resolved to their canonical names) into the entry namespace. Its dates are discarded since all entries share the dates of the carrier by definition. - uref_sub_extract() creates a standalone uref from an entry, duplicating its ubuf, inheriting the carrier dates and flags, and renaming the entry attributes back to plain names, converted back to shorthand storage where applicable so that legacy typed accessors (uref_flow_get_def() etc.) find them. - uref_sub_delete_attrs() and uref_sub_rename_attrs() manage entry namespaces; uref_sub_detach_ubuf() now uses them to delete the detached entry's attributes and renumber the following entries, keeping indices and attributes consistent. The intended model is that entries are merged into a carrier at one pipe boundary and extracted back at another, so that pipes in between never need per-entry attribute access. Co-Authored-By: Claude Fable 5 --- include/upipe/uref_sub.h | 326 ++++++++++++++++++++++++++++++++++++++- tests/uref_sub_test.c | 59 ++++++- 2 files changed, 376 insertions(+), 9 deletions(-) diff --git a/include/upipe/uref_sub.h b/include/upipe/uref_sub.h index 48693b754..d97719352 100644 --- a/include/upipe/uref_sub.h +++ b/include/upipe/uref_sub.h @@ -26,9 +26,11 @@ * that pipes deriving urefs through internal helpers (block splice/split, * picture field split) do not carry the chain. * - * Please note that indices of entries following a detached entry shift down - * by one, but per-entry attributes are not renumbered; it is the - * responsibility of the caller to delete or rewrite them. + * The intended model is that entries are merged into a carrier uref at one + * pipe boundary (@ref uref_sub_merge) and extracted back into standalone + * urefs at another (@ref uref_sub_extract); both round-trip all attributes + * generically between plain names and the per-entry namespace, so pipes in + * between never need per-entry attribute access. */ #ifndef _UPIPE_UREF_SUB_H_ @@ -41,10 +43,18 @@ extern "C" { #include "upipe/uref.h" #include "upipe/uref_attr.h" #include "upipe/ubuf.h" +#include "upipe/udict.h" #include "upipe/ulist.h" #include #include +#include +#include +#include + +/** @internal @This is the maximum length of a per-entry attribute name + * prefix, including the trailing zero. */ +#define UREF_SUB_PREFIX_SIZE 16 UREF_ATTR_UNSIGNED_VA(sub, flow_id, "sub[%" PRIu8"].f.id", sub ubuf flow ID, uint8_t index, index) @@ -105,11 +115,169 @@ static inline uint8_t uref_sub_attach_ubuf(struct uref *uref, return ulist_depth(&uref->sub_ubufs); } +/** @internal @This builds the attribute name prefix of an entry. + * + * @param buffer buffer of at least UREF_SUB_PREFIX_SIZE octets + * @param index entry index + * @return length of the prefix + */ +static inline size_t uref_sub_prefix(char *buffer, uint8_t index) +{ + return snprintf(buffer, UREF_SUB_PREFIX_SIZE, "sub[%" PRIu8 "].", index); +} + +/** @internal @This copies the value of an attribute to another name and + * type, possibly within the same udict. The value is staged in a temporary + * buffer because setting an attribute may reallocate the underlying storage. + * + * @param dst_uref destination uref + * @param dst_name destination attribute name (NULL if dst_type is a + * shorthand) + * @param dst_type destination attribute type + * @param src_uref source uref (may be the same as dst_uref) + * @param src_name source attribute name (NULL if src_type is a shorthand) + * @param src_type source attribute type + * @return an error code + */ +static inline int uref_sub_attr_transfer(struct uref *dst_uref, + const char *dst_name, + enum udict_type dst_type, + struct uref *src_uref, + const char *src_name, + enum udict_type src_type) +{ + size_t size; + const uint8_t *v; + UBASE_RETURN(udict_get(src_uref->udict, src_name, src_type, &size, &v)); + uint8_t *tmp = (uint8_t *)malloc(size); + if (unlikely(size != 0 && tmp == NULL)) + return UBASE_ERR_ALLOC; + if (size != 0) + memcpy(tmp, v, size); + + if (dst_uref->udict == NULL) { + dst_uref->udict = udict_alloc(dst_uref->mgr->udict_mgr, 0); + if (unlikely(dst_uref->udict == NULL)) { + free(tmp); + return UBASE_ERR_ALLOC; + } + } + uint8_t *new_v; + int err = udict_set(dst_uref->udict, dst_name, dst_type, size, &new_v); + if (likely(ubase_check(err)) && size != 0) + memcpy(new_v, tmp, size); + free(tmp); + return err; +} + +/** @internal @This checks whether a plain attribute name and base type + * correspond to a shorthand type, so that extracted attributes remain + * visible to shorthand accessors. + * + * @param uref pointer to uref structure (used to query the udict manager) + * @param name plain attribute name + * @param base_type base attribute type + * @return the shorthand type, or base_type if there is none + */ +static inline enum udict_type uref_sub_shorthand(struct uref *uref, + const char *name, + enum udict_type base_type) +{ + for (int type = UDICT_TYPE_SHORTHAND + 1; ; type++) { + const char *shorthand_name; + enum udict_type shorthand_base; + if (!ubase_check(udict_name(uref->udict, (enum udict_type)type, + &shorthand_name, &shorthand_base))) + return base_type; + if (shorthand_base == base_type && !strcmp(shorthand_name, name)) + return (enum udict_type)type; + } +} + +/** @This deletes all per-entry attributes of the given entry. + * + * @param uref pointer to uref structure + * @param index entry index + * @return an error code + */ +static inline int uref_sub_delete_attrs(struct uref *uref, uint8_t index) +{ + if (uref->udict == NULL) + return UBASE_ERR_NONE; + char prefix[UREF_SUB_PREFIX_SIZE]; + size_t prefix_len = uref_sub_prefix(prefix, index); + for ( ; ; ) { + const char *name = NULL; + enum udict_type type = UDICT_TYPE_END; + for ( ; ; ) { + udict_iterate(uref->udict, &name, &type); + if (unlikely(type == UDICT_TYPE_END)) + return UBASE_ERR_NONE; + if (name != NULL && !strncmp(name, prefix, prefix_len)) + break; + } + /* deletion invalidates the iteration, so restart it */ + UBASE_RETURN(udict_delete(uref->udict, type, name)); + } +} + +/** @This renames all per-entry attributes of an entry to another index. + * Existing attributes of the destination index are overwritten on name + * collision, so the caller should make sure the destination namespace is + * free. + * + * @param uref pointer to uref structure + * @param from source entry index + * @param to destination entry index + * @return an error code + */ +static inline int uref_sub_rename_attrs(struct uref *uref, uint8_t from, + uint8_t to) +{ + if (uref->udict == NULL || from == to) + return UBASE_ERR_NONE; + char from_prefix[UREF_SUB_PREFIX_SIZE], to_prefix[UREF_SUB_PREFIX_SIZE]; + size_t from_len = uref_sub_prefix(from_prefix, from); + size_t to_len = uref_sub_prefix(to_prefix, to); + for ( ; ; ) { + const char *name = NULL; + enum udict_type type = UDICT_TYPE_END; + for ( ; ; ) { + udict_iterate(uref->udict, &name, &type); + if (unlikely(type == UDICT_TYPE_END)) + return UBASE_ERR_NONE; + if (name != NULL && !strncmp(name, from_prefix, from_len)) + break; + } + /* copy both names as setting an attribute may reallocate the + * storage the iterated name points into */ + size_t suffix_len = strlen(name + from_len); + char *old_name = (char *)malloc(from_len + suffix_len + 1 + + to_len + suffix_len + 1); + if (unlikely(old_name == NULL)) + return UBASE_ERR_ALLOC; + char *new_name = old_name + from_len + suffix_len + 1; + memcpy(old_name, name, from_len + suffix_len + 1); + memcpy(new_name, to_prefix, to_len); + memcpy(new_name + to_len, name + from_len, suffix_len + 1); + + int err = uref_sub_attr_transfer(uref, new_name, type, + uref, old_name, type); + if (likely(ubase_check(err))) + err = udict_delete(uref->udict, type, old_name); + free(old_name); + if (unlikely(!ubase_check(err))) + return err; + /* the transfer invalidates the iteration, so restart it */ + } +} + /** @This detaches the ubuf at the given entry from a uref. The returned * ubuf must be freed or re-attached at some point, otherwise it will leak. - * Entry 0 detaches the primary ubuf (see @ref uref_detach_ubuf). Indices of - * the following entries shift down by one, but per-entry attributes are not - * renumbered. + * Entry 0 detaches the primary ubuf (see @ref uref_detach_ubuf) and leaves + * attributes untouched. For other entries, the entry's attributes are + * deleted and the attributes of the following entries are renumbered to + * match their new indices (best effort in case of allocation failure). * * @param uref pointer to uref structure * @param index entry index @@ -124,6 +292,11 @@ static inline struct ubuf *uref_sub_detach_ubuf(struct uref *uref, if (uchain == NULL) return NULL; ulist_delete(uchain); + + uref_sub_delete_attrs(uref, index); + uint8_t last = ulist_depth(&uref->sub_ubufs) + 1; + for (uint8_t i = index; i < last; i++) + uref_sub_rename_attrs(uref, i + 1, i); return ubuf_from_uchain(uchain); } @@ -181,6 +354,147 @@ static inline struct ubuf *uref_sub_find_flow_id(struct uref *uref, return NULL; } +/** @This extracts an entry of a uref as a standalone uref, without + * modifying the source. The new uref references a duplicate of the entry's + * ubuf, inherits the dates and flags of the source uref (which all entries + * share by definition), and its plain attributes are the per-entry + * attributes of the entry, renamed from the "sub[i]." namespace and + * converted back to shorthands where applicable. For entry 0 the plain + * attributes of the source are also copied. Per-entry attributes of other + * entries are not carried. + * + * @param uref pointer to uref structure + * @param index entry index + * @return extracted uref, or NULL if the entry does not exist or in case of + * allocation failure + */ +static inline struct uref *uref_sub_extract(struct uref *uref, uint8_t index) +{ + if (index != 0 && ulist_at(&uref->sub_ubufs, index - 1) == NULL) + return NULL; + struct uref *new_uref = uref_sibling_alloc(uref); + if (unlikely(new_uref == NULL)) + return NULL; + + new_uref->flags = uref->flags; + new_uref->date_sys = uref->date_sys; + new_uref->date_prog = uref->date_prog; + new_uref->date_orig = uref->date_orig; + new_uref->dts_pts_delay = uref->dts_pts_delay; + new_uref->cr_dts_delay = uref->cr_dts_delay; + new_uref->rap_cr_delay = uref->rap_cr_delay; + new_uref->priv = uref->priv; + + struct ubuf *ubuf = uref_sub_get(uref, index); + if (ubuf != NULL) { + new_uref->ubuf = ubuf_dup(ubuf); + if (unlikely(new_uref->ubuf == NULL)) + goto uref_sub_extract_err; + } + + if (uref->udict != NULL) { + char prefix[UREF_SUB_PREFIX_SIZE]; + size_t prefix_len = uref_sub_prefix(prefix, index); + const char *name = NULL; + enum udict_type type = UDICT_TYPE_END; + for ( ; ; ) { + udict_iterate(uref->udict, &name, &type); + if (unlikely(type == UDICT_TYPE_END)) + break; + + int err = UBASE_ERR_NONE; + if (name != NULL && !strncmp(name, prefix, prefix_len)) { + const char *plain = name + prefix_len; + enum udict_type plain_type = + uref_sub_shorthand(uref, plain, type); + err = uref_sub_attr_transfer(new_uref, + plain_type > UDICT_TYPE_SHORTHAND ? NULL : plain, + plain_type, uref, name, type); + } else if (index == 0 && + (name == NULL || strncmp(name, "sub[", 4))) + err = uref_sub_attr_transfer(new_uref, name, type, + uref, name, type); + if (unlikely(!ubase_check(err))) + goto uref_sub_extract_err; + } + } + return new_uref; + +uref_sub_extract_err: + uref_free(new_uref); + return NULL; +} + +/** @This merges a standalone uref into a carrier uref as a new entry. On + * success the merged uref is entirely consumed: its ubuf becomes the new + * entry, its plain attributes (including shorthands) are copied into the + * "sub[i]." namespace of the carrier, and its dates are discarded since all + * entries share the dates of the carrier by definition. The merged uref + * must have a ubuf and must not itself carry additional ubufs; in that case + * (and on allocation failure) an error is returned and the merged uref + * remains owned by the caller. + * + * @param uref pointer to carrier uref structure + * @param sub_uref uref to merge as a new entry + * @param index_p filled in with the entry index, may be NULL + * @return an error code + */ +static inline int uref_sub_merge(struct uref *uref, struct uref *sub_uref, + uint8_t *index_p) +{ + if (unlikely(sub_uref->ubuf == NULL || + !ulist_empty(&sub_uref->sub_ubufs))) + return UBASE_ERR_INVALID; + + uint8_t index = ulist_depth(&uref->sub_ubufs) + 1; + char prefix[UREF_SUB_PREFIX_SIZE]; + size_t prefix_len = uref_sub_prefix(prefix, index); + + if (sub_uref->udict != NULL) { + const char *name = NULL; + enum udict_type type = UDICT_TYPE_END; + for ( ; ; ) { + udict_iterate(sub_uref->udict, &name, &type); + if (unlikely(type == UDICT_TYPE_END)) + break; + + const char *plain = name; + enum udict_type base = type; + if (plain == NULL) { + if (unlikely(!ubase_check(udict_name(sub_uref->udict, type, + &plain, &base)))) { + uref_sub_delete_attrs(uref, index); + return UBASE_ERR_INVALID; + } + } else if (!strncmp(plain, "sub[", 4)) + /* per-entry namespaces cannot be nested */ + continue; + + size_t plain_len = strlen(plain); + char *new_name = (char *)malloc(prefix_len + plain_len + 1); + if (unlikely(new_name == NULL)) { + uref_sub_delete_attrs(uref, index); + return UBASE_ERR_ALLOC; + } + memcpy(new_name, prefix, prefix_len); + memcpy(new_name + prefix_len, plain, plain_len + 1); + int err = uref_sub_attr_transfer(uref, new_name, base, + sub_uref, name, type); + free(new_name); + if (unlikely(!ubase_check(err))) { + uref_sub_delete_attrs(uref, index); + return err; + } + } + } + + uref_sub_attach_ubuf(uref, uref_detach_ubuf(sub_uref)); + uref_free(sub_uref); + if (index_p != NULL) + *index_p = index; + return UBASE_ERR_NONE; +} + #ifdef __cplusplus } #endif diff --git a/tests/uref_sub_test.c b/tests/uref_sub_test.c index 51fbaf495..8f9e3cbc5 100644 --- a/tests/uref_sub_test.c +++ b/tests/uref_sub_test.c @@ -21,6 +21,7 @@ #include "upipe/ubuf_block_mem.h" #include "upipe/uref.h" #include "upipe/uref_std.h" +#include "upipe/uref_flow.h" #include "upipe/uref_sub.h" #include @@ -135,13 +136,65 @@ int main(int argc, char **argv) ubase_nassert(uref_sub_replace_ubuf(uref, build_block(ubuf_mgr, 'X', 8), 42)); - /* detach an additional entry: following entries shift down */ + /* extract an entry as a standalone uref: per-entry attributes become + * plain attributes again, visible to shorthand accessors */ + uref->date_prog = 42; + struct uref *extracted = uref_sub_extract(uref, 1); + assert(extracted != NULL); + assert(uref_sub_count(extracted) == 1); + check_block(extracted->ubuf, 'B', 32); + assert(extracted->date_prog == 42); + ubase_assert(uref_flow_get_id(extracted, &v)); + assert(v == 0x101); + ubase_assert(uref_flow_get_def(extracted, &def)); + assert(!strcmp(def, "sound.s32.")); + uref_free(extracted); + + /* extract entry 0: its own namespace, without other entries' */ + extracted = uref_sub_extract(uref, 0); + assert(extracted != NULL); + check_block(extracted->ubuf, 'V', 64); + ubase_assert(uref_flow_get_id(extracted, &v)); + assert(v == 0x100); + ubase_nassert(uref_sub_get_flow_id(extracted, &v, 1)); + uref_free(extracted); + + assert(uref_sub_extract(uref, 42) == NULL); + + /* merge a standalone uref as a new entry */ + struct uref *sound = uref_alloc(mgr); + assert(sound != NULL); + ubase_nassert(uref_sub_merge(uref, sound, NULL)); /* no ubuf */ + uref_attach_ubuf(sound, build_block(ubuf_mgr, 'M', 24)); + ubase_assert(uref_flow_set_id(sound, 0x200)); + ubase_assert(uref_flow_set_def(sound, "sound.s16.")); + uint8_t merge_index = 0; + ubase_assert(uref_sub_merge(uref, sound, &merge_index)); + assert(merge_index == 3); + assert(uref_sub_count(uref) == 4); + check_block(uref_sub_find_flow_id(uref, 0x200, &merge_index), 'M', 24); + assert(merge_index == 3); + ubase_assert(uref_sub_get_def(uref, &def, 3)); + assert(!strcmp(def, "sound.s16.")); + + /* detach an additional entry: its attributes are deleted and following + * entries' attributes are renumbered */ struct ubuf *detached = uref_sub_detach_ubuf(uref, 1); check_block(detached, 'B', 32); ubuf_free(detached); - assert(uref_sub_count(uref) == 2); + assert(uref_sub_count(uref) == 3); check_block(uref_sub_get(uref, 1), 'S', 16); - assert(uref_sub_detach_ubuf(uref, 2) == NULL); + ubase_assert(uref_sub_get_flow_id(uref, &v, 1)); + assert(v == 0x102); + ubase_nassert(uref_sub_get_def(uref, &def, 1)); + ubase_assert(uref_sub_get_flow_id(uref, &v, 2)); + assert(v == 0x200); + ubase_assert(uref_sub_get_def(uref, &def, 2)); + assert(!strcmp(def, "sound.s16.")); + ubase_nassert(uref_sub_get_flow_id(uref, &v, 3)); + assert(uref_sub_find_flow_id(uref, 0x101, NULL) == NULL); + check_block(uref_sub_find_flow_id(uref, 0x200, NULL), 'M', 24); + assert(uref_sub_detach_ubuf(uref, 3) == NULL); /* legacy detach only touches the primary ubuf */ detached = uref_detach_ubuf(uref);