added Avro support for KafkaAdapter

.github/workflows/conda.yml

@@ -81,13 +81,24 @@ jobs:
         - uses: mamba-org/setup-micromamba@v2
           with:
             micromamba-version: '2.3.2-0'
-            environment-file: conda/dev-environment-unix.yml
+            environment-file: conda/dev-environment-linux.yml
             init-shell: >-
               bash
             cache-environment: true
             cache-downloads: true
             post-cleanup: 'all'
-          if: ${{ runner.os != 'Windows' }}
+          if: ${{ runner.os == 'Linux' }}
+
+        - uses: mamba-org/setup-micromamba@v2
+          with:
+            micromamba-version: '2.3.2-0'
+            environment-file: conda/dev-environment-osx.yml
+            init-shell: >-
+              bash
+            cache-environment: true
+            cache-downloads: true
+            post-cleanup: 'all'
+          if: ${{ runner.os == 'macOS' }}
 
         - uses: mamba-org/setup-micromamba@v2
           with:

conda/dev-environment-linux.yml

@@ -0,0 +1,63 @@
+name: csp
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+ - astor
+ - avrocpp
+ - bison
+ - brotli
+ - bump-my-version
+ - cmake
+ - codespell>=2.4,<2.5
+ - compilers
+ - cyrus-sasl
+ - deprecated
+ - docutils<0.22.1
+ - exprtk
+ - flex
+ - graphviz
+ - gtest
+ - httpx>=0.20,<1
+ - libarrow<23
+ - libboost>=1.80.0
+ - libboost-headers>=1.80.0
+ - librdkafka
+ - lz4-c
+ - make
+ - mamba
+ - mdformat>=0.7.19,<1.1
+ - mdformat-tables>=1,<1.1
+ - ninja
+ - numpy>=2
+ - pandas
+ - pillow
+ - polars
+ - psutil
+ - pyarrow>=15,<23
+ - pydantic>=2
+ - pytest
+ - pytest-asyncio
+ - pytest-cov
+ - pytest-sugar
+ - python>=3.10,<3.14
+ - python-build
+ - python-graphviz
+ - python-rapidjson
+ - pytz
+ - rapidjson
+ - requests
+ - ruamel.yaml
+ - ruff>=0.9,<0.15
+ - scikit-build
+ - setuptools>=69,<74
+ - sqlalchemy
+ - tar
+ - threadpoolctl
+ - tornado
+ - twine
+ - typing-extensions
+ - unzip
+ - wheel
+ - zip
+ - zlib

cpp/cmake/modules/FindAvro.cmake

@@ -0,0 +1,13 @@
+find_path(Avro_INCLUDE_DIR NAMES avro/Encoder.hh)
+find_library(Avro_LIBRARY NAMES avrocpp libavrocpp)
+
+if (NOT TARGET Avro::avrocpp)
+  add_library(Avro::avrocpp SHARED IMPORTED)
+  set_property(TARGET Avro::avrocpp PROPERTY
+          IMPORTED_LOCATION "${Avro_LIBRARY}")
+  target_include_directories(Avro::avrocpp INTERFACE ${Avro_INCLUDE_DIR})
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Avro DEFAULT_MSG Avro_LIBRARY Avro_INCLUDE_DIR)
+mark_as_advanced(Avro_INCLUDE_DIR Avro_LIBRARY Avro::avrocpp)

cpp/cmake/modules/FindDepsKafkaAdapter.cmake

@@ -2,11 +2,20 @@ cmake_minimum_required(VERSION 3.7.2)
 
 if (CSP_USE_VCPKG)
   find_package(RdKafka CONFIG REQUIRED)
-  if(NOT WIN32) 
+  find_package(unofficial-avro-cpp CONFIG REQUIRED)
+  set(CSP_HAVE_AVRO ON)
+  if(NOT WIN32)
     # Bad, but a temporary workaround for
     # https://github.com/microsoft/vcpkg/issues/40320
     link_directories(${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib)
   endif()
 else()
   find_package(RdKafka REQUIRED)
+  find_package(Avro QUIET)
+  if(Avro_FOUND)
+    set(CSP_HAVE_AVRO ON)
+  else()
+    set(CSP_HAVE_AVRO OFF)
+    message(STATUS "Avro not found - AVRO protocol support will be disabled")
+  endif()
 endif()

cpp/csp/adapters/kafka/CMakeLists.txt

@@ -24,6 +24,16 @@ find_package(DepsKafkaAdapter REQUIRED)
 
 target_link_libraries(csp_kafka_adapter PUBLIC csp_adapter_utils RdKafka::rdkafka RdKafka::rdkafka++)
 
+# Link Avro library if available
+if(CSP_HAVE_AVRO)
+  target_compile_definitions(csp_kafka_adapter PUBLIC CSP_HAVE_AVRO)
+  if(CSP_USE_VCPKG)
+    target_link_libraries(csp_kafka_adapter PUBLIC unofficial::avro-cpp::avrocpp)
+  else()
+    target_link_libraries(csp_kafka_adapter PUBLIC Avro::avrocpp)
+  endif()
+endif()
+
 install(TARGETS csp_kafka_adapter
         PUBLIC_HEADER DESTINATION include/csp/adapters/kafka
         RUNTIME DESTINATION ${CSP_RUNTIME_INSTALL_SUBDIR}

cpp/csp/adapters/kafka/KafkaPublisher.cpp

@@ -3,6 +3,9 @@
 #include <csp/adapters/kafka/KafkaPublisher.h>
 #include <csp/adapters/utils/MessageWriter.h>
 #include <csp/adapters/utils/JSONMessageWriter.h>
+#ifdef CSP_HAVE_AVRO
+#include <csp/adapters/utils/AvroMessageWriter.h>
+#endif
 
 #include <librdkafka/rdkafkacpp.h>
 
@@ -17,6 +20,10 @@ KafkaPublisher::KafkaPublisher( KafkaAdapterManager * mgr, const Dictionary & pr
     auto protocol = properties.get<std::string>( "protocol" );
     if( protocol == "JSON" )
         m_msgWriter = std::make_shared<utils::JSONMessageWriter>( properties );
+#ifdef CSP_HAVE_AVRO
+    else if( protocol == "AVRO" )
+        m_msgWriter = std::make_shared<utils::AvroMessageWriter>( properties );
+#endif
     else if( protocol != "RAW_BYTES" )
         CSP_THROW( NotImplemented, "msg protocol " << protocol << " not currently supported for kafka output adapters" );
 }