MINIFICPP-2859 Extend Conan usage and integrate it in Windows and MacOS CI jobs

.github/conan/compatibility.py

@@ -0,0 +1,95 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from conan.tools.build import supported_cppstd, supported_cstd
+
+
+def cppstd_compat(conanfile):
+    # It will try to find packages with all the cppstd versions
+    extension_properties = getattr(conanfile, "extension_properties", {})
+    compiler = conanfile.settings.get_safe("compiler")
+    compiler_version = conanfile.settings.get_safe("compiler.version")
+    cppstd = conanfile.settings.get_safe("compiler.cppstd")
+    if not compiler or not compiler_version:
+        return []
+    factors = []  # List of list, each sublist is a potential combination
+    if cppstd is not None and extension_properties.get("compatibility_cppstd") is not False:
+        cppstd_possible_values = supported_cppstd(conanfile)
+        if cppstd_possible_values is None:
+            conanfile.output.warning(f'No cppstd compatibility defined for compiler "{compiler}"')
+        else:  # The current cppst must be included in case there is other factor
+            factors.append([{"compiler.cppstd": v} for v in cppstd_possible_values])
+
+    cstd = conanfile.settings.get_safe("compiler.cstd")
+    if cstd is not None and extension_properties.get("compatibility_cstd") is not False:
+        cstd_possible_values = supported_cstd(conanfile)
+        if cstd_possible_values is None:
+            conanfile.output.warning(f'No cstd compatibility defined for compiler "{compiler}"')
+        else:
+            factors.append([{"compiler.cstd": v} for v in cstd_possible_values if v != cstd])
+    return factors
+
+
+def compatibility(conanfile):
+    # By default, different compiler.cppstd are compatible
+    # factors is a list of lists
+    factors = cppstd_compat(conanfile)
+
+    # MSVC fallback compatibility
+    compiler = conanfile.settings.get_safe("compiler")
+    compiler_version = conanfile.settings.get_safe("compiler.version")
+    if compiler == "msvc":
+        msvc_fallbacks = {
+            "195": ["194", "193"],
+            "194": ["193"],
+        }.get(compiler_version, [])
+
+        if msvc_fallbacks:
+            factors.append([{"compiler.version": v} for v in msvc_fallbacks])
+
+    # macOS / apple-clang: accept any compiler.version >= 13 as compatible
+    os_ = conanfile.settings.get_safe("os")
+    if os_ == "Macos" and compiler == "apple-clang" and compiler_version is not None:
+        try:
+            current_version = int(str(compiler_version).split(".")[0])
+        except ValueError:
+            current_version = None
+        if current_version is not None:
+            min_version = 13
+            max_version = 21  # support up to Xcode 27 with apple-clang 21 for now
+            candidate_versions = [
+                str(v) for v in range(min_version, max_version + 1)
+                if v != current_version
+            ]
+            factors.append([{"compiler.version": v} for v in candidate_versions])
+
+    combinations = _factors_combinations(factors)
+    return [{"settings": [(k, v) for k, v in comb.items()]} for comb in combinations]
+
+
+def _factors_combinations(factors):
+    combinations = []
+    for factor in factors:
+        if not combinations:
+            combinations = list(factor)
+            continue
+        new_combinations = []
+        for comb in combinations:
+            for f in factor:
+                new_comb = comb.copy()
+                new_comb.update(f)
+                new_combinations.append(new_comb)
+        combinations.extend(new_combinations)
+    return combinations

.github/workflows/ci.yml

@@ -13,6 +13,8 @@ env:
     -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
     -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON
   CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
+  CONAN_LOGIN_USERNAME_NIFI_CONAN: ${{ secrets.CONAN_USERNAME }}
+  CONAN_PASSWORD_NIFI_CONAN: ${{ secrets.CONAN_ACCESS_TOKEN }}
 jobs:
   macos_xcode:
     name: "macOS 26 aarch64"
@@ -63,6 +65,7 @@ jobs:
         -DSKIP_TESTS=OFF
         -DUSE_SHARED_LIBS=ON
         -DMINIFI_PERFORMANCE_TESTS=ON
+        -DUSE_CONAN=ON
     steps:
       - id: checkout
         uses: actions/checkout@v6
@@ -84,9 +87,17 @@ jobs:
           echo "PATH=/opt/homebrew/opt/ccache:/opt/homebrew/opt/ccache/bin:/opt/homebrew/opt/ccache/libexec:$PATH" >> $GITHUB_ENV
           echo "DYLD_LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV
           echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
+          mkdir -p ~/.conan2/extensions/plugins/compatibility
+          cp .github/conan/compatibility.py ~/.conan2/extensions/plugins/compatibility/compatibility.py
       - name: build
         run: |
-          python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && python main.py --noninteractive --skip-compiler-install --cmake-options="-DCMAKE_C_FLAGS=${CPPFLAGS} ${CFLAGS} -DCMAKE_CXX_FLAGS=${CPPFLAGS} ${CXXFLAGS}" --minifi-options="${MACOS_MINIFI_OPTIONS}"
+          python -m venv venv && source venv/bin/activate && pip install -r requirements.txt && \
+          python main.py --noninteractive --skip-compiler-install --cmake-options="-DCMAKE_C_FLAGS=${CPPFLAGS} ${CFLAGS} -DCMAKE_CXX_FLAGS=${CPPFLAGS} ${CXXFLAGS}" --minifi-options="${MACOS_MINIFI_OPTIONS}"
+        working-directory: bootstrap
+      - name: Upload conan packages
+        if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          source venv/bin/activate && conan remote login nifi-conan && conan upload "*" -r nifi-conan --confirm
         working-directory: bootstrap
       - name: cache save
         uses: actions/cache/save@v5
@@ -171,6 +182,7 @@ jobs:
         -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=ON
         -DUSE_SHARED_LIBS=OFF
         -DMINIFI_PERFORMANCE_TESTS=ON
+        -DUSE_CONAN=ON
     steps:
       - name: Support longpaths
         run: git config --system core.longpaths true
@@ -192,14 +204,27 @@ jobs:
           if ((Get-FileHash 'sqliteodbc_w64.exe').Hash -ne "a4804e4f54f42c721df1323c5fcac101a8c7a577e7f20979227324ceab572d51") {Write "Hash mismatch"; Exit 1}
           Start-Process -FilePath ".\sqliteodbc_w64.exe" -ArgumentList "/S" -Wait
         shell: powershell
+      - name: Copy conan compatibility.py
+        shell: powershell
+        run: |
+          $compatDir = Join-Path $env:USERPROFILE ".conan2\extensions\plugins\compatibility"
+          New-Item -ItemType Directory -Force -Path $compatDir | Out-Null
+          Copy-Item ".github\conan\compatibility.py" (Join-Path $compatDir "compatibility.py")
       - name: Zero sccache stats
         run: sccache --zero-stats
         shell: bash
       - name: build
+        working-directory: bootstrap
         run: |
-          python -m venv venv & venv\Scripts\activate & pip install -r requirements.txt & python main.py --noninteractive --skip-compiler-install --minifi-options="%WINDOWS_MINIFI_OPTIONS%" --cmake-options="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
+          python -m venv venv && venv\Scripts\activate && pip install -r requirements.txt && ^
+          python main.py --noninteractive --skip-compiler-install --minifi-options="%WINDOWS_MINIFI_OPTIONS%" --cmake-options="-DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache"
         shell: cmd
+      - name: Upload conan packages
+        if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
         working-directory: bootstrap
+        run: |
+          venv\Scripts\activate && conan remote login nifi-conan && conan upload "*" -r nifi-conan --confirm
+        shell: cmd
       - name: Save cache
         uses: actions/cache/save@v5
         if: always()

CMakeLists.txt

@@ -87,8 +87,12 @@ endif()
 # Use ccache if present
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
-    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+    if(NOT CMAKE_C_COMPILER_LAUNCHER)
+        set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_FOUND}")
+    endif()
+    if(NOT CMAKE_CXX_COMPILER_LAUNCHER)
+        set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_FOUND}")
+    endif()
     message("-- Found ccache: ${CCACHE_FOUND}")
 endif(CCACHE_FOUND)
 
@@ -197,6 +201,7 @@ set(PASSTHROUGH_CMAKE_ARGS -DANDROID_ABI=${ANDROID_ABI}
     -DANDROID_NDK=${ANDROID_NDK}
     -DCMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
     -DCMAKE_POLICY_VERSION_MINIMUM=${CMAKE_POLICY_VERSION_MINIMUM}
+    -DCMAKE_POLICY_DEFAULT_CMP0091=NEW
     -DCMAKE_C_FLAGS=${PASSTHROUGH_CMAKE_C_FLAGS}
     -DCMAKE_CXX_FLAGS=${PASSTHROUGH_CMAKE_CXX_FLAGS}
     -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
@@ -221,12 +226,12 @@ set(MINIFI_CPP_COMPILE_DEFINITIONS "")
 
 if(CUSTOM_MALLOC)
     if (CUSTOM_MALLOC STREQUAL jemalloc)
-        include(BundledJemalloc)
-        use_bundled_jemalloc(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-        set(CUSTOM_MALLOC_LIB JeMalloc::JeMalloc)
+        include(GetJemalloc)
+        get_jemalloc(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+        set(CUSTOM_MALLOC_LIB jemalloc::jemalloc)
     elseif (CUSTOM_MALLOC STREQUAL mimalloc)
-        include(MiMalloc)
-        set(CUSTOM_MALLOC_LIB mimalloc)
+        include(GetMiMalloc)
+        set(CUSTOM_MALLOC_LIB mimalloc-static)
     elseif (CUSTOM_MALLOC STREQUAL rpmalloc)
         include(RpMalloc)
         set(CUSTOM_MALLOC_LIB rpmalloc)
@@ -259,8 +264,8 @@ if(NOT WIN32)
 endif()
 
 # libsodium
-include(BundledLibSodium)
-use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
+include(GetLibSodium)
+get_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
 
 list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS SODIUM_STATIC=1)
 list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS "RPM_CONFIG_DIR=\"/${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME}\"")
@@ -283,7 +288,7 @@ include(GetSpdlog)
 get_spdlog()
 
 # yaml-cpp
-include(YamlCpp)
+include(GetYamlCpp)
 
 # concurrentqueue
 add_library(concurrentqueue INTERFACE)
@@ -295,7 +300,7 @@ target_include_directories(RapidJSON SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DI
 target_compile_definitions(RapidJSON INTERFACE RAPIDJSON_HAS_STDSTRING)
 
 # gsl-lite
-include(GslLite)
+include(GetGslLite)
 
 # Add necessary definitions based on the value of STRICT_GSL_CHECKS, see gsl-lite README for more details
 list(APPEND GslDefinitions gsl_CONFIG_DEFAULTS_VERSION=1)
@@ -314,13 +319,17 @@ endif()
 if (STRICT_GSL_CHECKS STREQUAL "DEBUG_ONLY")
     list(APPEND GslDefinitions $<$<NOT:$<CONFIG:Debug>>:${GslDefinitionsNonStrict}>)
 endif()
-target_compile_definitions(gsl-lite INTERFACE ${GslDefinitions})
 
-# date
-include(Date)
+get_target_property(_gsl_lite_real_target gsl-lite::gsl-lite ALIASED_TARGET)
+if (NOT _gsl_lite_real_target)
+    set(_gsl_lite_real_target gsl-lite::gsl-lite)
+endif()
+target_compile_definitions(${_gsl_lite_real_target} INTERFACE ${GslDefinitions})
 
-# magic_enum
-include(MagicEnum)
+include(Date)
+include(GetMagicEnum)
+include(GetRangeV3)
+include(GetAsio)
 
 # Setup warning flags
 if(MSVC)
@@ -371,7 +380,7 @@ endif()
 ## Add extensions
 
 # PugiXML required for standard processors and WEL extension
-include(PugiXml)
+include(GetPugiXml)
 
 file(GLOB extension-directories "extensions/*")
 foreach(extension-dir ${extension-directories})

CONAN.md

@@ -36,7 +36,11 @@ source, it uses CMake to install the third party external libraries that are ena
 
 ## Build MiNiFi C++ with Conan
 
-### Install Conan
+### Using the bootstrap python script
+
+The easiest way to build MiNiFi C++ using Conan is by using the the bootstrap script. Run the bootstrap script either with `bootstrap/py_bootstrap.sh` or `bootstrap/py_bootstrap.bat` and the bootstrap installs the conan package manager automatically. In the bootstrap script under Build options menu select the `USE_CONAN` option. Running the one click build with the option selected, it will download the available thirdparty dependencies from the conan repository, otherwise it will build the thirdparty dependency from source (either with the available conan recipe or using the bundled cmake script).
+
+### Install Conan manually
 
 ~~~bash
 sudo pip install --force-reinstall -v "conan==2.28.1"
@@ -45,15 +49,29 @@ sudo pip install --force-reinstall -v "conan==2.28.1"
 conan profile detect
 ~~~
 
-### Create Custom RocksDB Conan Package
-The default RocksDB conan package is built with -fno-rtti, which makes it incompatible with MiNiFi. So we need to create a custom RocksDB conan package that is built with -frtti.
+### Create Custom Conan Package
+Some packages need custom versions built for MiNiFi specifically. This could be due patched source code, different compile options, or different dependency library versions.
+
+For example the default RocksDB conan package is built with -fno-rtti, which makes it incompatible with MiNiFi. So we need to create a custom RocksDB conan package that is built with -frtti.
 
 ~~~bash
 cd $HOME/nifi-minifi-cpp/thirdparty/rocksdb/all
 
 conan create . --user=minifi --channel=develop --test-folder="" --version=11.1.1 --profile=../../../etc/conan/profiles/release-linux
 ~~~
 
+If you want to use conan to build these custom libraries as part of the MiNiFi build you only need to export these libraries instead of creating them.
+
+~~~bash
+conan export $HOME/nifi-minifi-cpp/thirdparty/rocksdb/all --version=11.1.1 --user=minifi --channel=develop
+~~~
+
+You can add the official NiFi remote Conan repository to download the custom Conan packages and recipes manually:
+
+~~~bash
+conan remote add nifi-conan https://apache.jfrog.io/artifactory/api/conan/nifi-conan
+~~~
+
 ### Build MiNiFi
 
 ~~~bash
@@ -68,6 +86,12 @@ conan install . --build=missing --output-folder=build_conan --profile=etc/conan/
 conan build . --output-folder=build_conan --profile=etc/conan/profiles/release-linux
 ~~~
 
+After `conan install` command you can also use the conan generated `conan_toolchain.cmake` file with the CMake command to build with Conan packages:
+
+~~~bash
+cmake -DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake .. && ninja
+~~~
+
 - **NOTE**: After building MiNiFi, we must have the MINIFI_HOME environment variable created in order to successfully run the minifi binary executable.
 
 - **NOTE**: When we install the prebuilt conan package representing the MiNiFi third party libraries, we add the `--build=missing` in case some of the prebuilt missing conan packages are not found from conancenter, jfrog, or one of our conan repositories, then conan will build the conan packages from source. We can upload new prebuilt conan packages by **[conan upload](https://docs.conan.io/2/reference/commands/upload.html)**.

LICENSE

@@ -205,7 +205,6 @@ APACHE NIFI - MINIFI C++ SUBCOMPONENTS:
 
 The Apache NiFi - MiNiFi C++ project contains subcomponents licensed under the Apache License, Version 2.0:
 This product bundles 'Simple-Windows-Posix-Semaphore' which is available under an ALv2 license
-This project bundles 'mbedTLS' which is available under an ALv2 license
 This project bundles 'RocksDB' which is available under an ALv2 license
 This project bundles 'AWS SDK for C++' which is available under an ALv2 license
 This project bundles 'C++ Client Libraries for Google Cloud Services' which is available under an ALv2 license
@@ -586,40 +585,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
-This product bundles 'libuvc' which is available under a BSD license.
-
-Software License Agreement (BSD License)
-
-Copyright (C) 2010-2015 Ken Tossell
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
- * Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-   copyright notice, this list of conditions and the following
-   disclaimer in the documentation and/or other materials provided
-   with the distribution.
- * Neither the name of the author nor other contributors may be
-   used to endorse or promote products derived from this software
-   without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
-ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
 This product bundles 'JsonCpp' which is available under a MIT license.
 
 The JsonCpp library's source code, including accompanying documentation,