diff --git a/.dockerignore b/.dockerignore index 6a80bd238a..ecb09e3fc3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,6 +39,7 @@ logs **/cmake-build-* docker/* !docker/conf +!docker/settings_user.yml target *.pyc *.swp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df87f9a4b6..c6a5426d53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,13 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} env: - DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ - -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ - -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache CONAN_LOGIN_USERNAME_NIFI_CONAN: ${{ secrets.CONAN_USERNAME }} CONAN_PASSWORD_NIFI_CONAN: ${{ secrets.CONAN_ACCESS_TOKEN }} + DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ + -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ + -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DENABLE_LLAMACPP=ON -DDOCKER_BUILD_ONLY=ON -DMINIFI_PERFORMANCE_TESTS=ON -DDOCKER_USE_CONAN=ON \ + -DDOCKER_NIFI_CONAN_USER=${{ secrets.CONAN_USERNAME }} -DDOCKER_NIFI_CONAN_PASSWORD=${{ secrets.CONAN_ACCESS_TOKEN }} jobs: macos_xcode: name: "macOS 26 aarch64" @@ -294,6 +295,7 @@ jobs: -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF -DUSE_SHARED_LIBS=ON -DMINIFI_PERFORMANCE_TESTS=ON + -DUSE_CONAN=ON steps: - id: checkout uses: actions/checkout@v6 @@ -334,6 +336,11 @@ jobs: && pip install -r requirements.txt \ && python main.py --noninteractive --skip-compiler-install --cmake-options="-DSTRICT_GSL_CHECKS=AUDIT -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" --minifi-options="${UBUNTU_CLANG_MINIFI_OPTIONS}" working-directory: bootstrap + - name: Upload conan packages + if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + source venv/bin/activate && conan remote login nifi-conan && conan upload "*" -r nifi-conan --confirm + working-directory: bootstrap - id: cache_save uses: actions/cache/save@v5 if: always() @@ -443,7 +450,8 @@ jobs: mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DMINIFI_FAIL_ON_WARNINGS=OFF -DENABLE_AWS=ON -DENABLE_AZURE=ON \ -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_KAFKA=ON -DENABLE_MQTT=ON -DENABLE_OPC=ON -DENABLE_OPS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \ - -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_LLAMACPP=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky-test + -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_LLAMACPP=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} \ + -DDOCKER_USE_CONAN=ON -DDOCKER_NIFI_CONAN_USER=${{ env.CONAN_LOGIN_USERNAME_NIFI_CONAN }} -DDOCKER_NIFI_CONAN_PASSWORD=${{ env.CONAN_PASSWORD_NIFI_CONAN }} .. && make rocky-test - name: cache save uses: actions/cache/save@v5 if: always() diff --git a/bootstrap/cli.py b/bootstrap/cli.py index c014630425..de3e30124e 100644 --- a/bootstrap/cli.py +++ b/bootstrap/cli.py @@ -76,6 +76,8 @@ def run_conan_install(minifi_options: MinifiOptions, package_manager: PackageMan return False compiler_settings = " -s:a compiler.cppstd=23" + if "-stdlib=libc++" in os.environ.get("CXXFLAGS", ""): + compiler_settings += " -s:a compiler.libcxx=libc++" generator_setting = " -c tools.cmake.cmaketoolchain:generator=Ninja" if minifi_options.use_ninja.value == "ON" else "" conan_remote_add_cmd = "conan remote add nifi-conan https://apache.jfrog.io/artifactory/api/conan/nifi-conan --force" if not package_manager.run_cmd(conan_remote_add_cmd): @@ -83,6 +85,9 @@ def run_conan_install(minifi_options: MinifiOptions, package_manager: PackageMan return False build_cmd = f"conan install {minifi_options.source_dir} --output-folder={minifi_options.build_dir} --build=missing {conan_options} " \ f"--settings=build_type={minifi_options.build_type.value}{generator_setting}{compiler_settings}" + if platform.system() == "Linux": + # Conan's prebuilt m4 binary fails autom4te's version probe in some Linux build containers (e.g. RockyLinux 8), so always build m4 from source on Linux. + build_cmd += " --build=m4/*" res = package_manager.run_cmd(build_cmd) print("Conan install was successful" if res else "Conan install was unsuccessful") return res @@ -138,6 +143,11 @@ def do_one_click_configuration(minifi_options: MinifiOptions, package_manager: P return True +def do_one_click_conan_install(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: + assert run_conan_install(minifi_options, package_manager) + return True + + def main_menu(minifi_options: MinifiOptions, package_manager: PackageManager): done = False while not done: diff --git a/bootstrap/main.py b/bootstrap/main.py index af9d1eaf53..53be05dfab 100644 --- a/bootstrap/main.py +++ b/bootstrap/main.py @@ -17,7 +17,7 @@ import argparse import pathlib -from cli import main_menu, do_one_click_build, do_one_click_configuration +from cli import main_menu, do_one_click_build, do_one_click_configuration, do_one_click_conan_install from minifi_option import parse_minifi_options from package_manager import get_package_manager @@ -35,6 +35,8 @@ help="Initiates the one click build") parser.add_argument('--run-configuration', action="store_true", default=False, help="Runs configuration") + parser.add_argument('--run-conan-install', action="store_true", default=False, + help="Runs conan install") args = parser.parse_args() no_confirm = args.noconfirm or args.noninteractive @@ -62,5 +64,8 @@ do_one_click_build(minifi_options, package_manager) elif args.run_configuration: do_one_click_configuration(minifi_options, package_manager) + elif args.run_conan_install: + minifi_options.build_options["USE_CONAN"].value = "ON" + do_one_click_conan_install(minifi_options, package_manager) else: main_menu(minifi_options, package_manager) diff --git a/bootstrap/minifi_option.py b/bootstrap/minifi_option.py index ebcdb8e22f..32168d2b74 100644 --- a/bootstrap/minifi_option.py +++ b/bootstrap/minifi_option.py @@ -31,7 +31,7 @@ def __init__(self, cache_values: Dict[str, CMakeCacheValue]): self.build_type = CMakeCacheValue("Specifies the build type on single-configuration generators", "CMAKE_BUILD_TYPE", "STRING", "Release") self.build_type.possible_values = ["Release", "Debug", "RelWithDebInfo", "MinSizeRel"] - additional_build_options = ["DOCKER_BUILD_ONLY", "DOCKER_SKIP_TESTS", "DOCKER_CREATE_RPM", "SKIP_TESTS", "PORTABLE"] + additional_build_options = ["DOCKER_BUILD_ONLY", "DOCKER_USE_CONAN", "DOCKER_SKIP_TESTS", "DOCKER_CREATE_RPM", "SKIP_TESTS", "PORTABLE"] self.use_ninja = CMakeCacheValue("Specifies if CMake should use the Ninja generator or the system default", "USE_NINJA", "BOOL", "ON") self.use_conan = CMakeCacheValue("Specifies if CMake should use Conan package manager", "USE_CONAN", "BOOL", "OFF") if "USE_NINJA" in cache_values: diff --git a/bootstrap/package_manager.py b/bootstrap/package_manager.py index 91f3586f24..04a6794cef 100644 --- a/bootstrap/package_manager.py +++ b/bootstrap/package_manager.py @@ -204,6 +204,25 @@ def install_compiler(self) -> str: return "" +class ApkPackageManager(PackageManager): + def __init__(self, no_confirm): + PackageManager.__init__(self, no_confirm) + + def install(self, dependencies: Dict[str, Set[str]]) -> bool: + return self._install(dependencies=dependencies, + install_cmd="sudo apk add --no-cache", + replace_dict={"libarchive": {"libarchive-dev"}, + "python": {"python3-dev"}}) + + def _get_installed_packages(self) -> Set[str]: + result = subprocess.run(['apk', 'info'], text=True, capture_output=True, check=True) + return set(result.stdout.splitlines()) + + def install_compiler(self) -> str: + self.install({"compiler": {"g++"}}) + return "" + + def _get_vs_dev_cmd_path(vs_where_location: VsWhereLocation): if vs_where_location == VsWhereLocation.CHOCO: vs_where_path = "vswhere" @@ -326,6 +345,8 @@ def get_package_manager(no_confirm: bool) -> PackageManager: return DnfPackageManager(no_confirm, False) elif "opensuse" in distro_id: return ZypperPackageManager(no_confirm) + elif "alpine" in distro_id: + return ApkPackageManager(no_confirm) else: sys.exit(f"Unsupported platform {distro_id} exiting") elif platform_system == "Windows": diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 856832617f..e54da6750b 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -36,6 +36,9 @@ add_custom_target( -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS} -c DOCKER_PUSH=${DOCKER_PUSH} -c DOCKER_TAGS=${DOCKER_TAGS} + -c DOCKER_USE_CONAN=${DOCKER_USE_CONAN} + -c DOCKER_NIFI_CONAN_USER=${DOCKER_NIFI_CONAN_USER} + -c DOCKER_NIFI_CONAN_PASSWORD=${DOCKER_NIFI_CONAN_PASSWORD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) # Create minimal docker image @@ -80,26 +83,11 @@ add_custom_target( -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS} -c DOCKER_PUSH=${DOCKER_PUSH} -c DOCKER_TAGS=${DOCKER_TAGS} + -c DOCKER_USE_CONAN=${DOCKER_USE_CONAN} + -c DOCKER_NIFI_CONAN_USER=${DOCKER_NIFI_CONAN_USER} + -c DOCKER_NIFI_CONAN_PASSWORD=${DOCKER_NIFI_CONAN_PASSWORD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) -add_custom_target( - centos - COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh - -u 1000 - -g 1000 - -v ${PROJECT_VERSION_STR} - -o ${MINIFI_DOCKER_OPTIONS_STR} - -l ${CMAKE_BINARY_DIR} - -d centos - -c BUILD_NUMBER=${BUILD_NUMBER} - -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION} - -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS} - -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS} - -c DOCKER_PUSH=${DOCKER_PUSH} - -c DOCKER_TAGS=${DOCKER_TAGS} - -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( rocky-test @@ -117,6 +105,9 @@ add_custom_target( -c DOCKER_PUSH=${DOCKER_PUSH} -c DOCKER_TAGS=${DOCKER_TAGS} -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} + -c DOCKER_USE_CONAN=${DOCKER_USE_CONAN} + -c DOCKER_NIFI_CONAN_USER=${DOCKER_NIFI_CONAN_USER} + -c DOCKER_NIFI_CONAN_PASSWORD=${DOCKER_NIFI_CONAN_PASSWORD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) @@ -138,6 +129,9 @@ add_custom_target( -c DOCKER_PUSH=${DOCKER_PUSH} -c DOCKER_TAGS=${DOCKER_TAGS} -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} + -c DOCKER_USE_CONAN=${DOCKER_USE_CONAN} + -c DOCKER_NIFI_CONAN_USER=${DOCKER_NIFI_CONAN_USER} + -c DOCKER_NIFI_CONAN_PASSWORD=${DOCKER_NIFI_CONAN_PASSWORD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) set(ENABLED_TAGS "CORE") diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 8094945b44..2601865921 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -46,6 +46,9 @@ add_minifi_option(CI_BUILD "Build is used for CI." OFF) add_minifi_option(SKIP_TESTS "Skips building all tests." OFF) add_minifi_option(MINIFI_PERFORMANCE_TESTS "Build performance tests" OFF) add_minifi_option(DOCKER_BUILD_ONLY "Disables all targets except docker build scripts. Ideal for systems without an up-to-date compiler." OFF) +add_minifi_option(DOCKER_USE_CONAN "Use Conan package manager in docker build." OFF) +add_minifi_option(DOCKER_NIFI_CONAN_USER "Username for the NiFi Conan repository." "") +add_minifi_option(DOCKER_NIFI_CONAN_PASSWORD "Password for the NiFi Conan repository." "") add_minifi_option(DOCKER_SKIP_TESTS "Skip building tests in docker image targets." ON) add_minifi_option(DOCKER_CREATE_RPM "Create an RPM package in the rocky docker build target." ON) add_minifi_option(DOCKER_PUSH "Push created images to the specified tags" OFF) @@ -194,7 +197,11 @@ add_minifi_multi_option(MINIFI_AWS_SDK_CPP_SOURCE "Retrieves AWS SDK for C++ fro function(get_minifi_docker_options RET_VALUE) set(MINIFI_DOCKER_OPTIONS_STR ${MINIFI_EXTERNAL_DOCKER_OPTIONS_STR}) foreach(MINIFI_OPTION ${MINIFI_OPTIONS}) - if (MINIFI_OPTION STREQUAL "SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_BUILD_ONLY" OR MINIFI_OPTION STREQUAL "DOCKER_SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_PUSH") + if (MINIFI_OPTION STREQUAL "SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_BUILD_ONLY" OR MINIFI_OPTION STREQUAL "DOCKER_USE_CONAN" OR MINIFI_OPTION STREQUAL "DOCKER_NIFI_CONAN_USER" OR + MINIFI_OPTION STREQUAL "DOCKER_NIFI_CONAN_PASSWORD" OR MINIFI_OPTION STREQUAL "DOCKER_SKIP_TESTS" OR MINIFI_OPTION STREQUAL "DOCKER_PUSH") + continue() + endif() + if (MINIFI_OPTION MATCHES "^MINIFI_.*_SOURCE$") continue() endif() set(MINIFI_DOCKER_OPTIONS_STR "${MINIFI_DOCKER_OPTIONS_STR} -D${MINIFI_OPTION}=${${MINIFI_OPTION}}") diff --git a/cmake/PahoMqttC.cmake b/cmake/PahoMqttC.cmake index 120a836bee..8422b75c27 100644 --- a/cmake/PahoMqttC.cmake +++ b/cmake/PahoMqttC.cmake @@ -24,7 +24,7 @@ set(PAHO_ENABLE_TESTING OFF CACHE BOOL "" FORCE) set(PAHO_WITH_SSL ON CACHE BOOL "" FORCE) set(PAHO_HIGH_PERFORMANCE ON CACHE BOOL "" FORCE) -set(PATCH_FILE_1 "${CMAKE_SOURCE_DIR}/thirdparty/paho-mqtt/cmake-openssl.patch") +set(PATCH_FILE_1 "${CMAKE_SOURCE_DIR}/thirdparty/paho-mqtt/all/patches/cmake-openssl.patch") set(PC ${Bash_EXECUTABLE} -c "set -x &&\ (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE_1}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE_1}\\\")") diff --git a/conanfile.py b/conanfile.py index d688e479fb..e6052d5693 100644 --- a/conanfile.py +++ b/conanfile.py @@ -76,7 +76,7 @@ def requirements(self): if self.options.enable_all or self.options.enable_prometheus: self.requires("prometheus-cpp/1.3.0") if self.options.enable_all or self.options.enable_mqtt: - self.requires("paho-mqtt-c/1.3.16") + self.requires("paho-mqtt-c/1.3.16@minifi/develop") if self.options.enable_all or self.options.enable_couchbase: self.requires("couchbase_cxx_client/1.3.1@minifi/develop") self.requires("ms-gsl/4.0.0") @@ -141,10 +141,15 @@ def configure(self): setattr(self.options["aws-sdk-cpp"], "text-to-speech", False) if self.options.enable_all or self.options.get_safe("enable_llamacpp"): self.options["llama-cpp"].portable = self.options.portable + if self.options.enable_all or self.options.get_safe("enable_mqtt"): + self.options["paho-mqtt-c"].high_performance = True def generate(self): tc = CMakeToolchain(self) tc.variables["MINIFI_DEFAULT_DEPENDENCY_SOURCE"] = "CONAN" + + # Drop hardcoded install dir from Conan's generated toolchain to detect the correct install dir for RPM package installation + tc.blocks.remove("output_dirs") tc.generate() def build(self): diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh index ea038bfbb6..c17a606341 100755 --- a/docker/DockerBuild.sh +++ b/docker/DockerBuild.sh @@ -30,6 +30,9 @@ DISTRO_NAME= BUILD_NUMBER= DOCKER_CCACHE_DUMP_LOCATION= DOCKER_SKIP_TESTS=ON +DOCKER_USE_CONAN=OFF +DOCKER_NIFI_CONAN_USER= +DOCKER_NIFI_CONAN_PASSWORD= DOCKER_CREATE_RPM=ON CMAKE_BUILD_TYPE=Release PLATFORMS= @@ -44,7 +47,7 @@ function usage { echo "-p, --prefix Additional prefix added to the image tag" echo "-u, --uid User id to be used in the Docker image (default: 1000)" echo "-g, --gid Group id to be used in the Docker image (default: 1000)" - echo "-d, --distro-name Linux distribution build to be used for alternative builds (centos|rockylinux)" + echo "-d, --distro-name Linux distribution build to be used for alternative builds (rockylinux)" echo "-l --dump-location Path where to the output dump to be put" echo "-c --cmake-param CMake parameter passed in PARAM=value format" echo "-o --options Minifi options string" @@ -108,6 +111,12 @@ while [[ $# -gt 0 ]]; do DOCKER_SKIP_TESTS="${ARR[1]}" elif [ "${ARR[0]}" == "DOCKER_CREATE_RPM" ]; then DOCKER_CREATE_RPM="${ARR[1]}" + elif [ "${ARR[0]}" == "DOCKER_USE_CONAN" ]; then + DOCKER_USE_CONAN="${ARR[1]}" + elif [ "${ARR[0]}" == "DOCKER_NIFI_CONAN_USER" ]; then + DOCKER_NIFI_CONAN_USER="${ARR[1]}" + elif [ "${ARR[0]}" == "DOCKER_NIFI_CONAN_PASSWORD" ]; then + DOCKER_NIFI_CONAN_PASSWORD="${ARR[1]}" elif [ "${ARR[0]}" == "CMAKE_BUILD_TYPE" ]; then CMAKE_BUILD_TYPE="${ARR[1]}" elif [ "${ARR[0]}" == "DOCKER_PLATFORMS" ] && [ -n "${ARR[1]}" ]; then @@ -202,7 +211,13 @@ BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}" "--build-arg" "DISTRO_NAME=${DISTRO_NAME}" "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}" "--build-arg" "DOCKER_CREATE_RPM=${DOCKER_CREATE_RPM}" + "--build-arg" "DOCKER_USE_CONAN=${DOCKER_USE_CONAN}" + "--build-arg" "DOCKER_NIFI_CONAN_USER=${DOCKER_NIFI_CONAN_USER}" "--build-arg" "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") +if [ -n "${DOCKER_NIFI_CONAN_PASSWORD}" ]; then + export DOCKER_NIFI_CONAN_PASSWORD + BUILD_ARGS+=("--secret" "id=nifi_conan_password,env=DOCKER_NIFI_CONAN_PASSWORD") +fi if [ -n "${DUMP_LOCATION}" ]; then BUILD_ARGS+=("--build-arg" "DOCKER_MAKE_TARGET=package") fi @@ -223,6 +238,7 @@ if [ -n "${DISTRO_NAME}" ]; then fi else if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then + echo docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" --target build -t minifi_build .. docker buildx build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" --target build -t minifi_build .. dump_ccache "minifi_build" "${DOCKER_CCACHE_DUMP_LOCATION}" fi diff --git a/docker/Dockerfile b/docker/Dockerfile index b050fa8a63..94a34e73ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,6 +31,8 @@ ARG GID=1000 ARG MINIFI_OPTIONS="" ARG CMAKE_BUILD_TYPE=Release ARG DOCKER_SKIP_TESTS=ON +ARG DOCKER_USE_CONAN=OFF +ARG DOCKER_NIFI_CONAN_USER= # Install the system dependencies needed for a build RUN apk --no-cache add gcc \ @@ -50,7 +52,11 @@ RUN apk --no-cache add gcc \ patch \ python3-dev \ doxygen \ - ccache + ccache \ + python3 \ + py3-pip \ + m4 \ + pkgconf ENV USER=minificpp ENV MINIFI_BASE_DIR=/opt/minifi @@ -64,11 +70,28 @@ COPY --chown=${USER}:${USER} . ${MINIFI_BASE_DIR} USER ${USER} -RUN mkdir ${MINIFI_BASE_DIR}/build +RUN --mount=type=secret,id=nifi_conan_password,mode=0444 if [ "${DOCKER_USE_CONAN}" == "ON" ]; then \ + set -e; \ + cd ${MINIFI_BASE_DIR}/bootstrap && \ + python3 -m venv venv && \ + source venv/bin/activate && \ + pip install -r requirements.txt && \ + conan profile detect --exist-ok && \ + cp ${MINIFI_BASE_DIR}/docker/settings_user.yml "$(conan config home)/settings_user.yml" && \ + printf '\nos.libc=musl\n' >> "$(conan config home)/profiles/default" && \ + python3 main.py --skip-compiler-install --run-conan-install --minifi-options="${MINIFI_OPTIONS}" && \ + if [ -n "${DOCKER_NIFI_CONAN_USER}" ] && [ -f /run/secrets/nifi_conan_password ]; then \ + conan remote login nifi-conan ${DOCKER_NIFI_CONAN_USER} -p "$(cat /run/secrets/nifi_conan_password)" || true; \ + conan upload "*" -r nifi-conan --confirm || true; \ + fi; \ + fi + +RUN mkdir -p ${MINIFI_BASE_DIR}/build WORKDIR ${MINIFI_BASE_DIR}/build RUN export PATH=/usr/lib64/ccache/bin${PATH:+:${PATH}} && \ export CCACHE_DIR=${MINIFI_BASE_DIR}/.ccache && \ - cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ + if [ "${DOCKER_USE_CONAN}" == "ON" ]; then CONAN_TOOLCHAIN="-DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake"; fi && \ + cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${CONAN_TOOLCHAIN:-} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ make -j "$(nproc)" package && \ tar -xzvf "${MINIFI_BASE_DIR}/build/nifi-minifi-cpp-${MINIFI_VERSION}.tar.gz" -C "${MINIFI_BASE_DIR}" @@ -93,7 +116,7 @@ ENV MINIFI_VERSIONED_HOME=${MINIFI_BASE_DIR}/nifi-minifi-cpp-${MINIFI_VERSION} RUN addgroup -g ${GID} ${USER} && adduser -u ${UID} -D -G ${USER} -g "" ${USER} && \ install -d -o ${USER} -g ${USER} ${MINIFI_BASE_DIR} && ln -s ${MINIFI_VERSIONED_HOME} ${MINIFI_HOME} && \ apk add --no-cache libstdc++ tzdata alpine-conf && \ - if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PYTHON_SCRIPTING=ON"; then apk add --no-cache python3; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON"; then apk add --no-cache python3; fi && \ setup-timezone -z UTC && \ apk del alpine-conf diff --git a/docker/rockylinux/Dockerfile b/docker/rockylinux/Dockerfile index 7af9a3de23..1e317a02d5 100644 --- a/docker/rockylinux/Dockerfile +++ b/docker/rockylinux/Dockerfile @@ -31,6 +31,8 @@ ARG CMAKE_BUILD_TYPE=Release ARG DOCKER_SKIP_TESTS=ON ARG DOCKER_MAKE_TARGET="all" ARG DOCKER_CREATE_RPM=ON +ARG DOCKER_USE_CONAN=OFF +ARG DOCKER_NIFI_CONAN_USER= ENV MINIFI_BASE_DIR=/opt/minifi ENV MINIFI_HOME=$MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION @@ -42,8 +44,9 @@ COPY . ${MINIFI_BASE_DIR} # Install the system dependencies needed for a build # ccache is in EPEL RUN dnf -y install epel-release && dnf -y install gcc-toolset-14 gcc-toolset-14-libatomic-devel sudo git which make libarchive ccache ca-certificates perl patch bison flex libtool cmake rpmdevtools && \ + if [ "${DOCKER_USE_CONAN}" == "ON" ]; then dnf -y install python3.9; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3-devel; fi && \ - if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi RUN cd $MINIFI_BASE_DIR && \ ln -s /usr/bin/ccache /usr/lib64/ccache/c++ @@ -54,18 +57,32 @@ RUN groupadd -g ${GID} ${USER} && useradd -g ${GID} ${USER} && \ USER ${USER} +RUN --mount=type=secret,id=nifi_conan_password,mode=0444 echo "DOCKER_USE_CONAN=${DOCKER_USE_CONAN}" && if [ "${DOCKER_USE_CONAN}" == "ON" ]; then \ + set -e; \ + cd ${MINIFI_BASE_DIR}/bootstrap && \ + python3.9 -m venv venv && \ + source /opt/rh/gcc-toolset-14/enable && \ + source venv/bin/activate && \ + pip3.9 install -r requirements.txt && \ + python3.9 main.py --skip-compiler-install --run-conan-install --minifi-options="${MINIFI_OPTIONS}" && \ + if [ -n "${DOCKER_NIFI_CONAN_USER}" ] && [ -f /run/secrets/nifi_conan_password ]; then \ + conan remote login nifi-conan ${DOCKER_NIFI_CONAN_USER} -p "$(cat /run/secrets/nifi_conan_password)" || true; \ + conan upload "*" -r nifi-conan --confirm || true; \ + fi; \ + fi + # Perform the build RUN cd $MINIFI_BASE_DIR && \ - mkdir build && \ + mkdir -p build && \ cd build && \ source /opt/rh/gcc-toolset-14/enable && \ export PATH=/usr/lib64/ccache${PATH:+:${PATH}} && \ export CCACHE_DIR=${MINIFI_BASE_DIR}/.ccache && \ - cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ + if [ "${DOCKER_USE_CONAN}" == "ON" ]; then CONAN_TOOLCHAIN="-DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake"; fi && \ + cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${CONAN_TOOLCHAIN:-} ${MINIFI_OPTIONS} -DPython_EXECUTABLE=/usr/bin/python3.6 -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ make -j "$(nproc)" ${DOCKER_MAKE_TARGET} && \ if [[ "${DOCKER_CREATE_RPM}" == "ON" ]] ; then \ cmake -DMINIFI_PACKAGING_TYPE=RPM .. && \ make -j "$(nproc)" package && \ ../packaging/rpm/check_rpm_contents.sh *.rpm expected-rpm-contents.txt; \ fi - diff --git a/docker/settings_user.yml b/docker/settings_user.yml new file mode 100644 index 0000000000..82e5ded266 --- /dev/null +++ b/docker/settings_user.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +os: + Linux: + libc: [null, "musl"] diff --git a/thirdparty/paho-mqtt/all/conandata.yml b/thirdparty/paho-mqtt/all/conandata.yml new file mode 100644 index 0000000000..839411e747 --- /dev/null +++ b/thirdparty/paho-mqtt/all/conandata.yml @@ -0,0 +1,8 @@ +sources: + "1.3.16": + url: "https://github.com/eclipse/paho.mqtt.c/archive/v1.3.16.tar.gz" + sha256: "8b960f51edc7e03507637d987882bc486d8f4be6e79431bf99e2763344fd14c5" +patches: + "1.3.16": + - patch_file: "patches/cmake-openssl.patch" + patch_type: "conan" diff --git a/thirdparty/paho-mqtt/all/conanfile.py b/thirdparty/paho-mqtt/all/conanfile.py new file mode 100644 index 0000000000..9cea38432e --- /dev/null +++ b/thirdparty/paho-mqtt/all/conanfile.py @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe based on https://github.com/conan-io/conan-center-index/blob/master/recipes/paho-mqtt-c/all/conanfile.py +# Removed anl library linkage in case musl is used instead of glibc, since it is not available in musl + +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import apply_conandata_patches, export_conandata_patches, get, copy, rm, replace_in_file +import os +import platform + +required_conan_version = ">=2.1" + + +class PahoMqttcConan(ConanFile): + name = "paho-mqtt-c" + description = "Eclipse Paho MQTT C client library for Linux, Windows and MacOS" + license = "EPL-2.0" + url = "https://github.com/conan-io/conan-center-index" + homepage = "https://github.com/eclipse/paho.mqtt.c" + topics = ("mqtt", "iot", "eclipse", "ssl", "tls", "paho") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "ssl": [True, False], + "asynchronous": [True, False], + "high_performance": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "ssl": True, + "asynchronous": True, + "high_performance": False, + } + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + self.settings.rm_safe("compiler.libcxx") + self.settings.rm_safe("compiler.cppstd") + + def layout(self): + cmake_layout(self, src_folder="src") + + def requirements(self): + if self.options.ssl: + # Headers are exposed https://github.com/eclipse/paho.mqtt.c/blob/f7799da95e347bbc930b201b52a1173ebbad45a7/src/SSLSocket.h#L29 + self.requires("openssl/3.6.2", transitive_headers=True) + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["PAHO_ENABLE_TESTING"] = False + tc.variables["PAHO_BUILD_DOCUMENTATION"] = False + tc.variables["PAHO_ENABLE_CPACK"] = False + tc.variables["PAHO_BUILD_DEB_PACKAGE"] = False + tc.variables["PAHO_BUILD_ASYNC"] = self.options.asynchronous + tc.variables["PAHO_BUILD_STATIC"] = not self.options.shared + tc.variables["PAHO_BUILD_SHARED"] = self.options.shared + tc.variables["PAHO_BUILD_SAMPLES"] = False + tc.variables["PAHO_WITH_SSL"] = self.options.ssl + if self.options.ssl: + tc.cache_variables["OPENSSL_SEARCH_PATH"] = self.dependencies["openssl"].package_folder.replace("\\", "/") + tc.cache_variables["OPENSSL_ROOT_DIR"] = self.dependencies["openssl"].package_folder.replace("\\", "/") + tc.variables["PAHO_HIGH_PERFORMANCE"] = self.options.high_performance + tc.cache_variables["CMAKE_POLICY_DEFAULT_CMP0042"] = "NEW" + tc.generate() + + deps = CMakeDeps(self) + deps.generate() + + def _patch_source(self): + apply_conandata_patches(self) + if not self.options.get_safe("fPIC", True): + replace_in_file(self, os.path.join(self.source_folder, "src", "CMakeLists.txt"), "POSITION_INDEPENDENT_CODE ON", "") + + def build(self): + self._patch_source() + cmake = CMake(self) + cmake.configure() + cmake.build(target=self._cmake_target) + + def package(self): + copy(self, "edl-v10", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "epl-v20", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "notice.html", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + + # Manually copy since the CMake installs everything + copy(self, pattern="MQTT*.h", src=os.path.join(self.source_folder, "src"), dst=os.path.join(self.package_folder, "include")) + + for suffix in ["lib", "a", "dylib"]: + copy(self, pattern=f"*.{suffix}", src=self.build_folder, dst=os.path.join(self.package_folder, "lib"), keep_path=False) + copy(self, pattern=f"*{self._lib_target}.so*", src=self.build_folder, dst=os.path.join(self.package_folder, "lib"), keep_path=False) + copy(self, pattern=f"*{self._lib_target}.dll", src=self.build_folder, dst=os.path.join(self.package_folder, "bin"), keep_path=False) + rm(self, "*.pdb", os.path.join(self.package_folder, "lib")) + rm(self, "*.pdb", os.path.join(self.package_folder, "bin")) + rm(self, "*.cmake", os.path.join(self.package_folder, "lib")) + + def package_info(self): + self.cpp_info.set_property("cmake_file_name", "eclipse-paho-mqtt-c") + self.cpp_info.set_property("cmake_target_name", f"eclipse-paho-mqtt-c::{self._cmake_target}") + + self.cpp_info.components["_paho-mqtt-c"].libs = [self._lib_target] + if self.settings.os == "Windows": + if not self.options.shared: + self.cpp_info.components["_paho-mqtt-c"].system_libs.append("ws2_32") + if self.settings.compiler == "gcc": + self.cpp_info.components["_paho-mqtt-c"].system_libs.extend( + ["wsock32", "uuid", "crypt32", "rpcrt4"]) + elif self.settings.os == "Linux": + libs = ["c", "dl", "pthread"] + if platform.libc_ver()[0] == "glibc": + libs.insert(0, "anl") + self.cpp_info.components["_paho-mqtt-c"].system_libs.extend(libs) + elif self.settings.os == "FreeBSD": + self.cpp_info.components["_paho-mqtt-c"].system_libs.extend(["compat", "pthread"]) + elif self.settings.os == "Android": + self.cpp_info.components["_paho-mqtt-c"].system_libs.extend(["c"]) + else: + self.cpp_info.components["_paho-mqtt-c"].system_libs.extend(["c", "pthread"]) + + if self.options.ssl: + self.cpp_info.components["_paho-mqtt-c"].requires = ["openssl::openssl"] + self.cpp_info.components["_paho-mqtt-c"].set_property("cmake_target_name", f"eclipse-paho-mqtt-c::{self._cmake_target}") + + @property + def _cmake_target(self): + target = "paho-mqtt3" + target += "a" if self.options.asynchronous else "c" + if self.options.ssl: + target += "s" + if not self.options.shared: + target += "-static" + return target + + @property + def _lib_target(self): + target = "paho-mqtt3" + target += "a" if self.options.asynchronous else "c" + if self.options.ssl: + target += "s" + if not self.options.shared: + # https://github.com/eclipse/paho.mqtt.c/blob/317fb008e1541838d1c29076d2bc5c3e4b6c4f53/src/CMakeLists.txt#L154 + if self.settings.os == "Windows": + target += "-static" + return target diff --git a/thirdparty/paho-mqtt/cmake-openssl.patch b/thirdparty/paho-mqtt/all/patches/cmake-openssl.patch similarity index 100% rename from thirdparty/paho-mqtt/cmake-openssl.patch rename to thirdparty/paho-mqtt/all/patches/cmake-openssl.patch diff --git a/thirdparty/paho-mqtt/config.yml b/thirdparty/paho-mqtt/config.yml new file mode 100644 index 0000000000..a2afb6a003 --- /dev/null +++ b/thirdparty/paho-mqtt/config.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +versions: + "1.3.16": + folder: "all"