structured logging: JSON and journald

compile_commands.json

@@ -0,0 +1 @@
+cmake/compile_commands.json

patch/civetweb/0001-Increase-niceness-of-all-civetweb-threads-as-DNS-ope.patch

@@ -2,14 +2,13 @@ diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb
 index da254bb4..b1e5b9a6 100644
 --- a/src/webserver/civetweb/civetweb.c
 +++ b/src/webserver/civetweb/civetweb.c
-@@ -244,6 +244,7 @@ static void DEBUG_TRACE_FUNC(const char *func,
+@@ -244,5 +244,6 @@ static void DEBUG_TRACE_FUNC(const char *func,
 
  #else
  #include "log.h"
 +#include <sys/resource.h>
  #define DEBUG_TRACE(fmt, ...)                                                  \
- 	if(debug_flags[DEBUG_WEBSERVER]) {\
- 		log_web("DEBUG: " fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__); }
+ 		log_web_debug(DEBUG_WEBSERVER, fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__);
 @@ -2890,6 +2891,9 @@ mg_set_thread_name(const char *name)
  	 */
  	(void)prctl(PR_SET_NAME, threadName, 0, 0, 0);

patch/civetweb/0001-Log-debug-messages-to-webserver.log-when-debug.webse.patch

@@ -2,16 +2,15 @@ diff --git a/src/webserver/civetweb/civetweb.c b/src/webserver/civetweb/civetweb
 index 3df8eab9..9b0c6308 100644
 --- a/src/webserver/civetweb/civetweb.c
 +++ b/src/webserver/civetweb/civetweb.c
-@@ -239,9 +239,10 @@ static void DEBUG_TRACE_FUNC(const char *func,
+@@ -239,9 +239,9 @@ static void DEBUG_TRACE_FUNC(const char *func,
  #endif
 
  #else
 +#include "log.h"
  #define DEBUG_TRACE(fmt, ...)                                                  \
 -	do {                                                                       \
 -	} while (0)
-+	if(debug_flags[DEBUG_WEBSERVER]) {\
-+		log_web("DEBUG: " fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__); }
++		log_web_debug(DEBUG_WEBSERVER, fmt " (%s:%d)", ##__VA_ARGS__, short_path(__FILE__), __LINE__);
  #endif /* DEBUG */
  #endif /* DEBUG_TRACE */
- 
+

src/CMakeLists.txt

@@ -409,6 +409,44 @@ else()
     target_compile_definitions(civetweb PRIVATE NO_SSL)
 endif()
 
+set(USE_SYSTEMD_JOURNAL "AUTO" CACHE STRING "Build FTL with systemd journal support (AUTO, DLOPEN, COMPILE, OFF)")
+set_property(CACHE USE_SYSTEMD_JOURNAL PROPERTY STRINGS AUTO DLOPEN COMPILE OFF)
+
+find_library(LIBSYSTEMD NAMES libsystemd${LIBRARY_SUFFIX} systemd)
+check_include_file("systemd/sd-journal.h" HAVE_SD_JOURNAL_H)
+find_library(LIBDL dl)
+if(USE_SYSTEMD_JOURNAL STREQUAL "OFF")
+    message(STATUS "Building FTL with systemd journal support: NO")
+elseif(USE_SYSTEMD_JOURNAL STREQUAL "COMPILE")
+    if(NOT LIBSYSTEMD OR NOT HAVE_SD_JOURNAL_H)
+        message(FATAL_ERROR "COMPILE mode requested but systemd headers/library not found")
+    endif()
+    message(STATUS "Building FTL with systemd journal support: COMPILE")
+    target_compile_definitions(core PRIVATE HAVE_SYSTEMD_JOURNAL)
+    target_link_libraries(pihole-FTL ${LIBSYSTEMD})
+elseif(USE_SYSTEMD_JOURNAL STREQUAL "DLOPEN")
+    if(NOT LIBDL)
+        message(FATAL_ERROR "DLOPEN mode requested but dl library not found")
+    endif()
+    target_link_libraries(pihole-FTL ${LIBDL})
+    message(STATUS "Building FTL with systemd journal support: DLOPEN")
+    target_compile_definitions(core PRIVATE DL_SYSTEMD_JOURNAL)
+elseif(USE_SYSTEMD_JOURNAL STREQUAL "AUTO")
+    if(LIBSYSTEMD AND HAVE_SD_JOURNAL_H)
+        message(STATUS "Building FTL with systemd journal support: AUTO → COMPILE mode")
+        target_compile_definitions(core PRIVATE HAVE_SYSTEMD_JOURNAL)
+        target_link_libraries(pihole-FTL ${LIBSYSTEMD})
+    elseif(LIBDL)
+        target_link_libraries(pihole-FTL ${LIBDL})
+        message(STATUS "Building FTL with systemd journal support: AUTO → DLOPEN fallback")
+        target_compile_definitions(core PRIVATE DL_SYSTEMD_JOURNAL)
+    else()
+        message(FATAL_ERROR "AUTO mode requested but neither LIBSYSTEMD nor LIBDL are available")
+    endif()
+else()
+    message(FATAL_ERROR "Invalid USE_SYSTEMD_JOURNAL: ${USE_SYSTEMD_JOURNAL}")
+endif()
+
 # After finishing building the FTL binary, we append the sha256sum of the binary
 # in raw form to itself and print the checksum to the console
 add_custom_command(TARGET pihole-FTL POST_BUILD COMMENT "Appending sha256sum to pihole-FTL"

src/args.c

@@ -80,6 +80,10 @@
 // test
 #include <sys/mman.h>
 
+#ifdef DL_SYSTEMD_JOURNAL
+#include <dlfcn.h>
+#endif
+
 // defined in dnsmasq.c
 extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal);
 
@@ -91,6 +95,7 @@ extern int sqlite3_rsync_main(int argc, char **argv);
 
 bool debug_mode = false;
 bool daemonmode = true, cli_mode = false;
+bool log_json = false, log_journal = false;
 int argc_dnsmasq = 0;
 const char** argv_dnsmasq = NULL;
 
@@ -219,6 +224,33 @@ static bool strStartsWithIgnoreCase(const char *input, const char *start)
 	return strncasecmp(input, start, strlen(start)) == 0;
 }
 
+#ifdef DL_SYSTEMD_JOURNAL
+static void *systemd_handle = NULL;
+sd_journal_send_t sd_journal_send = NULL;
+
+// Intentionally never dlclose() libsystemd
+// We open it once and use it until late shutdown
+// OS cleanup at exit is sufficient.
+static bool init_systemd_journal(void)
+{
+	systemd_handle = dlopen("libsystemd.so.0", RTLD_NOW | RTLD_LOCAL);
+	if (!systemd_handle)
+		return false;
+
+	dlerror();
+	sd_journal_send = (sd_journal_send_t)dlsym(systemd_handle, "sd_journal_send");
+	const char *err = dlerror();
+	if (err != NULL) {
+		dlclose(systemd_handle);
+		systemd_handle = NULL;
+		sd_journal_send = NULL;
+		return false;
+	}
+
+	return true;
+}
+#endif
+
 void parse_args(int argc, char *argv[])
 {
 	bool quiet = false;
@@ -1104,6 +1136,33 @@ void parse_args(int argc, char *argv[])
 			ok = true;
 		}
 
+		if(strcmp(argv[i], "--log-json") == 0)
+		{
+			log_json = true;
+			ok = true;
+		}
+
+		if(strcmp(argv[i], "--log-journal") == 0 && log_journal == false)
+		{
+#ifdef HAVE_SYSTEMD_JOURNAL
+			log_journal = true;
+			ok = true;
+#elif defined(DL_SYSTEMD_JOURNAL)
+
+			if(!init_systemd_journal())
+			{
+				printf("Error: Failed to load libsystemd. Logging to journal is not available.\n");
+				exit(EXIT_FAILURE);
+			}
+
+			log_journal = true;
+			ok = true;
+#else
+			printf("Error: FTL was compiled without systemd support. Logging to journal is not available.\n");
+			exit(EXIT_FAILURE);
+#endif
+		}
+
 		// Regex test mode
 		if(strcmp(argv[i], "regex-test") == 0)
 		{

src/args.h

@@ -13,6 +13,7 @@
 void parse_args(int argc, char *argv[]);
 
 extern bool daemonmode, cli_mode;
+extern bool log_json, log_journal;
 extern int argc_dnsmasq;
 extern const char ** argv_dnsmasq;
 
@@ -56,4 +57,9 @@ void test_dnsmasq_options(int argc, const char *argv[]);
 #define COL_CYAN	"\x1b[96m" // bright foreground color
 #define CLI_OVER	"\r\x1b[K" // go back to beginning of line and erase to end of line
 
+#ifdef DL_SYSTEMD_JOURNAL
+typedef int (*sd_journal_send_t)(const char *format, ...);
+extern sd_journal_send_t sd_journal_send;
+#endif
+
 #endif //ARGS_H

src/dnsmasq/dnsmasq.c

@@ -171,9 +171,14 @@ int main_dnsmasq (int argc, char **argv)
      ensure that we're not using those fds for real stuff. */
   for (i = 0; i < 3; i++)
     open("/dev/null", O_RDWR); 
+
+  /**** Pi-hole modification ****/
+  // don't close all fds, as this would close the journald socket
 
   /* Close any file descriptors we inherited apart from std{in|out|err} */
-  close_fds(max_fd, -1, -1, -1);
+  //close_fds(max_fd, -1, -1, -1); 
+
+  /*****************************/
 
 #ifndef HAVE_LINUX_NETWORK
 #  if !(defined(IP_RECVDSTADDR) && defined(IP_RECVIF) && defined(IP_SENDSRCADDR))
@@ -728,19 +733,24 @@ int main_dnsmasq (int argc, char **argv)
 
    log_err = log_start(ent_pw, err_pipe[1]);
 
-   if (!option_bool(OPT_DEBUG)) 
-     {       
-       /* open  stdout etc to /dev/null */
-       int nullfd = open("/dev/null", O_RDWR);
-       if (nullfd != -1)
-	 {
-	   dup2(nullfd, STDOUT_FILENO);
-	   dup2(nullfd, STDERR_FILENO);
-	   dup2(nullfd, STDIN_FILENO);
-	   close(nullfd);
-	 }
-     }
+   /**** Pi-hole modification ****/
+   // We want to log to stdout/stderr, so we are preventing dnsmasq from replacing them with /dev/null
 
+   //if (!option_bool(OPT_DEBUG)) 
+   //  {       
+   //    /* open  stdout etc to /dev/null */
+   //    int nullfd = open("/dev/null", O_RDWR);
+   //    if (nullfd != -1)
+   //      {
+   //        dup2(nullfd, STDOUT_FILENO);
+   //        dup2(nullfd, STDERR_FILENO);
+   //        dup2(nullfd, STDIN_FILENO);
+   //        close(nullfd);
+   //      }
+   //  }
+
+   /******************************/
+
    /* if we are to run scripts, we need to fork a helper before dropping root. */
   daemon->helperfd = -1;
 #ifdef HAVE_SCRIPT 

src/dnsmasq/log.c

@@ -327,7 +327,8 @@ void my_syslog(int priority, const char *format, ...)
   va_start(ap, format);
   len = vsnprintf(buffer, MAX_MESSAGE, format, ap) + 1u; /* include zero-terminator */
   va_end(ap);
-  FTL_dnsmasq_log(buffer, len > MAX_MESSAGE ? MAX_MESSAGE : len);
+  time(&time_now);
+  FTL_dnsmasq_log(buffer, priority, len > MAX_MESSAGE ? MAX_MESSAGE : len, time_now);
   /*******************************************************************************/
 
   if (echo_stderr) 
@@ -408,8 +409,10 @@ void my_syslog(int priority, const char *format, ...)
 	  for (tmp = entries; tmp->next; tmp = tmp->next);
 	  tmp->next = entry;
 	}
-
-      time(&time_now);
+      /************** Pi-hole modification **************/
+      // get the current time earlier, so JSON logging and logfile use the same timestamp
+      //time(&time_now);
+      /**************************************************/
       p = entry->payload;
       if (!log_to_file)
 	p += sprintf(p, "<%d>", priority | log_fac);

src/dnsmasq_interface.c

@@ -59,6 +59,12 @@
 // init_api_sessions()
 #include "api/api.h"
 
+#ifdef HAVE_SYSTEMD_JOURNAL
+#define SD_JOURNAL_SUPPRESS_LOCATION
+// sd_journal_send()
+#include <systemd/sd-journal.h>
+#endif
+
 // Public prototypes (defined in this file, called from other translation units)
 void FTL_dump_cache_stats(void);
 
@@ -1652,7 +1658,7 @@ static bool special_domain(const queriesData *query, const char *domain)
 	// _dns.resolver.arpa.     86400   IN      SVCB    2 dns.google. alpn="h2,h3" key7="/dns-query{?dns}"
 	//
 	// RFC 9462, Section 4 says:
-	// 
+	//
 	// If the recursive resolver that receives this query has no Designated
 	// Resolvers, it SHOULD return NODATA for queries to the "resolver.arpa"
 	// zone, to provide a consistent and accurate signal to clients that it
@@ -1902,7 +1908,7 @@ static bool FTL_check_blocking(const char *domainstr, queriesData *query, client
 	}
 
 	// when we reach this point: the query is not in FTL's cache (for this client)
-	
+
 	// Check exact whitelist for match
 	const char *blockedDomain = domainstr;
 	PERF_START(_pcb_allow);
@@ -4098,16 +4104,46 @@ static void _query_set_dnssec(queriesData *query, const enum dnssec_status dnsse
 }
 
 // Add dnsmasq log line to internal FIFO buffer (can be queried via the API)
-void FTL_dnsmasq_log(const char *payload, const int length)
+void FTL_dnsmasq_log(const char *payload, const int priority, const int length, const time_t now)
 {
 	// Lock SHM
 	lock_shm();
 
+	const char *prio = priostr(priority, DEBUG_GENERIC);
+
 	// Add to FIFO buffer
-	add_to_fifo_buffer(FIFO_DNSMASQ, payload, NULL, length);
+	add_to_fifo_buffer(FIFO_DNSMASQ, payload, prio, length);
 
 	// Unlock SHM
 	unlock_shm();
+
+	if(log_json && !daemonmode)
+	{
+		// Get and log PID of current process to avoid ambiguities when more than one
+		// pihole-FTL instance is logging to the same output
+		char idstr[42];
+		get_idstr(idstr, sizeof(idstr));
+
+		// JSON logging only for stdout
+		log_to_json(
+            		now,
+            		prio,
+            		"dnsmasq",
+            		idstr,
+            		payload
+        	);
+	}
+#if defined(HAVE_SYSTEMD_JOURNAL) || defined(DL_SYSTEMD_JOURNAL)
+	if(log_journal)
+	{
+		sd_journal_send(
+			"MESSAGE=%s", payload,
+			"PRIORITY=%i", priority,
+			"COMPONENT=dnsmasq",
+			NULL
+		);
+	}
+#endif
 }
 
 static const char *check_dnsmasq_name(const char *name)

src/dnsmasq_interface.h

@@ -16,6 +16,8 @@
 #include "edns0.h"
 #include "metrics.h"
 
+#include <time.h>
+
 enum protocol { TCP, UDP, INTERNAL };
 
 void FTL_hook(unsigned int flags, const char *name, const union all_addr *addr, char *arg, int id, unsigned short type, const char *file, const int line);
@@ -56,6 +58,6 @@ bool get_dnsmasq_debug(void) __attribute__ ((pure));
 // defined in src/dnsmasq/cache.c
 extern char *querystr(char *desc, unsigned short type);
 
-extern void FTL_dnsmasq_log(const char *payload, const int length);
+extern void FTL_dnsmasq_log(const char *payload, const int priority, const int length, const time_t now);
 
 #endif // DNSMASQ_INTERFACE_H

src/enums.h

@@ -159,7 +159,8 @@ enum debug_flag {
 	DEBUG_NETLINK,
 	DEBUG_TIMING,
 	DEBUG_PERFORMANCE,
-	DEBUG_MAX
+	DEBUG_MAX,
+	DEBUG_GENERIC // this is for getting a regular "DEBUG" string from priostr(); it is explicitly not selectable as a debug flag, only for carrying over the generic syslog level.
 } __attribute__ ((packed));
 
 enum events {