Known TLDs

- Use latest `clang-format`
- Fix #212:
  - Add config `knowntlds_file` to load known TLDs
  - Update static known TLDs
  - `query_classification_index`: Use global known TLDs
- `dnstap`: Fix compile warning when not `--enable-dnstap`

fmt.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-clang-format-4.0 \
+clang-format \
     -style=file \
     -i \
     src/*.c \

src/Makefile.am

@@ -41,7 +41,7 @@ dist_dsc_SOURCES = asn_index.h base64.h certain_qnames_index.h client_index.h \
   tld_index.h transport_index.h xmalloc.h response_time_index.h \
   pcap_layers/byteorder.h pcap_layers/pcap_layers.h \
   pcap-thread/pcap_thread.h \
-  dnstap.h input_mode.h
+  dnstap.h input_mode.h knowntlds.inc
 dsc_LDADD = $(PTHREAD_LIBS) $(libmaxminddb_LIBS) \
   $(libdnswire_LIBS) $(libuv_LIBS)
 man1_MANS = dsc.1

src/asn_index.h

@@ -39,8 +39,8 @@
 
 #include "dns_message.h"
 
-int asn_indexer(const dns_message*);
-int asn_iterator(const char** label);
+int  asn_indexer(const dns_message*);
+int  asn_iterator(const char** label);
 void asn_reset(void);
 void asn_init(void);
 

src/client_index.h

@@ -39,8 +39,8 @@
 
 #include "dns_message.h"
 
-int client_indexer(const dns_message*);
-int client_iterator(const char** label);
+int  client_indexer(const dns_message*);
+int  client_iterator(const char** label);
 void client_reset(void);
 
 #endif /* __dsc_client_index_h */

src/client_subnet_index.h

@@ -39,11 +39,11 @@
 
 #include "dns_message.h"
 
-int client_subnet_indexer(const dns_message*);
-int client_subnet_iterator(const char** label);
+int  client_subnet_indexer(const dns_message*);
+int  client_subnet_iterator(const char** label);
 void client_subnet_reset(void);
 void client_subnet_init(void);
-int client_subnet_v4_mask_set(const char* mask);
-int client_subnet_v6_mask_set(const char* mask);
+int  client_subnet_v4_mask_set(const char* mask);
+int  client_subnet_v6_mask_set(const char* mask);
 
 #endif /* __dsc_client_subnet_index_h */

src/config_hooks.c

@@ -43,6 +43,11 @@
 #include "pcap.h"
 #include "compat.h"
 #include "response_time_index.h"
+#include "input_mode.h"
+#include "dnstap.h"
+
+#include "knowntlds.inc"
+
 #if defined(HAVE_LIBGEOIP) && defined(HAVE_GEOIP_H)
 #define HAVE_GEOIP 1
 #include <GeoIP.h>
@@ -51,10 +56,6 @@
 #define HAVE_MAXMINDDB 1
 #include <maxminddb.h>
 #endif
-
-#include "input_mode.h"
-#include "dnstap.h"
-
 #include <unistd.h>
 #include <errno.h>
 #include <limits.h>
@@ -65,6 +66,7 @@
 #include <sys/types.h>
 #include <pwd.h>
 #include <grp.h>
+#include <ctype.h>
 
 extern int input_mode;
 extern int promisc_flag;
@@ -608,3 +610,69 @@ int set_response_time_bucket_size(const char* s)
     dsyslogf(LOG_INFO, "set response time bucket size to %d", bucket_size);
     return 1;
 }
+
+const char** KnownTLDS = KnownTLDS_static;
+
+int load_knowntlds(const char* file)
+{
+    FILE*   fp;
+    char *  buffer        = 0, *p;
+    size_t  bufsize       = 0;
+    char**  new_KnownTLDS = 0;
+    size_t  new_size      = 0;
+    ssize_t nread;
+
+    if (KnownTLDS != KnownTLDS_static) {
+        dsyslog(LOG_ERR, "Known TLDs already loaded once");
+        return 0;
+    }
+
+    if (!(fp = fopen(file, "r"))) {
+        dsyslogf(LOG_ERR, "unable to open %s", file);
+        return 0;
+    }
+
+    if (!(new_KnownTLDS = xrealloc(new_KnownTLDS, (new_size + 1) * sizeof(char*)))) {
+        dsyslog(LOG_ERR, "out of memory");
+        return 0;
+    }
+    new_KnownTLDS[new_size] = ".";
+    new_size++;
+
+    while ((nread = getline(&buffer, &bufsize, fp)) > 0) {
+        for (p = buffer; *p; p++) {
+            if (*p == '\r' || *p == '\n') {
+                *p = 0;
+                break;
+            }
+            *p = tolower(*p);
+        }
+        if (buffer[0] == '#') {
+            continue;
+        }
+
+        if (!(new_KnownTLDS = xrealloc(new_KnownTLDS, (new_size + 1) * sizeof(char*)))) {
+            dsyslog(LOG_ERR, "out of memory");
+            return 0;
+        }
+        new_KnownTLDS[new_size] = xstrdup(buffer);
+        if (!new_KnownTLDS[new_size]) {
+            dsyslog(LOG_ERR, "out of memory");
+            return 0;
+        }
+        new_size++;
+    }
+    free(buffer);
+    fclose(fp);
+
+    if (!(new_KnownTLDS = xrealloc(new_KnownTLDS, (new_size + 1) * sizeof(char*)))) {
+        dsyslog(LOG_ERR, "out of memory");
+        return 0;
+    }
+    new_KnownTLDS[new_size] = 0;
+
+    KnownTLDS = (const char**)new_KnownTLDS;
+    dsyslogf(LOG_INFO, "loaded %zd known TLDs from %s", new_size - 1, file);
+
+    return 1;
+}

src/config_hooks.h

@@ -45,37 +45,40 @@ enum dnstap_via {
     dnstap_via_udp,
 };
 
-int open_interface(const char* interface);
-int open_dnstap(enum dnstap_via via, const char* file_or_ip, const char* port, const char* user, const char* group, const char* umask);
-int set_bpf_program(const char* s);
-int add_local_address(const char* s, const char* m);
-int set_run_dir(const char* dir);
-int set_pid_file(const char* s);
-int set_statistics_interval(const char* s);
-int add_dataset(const char* name, const char* layer_ignored, const char* firstname, const char* firstindexer, const char* secondname, const char* secondindexer, const char* filtername, dataset_opt opts);
-int set_bpf_vlan_tag_byte_order(const char* which);
-int set_match_vlan(const char* s);
-int set_minfree_bytes(const char* s);
-int set_output_format(const char* output_format);
+extern const char** KnownTLDS;
+
+int  open_interface(const char* interface);
+int  open_dnstap(enum dnstap_via via, const char* file_or_ip, const char* port, const char* user, const char* group, const char* umask);
+int  set_bpf_program(const char* s);
+int  add_local_address(const char* s, const char* m);
+int  set_run_dir(const char* dir);
+int  set_pid_file(const char* s);
+int  set_statistics_interval(const char* s);
+int  add_dataset(const char* name, const char* layer_ignored, const char* firstname, const char* firstindexer, const char* secondname, const char* secondindexer, const char* filtername, dataset_opt opts);
+int  set_bpf_vlan_tag_byte_order(const char* which);
+int  set_match_vlan(const char* s);
+int  set_minfree_bytes(const char* s);
+int  set_output_format(const char* output_format);
 void set_dump_reports_on_exit(void);
-int set_geoip_v4_dat(const char* dat, int options);
-int set_geoip_v6_dat(const char* dat, int options);
-int set_geoip_asn_v4_dat(const char* dat, int options);
-int set_geoip_asn_v6_dat(const char* dat, int options);
-int set_asn_indexer_backend(enum geoip_backend backend);
-int set_country_indexer_backend(enum geoip_backend backend);
-int set_maxminddb_asn(const char* file);
-int set_maxminddb_country(const char* file);
-int set_pcap_buffer_size(const char* s);
+int  set_geoip_v4_dat(const char* dat, int options);
+int  set_geoip_v6_dat(const char* dat, int options);
+int  set_geoip_asn_v4_dat(const char* dat, int options);
+int  set_geoip_asn_v6_dat(const char* dat, int options);
+int  set_asn_indexer_backend(enum geoip_backend backend);
+int  set_country_indexer_backend(enum geoip_backend backend);
+int  set_maxminddb_asn(const char* file);
+int  set_maxminddb_country(const char* file);
+int  set_pcap_buffer_size(const char* s);
 void set_no_wait_interval(void);
-int set_pt_timeout(const char* s);
+int  set_pt_timeout(const char* s);
 void set_drop_ip_fragments(void);
-int set_dns_port(const char* s);
-int set_response_time_mode(const char* s);
-int set_response_time_max_queries(const char* s);
-int set_response_time_full_mode(const char* s);
-int set_response_time_max_seconds(const char* s);
-int set_response_time_max_sec_mode(const char* s);
-int set_response_time_bucket_size(const char* s);
+int  set_dns_port(const char* s);
+int  set_response_time_mode(const char* s);
+int  set_response_time_max_queries(const char* s);
+int  set_response_time_full_mode(const char* s);
+int  set_response_time_max_seconds(const char* s);
+int  set_response_time_max_sec_mode(const char* s);
+int  set_response_time_bucket_size(const char* s);
+int  load_knowntlds(const char* file);
 
 #endif /* __dsc_config_hooks_h */

src/country_index.h

@@ -39,8 +39,8 @@
 
 #include "dns_message.h"
 
-int country_indexer(const dns_message*);
-int country_iterator(const char** label);
+int  country_indexer(const dns_message*);
+int  country_iterator(const char** label);
 void country_reset(void);
 void country_init(void);
 

src/daemon.c

@@ -457,7 +457,7 @@ int main(int argc, char* argv[])
         daemonize();
     write_pid_file();
 
-/*
+    /*
      * Handle signal when using pthreads
      */
 
@@ -580,7 +580,7 @@ int main(int argc, char* argv[])
         if (0 == fork()) {
             struct sigaction action;
 
-/*
+            /*
              * Remove the blocking of signals
              */
 

src/dns_ip_version_index.h

@@ -39,8 +39,8 @@
 
 #include "dns_message.h"
 
-int dns_ip_version_indexer(const dns_message*);
-int dns_ip_version_iterator(const char** label);
+int  dns_ip_version_indexer(const dns_message*);
+int  dns_ip_version_iterator(const char** label);
 void dns_ip_version_reset(void);
 
 #endif /* __dsc_dns_ip_version_index_h */

src/dns_message.c

@@ -453,7 +453,7 @@ int add_qname_filter(const char* name, const char* pat)
     int           x;
     while ((*fl))
         fl = &((*fl)->next);
-    r      = xcalloc(1, sizeof(*r));
+    r = xcalloc(1, sizeof(*r));
     if (NULL == r) {
         dsyslogf(LOG_ERR, "Cant allocate memory for '%s' qname filter", name);
         return 0;

src/dns_message.h

@@ -93,16 +93,16 @@ struct dns_message {
     } edns;
 };
 
-void dns_message_handle(dns_message* m);
-int dns_message_add_array(const char* name, const char* fn, const char* fi, const char* sn, const char* si, const char* f, dataset_opt opts);
-void dns_message_flush_arrays(void);
-void dns_message_report(FILE* fp, md_array_printer* printer);
+void        dns_message_handle(dns_message* m);
+int         dns_message_add_array(const char* name, const char* fn, const char* fi, const char* sn, const char* si, const char* f, dataset_opt opts);
+void        dns_message_flush_arrays(void);
+void        dns_message_report(FILE* fp, md_array_printer* printer);
 void        dns_message_clear_arrays(void);
 const char* dns_message_QnameToNld(const char* qname, int nld);
 const char* dns_message_tld(dns_message* m);
-void dns_message_filters_init(void);
-void dns_message_indexers_init(void);
-int add_qname_filter(const char* name, const char* pat);
+void        dns_message_filters_init(void);
+void        dns_message_indexers_init(void);
+int         add_qname_filter(const char* name, const char* pat);
 
 #include <arpa/nameser.h>
 #ifdef HAVE_ARPA_NAMESER_COMPAT_H

src/dns_source_port_index.h

@@ -39,12 +39,12 @@
 
 #include "dns_message.h"
 
-int dns_source_port_indexer(const dns_message*);
-int dns_source_port_iterator(const char** label);
+int  dns_source_port_indexer(const dns_message*);
+int  dns_source_port_iterator(const char** label);
 void dns_source_port_reset(void);
 
-int dns_sport_range_indexer(const dns_message*);
-int dns_sport_range_iterator(const char** label);
+int  dns_sport_range_indexer(const dns_message*);
+int  dns_sport_range_iterator(const char** label);
 void dns_sport_range_reset(void);
 
 #endif /* __dsc_dns_source_port_index_h */

src/dnstap.c

@@ -814,14 +814,14 @@ static char*                 _sock_file = 0;
 
 extern int no_wait_interval;
 
+#ifdef USE_DNSTAP
 static void _atexit(void)
 {
-#ifdef USE_DNSTAP
     if (_sock_file) {
         unlink(_sock_file);
     }
-#endif
 }
+#endif
 
 void dnstap_init(enum dnstap_via via, const char* sock_or_host, int port, uid_t uid, gid_t gid, int mask)
 {

src/dsc.conf.5.in

@@ -331,6 +331,10 @@ The query will be counted as timed out.
 .TP
 \fBresponse_time_bucket_size\fR SIZE ;
 Control the size of bucket (microseconds) in bucket mode.
+.TP
+\fBknowntlds_file\fR FILE ;
+Load known TLDs from FILE, this should be or have the same format as
+.IR https://data.iana.org/TLD/tlds-alpha-by-domain.txt .
 .SH DATASETS
 A \fBdataset\fR is a 2-D array of counters.
 For example, you might have a dataset with \*(lqQuery Type\*(rq along one
@@ -948,6 +952,8 @@ output_format XML;
 #response_time_max_seconds 5;
 #response_time_max_sec_mode ceil;
 #response_time_bucket_size 100;
+
+#knowntlds_file file;
 .fi
 .SH FILES
 @etcdir@/dsc.conf

src/dsc.conf.sample.in

@@ -297,3 +297,9 @@ dataset direction_vs_ipproto ip Direction:ip_direction IPProto:ip_proto any;
 #  Control the size of bucket (microseconds) in bucket mode.
 #
 #response_time_bucket_size 100;
+
+# Known TLDs
+#
+#  Load known TLDs from a file, see https://data.iana.org/TLD/tlds-alpha-by-domain.txt
+#
+#knowntlds_file file;

src/hashtbl.h

@@ -47,8 +47,8 @@ typedef struct _hashitem {
 } hashitem;
 
 typedef unsigned int hashfunc(const void* key);
-typedef int hashkeycmp(const void* a, const void* b);
-typedef void hashfree(void* p);
+typedef int          hashkeycmp(const void* a, const void* b);
+typedef void         hashfree(void* p);
 
 typedef struct
 {
@@ -67,12 +67,12 @@ typedef struct
 } hashtbl;
 
 hashtbl* hash_create(int N, hashfunc*, hashkeycmp*, int use_arena, hashfree*, hashfree*);
-void hash_destroy(hashtbl*);
-int hash_add(const void* key, void* data, hashtbl*);
-void hash_remove(const void* key, hashtbl* tbl);
-void* hash_find(const void* key, hashtbl*);
-void  hash_iter_init(hashtbl*);
-void* hash_iterate(hashtbl*);
+void     hash_destroy(hashtbl*);
+int      hash_add(const void* key, void* data, hashtbl*);
+void     hash_remove(const void* key, hashtbl* tbl);
+void*    hash_find(const void* key, hashtbl*);
+void     hash_iter_init(hashtbl*);
+void*    hash_iterate(hashtbl*);
 
 /*
  * found in lookup3.c