zpcap

- Fix #177: Add `input.zpcap`: module for reading compressed PCAPs
- `examples/dumpdns.lua`: Add support for dumping compressed PCAPs

.lgtm.yml

@@ -13,6 +13,8 @@ extraction:
         - liblmdb-dev
         - libck-dev
         - libgnutls28-dev
+        - liblz4-dev
+        - libzstd-dev
     configure:
       command:
         - ./autogen.sh

README.md

@@ -47,16 +47,18 @@ architectures because of LuaJIT.
 - [liblmdb](https://github.com/LMDB/lmdb)
 - [libck](https://github.com/concurrencykit/ck)
 - [libgnutls](https://www.gnutls.org/)
+- [liblz4](http://www.lz4.org/)
+- [libzstd](http://www.zstd.net/)
 - [luajit](http://luajit.org/) (for building)
 - automake/autoconf/libtool/pkg-config (for building)
 
-Debian/Ubuntu: `apt-get install libluajit-5.1-dev libpcap-dev luajit liblmdb-dev libck-dev libgnutls28-dev`
+Debian/Ubuntu: `apt-get install libluajit-5.1-dev libpcap-dev luajit liblmdb-dev libck-dev libgnutls28-dev liblz4-dev libzstd-dev`
 
-CentOS: `yum install luajit-devel libpcap-devel lmdb-devel ck-devel gnutls-devel`
+CentOS: `yum install luajit-devel libpcap-devel lmdb-devel ck-devel gnutls-devel lz4-devel libzstd-devel`
 
-FreeBSD: `pkg install luajit libpcap lmdb gnutls concurrencykit`
+FreeBSD: `pkg install luajit libpcap lmdb gnutls concurrencykit zstd liblz4`
 
-OpenBSD: `pkg_add luajit gnutls` + manual install of libpcap, liblmdb and libck
+OpenBSD: `pkg_add luajit gnutls lz4 zstd` + manual install of libpcap, liblmdb and libck
 
 On some version of SUSE Linux Enterprise moonjit is used as an compatible
 alternative to luajit.

configure.ac

@@ -91,6 +91,8 @@ PKG_CHECK_MODULES([ck], [ck >= 0], [
   AC_CHECK_LIB([ck], [ck_array_init],, [AC_MSG_ERROR([libck not found])])
 ])
 AC_CHECK_LIB([gnutls], [gnutls_init],, [AC_MSG_ERROR([libgnutls not found])])
+PKG_CHECK_MODULES([liblz4], [liblz4 >= 1.8.0], [AC_DEFINE([HAVE_LZ4], [], [Use liblz4])],:)
+PKG_CHECK_MODULES([libzstd], [libzstd >= 1.3.0], [AC_DEFINE([HAVE_ZSTD], [], [Use libzstd])],:)
 
 # Checks for sizes
 AC_CHECK_SIZEOF([void*])

debian/control

@@ -5,7 +5,8 @@ Maintainer: Jerry Lundström <[email protected]>
 Build-Depends: debhelper (>= 8.0.0), build-essential, automake,
  autoconf (>= 2.64), libpcap-dev, netbase, libtool,
  libluajit-5.1-dev (>= 2.0.0), luajit (>= 2.0.0), pkg-config, liblmdb-dev,
- libck-dev, gcc (>= 4:5.0.0) | gcc-5, libgnutls28-dev
+ libck-dev, gcc (>= 4:5.0.0) | gcc-5, libgnutls28-dev, liblz4-dev,
+ libzstd-dev
 Standards-Version: 3.9.4
 Homepage: https://www.dns-oarc.net/tools/dnsjit
 Vcs-Git: https://github.com/DNS-OARC/dnsjit.git

examples/dumpdns.lua

@@ -1,18 +1,30 @@
 #!/usr/bin/env dnsjit
 local pcap = arg[2]
+local compression = arg[3]
 
 if pcap == nil then
-    print("usage: "..arg[1].." <pcap>")
+    print("usage: "..arg[1].." <pcap> [compression]")
     return
 end
 
 local object = require("dnsjit.core.objects")
 local input = require("dnsjit.input.pcap").new()
+local zinput = require("dnsjit.input.zpcap").new()
 local layer = require("dnsjit.filter.layer").new()
 local dns = require("dnsjit.core.object.dns").new()
 
-input:open_offline(pcap)
-layer:producer(input)
+if string.lower(string.sub(pcap, -4)) == ".zst" or compression == "zstd" then
+    zinput:zstd()
+    zinput:open(pcap)
+    layer:producer(zinput)
+elseif string.lower(string.sub(pcap, -4)) == ".lz4" or compression == "lz4" then
+    zinput:lz4()
+    zinput:open(pcap)
+    layer:producer(zinput)
+else
+    input:open_offline(pcap)
+    layer:producer(input)
+end
 local producer, ctx = layer:produce()
 
 while true do

rpm/dnsjit.spec

@@ -19,6 +19,8 @@ BuildRequires:  luajit-devel >= 2.0.0
 BuildRequires:  lmdb-devel
 BuildRequires:  ck-devel
 BuildRequires:  gnutls-devel
+BuildRequires:  lz4-devel
+BuildRequires:  libzstd-devel
 BuildRequires:  autoconf >= 2.64
 BuildRequires:  automake
 BuildRequires:  libtool

src/Makefile.am

@@ -27,7 +27,8 @@ AM_CFLAGS = -Werror=attributes \
   -I$(top_srcdir)/include \
   $(SIMD_FLAGS) $(CPUEXT_FLAGS) \
   $(PTHREAD_CFLAGS) \
-  $(luajit_CFLAGS)
+  $(luajit_CFLAGS) \
+  $(liblz4_CFLAGS) $(libzstd_CFLAGS)
 
 EXTRA_DIST = gen-manpage.lua gen-compat.lua gen-errno.sh dnsjit.1in
 
@@ -41,19 +42,19 @@ dnsjitincludedir = $(includedir)/dnsjit
 nobase_dnsjitinclude_HEADERS = globals.h version.h
 lua_hobjects = core/compat.luaho
 lua_objects = core.luao lib.luao input.luao filter.luao output.luao
-dnsjit_LDADD = $(PTHREAD_LIBS) $(luajit_LIBS)
+dnsjit_LDADD = $(PTHREAD_LIBS) $(luajit_LIBS) $(liblz4_LIBS) $(libzstd_LIBS)
 
 # C source and headers
-dnsjit_SOURCES += core/channel.c core/compat.c core/file.c core/log.c core/object.c core/object/dns.c core/object/ether.c core/object/gre.c core/object/icmp6.c core/object/icmp.c core/object/ieee802.c core/object/ip6.c core/object/ip.c core/object/linuxsll.c core/object/loop.c core/object/null.c core/object/payload.c core/object/pcap.c core/object/tcp.c core/object/udp.c core/producer.c core/receiver.c core/thread.c filter/copy.c filter/ipsplit.c filter/layer.c filter/split.c filter/timing.c input/fpcap.c input/mmpcap.c input/pcap.c lib/base64url.c lib/clock.c lib/trie.c output/dnscli.c output/pcap.c output/respdiff.c output/tcpcli.c output/tlscli.c output/udpcli.c
-nobase_dnsjitinclude_HEADERS += core/assert.h core/channel.h core/compat.h core/file.h core/log.h core/object/dns.h core/object/ether.h core/object/gre.h core/object.h core/object/icmp6.h core/object/icmp.h core/object/ieee802.h core/object/ip6.h core/object/ip.h core/object/linuxsll.h core/object/loop.h core/object/null.h core/object/payload.h core/object/pcap.h core/object/tcp.h core/object/udp.h core/producer.h core/receiver.h core/thread.h core/timespec.h filter/copy.h filter/ipsplit.h filter/layer.h filter/split.h filter/timing.h input/fpcap.h input/mmpcap.h input/pcap.h lib/base64url.h lib/clock.h lib/trie.h output/dnscli.h output/pcap.h output/respdiff.h output/tcpcli.h output/tlscli.h output/udpcli.h
+dnsjit_SOURCES += core/channel.c core/compat.c core/file.c core/log.c core/object.c core/object/dns.c core/object/ether.c core/object/gre.c core/object/icmp6.c core/object/icmp.c core/object/ieee802.c core/object/ip6.c core/object/ip.c core/object/linuxsll.c core/object/loop.c core/object/null.c core/object/payload.c core/object/pcap.c core/object/tcp.c core/object/udp.c core/producer.c core/receiver.c core/thread.c filter/copy.c filter/ipsplit.c filter/layer.c filter/split.c filter/timing.c input/fpcap.c input/mmpcap.c input/pcap.c input/zpcap.c lib/base64url.c lib/clock.c lib/trie.c output/dnscli.c output/pcap.c output/respdiff.c output/tcpcli.c output/tlscli.c output/udpcli.c
+nobase_dnsjitinclude_HEADERS += core/assert.h core/channel.h core/compat.h core/file.h core/log.h core/object/dns.h core/object/ether.h core/object/gre.h core/object.h core/object/icmp6.h core/object/icmp.h core/object/ieee802.h core/object/ip6.h core/object/ip.h core/object/linuxsll.h core/object/loop.h core/object/null.h core/object/payload.h core/object/pcap.h core/object/tcp.h core/object/udp.h core/producer.h core/receiver.h core/thread.h core/timespec.h filter/copy.h filter/ipsplit.h filter/layer.h filter/split.h filter/timing.h input/fpcap.h input/mmpcap.h input/pcap.h input/zpcap.h lib/base64url.h lib/clock.h lib/trie.h output/dnscli.h output/pcap.h output/respdiff.h output/tcpcli.h output/tlscli.h output/udpcli.h
 
 # Lua headers
-nobase_dnsjitinclude_HEADERS += core/channel.hh core/file.hh core/log.hh core/object/dns.hh core/object/ether.hh core/object/gre.hh core/object.hh core/object/icmp6.hh core/object/icmp.hh core/object/ieee802.hh core/object/ip6.hh core/object/ip.hh core/object/linuxsll.hh core/object/loop.hh core/object/null.hh core/object/payload.hh core/object/pcap.hh core/object/tcp.hh core/object/udp.hh core/producer.hh core/receiver.hh core/thread.hh core/timespec.hh filter/copy.hh filter/ipsplit.hh filter/layer.hh filter/split.hh filter/timing.hh input/fpcap.hh input/mmpcap.hh input/pcap.hh lib/base64url.hh lib/clock.hh lib/trie.hh output/dnscli.hh output/pcap.hh output/respdiff.hh output/tcpcli.hh output/tlscli.hh output/udpcli.hh
-lua_hobjects += core/channel.luaho core/file.luaho core/log.luaho core/object/dns.luaho core/object/ether.luaho core/object/gre.luaho core/object/icmp6.luaho core/object/icmp.luaho core/object/ieee802.luaho core/object/ip6.luaho core/object/ip.luaho core/object/linuxsll.luaho core/object/loop.luaho core/object.luaho core/object/null.luaho core/object/payload.luaho core/object/pcap.luaho core/object/tcp.luaho core/object/udp.luaho core/producer.luaho core/receiver.luaho core/thread.luaho core/timespec.luaho filter/copy.luaho filter/ipsplit.luaho filter/layer.luaho filter/split.luaho filter/timing.luaho input/fpcap.luaho input/mmpcap.luaho input/pcap.luaho lib/base64url.luaho lib/clock.luaho lib/trie.luaho output/dnscli.luaho output/pcap.luaho output/respdiff.luaho output/tcpcli.luaho output/tlscli.luaho output/udpcli.luaho
+nobase_dnsjitinclude_HEADERS += core/channel.hh core/file.hh core/log.hh core/object/dns.hh core/object/ether.hh core/object/gre.hh core/object.hh core/object/icmp6.hh core/object/icmp.hh core/object/ieee802.hh core/object/ip6.hh core/object/ip.hh core/object/linuxsll.hh core/object/loop.hh core/object/null.hh core/object/payload.hh core/object/pcap.hh core/object/tcp.hh core/object/udp.hh core/producer.hh core/receiver.hh core/thread.hh core/timespec.hh filter/copy.hh filter/ipsplit.hh filter/layer.hh filter/split.hh filter/timing.hh input/fpcap.hh input/mmpcap.hh input/pcap.hh input/zpcap.hh lib/base64url.hh lib/clock.hh lib/trie.hh output/dnscli.hh output/pcap.hh output/respdiff.hh output/tcpcli.hh output/tlscli.hh output/udpcli.hh
+lua_hobjects += core/channel.luaho core/file.luaho core/log.luaho core/object/dns.luaho core/object/ether.luaho core/object/gre.luaho core/object/icmp6.luaho core/object/icmp.luaho core/object/ieee802.luaho core/object/ip6.luaho core/object/ip.luaho core/object/linuxsll.luaho core/object/loop.luaho core/object.luaho core/object/null.luaho core/object/payload.luaho core/object/pcap.luaho core/object/tcp.luaho core/object/udp.luaho core/producer.luaho core/receiver.luaho core/thread.luaho core/timespec.luaho filter/copy.luaho filter/ipsplit.luaho filter/layer.luaho filter/split.luaho filter/timing.luaho input/fpcap.luaho input/mmpcap.luaho input/pcap.luaho input/zpcap.luaho lib/base64url.luaho lib/clock.luaho lib/trie.luaho output/dnscli.luaho output/pcap.luaho output/respdiff.luaho output/tcpcli.luaho output/tlscli.luaho output/udpcli.luaho
 
 # Lua sources
-dist_dnsjit_SOURCES += core/channel.lua core/compat.lua core/file.lua core/loader.lua core/log.lua core/object/dns/label.lua core/object/dns.lua core/object/dns/q.lua core/object/dns/rr.lua core/object/ether.lua core/object/gre.lua core/object/icmp6.lua core/object/icmp.lua core/object/ieee802.lua core/object/ip6.lua core/object/ip.lua core/object/linuxsll.lua core/object/loop.lua core/object.lua core/object/null.lua core/object/payload.lua core/object/pcap.lua core/objects.lua core/object/tcp.lua core/object/udp.lua core/producer.lua core/receiver.lua core/thread.lua core/timespec.lua filter/copy.lua filter/ipsplit.lua filter/layer.lua filter/split.lua filter/timing.lua input/fpcap.lua input/mmpcap.lua input/pcap.lua input/zero.lua lib/base64url.lua lib/clock.lua lib/getopt.lua lib/ip.lua lib/parseconf.lua lib/trie/iter.lua lib/trie.lua lib/trie/node.lua output/dnscli.lua output/null.lua output/pcap.lua output/respdiff.lua output/tcpcli.lua output/tlscli.lua output/udpcli.lua
-lua_objects += core/channel.luao core/compat.luao core/file.luao core/loader.luao core/log.luao core/object/dns/label.luao core/object/dns.luao core/object/dns/q.luao core/object/dns/rr.luao core/object/ether.luao core/object/gre.luao core/object/icmp6.luao core/object/icmp.luao core/object/ieee802.luao core/object/ip6.luao core/object/ip.luao core/object/linuxsll.luao core/object/loop.luao core/object.luao core/object/null.luao core/object/payload.luao core/object/pcap.luao core/objects.luao core/object/tcp.luao core/object/udp.luao core/producer.luao core/receiver.luao core/thread.luao core/timespec.luao filter/copy.luao filter/ipsplit.luao filter/layer.luao filter/split.luao filter/timing.luao input/fpcap.luao input/mmpcap.luao input/pcap.luao input/zero.luao lib/base64url.luao lib/clock.luao lib/getopt.luao lib/ip.luao lib/parseconf.luao lib/trie/iter.luao lib/trie.luao lib/trie/node.luao output/dnscli.luao output/null.luao output/pcap.luao output/respdiff.luao output/tcpcli.luao output/tlscli.luao output/udpcli.luao
+dist_dnsjit_SOURCES += core/channel.lua core/compat.lua core/file.lua core/loader.lua core/log.lua core/object/dns/label.lua core/object/dns.lua core/object/dns/q.lua core/object/dns/rr.lua core/object/ether.lua core/object/gre.lua core/object/icmp6.lua core/object/icmp.lua core/object/ieee802.lua core/object/ip6.lua core/object/ip.lua core/object/linuxsll.lua core/object/loop.lua core/object.lua core/object/null.lua core/object/payload.lua core/object/pcap.lua core/objects.lua core/object/tcp.lua core/object/udp.lua core/producer.lua core/receiver.lua core/thread.lua core/timespec.lua filter/copy.lua filter/ipsplit.lua filter/layer.lua filter/split.lua filter/timing.lua input/fpcap.lua input/mmpcap.lua input/pcap.lua input/zero.lua input/zpcap.lua lib/base64url.lua lib/clock.lua lib/getopt.lua lib/ip.lua lib/parseconf.lua lib/trie/iter.lua lib/trie.lua lib/trie/node.lua output/dnscli.lua output/null.lua output/pcap.lua output/respdiff.lua output/tcpcli.lua output/tlscli.lua output/udpcli.lua
+lua_objects += core/channel.luao core/compat.luao core/file.luao core/loader.luao core/log.luao core/object/dns/label.luao core/object/dns.luao core/object/dns/q.luao core/object/dns/rr.luao core/object/ether.luao core/object/gre.luao core/object/icmp6.luao core/object/icmp.luao core/object/ieee802.luao core/object/ip6.luao core/object/ip.luao core/object/linuxsll.luao core/object/loop.luao core/object.luao core/object/null.luao core/object/payload.luao core/object/pcap.luao core/objects.luao core/object/tcp.luao core/object/udp.luao core/producer.luao core/receiver.luao core/thread.luao core/timespec.luao filter/copy.luao filter/ipsplit.luao filter/layer.luao filter/split.luao filter/timing.luao input/fpcap.luao input/mmpcap.luao input/pcap.luao input/zero.luao input/zpcap.luao lib/base64url.luao lib/clock.luao lib/getopt.luao lib/ip.luao lib/parseconf.luao lib/trie/iter.luao lib/trie.luao lib/trie/node.luao output/dnscli.luao output/null.luao output/pcap.luao output/respdiff.luao output/tcpcli.luao output/tlscli.luao output/udpcli.luao
 
 dnsjit_LDFLAGS = -Wl,-E
 dnsjit_LDADD += $(lua_hobjects) $(lua_objects)
@@ -63,7 +64,7 @@ man1_MANS = dnsjit.1
 CLEANFILES += $(man1_MANS)
 
 man3_MANS = dnsjit.core.3 dnsjit.lib.3 dnsjit.input.3 dnsjit.filter.3 dnsjit.output.3
-man3_MANS += dnsjit.core.channel.3 dnsjit.core.compat.3 dnsjit.core.file.3 dnsjit.core.loader.3 dnsjit.core.log.3 dnsjit.core.object.3 dnsjit.core.object.dns.3 dnsjit.core.object.dns.label.3 dnsjit.core.object.dns.q.3 dnsjit.core.object.dns.rr.3 dnsjit.core.object.ether.3 dnsjit.core.object.gre.3 dnsjit.core.object.icmp.3 dnsjit.core.object.icmp6.3 dnsjit.core.object.ieee802.3 dnsjit.core.object.ip.3 dnsjit.core.object.ip6.3 dnsjit.core.object.linuxsll.3 dnsjit.core.object.loop.3 dnsjit.core.object.null.3 dnsjit.core.object.payload.3 dnsjit.core.object.pcap.3 dnsjit.core.objects.3 dnsjit.core.object.tcp.3 dnsjit.core.object.udp.3 dnsjit.core.producer.3 dnsjit.core.receiver.3 dnsjit.core.thread.3 dnsjit.core.timespec.3 dnsjit.filter.copy.3 dnsjit.filter.ipsplit.3 dnsjit.filter.layer.3 dnsjit.filter.split.3 dnsjit.filter.timing.3 dnsjit.input.fpcap.3 dnsjit.input.mmpcap.3 dnsjit.input.pcap.3 dnsjit.input.zero.3 dnsjit.lib.base64url.3 dnsjit.lib.clock.3 dnsjit.lib.getopt.3 dnsjit.lib.ip.3 dnsjit.lib.parseconf.3 dnsjit.lib.trie.3 dnsjit.lib.trie.iter.3 dnsjit.lib.trie.node.3 dnsjit.output.dnscli.3 dnsjit.output.null.3 dnsjit.output.pcap.3 dnsjit.output.respdiff.3 dnsjit.output.tcpcli.3 dnsjit.output.tlscli.3 dnsjit.output.udpcli.3
+man3_MANS += dnsjit.core.channel.3 dnsjit.core.compat.3 dnsjit.core.file.3 dnsjit.core.loader.3 dnsjit.core.log.3 dnsjit.core.object.3 dnsjit.core.object.dns.3 dnsjit.core.object.dns.label.3 dnsjit.core.object.dns.q.3 dnsjit.core.object.dns.rr.3 dnsjit.core.object.ether.3 dnsjit.core.object.gre.3 dnsjit.core.object.icmp.3 dnsjit.core.object.icmp6.3 dnsjit.core.object.ieee802.3 dnsjit.core.object.ip.3 dnsjit.core.object.ip6.3 dnsjit.core.object.linuxsll.3 dnsjit.core.object.loop.3 dnsjit.core.object.null.3 dnsjit.core.object.payload.3 dnsjit.core.object.pcap.3 dnsjit.core.objects.3 dnsjit.core.object.tcp.3 dnsjit.core.object.udp.3 dnsjit.core.producer.3 dnsjit.core.receiver.3 dnsjit.core.thread.3 dnsjit.core.timespec.3 dnsjit.filter.copy.3 dnsjit.filter.ipsplit.3 dnsjit.filter.layer.3 dnsjit.filter.split.3 dnsjit.filter.timing.3 dnsjit.input.fpcap.3 dnsjit.input.mmpcap.3 dnsjit.input.pcap.3 dnsjit.input.zero.3 dnsjit.input.zpcap.3 dnsjit.lib.base64url.3 dnsjit.lib.clock.3 dnsjit.lib.getopt.3 dnsjit.lib.ip.3 dnsjit.lib.parseconf.3 dnsjit.lib.trie.3 dnsjit.lib.trie.iter.3 dnsjit.lib.trie.node.3 dnsjit.output.dnscli.3 dnsjit.output.null.3 dnsjit.output.pcap.3 dnsjit.output.respdiff.3 dnsjit.output.tcpcli.3 dnsjit.output.tlscli.3 dnsjit.output.udpcli.3
 CLEANFILES += *.3in $(man3_MANS)
 
 .lua.luao:
@@ -237,6 +238,9 @@ dnsjit.input.pcap.3in: input/pcap.lua gen-manpage.lua
 dnsjit.input.zero.3in: input/zero.lua gen-manpage.lua
 	$(LUAJIT) "$(srcdir)/gen-manpage.lua" "$(srcdir)/input/zero.lua" > "$@"
 
+dnsjit.input.zpcap.3in: input/zpcap.lua gen-manpage.lua
+	$(LUAJIT) "$(srcdir)/gen-manpage.lua" "$(srcdir)/input/zpcap.lua" > "$@"
+
 dnsjit.lib.base64url.3in: lib/base64url.lua gen-manpage.lua
 	$(LUAJIT) "$(srcdir)/gen-manpage.lua" "$(srcdir)/lib/base64url.lua" > "$@"
 

src/gen-makefile.sh

@@ -29,7 +29,8 @@ AM_CFLAGS = -Werror=attributes \
   -I$(top_srcdir)/include \
   $(SIMD_FLAGS) $(CPUEXT_FLAGS) \
   $(PTHREAD_CFLAGS) \
-  $(luajit_CFLAGS)
+  $(luajit_CFLAGS) \
+  $(liblz4_CFLAGS) $(libzstd_CFLAGS)
 
 EXTRA_DIST = gen-manpage.lua gen-compat.lua gen-errno.sh dnsjit.1in
 
@@ -43,7 +44,7 @@ dnsjitincludedir = $(includedir)/dnsjit
 nobase_dnsjitinclude_HEADERS = globals.h version.h
 lua_hobjects = core/compat.luaho
 lua_objects = core.luao lib.luao input.luao filter.luao output.luao
-dnsjit_LDADD = $(PTHREAD_LIBS) $(luajit_LIBS)
+dnsjit_LDADD = $(PTHREAD_LIBS) $(luajit_LIBS) $(liblz4_LIBS) $(libzstd_LIBS)
 
 # C source and headers';
 

src/input.lua

@@ -25,5 +25,6 @@ module(...,package.seeall)
 -- dnsjit.input.fpcap (3),
 -- dnsjit.input.mmpcap (3),
 -- dnsjit.input.pcap (3),
--- dnsjit.input.zero (3)
+-- dnsjit.input.zero (3),
+-- dnsjit.input.zpcap (3)
 return