diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..9ccbba5 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) + +project(RESP LANGUAGES C) + +list(PREPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +find_package(SpicyPlugin REQUIRED) + +# Set minimum versions that this plugin needs. Make sure to use "x.y.z" format. +# spicy_require_version("1.2.0") +# spicy_plugin_require_version("0.99.0") +# zeek_require_version("5.0.0") + +if (NOT CMAKE_BUILD_TYPE) + # Default to release build. + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") +endif () + +add_subdirectory(analyzer) diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..ea26df0 --- /dev/null +++ b/COPYING @@ -0,0 +1 @@ +TODO: Please provide licensing information here. diff --git a/README b/README new file mode 100644 index 0000000..945872d --- /dev/null +++ b/README @@ -0,0 +1,13 @@ +# Spicy-based RESP analyzer + +Parses the [Redis serialization protocol](https://redis.io/docs/latest/develop/reference/protocol-spec/) (RESP). + +## Usage + +Currently, the package sucks. The following is usage from the `analyzer` directory: + +1) Grab a PCAP (like [redis.pcap](https://github.com/macbre/data-flow-graph/blob/master/sources/pcap/redis.pcap)) +2) Compile the code so Zeek can use it: `spicyz -o resp.hlto resp.spicy resp.evt zeek_analyzer.spicy` +3) See some output via Zeek: `zeek -C -r redis.pcap resp.hlto` + +This will be updated as it's better :) diff --git a/analyzer/CMakeLists.txt b/analyzer/CMakeLists.txt new file mode 100644 index 0000000..28afeb1 --- /dev/null +++ b/analyzer/CMakeLists.txt @@ -0,0 +1,5 @@ +spicy_add_analyzer( + NAME RESP + PACKAGE_NAME spicy-resp + SOURCES resp.spicy resp.evt zeek_resp.spicy +) diff --git a/resp.evt b/analyzer/resp.evt similarity index 66% rename from resp.evt rename to analyzer/resp.evt index f1ab89a..3377bb5 100644 --- a/resp.evt +++ b/analyzer/resp.evt @@ -5,4 +5,4 @@ protocol analyzer spicy::RESP over TCP: import RESP; import Zeek_RESP; -on RESP::Data -> event resp::data($conn, Zeek_RESP::create_data(self)); +on RESP::Data -> event RESP::data($conn, Zeek_RESP::create_data(self)); diff --git a/resp.spicy b/analyzer/resp.spicy similarity index 99% rename from resp.spicy rename to analyzer/resp.spicy index 156d501..7351503 100644 --- a/resp.spicy +++ b/analyzer/resp.spicy @@ -1,7 +1,5 @@ module RESP; -import spicy; - public type Data = unit { ty: uint8 &convert=DataType($$); switch ( self.ty ) { diff --git a/zeek_analyzer.spicy b/analyzer/zeek_resp.spicy similarity index 91% rename from zeek_analyzer.spicy rename to analyzer/zeek_resp.spicy index bb9939a..baeec65 100644 --- a/zeek_analyzer.spicy +++ b/analyzer/zeek_resp.spicy @@ -1,9 +1,12 @@ +# Set up protocol confirmation/rejection for analyzers, as well as any further +# Zeek-specific analysis. + module Zeek_RESP; import RESP; -import spicy; import zeek; +# Any error bubbling up to the top unit will trigger a protocol rejection. on RESP::Data::%done { print self; zeek::confirm_protocol(); diff --git a/cmake/FindSpicyPlugin.cmake b/cmake/FindSpicyPlugin.cmake new file mode 100644 index 0000000..67cbd47 --- /dev/null +++ b/cmake/FindSpicyPlugin.cmake @@ -0,0 +1,78 @@ +# Find the Spicy plugin to get access to the infrastructure it provides. +# +# While most of the actual CMake logic for building analyzers comes with the Spicy +# plugin for Zeek, this code bootstraps us by asking "spicyz" for the plugin's +# location. Either make sure that "spicyz" is in PATH, set the environment +# variable SPICYZ to point to its location, or set variable ZEEK_SPICY_ROOT +# in either CMake or environment to point to its installation or build +# directory. +# +# This exports: +# +# SPICY_PLUGIN_FOUND True if plugin and all dependencies were found +# SPICYZ Path to spicyz +# SPICY_PLUGIN_VERSION Version string of plugin +# SPICY_PLUGIN_VERSION_NUMBER Numerical version number of plugin + +# Runs `spicyz` with the flags given as second argument and stores the output in the variable named +# by the first argument. +function (run_spicycz output) + execute_process(COMMAND "${SPICYZ}" ${ARGN} OUTPUT_VARIABLE output_ + OUTPUT_STRIP_TRAILING_WHITESPACE) + + string(STRIP "${output_}" output_) + set(${output} "${output_}" PARENT_SCOPE) +endfunction () + +# Checks that the Spicy plugin version it at least the given version. +function (spicy_plugin_require_version version) + string(REGEX MATCH "([0-9]*)\.([0-9]*)\.([0-9]*).*" _ ${version}) + math(EXPR version_number "${CMAKE_MATCH_1} * 10000 + ${CMAKE_MATCH_2} * 100 + ${CMAKE_MATCH_3}") + + if ("${SPICY_PLUGIN_VERSION_NUMBER}" LESS "${version_number}") + message(FATAL_ERROR "Package requires at least Spicy plugin version ${version}, " + "have ${SPICY_PLUGIN_VERSION}") + endif () +endfunction () + +### +### Main +### + +if (NOT SPICYZ) + set(SPICYZ "$ENV{SPICYZ}") +endif () + +if (NOT SPICYZ) + # Support an in-tree Spicy build. + find_program( + spicyz spicyz + HINTS ${ZEEK_SPICY_ROOT}/bin ${ZEEK_SPICY_ROOT}/build/bin $ENV{ZEEK_SPICY_ROOT}/bin + $ENV{ZEEK_SPICY_ROOT}/build/bin ${PROJECT_SOURCE_DIR}/../../build/bin) + set(SPICYZ "${spicyz}") +endif () + +message(STATUS "spicyz: ${SPICYZ}") + +if (SPICYZ) + set(SPICYZ "${SPICYZ}" CACHE PATH "" FORCE) # make sure it's in the cache + + run_spicycz(SPICY_PLUGIN_VERSION "--version") + run_spicycz(SPICY_PLUGIN_VERSION_NUMBER "--version-number") + message(STATUS "Zeek plugin version: ${SPICY_PLUGIN_VERSION}") + + run_spicycz(spicy_plugin_path "--print-plugin-path") + set(spicy_plugin_cmake_path "${spicy_plugin_path}/cmake") + message(STATUS "Zeek plugin CMake path: ${spicy_plugin_cmake_path}") + + list(PREPEND CMAKE_MODULE_PATH "${spicy_plugin_cmake_path}") + find_package(Zeek REQUIRED) + find_package(Spicy REQUIRED) + zeek_print_summary() + spicy_print_summary() + + include(ZeekSpicyAnalyzerSupport) +endif () + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(SpicyPlugin DEFAULT_MSG SPICYZ ZEEK_FOUND) diff --git a/resp.zeek b/resp.zeek deleted file mode 100644 index 357d22a..0000000 --- a/resp.zeek +++ /dev/null @@ -1,18 +0,0 @@ -type RESPData: record { - simple_string: string &optional; - simple_error: string &optional; - i: int &optional; - bulk_string: string &optional; - #array: - is_null: bool; - boolean: bool &optional; - #double_: double &optional; - big_num: string &optional; - bulk_error: string &optional; - verbatim_string: string &optional; -}; - -event resp::data(c: connection, data: RESPData) - { - print "RESP data", c$id, data; - } diff --git a/scripts/__load__.zeek b/scripts/__load__.zeek new file mode 100644 index 0000000..fd8a18b --- /dev/null +++ b/scripts/__load__.zeek @@ -0,0 +1,2 @@ +@load ./main +@load-sigs ./dpd diff --git a/scripts/dpd.sig b/scripts/dpd.sig new file mode 100644 index 0000000..6072e1a --- /dev/null +++ b/scripts/dpd.sig @@ -0,0 +1,8 @@ +# TODO: Use this file to optionally declare signatures activating your analyzer +# (instead of, or in addition to, using a well-known port). +# +# signature dpd_resp { +# ip-proto == tcp +# payload /^\x11\x22\x33\x44/ # TODO: Detect your protocol here. +# enable "spicy_RESP" +# } diff --git a/scripts/main.zeek b/scripts/main.zeek new file mode 100644 index 0000000..63ebf78 --- /dev/null +++ b/scripts/main.zeek @@ -0,0 +1,103 @@ +@load base/protocols/conn/removal-hooks + +module RESP; + +export { + ## Log stream identifier. + redef enum Log::ID += { LOG }; + + ## The ports to register RESP for. + const ports = { + # TODO: Replace with actual port(s). + 12345/tcp, + } &redef; + + ## Record type containing the column fields of the RESP log. + type Info: record { + ## Timestamp for when the activity happened. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + + # TODO: Adapt subsequent fields as needed. + + ## Request-side payload. + request: string &optional &log; + ## Response-side payload. + reply: string &optional &log; + }; + + ## A default logging policy hook for the stream. + global log_policy: Log::PolicyHook; + + ## Default hook into RESP logging. + global log_resp: event(rec: Info); + + ## RESP finalization hook. + global finalize_resp: Conn::RemovalHook; +} + +redef record connection += { + resp: Info &optional; +}; + +redef likely_server_ports += { ports }; + +# TODO: If you're going to send file data into the file analysis framework, you +# need to provide a file handle function. This is a simple example that's +# sufficient if the protocol only transfers a single, complete file at a time. +# +# function get_file_handle(c: connection, is_orig: bool): string +# { +# return cat(Analyzer::ANALYZER_RESP, c$start_time, c$id, is_orig); +# } + +event zeek_init() &priority=5 + { + Log::create_stream(RESP::LOG, [$columns=Info, $ev=log_resp, $path="resp", $policy=log_policy]); + + Analyzer::register_for_ports(Analyzer::ANALYZER_RESP, ports); + + # TODO: To activate the file handle function above, uncomment this. + # Files::register_protocol(Analyzer::ANALYZER_RESP, [$get_file_handle=RESP::get_file_handle ]); + } + +# Initialize logging state. +hook set_session(c: connection) + { + if ( c?$resp ) + return; + + c$resp = Info($ts=network_time(), $uid=c$uid, $id=c$id); + Conn::register_removal_hook(c, finalize_resp); + } + +function emit_log(c: connection) + { + if ( ! c?$resp ) + return; + + Log::write(RESP::LOG, c$resp); + delete c$resp; + } + +# Example event defined in resp.evt. +event RESP::data(c: connection, is_orig: bool, payload: string) + { + hook set_session(c); + + local info = c$resp; + if ( is_orig ) + info$request = payload; + else + info$reply = payload; + } + +hook finalize_resp(c: connection) + { + # TODO: For UDP protocols, you may want to do this after every request + # and/or reply. + emit_log(c); + } diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 0000000..fc422ef --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,2 @@ +.btest.failed.dat +.tmp diff --git a/testing/Baseline/tests.run-pcap/conn.log.filtered b/testing/Baseline/tests.run-pcap/conn.log.filtered new file mode 100644 index 0000000..1e6917b --- /dev/null +++ b/testing/Baseline/tests.run-pcap/conn.log.filtered @@ -0,0 +1,2 @@ +ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.9.63 63526 54.175.222.246 80 tcp http 0.755677 207 489 SF 0 ShADTadFf 7 790 4 705 - diff --git a/testing/Baseline/tests.run-pcap/output b/testing/Baseline/tests.run-pcap/output new file mode 100644 index 0000000..781c9fc --- /dev/null +++ b/testing/Baseline/tests.run-pcap/output @@ -0,0 +1,2 @@ +Hello world! +Goodbye world! diff --git a/testing/Baseline/tests.standalone/output b/testing/Baseline/tests.standalone/output new file mode 100644 index 0000000..d4cb061 --- /dev/null +++ b/testing/Baseline/tests.standalone/output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +RESP::Data { + payload: test string +} diff --git a/testing/Baseline/tests.trace/output b/testing/Baseline/tests.trace/output new file mode 100644 index 0000000..dbe7d21 --- /dev/null +++ b/testing/Baseline/tests.trace/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Testing RESP: [request] [orig_h=127.0.0.1, orig_p=64091/tcp, resp_h=127.0.0.1, resp_p=12345/tcp] Hello, Spicy! +Testing RESP: [reply] [orig_h=127.0.0.1, orig_p=64091/tcp, resp_h=127.0.0.1, resp_p=12345/tcp] Hello, back! diff --git a/testing/Baseline/tests.trace/resp.log b/testing/Baseline/tests.trace/resp.log new file mode 100644 index 0000000..805e618 --- /dev/null +++ b/testing/Baseline/tests.trace/resp.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path resp +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p request reply +#types time string addr port addr port string string +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 127.0.0.1 64091 127.0.0.1 12345 Hello, Spicy! Hello, back! +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/Files/random.seed b/testing/Files/random.seed new file mode 100644 index 0000000..6956a2c --- /dev/null +++ b/testing/Files/random.seed @@ -0,0 +1,21 @@ +2983378351 +1299727368 +0 +310447 +0 +1409073626 +3975311262 +34130240 +1450515018 +1466150520 +1342286698 +1193956778 +2188527278 +3361989254 +3912865238 +3596260151 +517973768 +1462428821 +0 +2278350848 +32767 diff --git a/testing/Makefile b/testing/Makefile new file mode 100644 index 0000000..b43cce5 --- /dev/null +++ b/testing/Makefile @@ -0,0 +1,3 @@ + +test: + @btest -c btest.cfg diff --git a/testing/Scripts/README b/testing/Scripts/README new file mode 100644 index 0000000..8098fa4 --- /dev/null +++ b/testing/Scripts/README @@ -0,0 +1,7 @@ +Place helper scripts, such a btest-diff canonifiers, in this directory. +Note that Zeek versions 4.1 and newer include their btest tooling as part +of the installation. Take a look at the folder reported via + + zeek-config --btest_tools_dir + +for scripts, PRNG seeds, and pcaps you might be able to reuse. diff --git a/testing/Scripts/diff-remove-timestamps b/testing/Scripts/diff-remove-timestamps new file mode 100755 index 0000000..f9b986f --- /dev/null +++ b/testing/Scripts/diff-remove-timestamps @@ -0,0 +1,6 @@ +#! /usr/bin/env bash +# +# Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs). + +sed -E 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | + sed -E 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' diff --git a/testing/Scripts/get-zeek-env b/testing/Scripts/get-zeek-env new file mode 100755 index 0000000..d92cb9e --- /dev/null +++ b/testing/Scripts/get-zeek-env @@ -0,0 +1,40 @@ +#! /bin/sh +# +# BTest helper for getting values for Zeek-related environment variables. + +# shellcheck disable=SC2002 +base="$(dirname "$0")" +zeek_dist=$(cat "${base}/../../build/CMakeCache.txt" 2>/dev/null | grep ZEEK_DIST | cut -d = -f 2) + +if [ -n "${zeek_dist}" ]; then + if [ "$1" = "zeekpath" ]; then + "${zeek_dist}/build/zeek-path-dev" + elif [ "$1" = "zeek_plugin_path" ]; then + (cd "${base}/../.." && pwd) + elif [ "$1" = "path" ]; then + echo "${zeek_dist}/build/src:${zeek_dist}/aux/btest:${base}/:${zeek_dist}/aux/zeek-cut:$PATH" + else + echo "usage: $(basename "$0") " >&2 + exit 1 + fi +else + # Use Zeek installation for testing. In this case zeek-config must be in PATH. + if ! which zeek-config >/dev/null 2>&1; then + echo "zeek-config not found" >&2 + exit 1 + fi + + if [ "$1" = "zeekpath" ]; then + zeek-config --zeekpath + elif [ "$1" = "zeek_plugin_path" ]; then + # Combine the local tree and the system-wide path. This allows + # us to test on a local build or an installation made via zkg, + # which squirrels away the build. + echo "$(cd "${base}/../.." && pwd)/build:$(zeek-config --plugin_dir)" + elif [ "$1" = "path" ]; then + echo "${PATH}" + else + echo "usage: $(basename "$0") " >&2 + exit 1 + fi +fi diff --git a/testing/Traces/tcp-port-12345.pcap b/testing/Traces/tcp-port-12345.pcap new file mode 100644 index 0000000..279e3f5 Binary files /dev/null and b/testing/Traces/tcp-port-12345.pcap differ diff --git a/testing/Traces/udp-port-12345.pcap b/testing/Traces/udp-port-12345.pcap new file mode 100644 index 0000000..60feed7 Binary files /dev/null and b/testing/Traces/udp-port-12345.pcap differ diff --git a/testing/btest.cfg b/testing/btest.cfg new file mode 100644 index 0000000..8447470 --- /dev/null +++ b/testing/btest.cfg @@ -0,0 +1,27 @@ +[btest] +TestDirs = tests +TmpDir = %(testbase)s/.tmp +BaselineDir = %(testbase)s/Baseline +IgnoreDirs = .tmp +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store + +[environment] +ZEEKPATH=`%(testbase)s/Scripts/get-zeek-env zeekpath` +ZEEK_PLUGIN_PATH=`%(testbase)s/Scripts/get-zeek-env zeek_plugin_path` +ZEEK_SEED_FILE=%(testbase)s/Files/random.seed +PATH=`%(testbase)s/Scripts/get-zeek-env path` +PACKAGE=%(testbase)s/../scripts +TZ=UTC +LC_ALL=C +TRACES=%(testbase)s/Traces +TMPDIR=%(testbase)s/.tmp +TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-remove-timestamps +DIST=%(testbase)s/.. +# Set compilation-related variables to well-defined state. +CC= +CXX= +CFLAGS= +CPPFLAGS= +CXXFLAGS= +LDFLAGS= +DYLDFLAGS= diff --git a/testing/tests/availability.zeek b/testing/tests/availability.zeek new file mode 100644 index 0000000..0566185 --- /dev/null +++ b/testing/tests/availability.zeek @@ -0,0 +1,3 @@ +# @TEST-DOC: Check that the RESP analyzer is available. +# +# @TEST-EXEC: zeek -NN | grep -Eqi 'ANALYZER_RESP' diff --git a/testing/tests/standalone.spicy b/testing/tests/standalone.spicy new file mode 100644 index 0000000..eba2e57 --- /dev/null +++ b/testing/tests/standalone.spicy @@ -0,0 +1,7 @@ +# @TEST-DOC: Test parsing behavior of RESP. +# +# @TEST-EXEC: spicyc ${DIST}/analyzer/resp.spicy -j -d -o resp.hlto +# +# TODO: Add/adapt standalone parsing tests here. +# @TEST-EXEC: printf "test string" | spicy-dump -p RESP::Data resp.hlto >output 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output diff --git a/testing/tests/trace.zeek b/testing/tests/trace.zeek new file mode 100644 index 0000000..13bb81b --- /dev/null +++ b/testing/tests/trace.zeek @@ -0,0 +1,13 @@ +# @TEST-DOC: Test Zeek parsing a trace file through the RESP analyzer. +# +# @TEST-EXEC: zeek -Cr ${TRACES}/tcp-port-12345.pcap ${PACKAGE} %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: btest-diff resp.log + +# TODO: Adapt as suitable. The example only checks the output of the event +# handlers. + +event RESP::message(c: connection, is_orig: bool, payload: string) + { + print fmt("Testing RESP: [%s] %s %s", (is_orig ? "request" : "reply"), c$id, payload); + } diff --git a/zkg.meta b/zkg.meta new file mode 100644 index 0000000..d96f4d0 --- /dev/null +++ b/zkg.meta @@ -0,0 +1,23 @@ +[package] +script_dir = scripts +test_command = cd testing && btest -c btest.cfg +summary = TODO: A summary of spicy-resp in one line +description = TODO: A more detailed description of spicy-resp. + It can span multiple lines, with this indentation. +depends = + zeek >=4.0.0 +build_command = mkdir -p build && cd build && SPICYZ=$(command -v spicyz || echo %(package_base)s/spicy-plugin/build/bin/spicyz) cmake .. && cmake --build . + +[template] +source = https://github.com/zeek/package-template +version = v3.1.4 +zkg_version = 3.0.1-33 +features = spicy-protocol-analyzer + +[template_vars] +name = spicy-resp +analyzer = RESP +protocol = TCP +unit_orig = Data +unit_resp = Data +