Makefile: split makefile in parts in .mk/ folder

.mk/api.mk

@@ -0,0 +1,14 @@
+.typescript: statics/js/bundle.js js/browser.js js/api.js
+
+js/api.js: js/api.ts
+	cd js && npm install && PATH=`npm bin`:$$PATH tsc --module commonjs --target ES5 api.ts
+
+js/browser.js: js/browser.ts js/api.ts
+	cd js && npm install && PATH=`npm bin`:$$PATH tsc --module commonjs --target ES5 browser.ts
+
+statics/js/bundle.js: js/browser.js
+	cd js && PATH=`npm bin`:$$PATH browserify browser.js -o ../statics/js/bundle.js
+
+.PHONY: .typescript.clean
+.typescript.clean:
+	make -C js clean

.mk/bench.mk

@@ -0,0 +1,17 @@
+.PHONY: bench
+bench:	skydive bench.flow
+
+flow/pcaptraces/201801011400.pcap.gz:
+	aria2c -s 16 -x 16 -o $@ "http://mawi.nezu.wide.ad.jp/mawi/samplepoint-F/2018/201801011400.pcap.gz"
+
+flow/pcaptraces/%.pcap: flow/pcaptraces/%.pcap.gz
+	gunzip -fk $<
+
+flow/pcaptraces/201801011400.small.pcap: flow/pcaptraces/201801011400.pcap
+	tcpdump -r $< -w $@ "ip host 203.82.244.188"
+
+bench.flow.traces: flow/pcaptraces/201801011400.small.pcap
+
+bench.flow: bench.flow.traces
+	$(GO) test -bench=. ${SKYDIVE_GITHUB}/flow
+

.mk/bindata.mk

@@ -0,0 +1,28 @@
+STATIC_FILES=$(shell find statics -type f \( ! -iname "bindata.go" ! -iname "bundle.js" \))
+GO_BINDATA_GITHUB:=github.com/jteeuwen/go-bindata/go-bindata
+EXTRABINDATA:=
+
+ifeq ($(WITH_EBPF), true)
+  EXTRABINDATA+=probe/ebpf/*.o
+endif
+
+BINDATA_DIRS := \
+	js/*.js \
+	rbac/policy.csv \
+	statics/index.html \
+	statics/css/* \
+	statics/css/themes/*/* \
+	statics/fonts/* \
+	statics/img/* \
+	statics/js/* \
+	statics/schemas/* \
+	statics/workflows/*.yaml \
+	${EXTRABINDATA}
+
+.PHONY: .bindata
+.bindata: statics/bindata.go
+
+statics/bindata.go: statics/js/bundle.js $(EBPF_PROBES) $(STATIC_FILES)
+	echo $(STATIC_FILES)
+	go run ${GO_BINDATA_GITHUB} ${GO_BINDATA_FLAGS} -nometadata -o statics/bindata.go -pkg=statics -ignore=bindata.go $(BINDATA_DIRS)
+	gofmt -w -s statics/bindata.go

.mk/check.mk

@@ -0,0 +1,40 @@
+
+.PHONY: fmt
+fmt: genlocalfiles
+	@echo "+ $@"
+	@test -z "$$($(GO) fmt)" || \
+		(echo "+ please format Go code with 'gofmt -s'" && /bin/false)
+
+.PHONY: vet
+vet:
+	@echo "+ $@"
+	test -z "$$($(GO) tool vet $$( \
+			$(GO) list ./... \
+			| perl -pe 's|$(SKYDIVE_GITHUB)/?||g' \
+			| grep -v '^tests') 2>&1 \
+		| tee /dev/stderr \
+		| grep -v '^flow/probes/afpacket/' \
+		| grep -v 'exit status 1' \
+		)"
+
+.PHONY: check
+check: lint
+	# check if Go modules are in sync
+	# $(GO) mod tidy
+	# @test -z "$$(git diff go.mod go.sum)" || \
+	#	(echo -e "Go modules of sync:\n$$(git diff go.mod go.sum)" && /bin/false)
+	nbnotcomment=$$(grep '"linter":"golint"' lint.json | wc -l); \
+	if [ $$nbnotcomment -gt 0 ]; then \
+		cat lint.json; \
+		echo "===> You should comment you code <==="; \
+		exit 1; \
+	fi
+
+.PHONY: golangci-lint
+golangci-lint:
+	@echo "+ $@"
+	go run github.com/golangci/golangci-lint/cmd/golangci-lint run ${GOMETALINTER_FLAGS} -e '.*\.pb.go' -e '.*\._easyjson.go' -e '.*\._gendecoder.go' -e 'statics/bindata.go' --skip-dirs=statics --deadline 10m --out-format=json ./... | tee lint.json || true
+
+.PHONY: lint
+lint:
+	make golangci-lint GOMETALINTER_FLAGS="--disable-all --enable=golint"

.mk/contrib.mk

@@ -0,0 +1,40 @@
+.PHONY: contribs.clean
+contribs.clean: contrib.exporters.clean contrib.snort.clean contrib.collectd.clean
+
+.PHONY: contribs.test
+contribs.test: contrib.exporters.test
+
+.PHONY: contribs
+contribs: contrib.exporters contrib.snort contrib.collectd
+
+.PHONY: contribs.static
+contribs.static:
+	$(MAKE) -C contrib/exporters static
+
+.PHONY: contrib.exporters.clean
+contrib.exporters.clean:
+	$(MAKE) -C contrib/exporters clean
+
+.PHONY: contrib.exporters
+contrib.exporters: genlocalfiles
+	$(MAKE) -C contrib/exporters
+
+.PHONY: contrib.exporters.test
+contrib.exporters.test: genlocalfiles
+	$(MAKE) -C contrib/exporters test
+
+.PHONY: contribs.snort.clean
+contrib.snort.clean:
+	$(MAKE) -C contrib/snort clean
+
+.PHONY: contrib.snort
+contrib.snort:genlocalfiles
+	$(MAKE) -C contrib/snort
+
+.PHONY: contrib.collectd.clean
+contrib.collectd.clean:
+	$(MAKE) -C contrib/collectd clean
+
+.PHONY: contrib.collectd
+contrib.collectd: genlocalfiles
+	$(MAKE) -C contrib/collectd

.mk/debug.mk

@@ -0,0 +1,26 @@
+
+DLV_FLAGS=--check-go-version=false
+
+ifeq (${DEBUG}, true)
+define skydive_run
+sudo -E $$(which dlv) $(DLV_FLAGS) exec $$(which skydive) -- $1 -c skydive.yml
+endef
+else
+define skydive_run
+sudo -E $$(which skydive) $1 -c skydive.yml
+endef
+endif
+
+ifeq (${DEBUG}, true)
+  GOFLAGS=-gcflags='-N -l'
+  GO_BINDATA_FLAGS+=-debug
+  export DEBUG
+endif
+
+.PHONY: debug.agent
+run.agent:
+	$(call skydive_run,agent)
+
+.PHONY: debug.analyzer
+run.analyzer:
+	$(call skydive_run,analyzer)

.mk/dist.mk

@@ -0,0 +1,52 @@
+BOOTSTRAP:=contrib/packaging/rpm/generate-skydive-bootstrap.sh
+BOOTSTRAP_ARGS?=
+DOCKER_IMAGE?=skydive/skydive
+DOCKER_TAG?=devel
+DESTDIR?=$(shell pwd)
+SKYDIVE_TAR_INPUT:= \
+	vendor \
+	statics/bindata.go \
+	$(GEN_PROTO_FILES) \
+	$(GEN_DECODER_FILES) \
+	$(GEN_EASYJSON_FILES)
+
+SKYDIVE_TAR:=${DESTDIR}/$(SKYDIVE_PKG).tar
+
+define TAR_CMD
+tar $1 -f $(SKYDIVE_TAR) --transform="s||$(SKYDIVE_PATH)|" $2
+endef
+
+define TAR_APPEND
+$(call TAR_CMD,--append,$(SKYDIVE_TAR_INPUT))
+endef
+
+.PHONY: vendor
+vendor:
+ifeq (${GO111MODULE}, on)
+	go mod vendor
+endif
+
+.PHONY: localdist
+localdist: genlocalfiles vendor
+	git ls-files | $(call TAR_CMD,--create,--files-from -)
+	$(call TAR_APPEND,)
+	gzip -f $(SKYDIVE_TAR)
+
+.PHONY: dist
+dist: genlocalfiles vendor
+	git archive -o $(SKYDIVE_TAR) --prefix $(SKYDIVE_PATH) HEAD
+	$(call TAR_APPEND,)
+	gzip -f $(SKYDIVE_TAR)
+
+.PHONY: srpm
+srpm:
+	$(BOOTSTRAP) -s ${BOOTSTRAP_ARGS}
+
+.PHONY: rpm
+rpm:
+	$(BOOTSTRAP) -b ${BOOTSTRAP_ARGS}
+
+.PHONY: docker-image
+docker-image: static
+	cp $$GOPATH/bin/skydive contrib/docker/skydive.$$(uname -m)
+	docker build -t ${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg ARCH=$$(uname -m) -f contrib/docker/Dockerfile contrib/docker/

.mk/easyjson.mk

@@ -0,0 +1,13 @@
+GEN_EASYJSON_FILES = $(patsubst %.go,%_easyjson.go,$(shell git grep //go:generate | grep "easyjson" | grep -v Makefile | cut -d ":" -f 1))
+GEN_EASYJSON_FILES += flow/flow.pb_easyjson.go
+
+%_easyjson.go: %.go
+	go generate -run easyjson $<
+
+.PHONY: .easyjson
+.easyjson: flow/flow.pb_easyjson.go $(GEN_EASYJSON_FILES)
+
+.PHONY: .easyjson.clean
+.easyjson.clean:
+	find . \( -name *_easyjson.go ! -path './vendor/*' \) -exec rm {} \;
+

.mk/gendecoder.mk

@@ -0,0 +1,12 @@
+GEN_DECODER_FILES = $(patsubst %.go,%_gendecoder.go,$(shell git grep //go:generate | grep "gendecoder" | grep -v Makefile | cut -d ":" -f 1))
+GEN_DECODER_FILES += flow/flow.pb_gendecoder.go
+
+%_gendecoder.go: %.go
+	go generate -run gendecoder $<
+
+.PHONY: .gendecoder
+.gendecoder: $(GEN_DECODER_FILES)
+
+.PHONY: .gendecoder.clean
+.gendecoder.clean:
+	find . \( -name *_gendecoder.go ! -path './vendor/*' \) -exec rm {} \;

.mk/proto.mk

@@ -0,0 +1,55 @@
+define PROTOC_GEN
+go get -u ${PROTOC_GEN_GOFAST_GITHUB}
+go get -u ${PROTOC_GEN_GO_GITHUB}
+protoc -I. -Iflow/layers -Ivendor/github.com/gogo/protobuf --plugin=$${GOPATH}/bin/protoc-gen-gogofaster --gogofaster_out $$GOPATH/src $1
+endef
+
+GEN_PROTO_FILES = $(patsubst %.proto,%.pb.go,$(shell find . -name *.proto | grep -v ^./vendor))
+
+%.pb.go: %.proto
+	$(call PROTOC_GEN,$<)
+
+flow/flow.pb.go: flow/flow.proto filters/filters.proto
+	$(call PROTOC_GEN,flow/flow.proto)
+
+	# always export flow.ParentUUID as we need to store this information to know
+	# if it's a Outer or Inner packet.
+	sed -e 's/ParentUUID\(.*\),omitempty\(.*\)/ParentUUID\1\2/' \
+		-e 's/Protocol\(.*\),omitempty\(.*\)/Protocol\1\2/' \
+		-e 's/ICMPType\(.*\),omitempty\(.*\)/ICMPType\1\2/' \
+		-e 's/int64\(.*\),omitempty\(.*\)/int64\1\2/' \
+		-i $@
+	# add omitempty to RTT as it is not always filled
+	sed -e 's/json:"RTT"/json:"RTT,omitempty"/' -i $@
+	# do not export LastRawPackets used internally
+	sed -e 's/json:"LastRawPackets,omitempty"/json:"-"/g' -i $@
+	# add flowState to flow generated struct
+	sed -e 's/type Flow struct {/type Flow struct { XXX_state flowState `json:"-"`/' -i $@
+	# to fix generated layers import
+	sed -e 's/layers "flow\/layers"/layers "github.com\/skydive-project\/skydive\/flow\/layers"/' -i $@
+	sed -e 's/type FlowMetric struct {/\/\/ gendecoder\ntype FlowMetric struct {/' -i $@
+	sed -e 's/type FlowLayer struct {/\/\/ gendecoder\ntype FlowLayer struct {/' -i $@
+	sed -e 's/type TransportLayer struct {/\/\/ gendecoder\ntype TransportLayer struct {/' -i $@
+	sed -e 's/type ICMPLayer struct {/\/\/ gendecoder\ntype ICMPLayer struct {/' -i $@
+	sed -e 's/type IPMetric struct {/\/\/ gendecoder\ntype IPMetric struct {/' -i $@
+	sed -e 's/type TCPMetric struct {/\/\/ gendecoder\ntype TCPMetric struct {/' -i $@
+	# This is to allow calling go generate on flow/flow.pb.go
+	sed -e 's/DO NOT EDIT./DO NOT MODIFY/' -i $@
+	sed '1 i //go:generate go run github.com/skydive-project/skydive/scripts/gendecoder' -i $@
+	gofmt -s -w $@
+
+flow/flow.pb_easyjson.go: flow/flow.pb.go
+	go run github.com/safchain/easyjson/easyjson -all $<
+
+websocket/structmessage.pb.go: websocket/structmessage.proto
+	$(call PROTOC_GEN,$<)
+
+	sed -e 's/type StructMessage struct {/type StructMessage struct { XXX_state structMessageState `json:"-"`/' -i websocket/structmessage.pb.go
+	gofmt -s -w $@
+
+.proto: vendor $(GEN_PROTO_FILES)
+
+.PHONY: .proto.clean
+.proto.clean:
+	find . \( -name *.pb.go ! -path './vendor/*' \) -exec rm {} \;
+	rm -rf flow/layers/generated.proto

.mk/static.mk

@@ -0,0 +1,52 @@
+STATIC_DIR?=
+STATIC_LIBS?=
+
+OS_RHEL := $(shell test -f /etc/redhat-release && echo -n Y)
+ifeq ($(OS_RHEL),Y)
+	STATIC_DIR := /usr/lib64
+	STATIC_LIBS := \
+		libz.a \
+		liblzma.a \
+		libm.a
+endif
+
+OS_DEB := $(shell test -f /etc/debian_version && echo -n Y)
+ifeq ($(OS_DEB),Y)
+	STATIC_DIR := $(shell dpkg-architecture -c 'sh' -c 'echo /usr/lib/$$DEB_HOST_MULTIARCH')
+	STATIC_LIBS := \
+		libz.a \
+		liblzma.a \
+		libc.a \
+		libdl.a \
+		libpthread.a \
+		libc++.a \
+		libm.a
+endif
+
+ifeq ($(WITH_OPENCONTRAIL), true)
+  BUILD_TAGS+=opencontrail
+ifeq ($(OS_RHEL),Y)
+  STATIC_LIBS+=libxml2.a
+endif
+ifeq ($(OS_DEB),Y)
+  STATIC_LIBS+=libicuuc.a \
+               libicudata.a \
+               libxml2.a
+endif
+endif
+
+STATIC_LIBS_ABS := $(addprefix $(STATIC_DIR)/,$(STATIC_LIBS))
+STATIC_BUILD_TAGS := $(filter-out libvirt,$(BUILD_TAGS))
+
+.PHONY: static
+static: skydive.clean genlocalfiles
+	$(MAKE) compile.static WITH_LIBVIRT_GO=false
+	$(MAKE) contribs.static
+
+.PHONY: compile.static
+compile.static:
+	CGO_CFLAGS_ALLOW='.*' CGO_LDFLAGS_ALLOW='.*' $(GO) install \
+		-ldflags="${LDFLAGS} -B $(BUILD_ID) -X $(SKYDIVE_GITHUB_VERSION) '-extldflags=-static $(STATIC_LIBS_ABS)'" \
+		${GOFLAGS} \
+		${VERBOSE_FLAGS} -tags "netgo ${STATIC_BUILD_TAGS}" \
+		-installsuffix netgo || true