Rebar3 as primary build

.gitignore

@@ -1,13 +1,13 @@
 doc/
 erl_crash.dump
+*.crashdump
 /deps/
 /logs/
 ebin/
 cover/
 test/*.beam
 test/ct.cover.spec
 brod.d
-scripts/brod_cli
 .brod.plt
 .erlang.mk
 *.coverdata

.travis.yml

@@ -5,8 +5,17 @@ sudo: required
 services:
   - docker
 
+# install latest rebar3 because the one travis provids doesn't support cover_export_name option
 before_install:
+  - |
+    git clone https://github.com/erlang/rebar3.git
+    pushd .
+    cd rebar3
+    ./bootstrap
+    sudo mv rebar3 /usr/local/bin/
+    popd
   - git clone https://github.com/inaka/elvis.git; cd elvis; rebar3 escriptize; sudo cp _build/default/bin/elvis /usr/bin; cd ..
+  - sudo docker info
 
 notifications:
   email: false
@@ -17,26 +26,25 @@ otp_release:
   - 19.3
 
 script:
-  - set -e
+  # check style first
   - elvis rock
 
-  - echo "verifying rebar compilation"
+  # test compilation with rebar
   - rebar get-deps
   - rebar compile
 
+  # test with rebar3
   - make distclean
-  - echo "verifying rebar3 compilation"
   - rebar3 compile
 
   - make distclean
-  - make rel
-  - make edoc
-  - make xref
+  - rebar3 edoc
+  - rebar3 xref
   - |
     OTP_VSN=`erl -noshell -eval 'io:format(erlang:system_info(otp_release)), halt(0)'`
     if [ $OTP_VSN -eq 21 ]; then
       # run dialyzer only in otp 21
-      make dialyze
+      rebar3 dialyzer
       # test against kafka 0.9 in otp 21
       export KAFKA_VERSION="0.9"
     fi
@@ -54,3 +62,4 @@ script:
     export KAFKA_VERSION="1.1"
     make test-env
     make t
+    make cover

Makefile

@@ -1,62 +1,65 @@
+KAFKA_VERSION ?= 1.1
 PROJECT = brod
 PROJECT_DESCRIPTION = Kafka client library in Erlang
 PROJECT_VERSION = 3.6.0
 
 DEPS = supervisor3 kafka_protocol
-TEST_DEPS = docopt jsone meck proper
-REL_DEPS = docopt jsone
 
-COMMON_ERLC_OPTS = -Werror +warn_unused_vars +warn_shadow_vars +warn_unused_import +warn_obsolete_guard +debug_info -Dbuild_brod_cli
-ERLC_OPTS = $(COMMON_ERLC_OPTS)
-TEST_ERLC_OPTS = $(COMMON_ERLC_OPTS)
+ERLC_OPTS = -Werror +warn_unused_vars +warn_shadow_vars +warn_unused_import +warn_obsolete_guard +debug_info -Dbuild_brod_cli
 
-
-dep_supervisor3_commit = 1.1.5
-dep_kafka_protocol_commit = 2.1.0
+dep_supervisor3_commit = 1.1.7
+dep_kafka_protocol_commit = 2.1.1
 dep_kafka_protocol = git https://github.com/klarna/kafka_protocol.git $(dep_kafka_protocol_commit)
-dep_docopt = git https://github.com/zmstone/docopt-erl.git 0.1.3
-
-ERTS_VSN = $(shell erl -noshell -eval 'io:put_chars(erlang:system_info(version)), halt()')
-ESCRIPT_FILE = scripts/brod_cli
-ESCRIPT_EMU_ARGS = -sname brod_cli
 
 EDOC_OPTS = preprocess, {macros, [{build_brod_cli, true}]}
-COVER = true
-
-EUNIT_OPTS = verbose
-
-CT_OPTS = -ct_use_short_names true
-
-ERL_LIBS := $(ERL_LIBS):$(CURDIR)
 
 ## Make app the default target
 ## To avoid building a release when brod is used as a erlang.mk project's dependency
 app::
 
 include erlang.mk
 
-rel:: escript
-	@cp scripts/brod _rel/brod/bin/brod
-	@cp $(ESCRIPT_FILE) _rel/brod/bin/brod_cli
-	@ln -sf erts-$(ERTS_VSN) _rel/brod/erts
-	@tar -pczf _rel/brod.tar.gz -C _rel brod
+compile:
+	@rebar3 compile
 
 test-env:
 	@./scripts/setup-test-env.sh
 
-t: eunit ct
-	./scripts/cover-summary.escript eunit.coverdata ct.coverdata
+ut:
+	@rebar3 eunit -v --cover_export_name ut-$(KAFKA_VERSION)
+
+# version check, eunit and all common tests
+t: vsn-check ut
+	@rebar3 ct -v --cover_export_name ct-$(KAFKA_VERSION)
 
 vsn-check:
-	$(verbose) ./scripts/vsn-check.sh $(PROJECT_VERSION)
+	@./scripts/vsn-check.sh $(PROJECT_VERSION)
+
+distclean:: rebar-clean
+
+rebar-clean:
+	@rebar3 clean
+	@rm -rf _build
+	@rm -rf doc
+	@rm -f pipe.testdata
 
 hex-publish: distclean
-	$(verbose) rebar3 hex publish
+	@rebar3 hex publish
 
 ## tests that require kafka running at localhost
 INTEGRATION_CTS = brod_cg_commits brod_client brod_compression brod_consumer brod_producer brod_group_subscriber brod_topic_subscriber
 
 ## integration tests
-.PHONY: it
-it: eunit $(INTEGRATION_CTS:%=ct-%)
+it: ut $(INTEGRATION_CTS:%=it-%)
+
+$(INTEGRATION_CTS:%=it-%):
+	@rebar3 ct -v --suite=test/$(subst it-,,$@)_SUITE --cover_export_name $@-$(KAFKA_VERSION)
+
+## build escript and a releas, and copy escript to release bin dir
+brod-cli:
+	@rebar3 as brod_cli escriptize
+	@rebar3 as brod_cli release
+	@cp _build/brod_cli/bin/brod_cli _build/brod_cli/rel/brod/bin/
 
+cover:
+	@rebar3 cover -v

README.md

@@ -23,7 +23,7 @@ Why "brod"? [http://en.wikipedia.org/wiki/Max_Brod](http://en.wikipedia.org/wiki
 
 # Building and testing
 
-    make
+    make compile
     make test-env t # requires docker-compose in place
 
 # Working With Kafka 0.9.x or Earlier
@@ -47,29 +47,29 @@ e.g. in sys.config:
 
 Assuming kafka is running at `localhost:9092` and there is a topic named `test-topic`.
 
-Start Erlang shell by `make; erl -pa ebin -pa deps/*/ebin`, then paste lines below into shell:
+Start Erlang shell by `make compile; erl -pa _build/default/lib/*/ebin`, then paste lines below into shell:
 
 ```erlang
-rr(brod).
-{ok, _} = application:ensure_all_started(brod).
-KafkaBootstrapEndpoints = [{"localhost", 9092}].
-Topic = <<"test-topic">>.
-Partition = 0.
-ok = brod:start_client(KafkaBootstrapEndpoints, client1).
-ok = brod:start_producer(client1, Topic, _ProducerConfig = []).
-{ok, FirstOffset} = brod:produce_sync_offset(client1, Topic, Partition, <<"key1">>, <<"value1">>).
-ok = brod:produce_sync(client1, Topic, Partition, <<"key2">>, <<"value2">>).
-SubscriberCallbackFun = fun(Partition, Msg, ShellPid = CallbackState) -> ShellPid ! Msg, {ok, ack, CallbackState} end.
-Receive = fun() -> receive Msg -> Msg after 1000 -> timeout end end.
+rr(brod),
+{ok, _} = application:ensure_all_started(brod),
+KafkaBootstrapEndpoints = [{"localhost", 9092}],
+Topic = <<"test-topic">>,
+Partition = 0,
+ok = brod:start_client(KafkaBootstrapEndpoints, client1),
+ok = brod:start_producer(client1, Topic, _ProducerConfig = []),
+{ok, FirstOffset} = brod:produce_sync_offset(client1, Topic, Partition, <<"key1">>, <<"value1">>),
+ok = brod:produce_sync(client1, Topic, Partition, <<"key2">>, <<"value2">>),
+SubscriberCallbackFun = fun(Partition, Msg, ShellPid = CallbackState) -> ShellPid ! Msg, {ok, ack, CallbackState} end,
+Receive = fun() -> receive Msg -> Msg after 1000 -> timeout end end,
 brod_topic_subscriber:start_link(client1, Topic, Partitions=[Partition],
                                  _ConsumerConfig=[{begin_offset, FirstOffset}],
                                  _CommittdOffsets=[], message, SubscriberCallbackFun,
-                                 _CallbackState=self()).
-Receive().
-Receive().
+                                 _CallbackState=self()),
 AckCb = fun(Partition, BaseOffset) -> io:format(user, "\nProduced to partition ~p at base-offset ~p\n", [Partition, BaseOffset]) end,
 ok = brod:produce_cb(client1, Topic, Partition, <<>>, [{<<"key3">>, <<"value3">>}], AckCb).
-{ok, [Msg]} = brod:fetch(KafkaBootstrapEndpoints, Topic, Partition, FirstOffset + 2), Msg.
+Receive().
+Receive().
+{ok, {_, [Msg]}} = brod:fetch(KafkaBootstrapEndpoints, Topic, Partition, FirstOffset + 2), Msg.
 ```
 
 Example outputs:
@@ -413,80 +413,73 @@ brod:fetch(Hosts, Topic, Partition, 1).
 This will build a self-contained binary with brod application
 
 ```bash
-make rel
-./_rel/brod/bin/brod --help
+make brod-cli
+_build/brod_cli/rel/brod/bin/brod -h
 ```
 
 Disclaimer: This script is NOT designed for use cases where fault-tolerance is a hard requirement.
 As it may crash when e.g. kafka cluster is temporarily unreachable,
 or (for fetch command) when the parition leader migrates to another broker in the cluster.
 
-Start an Erlang shell with brod started
-
-```bash
-./_rel/brod/bin/brod-i
-```
-
-## brod-cli examples:
+## brod-cli examples (with `alias brod=_build/brod_cli/rel/brod/bin/brod`):
 
 ### Fetch and print metadata
 ```
-./scripts/brod meta -b localhost
+brod meta -b localhost
 ```
 
 ### Produce a Message
 
 ```
-./scripts/brod send -b localhost -t test-topic -p 0 -k "key" -v "value"
+brod send -b localhost -t test-topic -p 0 -k "key" -v "value"
 
 ```
 
 ### Fetch a Message
 
 ```
-./scripts/brod fetch -b localhost -t test-topic -p 0 --fmt 'io:format("offset=~p, ts=~p, key=~s, value=~s\n", [Offset, Ts, Key, Value])'
+brod fetch -b localhost -t test-topic -p 0 --fmt 'io:format("offset=~p, ts=~p, key=~s, value=~s\n", [Offset, Ts, Key, Value])'
 ```
 
 Bound variables to be used in `--fmt` expression:
 
 - `Offset`: Message offset
 - `Key`: Kafka key
 - `Value`: Kafka Value
-- `CRC`: Message CRC
 - `TsType`: Timestamp type either `create` or `append`
 - `Ts`: Timestamp, `-1` as no value
 
 ### Stream Messages to Kafka
 
 Send `README.md` to kafka one line per kafka message
 ```
-./scripts/brod pipe -b localhost:9092 -t test-topic -p 0 -s @./README.md
+brod pipe -b localhost:9092 -t test-topic -p 0 -s @./README.md
 ```
 
 ### Resolve Offset
 
 ```
-./scripts/brod offset -b localhost:9092 -t test-topic -p 0
+brod offset -b localhost:9092 -t test-topic -p 0
 ```
 
 ### List or Describe Groups
 
 ```
 # List all groups
-./scripts/brod groups -b localhost:9092
+brod groups -b localhost:9092
 
 # Describe groups
-./scripts/brod groups -b localhost:9092 --ids group-1,group-2
+brod groups -b localhost:9092 --ids group-1,group-2
 ```
 
 ### Display Committed Offsets
 
 ```
 # all topics
-./scripts/brod commits -b localhost:9092 --id the-group-id --describe
+brod commits -b localhost:9092 --id the-group-id --describe
 
 # a specific topic
-./scripts/brod commits -b localhost:9092 --id the-group-id --describe --topic topic-name
+brod commits -b localhost:9092 --id the-group-id --describe --topic topic-name
 ```
 
 ### Commit Offsets
@@ -495,16 +488,16 @@ NOTE: This feature is designed for force overwriting commits, not for regular us
 
 ```
 # Commit 'latest' offsets of all partitions with 2 days retention
-./scripts/brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 2d
+brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 2d
 
 # Commit offset=100 for partition 0 and 200 for partition 1
-./scripts/brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets "0:100,1:200"
+brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets "0:100,1:200"
 
 # Use --retention 0 to delete commits (may linger in kafka before cleaner does its job)
-./scripts/brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 0
+brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 0
 
 # Try join an active consumer group using 'range' protocol and steal one partition assignment then commit offset=10000
-./scripts/brod commits -b localhost:9092 -i the-group-id -t topic-name -o "0:10000" --protocol range
+brod commits -b localhost:9092 -i the-group-id -t topic-name -o "0:10000" --protocol range
 ```
 
 ## TODOs

rebar.config

@@ -1,4 +1,9 @@
-{deps, [ {supervisor3, "1.1.5"}
-       , {kafka_protocol, "2.1.0"}
+{deps, [ {supervisor3, "1.1.7"}
+       , {kafka_protocol, "2.1.1"}
        ]}.
+{edoc_opts, [{preprocess, true}, {macros, [{build_brod_cli, true}]}]}.
+{cover_enabled, true}.
 {erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,warn_obsolete_guard,debug_info]}.
+{xref_checks, [undefined_function_calls, undefined_functions,
+               locals_not_used, deprecated_function_calls,
+               deprecated_functions]}.

rebar.config.script

@@ -1,8 +1,29 @@
 IsRebar3 = erlang:function_exported(rebar3, main, 1),
-
+DocoptUrl = "https://github.com/zmstone/docopt-erl.git",
+DocOptTag = "0.1.3",
+DocoptDep = {docopt, {git, DocoptUrl, {branch, DocOptTag}}},
+Profiles =
+  {profiles, [
+    {brod_cli, [
+      {deps, [jsone, DocoptDep]},
+      {erl_opts, [{d, build_brod_cli}]},
+      {escript_name, brod_cli},
+      {relx, [{release, {brod, "i"}, % release the interactive shell as brod-i
+               [brod, jsone, docopt]},
+              {include_erts, true},
+              {overlay, [{copy, "scripts/brod", "bin"},
+                         {copy, "{{lib_dirs}}/crc32cer/priv/crc32cer.so", "bin"},
+                         {copy, "{{lib_dirs}}/snappyer/priv/snappyer.so", "bin"}
+                        ]}
+             ]}]},
+    {test, [
+      {deps, [meck, proper, jsone, DocoptDep]},
+      {erl_opts, [{d, build_brod_cli}]}
+    ]}
+  ]},
 case IsRebar3 of
   true ->
-    CONFIG;
+    [Profiles | CONFIG];
   false ->
     URLs = [ {supervisor3, "https://github.com/klarna/supervisor3.git"}
            , {kafka_protocol, "https://github.com/klarna/kafka_protocol.git"}
@@ -18,3 +39,4 @@ case IsRebar3 of
         end, URLs),
    lists:keyreplace(deps, 1, CONFIG, {deps, Rebar2Deps})
 end.
+