Skip to content

Commit 687f519

Browse files
hlts2hlts2
authored
Introduce bufbuild (#19)
* feat: enable to create grpc using buf Signed-off-by: hlts2 <[email protected]> * fix: command to change directory structure Signed-off-by: hlts2 <[email protected]> * fix: add code generation step before e2e test Signed-off-by: hlts2 <[email protected]> * fix: delete shadow directory to prevent helm installation Signed-off-by: hlts2 <[email protected]> * fix: deleted unnecessary code Signed-off-by: hlts2 <[email protected]> * fix: debug step Signed-off-by: hlts2 <[email protected]> * fix: revert debug step Signed-off-by: hlts2 <[email protected]> --------- Signed-off-by: hlts2 <[email protected]>
1 parent 7eed7cc commit 687f519

File tree

8 files changed

+41
-220
lines changed

8 files changed

+41
-220
lines changed

‎.github/workflows/release.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ jobs:
4848
- name: python dependencies
4949
run: |
5050
pip install --upgrade grpcio-tools protobuf grpcio googleapis-common-protos
51-
## dirty hack:
51+
## dirty hack:
5252
sudo rm -rf /usr/lib/python3/dist-packages/validate.py
5353
curl -LO https://raw.githubusercontent.com/rinx/word2vecjson/master/data/wordvecs1000.json
5454
- name: run tests

‎Makefile

Lines changed: 28 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,18 @@ VALD_DIR = vald-origin
2727
VALD_SHA = VALD_SHA
2828
VALD_CLIENT_PYTHON_VERSION = VALD_CLIENT_PYTHON_VERSION
2929

30-
PWD := $(eval PWD := $(shell pwd))$(PWD)
31-
GOPATH := $(eval GOPATH := $(shell go env GOPATH))$(GOPATH)
30+
BINDIR ?= /usr/local/bin
3231

3332
PROTO_ROOT = $(VALD_DIR)/apis/proto
3433
PB2DIR_ROOT = src
3534

36-
SHADOW_ROOT = vald
35+
BUF_VERSION_URL := https://raw.githubusercontent.com/vdaas/vald/main/versions/BUF_VERSION
36+
BUF_CONFIGS = \
37+
$(PROTO_ROOT)/buf.yaml \
38+
$(PROTO_ROOT)/buf.lock
39+
40+
SHADOW_ROOT = vald
41+
SHADOW_PROTO_ROOT = $(SHADOW_ROOT)/$(SHADOW_ROOT)
3742

3843
PROTOS = \
3944
v1/agent/core/agent.proto \
@@ -48,21 +53,8 @@ PROTOS = \
4853
v1/vald/upsert.proto \
4954
v1/payload/payload.proto
5055
PROTOS := $(PROTOS:%=$(PROTO_ROOT)/%)
51-
SHADOWS = $(PROTOS:$(PROTO_ROOT)/%.proto=$(SHADOW_ROOT)/%.proto)
56+
SHADOWS = $(PROTOS:$(PROTO_ROOT)/%.proto=$(SHADOW_PROTO_ROOT)/%.proto)
5257
PB2PYS = $(PROTOS:$(PROTO_ROOT)/%.proto=$(PB2DIR_ROOT)/$(SHADOW_ROOT)/%_pb2.py)
53-
PB2PY_VALIDATE = $(PB2DIR_ROOT)/validate/validate_pb2.py
54-
PB2PY_GOOGLEAPIS = $(PB2DIR_ROOT)/googleapis/googleapis/google/api/annotations_pb2.py
55-
PB2PY_GOOGLERPCS = $(PB2DIR_ROOT)/googleapis/googleapis/google/rpc/status_pb2.py
56-
PB2PY_VTEXTPY = $(PB2DIR_ROOT)/vtproto/ext_pb2.py
57-
58-
PROTO_PATHS = \
59-
$(PWD) \
60-
$(PWD)/$(VALD_DIR) \
61-
$(PWD)/$(PROTO_ROOT) \
62-
$(GOPATH)/src \
63-
$(GOPATH)/src/github.com/googleapis/googleapis \
64-
$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate \
65-
$(GOPATH)/src/github.com/planetscale/vtprotobuf/include/github.com/planetscale/vtprotobuf
6658

6759
MAKELISTS = Makefile
6860

@@ -73,14 +65,6 @@ blue = /bin/echo -e "\x1b[34m\#\# $1\x1b[0m"
7365
pink = /bin/echo -e "\x1b[35m\#\# $1\x1b[0m"
7466
cyan = /bin/echo -e "\x1b[36m\#\# $1\x1b[0m"
7567

76-
define go-get
77-
GO111MODULE=on go get -u $1
78-
endef
79-
80-
define go-get-no-mod
81-
GO111MODULE=off go get -u $1
82-
endef
83-
8468
.PHONY: all
8569
## execute clean and proto
8670
all: clean proto
@@ -104,85 +88,28 @@ help:
10488
.PHONY: clean
10589
## clean
10690
clean:
107-
rm -rf $(PB2DIR_ROOT)/google $(PB2DIR_ROOT)/vald $(PB2DIR_ROOT)/validate $(PB2DIR_ROOT)/vtproto
91+
rm -rf $(PB2DIR_ROOT)/google $(PB2DIR_ROOT)/vald $(PB2DIR_ROOT)/buf
10892
rm -rf $(SHADOW_ROOT)
10993
rm -rf $(VALD_DIR)
11094

11195
.PHONY: proto
11296
## build proto
113-
proto: \
114-
$(PB2PYS) \
115-
$(PB2PY_VALIDATE) \
116-
$(PB2PY_GOOGLEAPIS) \
117-
$(PB2PY_GOOGLERPCS) \
118-
$(PB2PY_VTEXTPY)
97+
proto: $(PB2PYS)
98+
@$(call green, "generating pb2.py files...")
99+
cp -f $(BUF_CONFIGS) $(SHADOW_ROOT)
100+
buf generate --include-imports
119101

120102
$(PROTOS): $(VALD_DIR)
121103
$(SHADOWS): $(PROTOS)
122-
$(SHADOW_ROOT)/%.proto: $(PROTO_ROOT)/%.proto
104+
$(SHADOW_PROTO_ROOT)/%.proto: $(PROTO_ROOT)/%.proto
123105
mkdir -p $(dir $@)
124106
cp $< $@
125-
sed -i -e 's:^import "apis/proto/:import "$(SHADOW_ROOT)/:' $@
126-
sed -i -e 's:^import "github.com/envoyproxy/protoc-gen-validate/:import ":' $@
127-
sed -i -e 's:^import "github.com/googleapis/googleapis/:import ":' $@
128-
sed -i -e 's:^import "github.com/planetscale/vtprotobuf/include/github.com/planetscale/vtprotobuf/:import ":' $@
107+
sed -i -e 's:^import "v1:import "$(SHADOW_ROOT)/v1:' $@
129108

130109
$(PB2DIR_ROOT):
131110
mkdir -p $@
132111

133112
$(PB2PYS): proto/deps $(PB2DIR_ROOT) $(SHADOWS)
134-
$(PB2DIR_ROOT)/$(SHADOW_ROOT)/%_pb2.py: $(SHADOW_ROOT)/%.proto
135-
@$(call green, "generating pb2.py files...")
136-
$(PYTHON) \
137-
-m grpc_tools.protoc \
138-
$(PROTO_PATHS:%=-I %) \
139-
--python_out=$(PWD)/$(PB2DIR_ROOT) \
140-
--grpc_python_out=$(PWD)/$(PB2DIR_ROOT) \
141-
$<
142-
143-
$(PB2PY_VALIDATE): $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate
144-
@$(call green, "generating pb2.py files...")
145-
(cd $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate; \
146-
$(PYTHON) \
147-
-m grpc_tools.protoc \
148-
$(PROTO_PATHS:%=-I %) \
149-
-I $(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate \
150-
--python_out=$(PWD)/$(PB2DIR_ROOT) \
151-
--grpc_python_out=$(PWD)/$(PB2DIR_ROOT) \
152-
validate/validate.proto)
153-
154-
$(PB2PY_GOOGLEAPIS): $(GOPATH)/src/github.com/googleapis/googleapis
155-
@$(call green, "generating pb2.py files...")
156-
(cd $(GOPATH)/src/github.com/googleapis/googleapis; \
157-
$(PYTHON) \
158-
-m grpc_tools.protoc \
159-
$(PROTO_PATHS:%=-I %) \
160-
-I $(GOPATH)/src/github.com/googleapis/googleapis \
161-
--python_out=$(PWD)/$(PB2DIR_ROOT) \
162-
--grpc_python_out=$(PWD)/$(PB2DIR_ROOT) \
163-
google/api/annotations.proto)
164-
165-
$(PB2PY_GOOGLERPCS): $(GOPATH)/src/github.com/googleapis/googleapis
166-
@$(call green, "generating pb2.py files...")
167-
(cd $(GOPATH)/src/github.com/googleapis/googleapis; \
168-
$(PYTHON) \
169-
-m grpc_tools.protoc \
170-
$(PROTO_PATHS:%=-I %) \
171-
-I $(GOPATH)/src/github.com/googleapis/googleapis \
172-
--python_out=$(PWD)/$(PB2DIR_ROOT) \
173-
--grpc_python_out=$(PWD)/$(PB2DIR_ROOT) \
174-
google/rpc/status.proto)
175-
176-
$(PB2PY_VTEXTPY): $(GOPATH)/src/github.com/planetscale/vtprotobuf
177-
@$(call green, "generating pb2.py files...")
178-
(cd $(GOPATH)/src/github.com/planetscale/vtprotobuf; \
179-
$(PYTHON) \
180-
-m grpc_tools.protoc \
181-
$(PROTO_PATHS:%=-I %) \
182-
-I $(GOPATH)/src/github.com/planetscale/vtprotobuf/include/github.com/planetscale/vtprotobuf \
183-
--python_out=$(PWD)/$(PB2DIR_ROOT) \
184-
--grpc_python_out=$(PWD)/$(PB2DIR_ROOT) \
185-
vtproto/ext.proto)
186113

187114
$(VALD_DIR):
188115
git clone --depth 1 https://$(VALDREPO) $(VALD_DIR)
@@ -227,25 +154,15 @@ vald/client/python/version/update: vald
227154

228155
.PHONY: proto/deps
229156
## install proto deps
230-
proto/deps: \
231-
$(GOPATH)/src/github.com/googleapis/googleapis \
232-
$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate \
233-
$(GOPATH)/src/github.com/planetscale/vtprotobuf
234-
235-
$(GOPATH)/src/github.com/googleapis/googleapis:
236-
git clone \
237-
--depth 1 \
238-
https://github.com/googleapis/googleapis \
239-
$(GOPATH)/src/github.com/googleapis/googleapis
240-
241-
$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate:
242-
git clone \
243-
--depth 1 \
244-
https://github.com/envoyproxy/protoc-gen-validate \
245-
$(GOPATH)/src/github.com/envoyproxy/protoc-gen-validate
246-
247-
$(GOPATH)/src/github.com/planetscale/vtprotobuf:
248-
git clone \
249-
--depth 1 \
250-
https://github.com/planetscale/vtprotobuf \
251-
$(GOPATH)/src/github.com/planetscale/vtprotobuf
157+
proto/deps: buf/install
158+
159+
.PHONY: buf/install
160+
## install buf command.
161+
buf/install: $(BINDIR)/buf
162+
163+
$(BINDIR)/buf:
164+
@version=$$(curl -sSL $(BUF_VERSION_URL)); \
165+
curl -sSL \
166+
"https://github.com/bufbuild/buf/releases/download/$$version/buf-$(shell uname -s)-$(shell uname -m)" \
167+
-o "${BINDIR}/buf" && \
168+
chmod +x "${BINDIR}/buf"

‎buf.gen.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
version: v1
2+
managed:
3+
enabled: false
4+
plugins:
5+
- plugin: buf.build/protocolbuffers/python
6+
out: src
7+
- plugin: buf.build/grpc/python
8+
out: src

‎buf.work.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
version: v1
2+
directories:
3+
# A vald directory containing proto files is created by executing the make command.
4+
- vald

0 commit comments

Comments
 (0)