From 166369ac7d135acfc00af04edbd51457c1428868 Mon Sep 17 00:00:00 2001 From: Takuya Ueda Date: Thu, 28 Feb 2019 15:40:44 +0900 Subject: [PATCH 1/3] Use tenntenn/jsonschema --- json_column.go | 15 +++++++----- json_row.go | 62 ++++++++++++++++++++++-------------------------- json_row_test.go | 15 ++++++++++-- 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/json_column.go b/json_column.go index bce9f18..fd4cf4b 100644 --- a/json_column.go +++ b/json_column.go @@ -10,6 +10,7 @@ import ( "cloud.google.com/go/civil" "cloud.google.com/go/spanner" structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/tenntenn/jsonschema" gspanner "google.golang.org/genproto/googleapis/spanner/v1" ) @@ -93,7 +94,7 @@ func (c *JSONColumn) marshalList(t *gspanner.Type, l *structpb.ListValue) ([]int return vs, nil } -func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...JSONSchemaOption) error { +func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...jsonschema.Option) error { switch t.Code { default: @@ -124,7 +125,9 @@ func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...JSONSchem } for i := range options { - if err := (options[i])(o); err != nil { + var err error + o, err = (options[i])(o) + if err != nil { return err } } @@ -132,7 +135,7 @@ func (c *JSONColumn) schema(o JSONObject, t *gspanner.Type, options ...JSONSchem return nil } -func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, options ...JSONSchemaOption) error { +func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, options ...jsonschema.Option) error { required := make([]string, len(t.Fields)) properties := make(map[string]interface{}, len(t.Fields)) @@ -146,9 +149,9 @@ func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, opt ref: path.Join(parent.Ref(), "properties", f.Name), } - opts := make([]JSONSchemaOption, len(options)+1) + opts := make([]jsonschema.Option, len(options)+1) copy(opts, options) - opts[len(opts)-1] = ByJSONReference(o.Ref(), PropertyOrder(i)) + opts[len(opts)-1] = jsonschema.ByReference(o.Ref(), jsonschema.PropertyOrder(i)) if err := c.schema(o, f.Type, opts...); err != nil { return err @@ -164,7 +167,7 @@ func (c *JSONColumn) schemaStruct(parent JSONObject, t *gspanner.StructType, opt return nil } -func (c *JSONColumn) schemaArray(parent JSONObject, t *gspanner.Type, options ...JSONSchemaOption) error { +func (c *JSONColumn) schemaArray(parent JSONObject, t *gspanner.Type, options ...jsonschema.Option) error { o := &mapJSONObject{ m: map[string]interface{}{}, diff --git a/json_row.go b/json_row.go index 69500a9..3a54ab7 100644 --- a/json_row.go +++ b/json_row.go @@ -7,7 +7,7 @@ import ( "path" "cloud.google.com/go/spanner" - "github.com/minio/minio/pkg/wildcard" + "github.com/tenntenn/jsonschema" ) // JSONRow is an encodable type of spanner.Row. @@ -55,30 +55,8 @@ func (o *mapJSONObject) Ref() string { return o.ref } -// JSONSchemaOption is options for JSON Schema. -type JSONSchemaOption func(o JSONObject) error - -// ByJSONReference explicits refrence of adding option. -// It only supports refs which begins "#/". -func ByJSONReference(pattern string, opt JSONSchemaOption) JSONSchemaOption { - return func(o JSONObject) error { - if wildcard.MatchSimple(pattern, o.Ref()) { - return opt(o) - } - return nil - } -} - -// PropertyOrder is add propertyOrder to schema. -func PropertyOrder(order int) JSONSchemaOption { - return func(o JSONObject) error { - o.Set("propertyOrder", order) - return nil - } -} - // Schema writes JSON Schema of the row to writer w. -func (r *JSONRow) Schema(w io.Writer, options ...JSONSchemaOption) error { +func (r *JSONRow) JSONSchema(w io.Writer, options ...jsonschema.Option) error { type colSchema struct { Name string Schema string @@ -98,9 +76,9 @@ func (r *JSONRow) Schema(w io.Writer, options ...JSONSchemaOption) error { ref: path.Join("#/properties", names[i]), } - opts := make([]JSONSchemaOption, len(options)+1) + opts := make([]jsonschema.Option, len(options)+1) copy(opts, options) - opts[len(opts)-1] = ByJSONReference(o.Ref(), PropertyOrder(i)) + opts[len(opts)-1] = jsonschema.ByReference(o.Ref(), jsonschema.PropertyOrder(i)) if err := (*JSONColumn)(&col).schema(o, col.Type, opts...); err != nil { return err @@ -134,16 +112,34 @@ func (r *JSONRow) Schema(w io.Writer, options ...JSONSchemaOption) error { return nil } -// JSONRows convert []*spanner.Row to []*Row. -func JSONRows(rows []*spanner.Row) []*JSONRow { - if rows == nil { +// JSONRows is an encodable type of []*spanner.Row. +type JSONRows []*spanner.Row + +var _ json.Marshaler = (JSONRows)(nil) + +// At returns ith row as *JSONRow. +func (rs JSONRows) At(i int) *JSONRow { + return (*JSONRow)(rs[i]) +} + +func (rs JSONRows) toJSONRowSlice() []*JSONRow { + if rs == nil { return nil } - rs := make([]*JSONRow, len(rows)) - for i := range rows { - rs[i] = (*JSONRow)(rows[i]) + rows := make([]*JSONRow, len(rs)) + for i := range rs { + rows[i] = rs.At(i) } + return rows +} + +// MarshalJSON implements json.Marshaler +func (rs JSONRows) MarshalJSON() ([]byte, error) { + return json.Marshal(rs.toJSONRowSlice()) +} - return rs +// Schema writes JSON Schema of the rows to writer w. +func (rs JSONRows) JSONSchema(w io.Writer, options ...jsonschema.Option) error { + return jsonschema.Generate(w, rs, options...) } diff --git a/json_row_test.go b/json_row_test.go index 9f574e4..c4ee3ab 100644 --- a/json_row_test.go +++ b/json_row_test.go @@ -56,7 +56,18 @@ func TestRows(t *testing.T) { } } -func TestJSONRow_Schema(t *testing.T) { +type noopFormatCheker struct{} + +func (noopFormatCheker) IsFormat(_ string) bool { + return true +} + +func init() { + gojsonschema.FormatCheckers.Add("datetime", noopFormatCheker{}) + gojsonschema.FormatCheckers.Add("textarea", noopFormatCheker{}) +} + +func TestJSONRow_JSONSchema(t *testing.T) { type T struct { N int @@ -83,7 +94,7 @@ func TestJSONRow_Schema(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { var got bytes.Buffer - err := (*JSONRow)(tt.row).Schema(&got) + err := (*JSONRow)(tt.row).JSONSchema(&got) switch { case tt.isErr && err == nil: t.Errorf("expected error does not occur") From e23ec3778dcf0ad989a392a730cc107863d8c38f Mon Sep 17 00:00:00 2001 From: Takuya Ueda Date: Thu, 28 Feb 2019 15:41:52 +0900 Subject: [PATCH 2/3] Fix go.mod --- go.mod | 7 +------ go.sum | 10 ++++------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index f228c6a..2a71935 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,7 @@ module github.com/gcpug/hake require ( cloud.google.com/go v0.35.1 github.com/golang/protobuf v1.2.0 - github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 // indirect - github.com/minio/minio v0.0.0-20190212015826-b8955fe57772 - github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/tenntenn/jsonschema v0.0.1 github.com/xeipuuv/gojsonschema v1.1.0 - golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045 // indirect - google.golang.org/api v0.1.0 google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 ) diff --git a/go.sum b/go.sum index cbb0e61..086c0d0 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,6 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -53,8 +51,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= -github.com/minio/minio v0.0.0-20190212015826-b8955fe57772 h1:fLv8dS1IEDsjtpvfTUIxKn3nSvyqY9xykH5RSl3nCPI= -github.com/minio/minio v0.0.0-20190212015826-b8955fe57772/go.mod h1:lXcp05uxYaW99ebgI6ZKIGYU7tqZkM5xSsG0xRt4VIU= +github.com/minio/minio v0.0.0-20190216002119-b6c00405ec5c h1:csjhUhUbwprAuwFxy7sCm94CvV1s9qY8Y1gM5mpC4jA= +github.com/minio/minio v0.0.0-20190216002119-b6c00405ec5c/go.mod h1:lXcp05uxYaW99ebgI6ZKIGYU7tqZkM5xSsG0xRt4VIU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= @@ -93,6 +91,8 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tenntenn/jsonschema v0.0.1 h1:eIKG8Xiw3nWVILPd3UuGww88j80vGEbSSYNGdcIy89c= +github.com/tenntenn/jsonschema v0.0.1/go.mod h1:1b70ZBDcPdpWbr6rkPPuwyY/Lc+t0zYP/p+3DBTS+M0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -102,8 +102,6 @@ github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4m go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045 h1:Pn8fQdvx+z1avAi7fdM2kRYWQNxGlavNDSyzrQg2SsU= -golang.org/x/arch v0.0.0-20181203225421-5a4828bb7045/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From 0d5f7f33421c0d71d2637005ac0dba049e214cbc Mon Sep 17 00:00:00 2001 From: Takuya Ueda Date: Tue, 5 Mar 2019 20:26:27 +0900 Subject: [PATCH 3/3] Use tenntenn/jsonschema --- go.mod | 2 +- json_row.go | 3 ++- json_row_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2a71935..aa2e844 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/gcpug/hake require ( cloud.google.com/go v0.35.1 github.com/golang/protobuf v1.2.0 - github.com/tenntenn/jsonschema v0.0.1 + github.com/tenntenn/jsonschema v0.0.2 github.com/xeipuuv/gojsonschema v1.1.0 google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 ) diff --git a/json_row.go b/json_row.go index 3a54ab7..f957fa8 100644 --- a/json_row.go +++ b/json_row.go @@ -131,6 +131,7 @@ func (rs JSONRows) toJSONRowSlice() []*JSONRow { for i := range rs { rows[i] = rs.At(i) } + return rows } @@ -141,5 +142,5 @@ func (rs JSONRows) MarshalJSON() ([]byte, error) { // Schema writes JSON Schema of the rows to writer w. func (rs JSONRows) JSONSchema(w io.Writer, options ...jsonschema.Option) error { - return jsonschema.Generate(w, rs, options...) + return jsonschema.Generate(w, rs.toJSONRowSlice(), options...) } diff --git a/json_row_test.go b/json_row_test.go index c4ee3ab..b4b99ab 100644 --- a/json_row_test.go +++ b/json_row_test.go @@ -120,3 +120,58 @@ func TestJSONRow_JSONSchema(t *testing.T) { }) } } + +func TestJSONRows_JSONSchema(t *testing.T) { + + type T struct { + N int + S string + } + + type NT struct { + T T + } + + cases := []struct { + name string + rows []*spanner.Row + isErr bool + }{ + {"int", rows(t, []R{{"col1", 100}}), false}, + {"int int", rows(t, []R{{"col1", 100}, {"col1", 200}}), false}, + {"int string", rows(t, []R{{"col1", 100, "col2", "string"}}), false}, + {"nested struct", rows(t, []R{{"col1", 100, "col2", T{N: 100, S: ""}}}), false}, + {"timestamp", rows(t, []R{{"col1", 100, "col2", timestamp(t, "2002-10-02T10:00:00Z")}}), false}, + {"bytes", rows(t, []R{{"col1", []byte("test")}}), false}, + } + + for _, tt := range cases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var got bytes.Buffer + err := JSONRows(tt.rows).JSONSchema(&got) + switch { + case tt.isErr && err == nil: + t.Errorf("expected error does not occur") + case !tt.isErr && err != nil: + t.Errorf("unexpected error %v", err) + } + + l := gojsonschema.NewStringLoader(got.String()) + s, err := gojsonschema.NewSchema(l) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + rowJSON := toJSON(t, JSONRows(tt.rows)) + r, err := s.Validate(gojsonschema.NewStringLoader(rowJSON)) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + if !r.Valid() { + t.Errorf("invalid JSON Schema: %s", got.String()) + } + }) + } +}