Skip to content

Commit 71cf017

Browse files
shahidhk0x777
shahidhk
authored and
0x777
committed
add an api to dump postgres schema (close hasura#1939) (hasura#1967)
1 parent 8389a7e commit 71cf017

File tree

31 files changed

+749
-158
lines changed

31 files changed

+749
-158
lines changed

‎.circleci/config.yml

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ jobs:
157157
# build the server binary, and package into docker image
158158
build_server:
159159
docker:
160-
- image: hasura/graphql-engine-server-builder:20190314
160+
- image: hasura/graphql-engine-server-builder:20190415-1
161161
working_directory: ~/graphql-engine
162162
steps:
163163
- attach_workspace:
@@ -171,6 +171,11 @@ jobs:
171171
- restore_cache:
172172
keys:
173173
- server-app-cache-{{ .Branch }}-{{ .Revision }}
174+
- run:
175+
name: Install latest postgresql client tools
176+
command: |
177+
apt-get -y update
178+
apt-get -y install postgresql-client-11
174179
- run:
175180
name: Build the server
176181
working_directory: ./server
@@ -218,7 +223,7 @@ jobs:
218223
environment:
219224
PG_VERSION: "11_1"
220225
docker:
221-
- image: hasura/graphql-engine-server-builder:20190314
226+
- image: hasura/graphql-engine-server-builder:20190415-1
222227
# TODO: change this to circleci postgis when they have one for pg 11
223228
- image: mdillon/postgis:11-alpine
224229
<<: *test_pg_env
@@ -228,7 +233,7 @@ jobs:
228233
environment:
229234
PG_VERSION: "10_6"
230235
docker:
231-
- image: hasura/graphql-engine-server-builder:20190314
236+
- image: hasura/graphql-engine-server-builder:20190415-1
232237
- image: circleci/postgres:10.6-alpine-postgis
233238
<<: *test_pg_env
234239

@@ -237,7 +242,7 @@ jobs:
237242
environment:
238243
PG_VERSION: "9_6"
239244
docker:
240-
- image: hasura/graphql-engine-server-builder:20190314
245+
- image: hasura/graphql-engine-server-builder:20190415-1
241246
- image: circleci/postgres:9.6-alpine-postgis
242247
<<: *test_pg_env
243248

@@ -246,7 +251,7 @@ jobs:
246251
environment:
247252
PG_VERSION: "9_5"
248253
docker:
249-
- image: hasura/graphql-engine-server-builder:20190314
254+
- image: hasura/graphql-engine-server-builder:20190415-1
250255
- image: circleci/postgres:9.5-alpine-postgis
251256
<<: *test_pg_env
252257

‎.circleci/server-builder.dockerfile

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,15 @@ FROM debian:stretch-20190228-slim
55
ARG docker_ver="17.09.0-ce"
66
ARG resolver="lts-13.12"
77
ARG stack_ver="1.9.3"
8+
ARG postgres_ver="11"
89

910
# Install GNU make, curl, git and docker client. Required to build the server
1011
RUN apt-get -y update \
11-
&& apt-get install -y curl g++ gcc libc6-dev libpq-dev libffi-dev libgmp-dev make xz-utils zlib1g-dev git gnupg upx netcat python3 python3-pip \
12+
&& apt-get -y install curl gnupg2 \
13+
&& echo "deb http://apt.postgresql.org/pub/repos/apt/ stretch-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
14+
&& curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
15+
&& apt-get -y update \
16+
&& apt-get install -y g++ gcc libc6-dev libpq-dev libffi-dev libgmp-dev make xz-utils zlib1g-dev git gnupg upx netcat python3 python3-pip postgresql-client-${postgres_ver} postgresql-client-common \
1217
&& curl -Lo /tmp/docker-${docker_ver}.tgz https://download.docker.com/linux/static/stable/x86_64/docker-${docker_ver}.tgz \
1318
&& tar -xz -C /tmp -f /tmp/docker-${docker_ver}.tgz \
1419
&& mv /tmp/docker/* /usr/bin \

‎cli/commands/migrate_create.go

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,19 @@ import (
77

88
"github.com/ghodss/yaml"
99
"github.com/hasura/graphql-engine/cli"
10-
mig "github.com/hasura/graphql-engine/cli/migrate/cmd"
10+
"github.com/hasura/graphql-engine/cli/migrate"
1111
"github.com/pkg/errors"
12-
log "github.com/sirupsen/logrus"
1312
"github.com/spf13/cobra"
1413
"github.com/spf13/pflag"
1514
"github.com/spf13/viper"
15+
16+
mig "github.com/hasura/graphql-engine/cli/migrate/cmd"
17+
log "github.com/sirupsen/logrus"
1618
)
1719

20+
const migrateCreateCmdExamples = ` # Setup migration files for the first time by introspecting a server:
21+
hasura migrate create "init" --from-sever`
22+
1823
func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command {
1924
v := viper.New()
2025
opts := &migrateCreateOptions{
@@ -25,6 +30,7 @@ func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command {
2530
Use: "create [migration-name]",
2631
Short: "Create files required for a migration",
2732
Long: "Create sql and yaml files required for a migration",
33+
Example: migrateCreateCmdExamples,
2834
SilenceUsage: true,
2935
Args: cobra.ExactArgs(1),
3036
PreRunE: func(cmd *cobra.Command, args []string) error {
@@ -48,13 +54,17 @@ func newMigrateCreateCmd(ec *cli.ExecutionContext) *cobra.Command {
4854
}
4955
f := migrateCreateCmd.Flags()
5056
opts.flags = f
51-
f.StringVar(&opts.sqlFile, "sql-from-file", "", "path to an sql file which contains the up actions")
57+
f.BoolVar(&opts.fromServer, "from-server", false, "get SQL statements and hasura metadata from the server")
58+
f.StringVar(&opts.sqlFile, "sql-from-file", "", "path to an sql file which contains the SQL statements")
59+
f.BoolVar(&opts.sqlServer, "sql-from-server", false, "take pg_dump from server and save it as a migration")
60+
f.StringArrayVar(&opts.schemaNames, "schema", []string{"public"}, "name of Postgres schema to export as migration")
5261
f.StringVar(&opts.metaDataFile, "metadata-from-file", "", "path to a hasura metadata file to be used for up actions")
5362
f.BoolVar(&opts.metaDataServer, "metadata-from-server", false, "take metadata from the server and write it as an up migration file")
5463
f.String("endpoint", "", "http(s) endpoint for Hasura GraphQL Engine")
5564
f.String("admin-secret", "", "admin secret for Hasura GraphQL Engine")
5665
f.String("access-key", "", "access key for Hasura GraphQL Engine")
5766
f.MarkDeprecated("access-key", "use --admin-secret instead")
67+
5868
migrateCreateCmd.MarkFlagFilename("sql-from-file")
5969
migrateCreateCmd.MarkFlagFilename("metadata-from-file")
6070

@@ -73,25 +83,51 @@ type migrateCreateOptions struct {
7383
flags *pflag.FlagSet
7484

7585
// Flags
86+
fromServer bool
7687
sqlFile string
88+
sqlServer bool
7789
metaDataFile string
7890
metaDataServer bool
91+
schemaNames []string
7992
}
8093

8194
func (o *migrateCreateOptions) run() (version int64, err error) {
8295
timestamp := getTime()
8396
createOptions := mig.New(timestamp, o.name, o.EC.MigrationDir)
8497

98+
if o.fromServer {
99+
o.sqlServer = true
100+
o.metaDataServer = true
101+
}
102+
103+
if o.flags.Changed("metadata-from-file") && o.sqlServer {
104+
return 0, errors.New("only one sql type can be set")
105+
}
106+
if o.flags.Changed("metadata-from-file") && o.metaDataServer {
107+
return 0, errors.New("only one metadata type can be set")
108+
}
109+
110+
var migrateDrv *migrate.Migrate
111+
if o.sqlServer || o.metaDataServer {
112+
migrateDrv, err = newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version)
113+
if err != nil {
114+
return 0, errors.Wrap(err, "cannot create migrate instance")
115+
}
116+
}
117+
85118
if o.flags.Changed("sql-from-file") {
86119
// sql-file flag is set
87120
err := createOptions.SetSQLUpFromFile(o.sqlFile)
88121
if err != nil {
89122
return 0, errors.Wrap(err, "cannot set sql file")
90123
}
91124
}
92-
93-
if o.flags.Changed("metadata-from-file") && o.metaDataServer {
94-
return 0, errors.New("only one metadata type can be set")
125+
if o.sqlServer {
126+
data, err := migrateDrv.ExportSchemaDump(o.schemaNames)
127+
if err != nil {
128+
return 0, errors.Wrap(err, "cannot fetch schema dump")
129+
}
130+
createOptions.SetSQLUp(string(data))
95131
}
96132

97133
if o.flags.Changed("metadata-from-file") {
@@ -103,12 +139,6 @@ func (o *migrateCreateOptions) run() (version int64, err error) {
103139
}
104140

105141
if o.metaDataServer {
106-
// create new migrate instance
107-
migrateDrv, err := newMigrate(o.EC.MigrationDir, o.EC.ServerConfig.ParsedEndpoint, o.EC.ServerConfig.AdminSecret, o.EC.Logger, o.EC.Version)
108-
if err != nil {
109-
return 0, errors.Wrap(err, "cannot create migrate instance")
110-
}
111-
112142
// fetch metadata from server
113143
metaData, err := migrateDrv.ExportMetadata()
114144
if err != nil {
@@ -138,7 +168,7 @@ func (o *migrateCreateOptions) run() (version int64, err error) {
138168
}
139169
}
140170

141-
if !o.flags.Changed("sql-from-file") && !o.flags.Changed("metadata-from-file") && !o.metaDataServer {
171+
if !o.flags.Changed("sql-from-file") && !o.flags.Changed("metadata-from-file") && !o.metaDataServer && !o.sqlServer {
142172
// Set empty data for [up|down].yaml
143173
createOptions.MetaUp = []byte(`[]`)
144174
createOptions.MetaDown = []byte(`[]`)
@@ -153,7 +183,7 @@ func (o *migrateCreateOptions) run() (version int64, err error) {
153183
if err != nil {
154184
return 0, errors.Wrap(err, "error creating migration files")
155185
}
156-
return 0, nil
186+
return timestamp, nil
157187
}
158188

159189
func getTime() int64 {

‎cli/migrate/database/driver.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,8 @@ type Driver interface {
105105
SettingsDriver
106106

107107
MetadataDriver
108+
109+
SchemaDriver
108110
}
109111

110112
// Open returns a new driver instance.

‎cli/migrate/database/hasuradb/hasuradb.go

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ var (
3939
type Config struct {
4040
MigrationsTable string
4141
SettingsTable string
42-
URL *nurl.URL
42+
v1URL *nurl.URL
43+
schemDumpURL *nurl.URL
4344
Headers map[string]string
4445
isCMD bool
4546
}
@@ -117,11 +118,16 @@ func (h *HasuraDB) Open(url string, isCMD bool, logger *log.Logger) (database.Dr
117118
hx, err := WithInstance(&Config{
118119
MigrationsTable: DefaultMigrationsTable,
119120
SettingsTable: DefaultSettingsTable,
120-
URL: &nurl.URL{
121+
v1URL: &nurl.URL{
121122
Scheme: scheme,
122123
Host: hurl.Host,
123124
Path: path.Join(hurl.Path, "v1/query"),
124125
},
126+
schemDumpURL: &nurl.URL{
127+
Scheme: scheme,
128+
Host: hurl.Host,
129+
Path: path.Join(hurl.Path, "v1alpha1/pg_dump"),
130+
},
125131
isCMD: isCMD,
126132
Headers: headers,
127133
}, logger)
@@ -162,7 +168,7 @@ func (h *HasuraDB) UnLock() error {
162168
return nil
163169
}
164170

165-
resp, body, err := h.sendQuery(h.migrationQuery)
171+
resp, body, err := h.sendv1Query(h.migrationQuery)
166172
if err != nil {
167173
return err
168174
}
@@ -283,7 +289,7 @@ func (h *HasuraDB) getVersions() (err error) {
283289
}
284290

285291
// Send Query
286-
resp, body, err := h.sendQuery(query)
292+
resp, body, err := h.sendv1Query(query)
287293
if err != nil {
288294
return err
289295
}
@@ -368,7 +374,7 @@ func (h *HasuraDB) Reset() error {
368374
},
369375
}
370376

371-
resp, body, err := h.sendQuery(query)
377+
resp, body, err := h.sendv1Query(query)
372378
if err != nil {
373379
return err
374380
}
@@ -401,7 +407,7 @@ func (h *HasuraDB) ensureVersionTable() error {
401407
},
402408
}
403409

404-
resp, body, err := h.sendQuery(query)
410+
resp, body, err := h.sendv1Query(query)
405411
if err != nil {
406412
h.logger.Debug(err)
407413
return err
@@ -443,7 +449,7 @@ func (h *HasuraDB) ensureVersionTable() error {
443449
},
444450
}
445451

446-
resp, body, err = h.sendQuery(query)
452+
resp, body, err = h.sendv1Query(query)
447453
if err != nil {
448454
return err
449455
}
@@ -469,10 +475,30 @@ func (h *HasuraDB) ensureVersionTable() error {
469475
return nil
470476
}
471477

472-
func (h *HasuraDB) sendQuery(m interface{}) (resp *http.Response, body []byte, err error) {
478+
func (h *HasuraDB) sendv1Query(m interface{}) (resp *http.Response, body []byte, err error) {
479+
request := gorequest.New()
480+
481+
request = request.Post(h.config.v1URL.String()).Send(m)
482+
483+
for headerName, headerValue := range h.config.Headers {
484+
request.Set(headerName, headerValue)
485+
}
486+
487+
resp, body, errs := request.EndBytes()
488+
489+
if len(errs) == 0 {
490+
err = nil
491+
} else {
492+
err = errs[0]
493+
}
494+
495+
return resp, body, err
496+
}
497+
498+
func (h *HasuraDB) sendSchemaDumpQuery(m interface{}) (resp *http.Response, body []byte, err error) {
473499
request := gorequest.New()
474500

475-
request = request.Post(h.config.URL.String()).Send(m)
501+
request = request.Post(h.config.schemDumpURL.String()).Send(m)
476502

477503
for headerName, headerValue := range h.config.Headers {
478504
request.Set(headerName, headerValue)

‎cli/migrate/database/hasuradb/metadata.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ func (h *HasuraDB) ExportMetadata() (interface{}, error) {
1313
Args: HasuraArgs{},
1414
}
1515

16-
resp, body, err := h.sendQuery(query)
16+
resp, body, err := h.sendv1Query(query)
1717
if err != nil {
1818
h.logger.Debug(err)
1919
return nil, err
@@ -46,7 +46,7 @@ func (h *HasuraDB) ResetMetadata() error {
4646
Args: HasuraArgs{},
4747
}
4848

49-
resp, body, err := h.sendQuery(query)
49+
resp, body, err := h.sendv1Query(query)
5050
if err != nil {
5151
h.logger.Debug(err)
5252
return err
@@ -72,7 +72,7 @@ func (h *HasuraDB) ReloadMetadata() error {
7272
Args: HasuraArgs{},
7373
}
7474

75-
resp, body, err := h.sendQuery(query)
75+
resp, body, err := h.sendv1Query(query)
7676
if err != nil {
7777
h.logger.Debug(err)
7878
return err
@@ -106,7 +106,7 @@ func (h *HasuraDB) ApplyMetadata(data interface{}) error {
106106
},
107107
}
108108

109-
resp, body, err := h.sendQuery(query)
109+
resp, body, err := h.sendv1Query(query)
110110
if err != nil {
111111
h.logger.Debug(err)
112112
return err
@@ -151,7 +151,7 @@ func (h *HasuraDB) Query(data []interface{}) error {
151151
Args: data,
152152
}
153153

154-
resp, body, err := h.sendQuery(query)
154+
resp, body, err := h.sendv1Query(query)
155155
if err != nil {
156156
h.logger.Debug(err)
157157
return err

0 commit comments

Comments
 (0)