From 049a22eee82ed8ceb0c860b8fbd57b79663e5edb Mon Sep 17 00:00:00 2001 From: Eiko Thomas <13103714+OkieOth@users.noreply.github.com> Date: Mon, 16 Sep 2024 21:20:37 +0200 Subject: [PATCH 1/4] add version command --- cmd/schemaguesser/cmd/root.go | 1 + cmd/schemaguesser/cmd/version.go | 18 ++++++++++++++++++ version.txt | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 cmd/schemaguesser/cmd/version.go diff --git a/cmd/schemaguesser/cmd/root.go b/cmd/schemaguesser/cmd/root.go index eec4af6..c6ee484 100644 --- a/cmd/schemaguesser/cmd/root.go +++ b/cmd/schemaguesser/cmd/root.go @@ -20,6 +20,7 @@ var rootCmd = &cobra.Command{ func init() { rootCmd.AddCommand(schemaCmd) rootCmd.AddCommand(listCmd) + rootCmd.AddCommand(versionCmd) rootCmd.PersistentFlags().StringVar(&mongoHelper.ConStr, "con_str", "mongodb://{MONGO_USER}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/admin", "Connection string to mongodb") } diff --git a/cmd/schemaguesser/cmd/version.go b/cmd/schemaguesser/cmd/version.go new file mode 100644 index 0000000..7afba31 --- /dev/null +++ b/cmd/schemaguesser/cmd/version.go @@ -0,0 +1,18 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +const Version = "1.2.0" + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Shows the version of the program", + Long: "Shows the version of the program", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(Version) + }, +} diff --git a/version.txt b/version.txt index 45a1b3f..26aaba0 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.1.2 +1.2.0 From 104f2a2679ea03ca60b6a4735d12366b68d4770a Mon Sep 17 00:00:00 2001 From: Eiko Thomas <13103714+OkieOth@users.noreply.github.com> Date: Mon, 16 Sep 2024 21:33:02 +0200 Subject: [PATCH 2/4] replace version.txt with version.go --- buildImage.sh | 2 +- version.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 version.txt diff --git a/buildImage.sh b/buildImage.sh index 64c1d44..7cd59f0 100755 --- a/buildImage.sh +++ b/buildImage.sh @@ -4,7 +4,7 @@ scriptPos=${0%/*} imageBase=ghcr.io/okieoth/mschemaguesser -imageTag=`cat $scriptPos/version.txt | grep -P '\d+\.\d+\.\d+'` +imageTag=`cat $scriptPos/cmd/schemaguesser/cmd/version.go | grep "const Version =" | sed -e 's-.*= "--' -e 's-".*--'` imageName="$imageBase:$imageTag" diff --git a/version.txt b/version.txt deleted file mode 100644 index 26aaba0..0000000 --- a/version.txt +++ /dev/null @@ -1 +0,0 @@ -1.2.0 From 43114eb85ce77e10cb89393c4f6311c6ccca6dbe Mon Sep 17 00:00:00 2001 From: Eiko Thomas <13103714+OkieOth@users.noreply.github.com> Date: Mon, 16 Sep 2024 21:55:10 +0200 Subject: [PATCH 3/4] introduce mongo client as parameter --- cmd/schemaguesser/cmd/list.go | 59 ++++++++++----- cmd/schemaguesser/cmd/schema.go | 32 +++++--- cmd/schemaguesser/cmd/version.go | 2 +- internal/pkg/mongoHelper/mongoHelper.go | 77 +++++--------------- internal/pkg/mongoHelper/mongoHelper_test.go | 9 ++- 5 files changed, 86 insertions(+), 93 deletions(-) diff --git a/cmd/schemaguesser/cmd/list.go b/cmd/schemaguesser/cmd/list.go index 0189b3a..e33e0aa 100644 --- a/cmd/schemaguesser/cmd/list.go +++ b/cmd/schemaguesser/cmd/list.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "go.mongodb.org/mongo-driver/mongo" "okieoth/schemaguesser/internal/pkg/mongoHelper" @@ -27,7 +28,14 @@ var databasesCmd = &cobra.Command{ Short: "Names of databases available in the connected server", Long: "Prints a list of available databases i the connected server", Run: func(cmd *cobra.Command, args1 []string) { - dbs := mongoHelper.ReadDatabasesOrPanic() + client, err := mongoHelper.Connect(mongoHelper.ConStr) + if err != nil { + msg := fmt.Sprintf("Failed to connect to db: %v", err) + panic(msg) + } + defer mongoHelper.CloseConnection(client) + + dbs := mongoHelper.ReadDatabasesOrPanic(client) for _, s := range *dbs { fmt.Println(s) } @@ -39,10 +47,17 @@ var collectionsCmd = &cobra.Command{ Short: "Collections available in the specified database", Long: "Provides information about existing collections in the specified database", Run: func(cmd *cobra.Command, args []string) { + client, err := mongoHelper.Connect(mongoHelper.ConStr) + if err != nil { + msg := fmt.Sprintf("Failed to connect to db: %v", err) + panic(msg) + } + defer mongoHelper.CloseConnection(client) + if databaseName == "all" { - printAllCollections() + printAllCollections(client) } else { - printOneCollection(databaseName, false) + printOneCollection(client, databaseName, false) } }, } @@ -52,14 +67,20 @@ var indexesCmd = &cobra.Command{ Short: "Indexes to a given collection", Long: "Provides information about indexes of a given collection", Run: func(cmd *cobra.Command, args []string) { + client, err := mongoHelper.Connect(mongoHelper.ConStr) + if err != nil { + msg := fmt.Sprintf("Failed to connect to db: %v", err) + panic(msg) + } + defer mongoHelper.CloseConnection(client) if databaseName == "all" { - printIndexesForAllDatabases() + printIndexesForAllDatabases(client) } else { if collectionName == "all" { - printIndexesForAllCollections(databaseName) + printIndexesForAllCollections(client, databaseName) } else { - printIndexesForOneCollection(databaseName, collectionName, false) + printIndexesForOneCollection(client, databaseName, collectionName, false) } } }, @@ -76,8 +97,8 @@ func init() { indexesCmd.Flags().StringVar(&collectionName, "collection", "all", "Name of the collection to show the indexes. If 'all', then the collections of all databases are printed.") } -func printOneCollection(dbName string, verbose bool) { - collections := mongoHelper.ReadCollectionsOrPanic(dbName) +func printOneCollection(client *mongo.Client, dbName string, verbose bool) { + collections := mongoHelper.ReadCollectionsOrPanic(client, dbName) for _, s := range *collections { if verbose { fmt.Printf("Database: %s, Collection: %s\n", dbName, s) @@ -87,15 +108,15 @@ func printOneCollection(dbName string, verbose bool) { } } -func printAllCollections() { - dbs := mongoHelper.ReadDatabasesOrPanic() +func printAllCollections(client *mongo.Client) { + dbs := mongoHelper.ReadDatabasesOrPanic(client) for _, db := range *dbs { - printOneCollection(db, true) + printOneCollection(client, db, true) } } -func printIndexesForOneCollection(dbName string, collName string, verbose bool) { - indexes, err := mongoHelper.ListIndexes(mongoHelper.ConStr, dbName, collName) +func printIndexesForOneCollection(client *mongo.Client, dbName string, collName string, verbose bool) { + indexes, err := mongoHelper.ListIndexes(client, dbName, collName) if err != nil { msg := fmt.Sprintf("Error while reading indexes for collection (%s.%s): \n%v\n", dbName, collName, err) panic(msg) @@ -109,16 +130,16 @@ func printIndexesForOneCollection(dbName string, collName string, verbose bool) } } -func printIndexesForAllCollections(dbName string) { - collections := mongoHelper.ReadCollectionsOrPanic(dbName) +func printIndexesForAllCollections(client *mongo.Client, dbName string) { + collections := mongoHelper.ReadCollectionsOrPanic(client, dbName) for _, coll := range *collections { - printIndexesForOneCollection(dbName, coll, true) + printIndexesForOneCollection(client, dbName, coll, true) } } -func printIndexesForAllDatabases() { - dbs := mongoHelper.ReadDatabasesOrPanic() +func printIndexesForAllDatabases(client *mongo.Client) { + dbs := mongoHelper.ReadDatabasesOrPanic(client) for _, db := range *dbs { - printIndexesForAllCollections(db) + printIndexesForAllCollections(client, db) } } diff --git a/cmd/schemaguesser/cmd/schema.go b/cmd/schemaguesser/cmd/schema.go index d6d1089..3b7f81f 100644 --- a/cmd/schemaguesser/cmd/schema.go +++ b/cmd/schemaguesser/cmd/schema.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "go.mongodb.org/mongo-driver/mongo" "okieoth/schemaguesser/internal/pkg/mongoHelper" "okieoth/schemaguesser/internal/pkg/schema" @@ -22,13 +23,20 @@ var schemaCmd = &cobra.Command{ Short: "functions around the schemas", Long: "With this command you can create schemas out of mongodb collection", Run: func(cmd *cobra.Command, args []string) { + client, err := mongoHelper.Connect(mongoHelper.ConStr) + if err != nil { + msg := fmt.Sprintf("Failed to connect to db: %v", err) + panic(msg) + } + defer mongoHelper.CloseConnection(client) + if dbName == "all" { - printSchemasForAllDatabases() + printSchemasForAllDatabases(client) } else { if colName == "all" { - printSchemasForAllCollections(dbName) + printSchemasForAllCollections(client, dbName) } else { - printSchemaForOneCollection(dbName, colName, false) + printSchemaForOneCollection(client, dbName, colName, false) } } @@ -45,7 +53,7 @@ func init() { schemaCmd.Flags().Int32Var(&itemCount, "item_count", 100, "Number of collection entries used to build the schema") } -func printSchemaForOneCollection(dbName string, collName string, doRecover bool) { +func printSchemaForOneCollection(client *mongo.Client, dbName string, collName string, doRecover bool) { defer func() { if doRecover { if r := recover(); r != nil { @@ -53,7 +61,7 @@ func printSchemaForOneCollection(dbName string, collName string, doRecover bool) } } }() - bsonRaw, err := mongoHelper.QueryCollection(mongoHelper.ConStr, dbName, collName, int(itemCount)) + bsonRaw, err := mongoHelper.QueryCollection(client, dbName, collName, int(itemCount)) if err != nil { msg := fmt.Sprintf("Error while reading data for collection (%s.%s): \n%v\n", dbName, collName, err) panic(msg) @@ -66,7 +74,7 @@ func printSchemaForOneCollection(dbName string, collName string, doRecover bool) fmt.Printf("Error while processing bson for schema: %v", err) } } - if len(bsonRaw) > 0 { + if len(bsonRaw) > 0 { schema.ReduceTypes(&mainType, &otherComplexTypes) //schema.GuessDicts(&otherComplexTypes) schema.PrintSchema(dbName, collName, &mainType, &otherComplexTypes, outputDir) @@ -75,16 +83,16 @@ func printSchemaForOneCollection(dbName string, collName string, doRecover bool) } } -func printSchemasForAllCollections(dbName string) { - collections := mongoHelper.ReadCollectionsOrPanic(dbName) +func printSchemasForAllCollections(client *mongo.Client, dbName string) { + collections := mongoHelper.ReadCollectionsOrPanic(client, dbName) for _, coll := range *collections { - printSchemaForOneCollection(dbName, coll, true) + printSchemaForOneCollection(client, dbName, coll, true) } } -func printSchemasForAllDatabases() { - dbs := mongoHelper.ReadDatabasesOrPanic() +func printSchemasForAllDatabases(client *mongo.Client) { + dbs := mongoHelper.ReadDatabasesOrPanic(client) for _, db := range *dbs { - printSchemasForAllCollections(db) + printSchemasForAllCollections(client, db) } } diff --git a/cmd/schemaguesser/cmd/version.go b/cmd/schemaguesser/cmd/version.go index 7afba31..b3d5bd6 100644 --- a/cmd/schemaguesser/cmd/version.go +++ b/cmd/schemaguesser/cmd/version.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" ) -const Version = "1.2.0" +const Version = "2.0.0" var versionCmd = &cobra.Command{ Use: "version", diff --git a/internal/pkg/mongoHelper/mongoHelper.go b/internal/pkg/mongoHelper/mongoHelper.go index 7be8f55..0fa1042 100644 --- a/internal/pkg/mongoHelper/mongoHelper.go +++ b/internal/pkg/mongoHelper/mongoHelper.go @@ -21,25 +21,21 @@ func Connect(conStr string) (*mongo.Client, error) { return mongo.Connect(context.Background(), options.Client().ApplyURI(conStr2Use)) } +func CloseConnection(client *mongo.Client) { + if client == nil { + return + } + if err := client.Disconnect(context.Background()); err != nil { + println("Error while disconnect: %v", err) + } +} + func Dummy() { fmt.Println("'Dummy' is called") } -func ListDatabases(conStr string) ([]string, error) { +func ListDatabases(client *mongo.Client) ([]string, error) { var ret []string - client, err := Connect(conStr) - if err != nil { - return ret, err - } - - defer func() { - if client == nil { - return - } - if err = client.Disconnect(context.Background()); err != nil { - println("Error while disconnect: %v", err) - } - }() cursor, err := client.ListDatabases(context.Background(), bson.M{}) if err != nil { @@ -52,21 +48,8 @@ func ListDatabases(conStr string) ([]string, error) { return ret, nil } -func ListCollections(conStr string, databaseName string) ([]string, error) { +func ListCollections(client *mongo.Client, databaseName string) ([]string, error) { var ret []string - client, err := Connect(conStr) - if err != nil { - return ret, err - } - - defer func() { - if client == nil { - return - } - if err = client.Disconnect(context.Background()); err != nil { - println("Error while disconnect: %v", err) - } - }() db := client.Database(databaseName) cursor, err := db.ListCollectionNames(context.Background(), bson.M{}) @@ -80,21 +63,8 @@ func ListCollections(conStr string, databaseName string) ([]string, error) { return ret, nil } -func ListIndexes(conStr string, databaseName string, collectionName string) ([]string, error) { +func ListIndexes(client *mongo.Client, databaseName string, collectionName string) ([]string, error) { var ret []string - client, err := Connect(conStr) - if err != nil { - return ret, err - } - - defer func() { - if client == nil { - return - } - if err = client.Disconnect(context.Background()); err != nil { - println("Error while disconnect: %v", err) - } - }() db := client.Database(databaseName) collection := db.Collection(collectionName) @@ -113,21 +83,8 @@ func ListIndexes(conStr string, databaseName string, collectionName string) ([]s return ret, nil } -func QueryCollection(conStr string, databaseName string, collectionName string, itemCount int) ([]bson.Raw, error) { +func QueryCollection(client *mongo.Client, databaseName string, collectionName string, itemCount int) ([]bson.Raw, error) { var ret []bson.Raw - client, err := Connect(conStr) - if err != nil { - return ret, err - } - - defer func() { - if client == nil { - return - } - if err = client.Disconnect(context.Background()); err != nil { - println("Error while disconnect: %v", err) - } - }() db := client.Database(databaseName) collection := db.Collection(collectionName) @@ -147,8 +104,8 @@ func QueryCollection(conStr string, databaseName string, collectionName string, return ret, nil } -func ReadCollectionsOrPanic(dbName string) *[]string { - collections, err := ListCollections(ConStr, dbName) +func ReadCollectionsOrPanic(client *mongo.Client, dbName string) *[]string { + collections, err := ListCollections(client, dbName) if err != nil { msg := fmt.Sprintf("Error while reading collections for database (%s): \n%v\n", dbName, err) panic(msg) @@ -156,8 +113,8 @@ func ReadCollectionsOrPanic(dbName string) *[]string { return &collections } -func ReadDatabasesOrPanic() *[]string { - dbs, err := ListDatabases(ConStr) +func ReadDatabasesOrPanic(client *mongo.Client) *[]string { + dbs, err := ListDatabases(client) if err != nil { msg := fmt.Sprintf("Error while reading existing databases: \n%v\n", err) panic(msg) diff --git a/internal/pkg/mongoHelper/mongoHelper_test.go b/internal/pkg/mongoHelper/mongoHelper_test.go index 69afe31..8c24e89 100644 --- a/internal/pkg/mongoHelper/mongoHelper_test.go +++ b/internal/pkg/mongoHelper/mongoHelper_test.go @@ -8,7 +8,14 @@ import ( var conStr = "mongodb://{MONGO_USER}:{MONGO_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/admin" func TestListDbs_IT(t *testing.T) { - dbs, err := ListDatabases(conStr) + client, err := Connect(conStr) + defer CloseConnection(client) + + if err != nil { + t.Errorf("Failed to get client: %v", err) + return + } + dbs, err := ListDatabases(client) if err != nil { t.Errorf("Failed to list databases: %v", err) From 657faed8beb605623f288eab71fc33d49eedb645 Mon Sep 17 00:00:00 2001 From: Eiko Thomas <13103714+OkieOth@users.noreply.github.com> Date: Mon, 16 Sep 2024 22:01:37 +0200 Subject: [PATCH 4/4] make the schema creating functions in go routines --- cmd/schemaguesser/cmd/schema.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/schemaguesser/cmd/schema.go b/cmd/schemaguesser/cmd/schema.go index 3b7f81f..274ce98 100644 --- a/cmd/schemaguesser/cmd/schema.go +++ b/cmd/schemaguesser/cmd/schema.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "go.mongodb.org/mongo-driver/mongo" + "sync" "okieoth/schemaguesser/internal/pkg/mongoHelper" "okieoth/schemaguesser/internal/pkg/schema" @@ -85,14 +86,26 @@ func printSchemaForOneCollection(client *mongo.Client, dbName string, collName s func printSchemasForAllCollections(client *mongo.Client, dbName string) { collections := mongoHelper.ReadCollectionsOrPanic(client, dbName) + var wg sync.WaitGroup for _, coll := range *collections { - printSchemaForOneCollection(client, dbName, coll, true) + wg.Add(1) + go func() { + defer wg.Done() + printSchemaForOneCollection(client, dbName, coll, true) + }() } + wg.Wait() } func printSchemasForAllDatabases(client *mongo.Client) { dbs := mongoHelper.ReadDatabasesOrPanic(client) + var wg sync.WaitGroup for _, db := range *dbs { - printSchemasForAllCollections(client, db) + wg.Add(1) + go func() { + defer wg.Done() + printSchemasForAllCollections(client, db) + }() } + wg.Wait() }