use ini file + implement data generator (#1)

* Add SSH config type

* Add ini file generation + read configs from it

* Delete protobuf dependency

* Update dependencies

* temp commit

* more stuff

* add more stuff

* add more stuff

* fix gitignore

Author: oleewere

.gitignore

@@ -15,4 +15,5 @@
 
 .idea/
 
-native-solr-client
+go-solr-client
+solr-docs-generator

Gopkg.lock

@@ -24,11 +24,6 @@
 #   go-tests = true
 #   unused-packages = true
 
-
-[[constraint]]
-  name = "github.com/golang/protobuf"
-  version = "1.1.0"
-
 [[constraint]]
   name = "github.com/satori/go.uuid"
   version = "1.2.0"

Gopkg.toml

@@ -16,6 +16,9 @@ GIT_REV_SHORT = $(shell git rev-parse --short HEAD)
 install:
 	go install
 
+build-generator:
+	go build -ldflags "-X main.GitRevString=$(GIT_REV_SHORT) -X main.Version=$(VERSION) -X main.ActionType=generator" -o solr-docs-generator .
+
 build:
 	go build -ldflags "-X main.GitRevString=$(GIT_REV_SHORT) -X main.Version=$(VERSION)" .
 

Makefile

@@ -5,29 +5,23 @@ import (
 	"flag"
 	"fmt"
 	"github.com/oleewere/go-solr-client/solr"
-	"github.com/satori/go.uuid"
+	"log"
 )
 
 var Version string
 var GitRevString string
+var ActionType string
 
 func main() {
 
-	var collection string
-	var url string
-	var krb5Path string
-	var keytabPath string
-	var principal string
-	var realm string
 	var isVersionCheck bool
+	var iniFileLocation string
 
 	flag.BoolVar(&isVersionCheck, "version", false, "Print application version and git revision if available")
-	flag.StringVar(&url, "url", "http://localhost:8983", "URL name for Solr or Solr proxy")
-	flag.StringVar(&collection, "collection", "hadoop_logs", "Collection name for the Solr client")
-	flag.StringVar(&krb5Path, "krb-conf-path", "", "Kerberos config location")
-	flag.StringVar(&keytabPath, "keytab-path", "", "Kerberos keytab location")
-	flag.StringVar(&principal, "principal", "", "Kerberos principal")
-	flag.StringVar(&realm, "realm", "", "Kerberos Realm e.g.: EXAMPLE.COM")
+	flag.StringVar(&iniFileLocation, "ini-file", "", "INI config file location")
+	if len(ActionType) == 0 {
+		flag.StringVar(&ActionType, "action-type", "generator", "action")
+	}
 
 	flag.Parse()
 
@@ -40,14 +34,19 @@ func main() {
 		os.Exit(0)
 	}
 
-	fmt.Print("Start Solr Cloud Client ...\n")
-
-	securityConfig := solr.InitSecurityConfig(krb5Path, keytabPath, principal, realm)
-
-	solrConfig := solr.SolrConfig{url, "hadoop_logs", &securityConfig, "/solr",
-		solr.TLSConfig{}, true, 60}
-	solrClient, err := solr.NewSolrClient(url, collection, &solrConfig)
+	if len(iniFileLocation) == 0 {
+		log.Fatal("INI config file option (--ini-file) is missing.")
+	}
 
+	if _, err := os.Stat(iniFileLocation); os.IsNotExist(err) {
+		solr.GenerateIniFile(iniFileLocation)
+		os.Exit(0)
+	}
+	log.Println("Starting Solr Client ...")
+	solrConfig, sshConfig := solr.GenerateSolrConfig(iniFileLocation)
+	solr.GenerateSolrData(&solrConfig, &sshConfig, iniFileLocation)
+    /*
+	solrClient, err := solr.NewSolrClient(&solrConfig)
 	_, response, _ := solrClient.Query(nil)
 	docs := response.Response.Docs
 	for _, doc := range docs {
@@ -58,29 +57,5 @@ func main() {
 		}
 		fmt.Printf("----------------------")
 	}
-
-	putDocs := solr.SolrDocuments{
-		solr.SolrDocument{
-			"id":          uuid.NewV4(),
-			"log_message": "[email protected]",
-			"seq_num":     100,
-			"level":       "FATAL",
-			"logtime":     "2018-07-03T15:55:47.396Z",
-		},
-		solr.SolrDocument{
-			"id":          uuid.NewV4(),
-			"log_message": "[email protected]",
-			"seq_num":     1000,
-			"level":       "FATAL",
-			"logtime":     "2018-07-03T15:55:47.396Z",
-		},
-	}
-
-	solrClient.Update(putDocs, nil, false)
-
-	if err != nil {
-		fmt.Print(err)
-	}
-
-	os.Exit(0)
+    */
 }

main.go

@@ -0,0 +1,51 @@
+# Copyright 2018 Oliver Szabo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[security]
+kerberosEnabled = false
+kerberosKeytab = /tmp/solr.keytab
+kerberosPrincipal = solr/myhostname
+kerberosRealm = EXAMPLE.COM
+kerberosKrb5Path = /tmp/krb5.conf
+
+[solr]
+url = http://localhost:8983
+context = /solr
+collection = hadoop_logs
+ssl = false
+connection_timeout = 60
+
+[ssh]
+enabled = false
+username = root
+hostname = myremotehost
+private_key_path = /keys/private_key
+download_location = /tmp
+remote_krb5_conf = /etc/krb5.conf
+remote_keytab = /etc/security/keytabs/solr.service.keytab
+
+[generator]
+num_writes = 10
+num_docs_per_write = 1000
+cluster_field = cluster
+cluster_num = 10
+filterable_field = host
+filterable_field_num = 1000
+level_field = level
+level_values = INFO,DEBUG,FATAL,WARN,ERROR,UNKNOWN,TRACE
+type_field = type
+type_values = ambari_server,ambari_agent,ambari_config,ambari_eclipselink,hdfs_name_node,hdfs_secondary_name_node
+date_field = logtime
+message_fields = log_message
+num_fields = seq_num

solr-client-sample.cfg

@@ -0,0 +1,37 @@
+[security]
+kerberosEnabled   = true
+kerberosKeytab    = /tmp/solr.keytab
+kerberosPrincipal = infra-solr/c7402.ambari.apache.org
+kerberosRealm     = AMBARI.APACHE.ORG
+kerberosKrb5Path  = /tmp/krb5.conf
+
+[solr]
+url                = http://c7402.ambari.apache.org:8886
+context            = /solr
+collection         = hadoop_logs
+ssl                = false
+connection_timeout = 60
+
+[ssh]
+enabled           = true
+username          = root
+hostname          = c7402.ambari.apache.org
+private_key_path  = /Users/oliverszabo/Projects/ambari-vagrant/centos7.4/insecure_private_key
+download_location = /tmp
+remote_krb5_conf  = /etc/krb5.conf
+remote_keytab     = /etc/security/keytabs/ambari-infra-solr.service.keytab
+
+[generator]
+num_writes     = 10
+num_docs_per_write = 1000
+cluster_field = cluster
+cluster_num   = 10
+filterable_field = host
+filterable_field_num   = 1000
+level_field    = level
+level_values   = INFO,DEBUG,FATAL,WARN,ERROR,UNKNOWN,TRACE
+type_field     = type
+type_values    = ambari_server,ambari_agent,ambari_config,ambari_eclipselink,hdfs_name_node,hdfs_secondary_name_node
+date_field     = logtime
+message_fields = log_message
+num_fields     = seq_num

solr-client.cfg

@@ -28,7 +28,7 @@ import (
 	"time"
 )
 
-func NewSolrClient(url string, collection string, solrConfig *SolrConfig) (*SolrClient, error) {
+func NewSolrClient(solrConfig *SolrConfig) (*SolrClient, error) {
 	httpClient := &http.Client{
 		Transport: &http.Transport{
 			Proxy:                 http.ProxyFromEnvironment,
@@ -38,9 +38,6 @@ func NewSolrClient(url string, collection string, solrConfig *SolrConfig) (*Solr
 			ResponseHeaderTimeout: 10 * time.Second,
 		},
 	}
-	solrConfig.Collection = collection
-	solrConfig.Url = url
-	solrConfig.SolrUrlContext = "/solr"
 
 	securityConfig := solrConfig.SecurityConfig