Skip to content

Commit

Permalink
Update go-buffered-processor version
Browse files Browse the repository at this point in the history
  • Loading branch information
oleewere committed Dec 8, 2018
1 parent fdd4408 commit 3f87ead
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 40 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
.glide/

.idea/
dist
solr-client.cfg
go-solr-client
solr-docs-generator
6 changes: 3 additions & 3 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Gopkg.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

[[constraint]]
name = "github.com/oleewere/go-buffered-processor"
version = "0.1.0"
version = "1.0.0"

[prune]
go-tests = true
Expand Down
22 changes: 8 additions & 14 deletions solr/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,21 @@ import (

// SolrDataProcessor type for processing Solr data
type SolrDataProcessor struct {
BatchContext *processor.BatchContext
Mutex *sync.Mutex
SolrClient *SolrClient
Mutex *sync.Mutex
SolrClient *SolrClient
}

// Process send gathered data to Solr
func (p SolrDataProcessor) Process() error {
func (p SolrDataProcessor) Process(batchContext *processor.BatchContext) error {
log.Println("Processing...")
p.Mutex.Lock()
defer p.Mutex.Unlock()
_, _, err := p.SolrClient.Update(p.BatchContext.BufferData, nil, true)
_, _, err := p.SolrClient.Update(batchContext.BufferData, nil, true)
return err
}

// GetBatchContext gather batch context that will be used by processor
func (p SolrDataProcessor) GetBatchContext() *processor.BatchContext {
return p.BatchContext
}

// HandleError handle errors during time based buffer processing (it is not used by this generator)
func (p SolrDataProcessor) HandleError(err error) {
func (p SolrDataProcessor) HandleError(batchContext *processor.BatchContext, err error) {
fmt.Println(err)
}

Expand Down Expand Up @@ -113,18 +107,18 @@ func GenerateSolrData(solrConfig *SolrConfig, sshConfig *SSHConfig, iniFileLocat
batchContext.MaxRetries = 20
batchContext.RetryTimeInterval = 10

proc := SolrDataProcessor{SolrClient: solrClient, BatchContext: batchContext, Mutex: &sync.Mutex{}}
proc := SolrDataProcessor{SolrClient: solrClient, Mutex: &sync.Mutex{}}

for i := 1; i <= numWrites; i++ {
for j := 1; j <= docsPerWrite; j++ {
solrDoc := createRandomSolrDoc(clusterField, clusterNum, filterableField, filterableFieldNum, levelField, levels, typeField, types, dateField, messageFields, numFields)

processor.ProcessData(solrDoc, proc)
processor.ProcessData(solrDoc, batchContext, proc)
}
randomMsg := fmt.Sprintf("Sending %d documents to Solr: %d/%d ...", docsPerWrite, i, numWrites)
log.Println(randomMsg)
}
proc.Process()
proc.Process(batchContext)
log.Println("Solr random documents generation has finished.")
}

Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 3f87ead

Please sign in to comment.