version 0.3.1 - see changelog

Author: djcas9

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.3.1 2015-04-18
+
+  + Fix table sort - respect query order by first.
+  + perf limit added to query response. Limit by 10k.
+  + better query debug output for node scope queries.
+  + minor UI fixes.
+  + rename osquery.go to query.go
+
 ## 0.3.0 - 2015-04-16
 
   + fix bug with node template not showing os icon

envdb.go

@@ -14,7 +14,7 @@ import (
 
 const (
 	Name    = "envdb"
-	Version = "0.3.0"
+	Version = "0.3.1"
 
 	DefaultServerPort    = 3636
 	DefaultWebServerPort = 8080

http.go

@@ -66,24 +66,24 @@ func NewWebServer(webPort int, server *Server) {
 			Format: "json",
 		})
 
-		if len(data) != 1 {
+		if len(data.Results) != 1 {
 			d := QueryResults{}
 
 			err := errors.New(fmt.Sprintf("Node not found for id (%s).", sql.Id))
 			return d, err
 		}
 
-		tables = strings.Split(data[0].Results.(string), "\n")
+		tables = strings.Split(data.Results[0].Results.(string), "\n")
 
 		for i, t := range tables {
 			newT := strings.Replace(t, "  => ", "", -1)
 			tables[i] = newT
 		}
 
 		tables = tables[:len(tables)-1]
-		data[0].Results = tables
+		data.Results[0].Results = tables
 
-		return data[0], nil
+		return data.Results[0], data.Error
 	})
 
 	gotalk.Handle("table-info", func(sql SqlRequest) (QueryResults, error) {
@@ -92,14 +92,14 @@ func NewWebServer(webPort int, server *Server) {
 			Format: "json",
 		})
 
-		if len(data) != 1 {
+		if len(data.Results) != 1 {
 			d := QueryResults{}
 
 			err := errors.New(fmt.Sprintf("Node not found for id (%s).", sql.Id))
 			return d, err
 		}
 
-		return data[0], nil
+		return data.Results[0], data.Error
 	})
 
 	gotalk.Handle("query", func(sql SqlRequest) ([]QueryResults, error) {
@@ -108,7 +108,7 @@ func NewWebServer(webPort int, server *Server) {
 			Format: "json",
 		})
 
-		return data, nil
+		return data.Results, data.Error
 	})
 
 	gotalk.Handle("disconnect", func(id string) error {

node.go

@@ -64,10 +64,6 @@ func (self *Node) Handlers() {
 			self.Id = self.Config.Cache.Id
 		} else {
 
-			// dev - remove
-			// TODO: test for cache file remove
-			// Log.Fatalf("no cache file - exit for debug.")
-
 			id, uuerr := uuid.NewV4()
 			err = uuerr
 

osquery.go renamed to query.go

@@ -5,6 +5,8 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+
+	"github.com/nu7hatch/gouuid"
 )
 
 const (
@@ -24,6 +26,27 @@ type QueryResults struct {
 	Error    string      `json:"error"`
 }
 
+type Response struct {
+	Id      string         `json:"id"`
+	Results []QueryResults `json:"results"`
+	Total   int            `json:"total"`
+	Error   error          `json:"error"`
+}
+
+func NewResponse() *Response {
+	var id string
+
+	if uuid, err := uuid.NewV4(); err == nil {
+		id = uuid.String()
+	}
+
+	return &Response{
+		Id:    id,
+		Error: nil,
+		Total: 0,
+	}
+}
+
 func CheckOsQueryVersion(version string) bool {
 	if version == MinOsQueryVersion {
 		return true

query_database.go

@@ -72,7 +72,7 @@ AS listening ON process.pid = listening.pid;`,
 
 	q7 := QueryDb{
 		Name:  "Shell history",
-		Query: `SELECT * FROM shell_history`,
+		Query: `SELECT * FROM shell_history;`,
 		Type:  "all",
 	}
 

server.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -212,19 +213,51 @@ func (self *Server) Delete(id string) error {
 	return nil
 }
 
-func (self *Server) sendAll(in interface{}) []QueryResults {
-	// go func() {
+func ProcessResults(data []byte) (bool, []map[string]interface{}, []byte) {
+	var all = []map[string]interface{}{}
+	var returnData = []map[string]interface{}{}
+
+	if err := json.Unmarshal(data, &all); err == nil {
+		if len(all) > DefaultRowLimit {
+
+			Log.Debug("Results too large. sending first 2000.")
+
+			for i, d := range all {
+
+				if i >= DefaultRowLimit {
+					break
+				}
+
+				returnData = append(returnData, d)
+			}
+
+			if data, err := json.Marshal(returnData); err == nil {
+				return true, all, data
+			}
+
+		} else {
+			return false, all, data
+		}
+	} else {
+		return false, all, data
+	}
+
+	return false, all, data
+}
+
+func (self *Server) sendAll(in interface{}) *Response {
 	self.mu.RLock()
 	defer self.mu.RUnlock()
 
 	var wg sync.WaitGroup
 
-	var results []QueryResults
+	resp := NewResponse()
 
 	Log.Debug("Sending request to nodes.")
-
 	Log.Debugf("Request: %s", in.(Query).Sql)
 
+	var count int
+
 	for s, node := range self.Nodes {
 		wg.Add(1)
 
@@ -236,6 +269,8 @@ func (self *Server) sendAll(in interface{}) []QueryResults {
 			var data []byte
 			err := s.Request("query", in, &data)
 
+			_, all, _ := ProcessResults(data)
+
 			qr := QueryResults{
 				Id:       node.Id,
 				Name:     node.Name,
@@ -248,62 +283,78 @@ func (self *Server) sendAll(in interface{}) []QueryResults {
 			}
 
 			elapsed := time.Since(start)
-			Log.Debugf(" * %s (%s)", node.Name, elapsed)
 
-			results = append(results, qr)
+			count += len(all)
+
+			Log.Debugf("  * %s (%s)", node.Name, elapsed)
+			resp.Results = append(resp.Results, qr)
 		}(s, node)
 	}
 
 	Log.Debug("Waiting for all requests to return.")
 
 	wg.Wait()
 
-	Log.Debug("Sending results back to requester.")
+	resp.Total = count
+
+	if resp.Total > DefaultRowLimit {
+		resp.Error = errors.New(fmt.Sprintf("Results too large. (Limit: %d got %d)", DefaultRowLimit, resp.Total))
+	}
 
-	return results
-	// }()
+	Log.Debug("Sending results back to requester.")
+	return resp
 }
 
-func (self *Server) sendTo(id string, in interface{}) []QueryResults {
+func (self *Server) sendTo(id string, in interface{}) *Response {
 	self.mu.RLock()
 	defer self.mu.RUnlock()
 
-	var results []QueryResults
+	resp := NewResponse()
 
 	node, err := self.GetNodeById(id)
 
 	if err != nil {
-		return results
-	}
+		resp.Total = 0
 
-	Log.Debugf("%s: sending request.", node.Name)
-	Log.Debugf("%s: request: %s", node.Name, in.(Query).Sql)
+		return resp
+	}
 
 	start := time.Now()
 
 	var data []byte
 	err = node.Socket.Request("query", in, &data)
 
+	var newData []byte
+	over, all, cut := ProcessResults(data)
+
+	if over {
+		newData = cut
+	} else {
+		newData = data
+	}
+
+	resp.Total = len(all)
+
 	qr := QueryResults{
 		Id:       node.Id,
 		Name:     node.Name,
 		Hostname: node.Hostname,
-		Results:  string(data),
+		Results:  string(newData),
 	}
 
 	if err != nil {
 		qr.Error = err.Error()
 	}
 
-	results = append(results, qr)
+	resp.Results = append(resp.Results, qr)
 
 	elapsed := time.Since(start)
-	Log.Debugf("%s: done (%s)", node.Name, elapsed)
 
-	return results
+	Log.Debugf("\n  - Node: %s\n  - Request: %s\n  - Response Id: %s\n  - Total: %d\n  - Elapsed Time: %s", node.Name, in.(Query).Sql, resp.Id, resp.Total, elapsed)
+	return resp
 }
 
-func (self *Server) Send(id string, in interface{}) []QueryResults {
+func (self *Server) Send(id string, in interface{}) *Response {
 
 	if id == "all" {
 		return self.sendAll(in)

server_config.go

@@ -28,6 +28,8 @@ var (
 	DefaultPublicKeyPath  = ""
 	DefaultPrivateKeyPath = ""
 
+	DefaultRowLimit = 10000
+
 	SSL = false
 )
 

web/login.html

@@ -38,11 +38,10 @@
   </div>
 
   <script>
+    $("#user-email").focus();
     $("#login").show().stop().animate({
       opacity: 1,
-    }, 2000, function() {
-      $("#user-email").focus();
-    });
+    }, 2000);
   </script>
 
 </body>

web/public/css/envdb.css

@@ -287,7 +287,8 @@ border-radius: 100px;
 }
 
 #nodes li.current{
-  background-color: #6E4FC5;
+  /* background-color: #6E4FC5; */
+  background-color: #0077FF;
 }
 
 #nodes li.current .node-metadata{