Improve error logging for unresolved variables

cmd/nuclei/main.go

@@ -352,6 +352,7 @@ on extensive configurability, massive extensibility and ease of use.`)
  flagSet.BoolVarP(&options.EnablePprof, "enable-pprof", "ep", false, "enable pprof debugging server"),
  flagSet.CallbackVarP(printTemplateVersion, "templates-version", "tv", "shows the version of the installed nuclei-templates"),
  flagSet.BoolVarP(&options.HealthCheck, "health-check", "hc", false, "run diagnostic check up"),
+ flagSet.DynamicVarP(&options.ErrorLabels, "error-label", "elabel", []string{"UNV"}, "enable logging for specific error labels (comma-separated) (labels: UNV"),
  )
 
  flagSet.CreateGroup("update", "Update",

internal/runner/options.go

@@ -17,6 +17,7 @@ import (
  "github.com/projectdiscovery/gologger/formatter"
  "github.com/projectdiscovery/gologger/levels"
  "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine"
@@ -26,6 +27,7 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/markdown"
  "github.com/projectdiscovery/nuclei/v3/pkg/reporting/exporters/sarif"
  "github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
  "github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
  fileutil "github.com/projectdiscovery/utils/file"
  "github.com/projectdiscovery/utils/generic"
@@ -87,6 +89,8 @@ func ParseOptions(options *types.Options) {
  // Load the resolvers if user asked for them
  loadResolvers(options)
 
+ options.ErrorLabels = parseErrorLables(options.ErrorLabels)
+
  err := protocolinit.Init(options)
  if err != nil {
  gologger.Fatal().Msgf("Could not initialize protocols: %s\n", err)
@@ -430,3 +434,21 @@ func getBoolEnvValue(key string) bool {
  value := os.Getenv(key)
  return strings.EqualFold(value, "true")
 }
+
+func parseErrorLables(errorLabels []string) []string {
+ // Make error label entries in stats
+ for _, v := range elabel.ErrorLableMap {
+ stats.NewEntry(v.Name, v.Description)
+ }
+ if len(errorLabels) == 0 {
+ return errorLabels
+ }
+ var errLabels []string
+ for _, label := range errorLabels {
+ label = strings.ToLower(label)
+ if v, ok := elabel.ErrorLableMap[label]; ok {
+ errLabels = append(errLabels, v.Name)
+ }
+ }
+ return errLabels
+}

internal/runner/runner.go

@@ -17,6 +17,7 @@ import (
  uncoverlib "github.com/projectdiscovery/uncover"
  "github.com/projectdiscovery/utils/env"
  permissionutil "github.com/projectdiscovery/utils/permission"
+ stringsutil "github.com/projectdiscovery/utils/strings"
  updateutils "github.com/projectdiscovery/utils/update"
 
  "github.com/logrusorgru/aurora"
@@ -40,6 +41,7 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolinit"
@@ -492,6 +494,7 @@ func (r *Runner) RunEnumeration() error {
  }
  }
  r.progress.Stop()
+ r.displayErrorInfo()
 
  if executorOpts.InputHelper != nil {
  _ = executorOpts.InputHelper.Close()
@@ -615,6 +618,17 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) {
  }
 }
 
+func (r *Runner) displayErrorInfo() {
+ if !r.options.Verbose {
+ return
+ }
+ errLables := r.options.ErrorLabels
+ // if error label is not provided, display the stats for specific labels
+ if !stringsutil.ContainsAny(elabel.UnresolvedVariablesErrorLabel, errLables...) {
+ stats.Display(elabel.UnresolvedVariablesErrorLabel)
+ }
+}
+
 // SaveResumeConfig to file
 func (r *Runner) SaveResumeConfig(path string) error {
  resumeCfgClone := r.resumeCfg.Clone()

pkg/protocols/common/errors/label/error_label.go

@@ -0,0 +1,35 @@
+package label
+
+import (
+ "strings"
+)
+
+const (
+ UnresolvedVariablesErrorLabel = "unresolved-variables-error"
+)
+
+var (
+ ErrorLabels = []string{
+ UnresolvedVariablesErrorLabel,
+ }
+
+ ErrorLableMap = map[string]struct {
+ Name string
+ Description string
+ }{
+ "unv": {
+ Name: UnresolvedVariablesErrorLabel,
+ Description: "Failed %v requests due to unresolved variables. Use -elabel=UNV to enable unresolved variables logs.",
+ },
+ }
+)
+
+// Contains checks if the error string contains any of the provided labels, if yes returns matched label and true
+func Contains(errStr string, lables []string) (string, bool) {
+ for _, label := range lables {
+ if strings.Contains(errStr, label) {
+ return label, true
+ }
+ }
+ return "", false
+}

pkg/protocols/common/expressions/variables.go

@@ -7,6 +7,8 @@ import (
 
  "github.com/Knetic/govaluate"
  "github.com/projectdiscovery/nuclei/v3/pkg/operators/common/dsl"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
+ errorutil "github.com/projectdiscovery/utils/errors"
 )
 
 var (
@@ -38,7 +40,7 @@ func ContainsUnresolvedVariables(items ...string) error {
  unresolvedVariables = append(unresolvedVariables, match[1])
  }
  if len(unresolvedVariables) > 0 {
- return errors.New("unresolved variables found: " + strings.Join(unresolvedVariables, ","))
+ return errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: "+strings.Join(unresolvedVariables, ","))
  }
  }
 

pkg/protocols/common/expressions/variables_test.go

@@ -4,6 +4,8 @@ import (
  "errors"
  "testing"
 
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
+ errorutil "github.com/projectdiscovery/utils/errors"
  "github.com/stretchr/testify/require"
 )
 
@@ -12,11 +14,11 @@ func TestUnresolvedVariablesCheck(t *testing.T) {
  data string
  err error
  }{
- {"{{test}}", errors.New("unresolved variables found: test")},
- {"{{test}}/{{another}}", errors.New("unresolved variables found: test,another")},
+ {"{{test}}", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test")},
+ {"{{test}}/{{another}}", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test,another")},
  {"test", nil},
- {"%7b%7btest%7d%7d", errors.New("unresolved variables found: test")},
- {"%7B%7Bfirst%2Asecond%7D%7D", errors.New("unresolved variables found: first%2Asecond")},
+ {"%7b%7btest%7d%7d", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "unresolved variables found: test")},
+ {"%7B%7Bfirst%2Asecond%7D%7D", errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, errors.New("unresolved variables found: first%2Asecond").Error())},
  {"{{7*7}}", nil},
  {"{{'a'+'b'}}", nil},
  {"{{'a'}}", nil},

pkg/protocols/http/build_request.go

@@ -14,6 +14,7 @@ import (
 
  "github.com/projectdiscovery/gologger"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
@@ -34,7 +35,7 @@ import (
 // ErrEvalExpression
 var (
  ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions")
- ErrUnresolvedVars = errorutil.NewWithFmt("unresolved variables `%v` found in request")
+ UnresolvedErrFmt = "unresolved variables `%v` found in request"
 )
 
 // generatedRequest is a single generated request wrapped for a template request
@@ -208,7 +209,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
  }
 
  if err := expressions.ContainsUnresolvedVariables(parts[1]); err != nil {
- return nil, ErrUnresolvedVars.Msgf(parts[1])
+ return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, parts[1])
  }
 
  parsed, err := urlutil.ParseURL(parts[1], true)
@@ -229,7 +230,7 @@ func (r *requestGenerator) makeSelfContainedRequest(ctx context.Context, data st
  if err := expressions.ContainsUnresolvedVariables(data); err != nil {
  // early exit: if there are any unresolved variables in `path` after evaluation
  // then return early since this will definitely fail
- return nil, ErrUnresolvedVars.Msgf(data)
+ return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, data)
  }
 
  urlx, err := urlutil.ParseURL(data, true)
@@ -289,6 +290,10 @@ func (r *requestGenerator) generateRawRequest(ctx context.Context, rawRequest st
  return unsafeReq, nil
  }
 
+ if err := expressions.ContainsUnresolvedVariables(rawRequestData.FullURL); err != nil {
+ return nil, errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, UnresolvedErrFmt, rawRequestData.FullURL)
+ }
+
  urlx, err := urlutil.ParseURL(rawRequestData.FullURL, true)
  if err != nil {
  return nil, errorutil.NewWithErr(err).Msgf("failed to create request with url %v got %v", rawRequestData.FullURL, err).WithTag("raw")

pkg/protocols/http/request.go

@@ -24,6 +24,7 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/output"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/fuzz"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -36,7 +37,9 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signerpool"
  templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
  "github.com/projectdiscovery/nuclei/v3/pkg/types"
+ "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
  "github.com/projectdiscovery/rawhttp"
+ errorutil "github.com/projectdiscovery/utils/errors"
  "github.com/projectdiscovery/utils/reader"
  sliceutil "github.com/projectdiscovery/utils/slice"
  stringsutil "github.com/projectdiscovery/utils/strings"
@@ -368,6 +371,9 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
  return true, nil
  }
  request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
+ if lb, ok := elabel.Contains(err.Error(), elabel.ErrorLabels); ok {
+ stats.Increment(lb)
+ }
  return true, err
  }
 
@@ -462,7 +468,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa
 
 const drainReqSize = int64(8 * 1024)
 
-var errStopExecution = errors.New("stop execution due to unresolved variables")
+var errStopExecution = errorutil.NewWithTag(elabel.UnresolvedVariablesErrorLabel, "stop execution due to unresolved variables")
 
 // executeRequest executes the actual generated request and returns error if occurred
 func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error {
@@ -538,7 +544,10 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
  }
  } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user.
  if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck {
- gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
+ stats.Increment(elabel.UnresolvedVariablesErrorLabel)
+ if _, ok := elabel.Contains(varErr.Error(), request.options.Options.ErrorLabels); ok {
+ gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr)
+ }
  return errStopExecution
  }
  }

pkg/protocols/network/request.go

@@ -18,6 +18,7 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/output"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/eventcreator"
@@ -28,6 +29,7 @@ import (
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump"
  protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
  templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
+ "github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
  errorutil "github.com/projectdiscovery/utils/errors"
  mapsutil "github.com/projectdiscovery/utils/maps"
  "github.com/projectdiscovery/utils/reader"
@@ -245,7 +247,11 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac
  reqBuilder.Write(finalData)
 
  if err := expressions.ContainsUnresolvedVariables(string(finalData)); err != nil {
- gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err)
+ request.options.Progress.IncrementFailedRequestsBy(1)
+ stats.Increment(elabel.UnresolvedVariablesErrorLabel)
+ if _, ok := elabel.Contains(err.Error(), request.options.Options.ErrorLabels); ok {
+ gologger.Warning().Msgf("[%s] Could not make network request for %s: %v\n", request.options.TemplateID, actualAddress, err)
+ }
  return nil
  }
 

pkg/tmplexec/generic/exec.go

@@ -7,6 +7,7 @@ import (
  "github.com/projectdiscovery/gologger"
  "github.com/projectdiscovery/nuclei/v3/pkg/output"
  "github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+ elabel "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/errors/label"
  "github.com/projectdiscovery/nuclei/v3/pkg/scan"
 )
 
@@ -79,7 +80,13 @@ func (g *Generic) ExecuteWithResults(ctx *scan.ScanContext) error {
  if g.options.HostErrorsCache != nil {
  g.options.HostErrorsCache.MarkFailed(ctx.Input.MetaInput.ID(), err)
  }
- gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
+ if _, ok := elabel.Contains(err.Error(), g.options.Options.ErrorLabels); ok {
+ gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
+ }
+ // TODO: remove logging for otherthan unresolved variables
+ if _, ok := elabel.Contains(err.Error(), []string{elabel.UnresolvedVariablesErrorLabel}); !ok {
+ gologger.Warning().Msgf("[%s] Could not execute request for %s: %s\n", g.options.TemplateID, ctx.Input.MetaInput.PrettyPrint(), err)
+ }
  }
  // If a match was found and stop at first match is set, break out of the loop and return
  if g.results.Load() && (g.options.StopAtFirstMatch || g.options.Options.StopAtFirstMatch) {

pkg/types/types.go

@@ -96,6 +96,8 @@ type Options struct {
  TraceLogFile string
  // ErrorLogFile specifies a file to write with the errors of all requests
  ErrorLogFile string
+ // ErrorLabels is the list of error labels to match and enable logging
+ ErrorLabels []string
  // ReportingDB is the db for report storage as well as deduplication
  ReportingDB string
  // ReportingConfig is the config file for nuclei reporting module