Skip to content

Commit a105306

Browse files
Ice3man543Ice3man543
committed
feat: added simpler stats tracking system
1 parent 727ff90 commit a105306

15 files changed

Lines changed: 756 additions & 408 deletions

File tree

‎cmd/nuclei/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,7 @@ on extensive configurability, massive extensibility and ease of use.`)
367367
flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"),
368368
flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"),
369369
flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"),
370-
flagSet.StringVarP(&options.DASTScanName, "dast-scan-report", "dtr", "", "write dast scan report to file"),
370+
flagSet.BoolVarP(&options.DASTReport, "dast-report", "drg", false, "write dast scan report to file"),
371371
flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"),
372372
flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"),
373373
flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"),

‎internal/runner/runner.go‎

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ type Runner struct {
9797
tmpDir string
9898
parser parser.Parser
9999
httpApiEndpoint *httpapi.Server
100+
fuzzStats *fuzzStats.Tracker
100101
dastServer *server.DASTServer
101102
}
102103

@@ -244,14 +245,6 @@ func New(options *types.Options) (*Runner, error) {
244245
}
245246
runner.inputProvider = inputProvider
246247

247-
// Create the output file if asked
248-
outputWriter, err := output.NewStandardWriter(options)
249-
if err != nil {
250-
return nil, errors.Wrap(err, "could not create output file")
251-
}
252-
// setup a proxy writer to automatically upload results to PDCP
253-
runner.output = runner.setupPDCPUpload(outputWriter)
254-
255248
if options.JSONL && options.EnableProgressBar {
256249
options.StatsJSON = true
257250
}
@@ -344,6 +337,42 @@ func New(options *types.Options) (*Runner, error) {
344337
runner.tmpDir = tmpDir
345338
}
346339

340+
if options.DASTReport || options.DASTServer {
341+
var err error
342+
runner.fuzzStats, err = fuzzStats.NewTracker()
343+
if err != nil {
344+
return nil, errors.Wrap(err, "could not create fuzz stats db")
345+
}
346+
if !options.DASTServer {
347+
dastServer, err := server.NewStatsServer(runner.fuzzStats)
348+
if err != nil {
349+
return nil, errors.Wrap(err, "could not create dast server")
350+
}
351+
runner.dastServer = dastServer
352+
}
353+
}
354+
355+
// Create the output file if asked
356+
outputWriter, err := output.NewStandardWriter(options)
357+
if err != nil {
358+
return nil, errors.Wrap(err, "could not create output file")
359+
}
360+
if runner.fuzzStats != nil {
361+
outputWriter.RequestHook = func(request *output.JSONLogRequest) {
362+
if request.Error == "none" || request.Error == "" {
363+
return
364+
}
365+
runner.fuzzStats.RecordErrorEvent(fuzzStats.ErrorEvent{
366+
TemplateID: request.Template,
367+
URL: request.Input,
368+
Error: request.Error,
369+
})
370+
}
371+
}
372+
373+
// setup a proxy writer to automatically upload results to PDCP
374+
runner.output = runner.setupPDCPUpload(outputWriter)
375+
347376
return runner, nil
348377
}
349378

@@ -453,6 +482,7 @@ func (r *Runner) RunEnumeration() error {
453482
Colorizer: r.colorizer,
454483
Parser: r.parser,
455484
TemporaryDirectory: r.tmpDir,
485+
FuzzStatsDB: r.fuzzStats,
456486
}
457487
dastServer, err := server.New(&server.Options{
458488
Address: r.options.DASTServerAddress,
@@ -513,13 +543,6 @@ func (r *Runner) RunEnumeration() error {
513543
FuzzParamsFrequency: fuzzFreqCache,
514544
GlobalMatchers: globalmatchers.New(),
515545
}
516-
if r.options.DASTScanName != "" {
517-
var err error
518-
executorOpts.FuzzStatsDB, err = fuzzStats.NewTracker(r.options.DASTScanName)
519-
if err != nil {
520-
return errors.Wrap(err, "could not create fuzz stats db")
521-
}
522-
}
523546

524547
if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) {
525548
// Go StdLib style experimental/debug feature switch
@@ -663,6 +686,12 @@ func (r *Runner) RunEnumeration() error {
663686
Retries: r.options.Retries,
664687
}, "")
665688

689+
if r.dastServer != nil {
690+
if err := r.dastServer.Start(); err != nil {
691+
r.dastServer.Start()
692+
}
693+
}
694+
666695
enumeration := false
667696
var results *atomic.Bool
668697
results, err = r.runStandardEnumeration(executorOpts, store, executorEngine)

‎internal/server/nuclei_sdk.go‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/logrusorgru/aurora"
1010
"github.com/projectdiscovery/gologger"
1111
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency"
12+
"github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
1213
"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
1314
"github.com/projectdiscovery/nuclei/v3/pkg/input/provider/http"
1415
"github.com/projectdiscovery/nuclei/v3/pkg/projectfile"
@@ -20,7 +21,6 @@ import (
2021
"github.com/projectdiscovery/nuclei/v3/pkg/catalog"
2122
"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
2223
"github.com/projectdiscovery/nuclei/v3/pkg/core"
23-
fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats"
2424
"github.com/projectdiscovery/nuclei/v3/pkg/input"
2525
"github.com/projectdiscovery/nuclei/v3/pkg/loader/parser"
2626
parsers "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow"
@@ -54,6 +54,7 @@ type NucleiExecutorOptions struct {
5454
Interactsh *interactsh.Client
5555
ProjectFile *projectfile.ProjectFile
5656
Browser *browserEngine.Browser
57+
FuzzStatsDB *stats.Tracker
5758
Colorizer aurora.Aurora
5859
Parser parser.Parser
5960
TemporaryDirectory string
@@ -83,13 +84,7 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) {
8384
Parser: opts.Parser,
8485
FuzzParamsFrequency: fuzzFreqCache,
8586
GlobalMatchers: globalmatchers.New(),
86-
}
87-
if opts.Options.DASTScanName != "" {
88-
var err error
89-
executorOpts.FuzzStatsDB, err = fuzzStats.NewTracker(opts.Options.DASTScanName)
90-
if err != nil {
91-
return nil, errors.Wrap(err, "could not create fuzz stats db")
92-
}
87+
FuzzStatsDB: opts.FuzzStatsDB,
9388
}
9489

9590
if opts.Options.ShouldUseHostError() {

‎internal/server/requests_worker.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@ import (
99
)
1010

1111
func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) {
12+
defer s.endpointsInQueue.Add(-1)
13+
1214
parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL)
1315
if err != nil {
1416
gologger.Warning().Msgf("Could not parse raw request: %s\n", err)
1517
return
1618
}
1719

1820
if parsedReq.URL.Scheme != "http" && parsedReq.URL.Scheme != "https" {
21+
gologger.Warning().Msgf("Invalid scheme: %s\n", parsedReq.URL.Scheme)
1922
return
2023
}
2124

@@ -43,12 +46,11 @@ func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) {
4346

4447
gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String())
4548

46-
// Fuzz the request finally
47-
s.fuzzRequest(req)
48-
}
49+
s.endpointsBeingTested.Add(1)
50+
defer s.endpointsBeingTested.Add(-1)
4951

50-
func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) {
51-
err := s.nucleiExecutor.ExecuteScan(req)
52+
// Fuzz the request finally
53+
err = s.nucleiExecutor.ExecuteScan(req)
5254
if err != nil {
5355
gologger.Warning().Msgf("Could not run nuclei: %s\n", err)
5456
return

0 commit comments

Comments
 (0)