Skip to content

Commit e96320e

Browse files
dogancanbakirdogancanbakir
committed
add request tracking to -stats flag
Adds request counting to all sources to help users monitor API usage and debug quota consumption issues like #1562. Changes: - Add Requests field to Statistics struct - Track HTTP requests in all 53 sources - Display request count in stats output Closes #1698
1 parent c1977c0 commit e96320e

54 files changed

Lines changed: 221 additions & 2 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pkg/runner/stats.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,12 @@ func printStatistics(stats map[string]subscraping.Statistics) {
2424
if sourceStats.Skipped {
2525
skipped = append(skipped, fmt.Sprintf(" %s", source))
2626
} else {
27-
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Errors))
27+
lines = append(lines, fmt.Sprintf(" %-20s %-10s %10d %10d %10d", source, sourceStats.TimeTaken.Round(time.Millisecond).String(), sourceStats.Results, sourceStats.Requests, sourceStats.Errors))
2828
}
2929
}
3030

3131
if len(lines) > 0 {
32-
gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56))
32+
gologger.Print().Msgf("\n Source Duration Results Requests Errors\n%s\n", strings.Repeat("─", 68))
3333
gologger.Print().Msg(strings.Join(lines, "\n"))
3434
gologger.Print().Msgf("\n")
3535
}

‎pkg/subscraping/sources/alienvault/alienvault.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ type Source struct {
2323
timeTaken time.Duration
2424
results int
2525
errors int
26+
requests int
2627
apiKeys []string
2728
skipped bool
2829
}
@@ -32,6 +33,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3233
results := make(chan subscraping.Result)
3334
s.errors = 0
3435
s.results = 0
36+
s.requests = 0
3537

3638
go func() {
3739
defer func(startTime time.Time) {
@@ -45,6 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4547
return
4648
}
4749

50+
s.requests++
4851
resp, err := session.Get(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain), "",
4952
map[string]string{"Authorization": "Bearer " + randomApiKey})
5053
if err != nil && resp == nil {
@@ -110,6 +113,7 @@ func (s *Source) Statistics() subscraping.Statistics {
110113
return subscraping.Statistics{
111114
Errors: s.errors,
112115
Results: s.results,
116+
Requests: s.requests,
113117
TimeTaken: s.timeTaken,
114118
Skipped: s.skipped,
115119
}

‎pkg/subscraping/sources/anubis/anubis.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,23 @@ type Source struct {
1717
timeTaken time.Duration
1818
errors int
1919
results int
20+
requests int
2021
}
2122

2223
// Run function returns all subdomains found with the service
2324
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
2425
results := make(chan subscraping.Result)
2526
s.errors = 0
2627
s.results = 0
28+
s.requests = 0
2729

2830
go func() {
2931
defer func(startTime time.Time) {
3032
s.timeTaken = time.Since(startTime)
3133
close(results)
3234
}(time.Now())
3335

36+
s.requests++
3437
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain))
3538
if err != nil {
3639
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -94,6 +97,7 @@ func (s *Source) Statistics() subscraping.Statistics {
9497
return subscraping.Statistics{
9598
Errors: s.errors,
9699
Results: s.results,
100+
Requests: s.requests,
97101
TimeTaken: s.timeTaken,
98102
}
99103
}

‎pkg/subscraping/sources/bevigil/bevigil.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ type Source struct {
2121
timeTaken time.Duration
2222
errors int
2323
results int
24+
requests int
2425
skipped bool
2526
}
2627

2728
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
2829
results := make(chan subscraping.Result)
2930
s.errors = 0
3031
s.results = 0
32+
s.requests = 0
3133

3234
go func() {
3335
defer func(startTime time.Time) {
@@ -43,6 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4345

4446
getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain)
4547

48+
s.requests++
4649
resp, err := session.Get(ctx, getUrl, "", map[string]string{
4750
"X-Access-Token": randomApiKey, "User-Agent": "subfinder",
4851
})
@@ -106,6 +109,7 @@ func (s *Source) Statistics() subscraping.Statistics {
106109
return subscraping.Statistics{
107110
Errors: s.errors,
108111
Results: s.results,
112+
Requests: s.requests,
109113
TimeTaken: s.timeTaken,
110114
Skipped: s.skipped,
111115
}

‎pkg/subscraping/sources/bufferover/bufferover.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type Source struct {
2727
timeTaken time.Duration
2828
errors int
2929
results int
30+
requests int
3031
skipped bool
3132
}
3233

@@ -35,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3536
results := make(chan subscraping.Result)
3637
s.errors = 0
3738
s.results = 0
39+
s.requests = 0
3840

3941
go func() {
4042
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5557
}
5658

5759
func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) {
60+
s.requests++
5861
resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey})
5962

6063
if err != nil && resp == nil {
@@ -131,6 +134,7 @@ func (s *Source) Statistics() subscraping.Statistics {
131134
return subscraping.Statistics{
132135
Errors: s.errors,
133136
Results: s.results,
137+
Requests: s.requests,
134138
TimeTaken: s.timeTaken,
135139
Skipped: s.skipped,
136140
}

‎pkg/subscraping/sources/builtwith/builtwith.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ type Source struct {
3535
timeTaken time.Duration
3636
errors int
3737
results int
38+
requests int
3839
skipped bool
3940
}
4041

@@ -43,6 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4344
results := make(chan subscraping.Result)
4445
s.errors = 0
4546
s.results = 0
47+
s.requests = 0
4648

4749
go func() {
4850
defer func(startTime time.Time) {
@@ -55,6 +57,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5557
return
5658
}
5759

60+
s.requests++
5861
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain))
5962
if err != nil {
6063
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -112,6 +115,7 @@ func (s *Source) Statistics() subscraping.Statistics {
112115
return subscraping.Statistics{
113116
Errors: s.errors,
114117
Results: s.results,
118+
Requests: s.requests,
115119
TimeTaken: s.timeTaken,
116120
Skipped: s.skipped,
117121
}

‎pkg/subscraping/sources/c99/c99.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ type Source struct {
1818
timeTaken time.Duration
1919
errors int
2020
results int
21+
requests int
2122
skipped bool
2223
}
2324

@@ -36,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3637
results := make(chan subscraping.Result)
3738
s.errors = 0
3839
s.results = 0
40+
s.requests = 0
3941

4042
go func() {
4143
defer func(startTime time.Time) {
@@ -50,6 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
5052
}
5153

5254
searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain)
55+
s.requests++
5356
resp, err := session.SimpleGet(ctx, searchURL)
5457
if err != nil {
5558
session.DiscardHTTPResponse(resp)
@@ -119,6 +122,7 @@ func (s *Source) Statistics() subscraping.Statistics {
119122
return subscraping.Statistics{
120123
Errors: s.errors,
121124
Results: s.results,
125+
Requests: s.requests,
122126
TimeTaken: s.timeTaken,
123127
Skipped: s.skipped,
124128
}

‎pkg/subscraping/sources/censys/censys.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ type Source struct {
7575
timeTaken time.Duration
7676
errors int
7777
results int
78+
requests int
7879
skipped bool
7980
}
8081

@@ -83,6 +84,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
8384
results := make(chan subscraping.Result)
8485
s.errors = 0
8586
s.results = 0
87+
s.requests = 0
8688

8789
go func() {
8890
defer func(startTime time.Time) {
@@ -136,6 +138,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
136138
headers[orgIDHeader] = randomApiKey.orgID
137139
}
138140

141+
s.requests++
139142
resp, err := session.HTTPRequest(
140143
ctx,
141144
http.MethodPost,
@@ -220,6 +223,7 @@ func (s *Source) Statistics() subscraping.Statistics {
220223
return subscraping.Statistics{
221224
Errors: s.errors,
222225
Results: s.results,
226+
Requests: s.requests,
223227
TimeTaken: s.timeTaken,
224228
Skipped: s.skipped,
225229
}

‎pkg/subscraping/sources/certspotter/certspotter.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ type Source struct {
2222
timeTaken time.Duration
2323
errors int
2424
results int
25+
requests int
2526
skipped bool
2627
}
2728

@@ -30,6 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
3031
results := make(chan subscraping.Result)
3132
s.errors = 0
3233
s.results = 0
34+
s.requests = 0
3335

3436
go func() {
3537
defer func(startTime time.Time) {
@@ -46,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
4648
headers := map[string]string{"Authorization": "Bearer " + randomApiKey}
4749
cookies := ""
4850

51+
s.requests++
4952
resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers)
5053
if err != nil {
5154
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -88,6 +91,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
8891
}
8992
reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id)
9093

94+
s.requests++
9195
resp, err := session.Get(ctx, reqURL, cookies, headers)
9296
if err != nil {
9397
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
@@ -152,6 +156,7 @@ func (s *Source) Statistics() subscraping.Statistics {
152156
return subscraping.Statistics{
153157
Errors: s.errors,
154158
Results: s.results,
159+
Requests: s.requests,
155160
TimeTaken: s.timeTaken,
156161
Skipped: s.skipped,
157162
}

‎pkg/subscraping/sources/chaos/chaos.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type Source struct {
1616
timeTaken time.Duration
1717
errors int
1818
results int
19+
requests int
1920
skipped bool
2021
}
2122

@@ -24,6 +25,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session)
2425
results := make(chan subscraping.Result)
2526
s.errors = 0
2627
s.results = 0
28+
s.requests = 0
2729

2830
go func() {
2931
defer func(startTime time.Time) {
@@ -38,6 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, _ *subscraping.Session)
3840
}
3941

4042
chaosClient := chaos.New(randomApiKey)
43+
s.requests++
4144
for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{
4245
Domain: domain,
4346
}) {
@@ -86,6 +89,7 @@ func (s *Source) Statistics() subscraping.Statistics {
8689
return subscraping.Statistics{
8790
Errors: s.errors,
8891
Results: s.results,
92+
Requests: s.requests,
8993
TimeTaken: s.timeTaken,
9094
Skipped: s.skipped,
9195
}

0 commit comments

Comments
 (0)