Skip to content

Commit f9d3a0e

Browse files
authored
zoekt: add fgprof for full profiling (#614)
Useful in local testing to capture both on and off cpu time spent. Should consider shipping this in the webserver as well. Test Plan: ran zoekt with -full_profile flag and inspected output in pprof.
1 parent 008a775 commit f9d3a0e

5 files changed

Lines changed: 89 additions & 25 deletions

File tree

cmd/zoekt/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ go_library(
99
"//:zoekt",
1010
"//query",
1111
"//shards",
12+
"@com_github_felixge_fgprof//:fgprof",
1213
],
1314
)
1415

cmd/zoekt/main.go

Lines changed: 57 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,14 @@ import (
1818
"context"
1919
"flag"
2020
"fmt"
21+
"io"
2122
"log"
2223
"os"
2324
"path/filepath"
2425
"runtime/pprof"
2526
"time"
2627

28+
"github.com/felixge/fgprof"
2729
"github.com/sourcegraph/zoekt"
2830
"github.com/sourcegraph/zoekt/query"
2931
"github.com/sourcegraph/zoekt/shards"
@@ -76,11 +78,57 @@ func loadShard(fn string, verbose bool) (zoekt.Searcher, error) {
7678
return s, nil
7779
}
7880

81+
func profile(path string, duration time.Duration, start func(io.Writer) (stop func())) func() bool {
82+
if path == "" {
83+
return func() bool { return false }
84+
}
85+
86+
f, err := os.Create(path)
87+
if err != nil {
88+
log.Fatal(err)
89+
}
90+
91+
t := time.Now()
92+
stop := start(f)
93+
94+
return func() bool {
95+
if time.Since(t) < duration {
96+
return true
97+
}
98+
stop()
99+
f.Close()
100+
return false
101+
}
102+
}
103+
104+
func startCPUProfile(path string, duration time.Duration) func() bool {
105+
return profile(path, duration, func(w io.Writer) func() {
106+
if err := pprof.StartCPUProfile(w); err != nil {
107+
log.Fatal(err)
108+
}
109+
110+
return pprof.StopCPUProfile
111+
})
112+
}
113+
114+
func startFullProfile(path string, duration time.Duration) func() bool {
115+
return profile(path, duration, func(w io.Writer) func() {
116+
stop := fgprof.Start(w, fgprof.FormatPprof)
117+
118+
return func() {
119+
if err := stop(); err != nil {
120+
log.Fatal(err)
121+
}
122+
}
123+
})
124+
}
125+
79126
func main() {
80127
shard := flag.String("shard", "", "search in a specific shard")
81128
index := flag.String("index_dir",
82129
filepath.Join(os.Getenv("HOME"), ".zoekt"), "search for index files in `directory`")
83130
cpuProfile := flag.String("cpu_profile", "", "write cpu profile to `file`")
131+
fullProfile := flag.String("full_profile", "", "write full profile to `file`")
84132
profileTime := flag.Duration("profile_time", time.Second, "run this long to gather stats.")
85133
verbose := flag.Bool("v", false, "print some background data")
86134
withRepo := flag.Bool("r", false, "print the repo before the file name")
@@ -124,35 +172,19 @@ func main() {
124172

125173
var sOpts zoekt.SearchOptions
126174
sres, err := searcher.Search(context.Background(), query, &sOpts)
127-
if *cpuProfile != "" {
128-
// If profiling, do it another time so we measure with
129-
// warm caches.
130-
f, err := os.Create(*cpuProfile)
131-
if err != nil {
132-
log.Fatal(err)
133-
}
134-
defer f.Close()
135-
if *verbose {
136-
log.Println("Displaying matches...")
137-
}
138-
139-
t := time.Now()
140-
if err := pprof.StartCPUProfile(f); err != nil {
141-
log.Fatal(err)
142-
}
143-
for {
144-
sres, _ = searcher.Search(context.Background(), query, &sOpts)
145-
if time.Since(t) > *profileTime {
146-
break
147-
}
148-
}
149-
pprof.StopCPUProfile()
150-
}
151-
152175
if err != nil {
153176
log.Fatal(err)
154177
}
155178

179+
// If profiling, do it another time so we measure with
180+
// warm caches.
181+
for run := startCPUProfile(*cpuProfile, *profileTime); run(); {
182+
sres, _ = searcher.Search(context.Background(), query, &sOpts)
183+
}
184+
for run := startFullProfile(*fullProfile, *profileTime); run(); {
185+
sres, _ = searcher.Search(context.Background(), query, &sOpts)
186+
}
187+
156188
displayMatches(sres.Files, pat, *withRepo, *list)
157189
if *verbose {
158190
log.Printf("stats: %#v", sres.Stats)

deps.bzl

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,13 @@ def go_dependencies():
170170
sum = "h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=",
171171
version = "v2.2.0",
172172
)
173+
go_repository(
174+
name = "com_github_chzyer_logex",
175+
build_file_proto_mode = "disable_global",
176+
importpath = "github.com/chzyer/logex",
177+
sum = "h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=",
178+
version = "v1.1.10",
179+
)
173180

174181
go_repository(
175182
name = "com_github_chzyer_readline",
@@ -178,6 +185,13 @@ def go_dependencies():
178185
sum = "h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=",
179186
version = "v1.5.1",
180187
)
188+
go_repository(
189+
name = "com_github_chzyer_test",
190+
build_file_proto_mode = "disable_global",
191+
importpath = "github.com/chzyer/test",
192+
sum = "h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=",
193+
version = "v0.0.0-20180213035817-a1ea475d72b1",
194+
)
181195

182196
go_repository(
183197
name = "com_github_client9_misspell",
@@ -346,6 +360,14 @@ def go_dependencies():
346360
sum = "h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=",
347361
version = "v1.1.0",
348362
)
363+
go_repository(
364+
name = "com_github_felixge_fgprof",
365+
build_file_proto_mode = "disable_global",
366+
importpath = "github.com/felixge/fgprof",
367+
sum = "h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=",
368+
version = "v0.9.3",
369+
)
370+
349371
go_repository(
350372
name = "com_github_flosch_pongo2_v4",
351373
build_file_proto_mode = "disable_global",

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/andygrunwald/go-gerrit v0.0.0-20230628115649-c44fe2fbf2ca
77
github.com/bmatcuk/doublestar v1.3.4
88
github.com/edsrzf/mmap-go v1.1.0
9+
github.com/felixge/fgprof v0.9.3
910
github.com/fsnotify/fsnotify v1.6.0
1011
github.com/gfleury/go-bitbucket-v1 v0.0.0-20230626192437-8d7be5866751
1112
github.com/go-enry/go-enry/v2 v2.8.4

go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
4646
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
4747
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
4848
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
49+
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
50+
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
51+
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
4952
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
5053
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
5154
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
@@ -81,6 +84,8 @@ github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byA
8184
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
8285
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
8386
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
87+
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
88+
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
8489
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
8590
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
8691
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
@@ -155,6 +160,7 @@ github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/
155160
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
156161
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
157162
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
163+
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
158164
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs=
159165
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA=
160166
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
@@ -180,6 +186,7 @@ github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3
180186
github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
181187
github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
182188
github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
189+
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
183190
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
184191
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
185192
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -427,6 +434,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w
427434
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
428435
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
429436
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
437+
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
430438
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
431439
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
432440
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

0 commit comments

Comments
 (0)