Skip to content

Commit 7730954

Browse files
andrewvcmergify-bot
andrewvc
authored and
mergify-bot
committed
[Heartbeat] Add httpcommon options to ZipURL (#27699)
Support httpcommon options for zip_url sources for browser based monitors. This adds custom SSL, proxy, and timeout options. Fixes #27597 (cherry picked from commit acb4fd5)
1 parent e391d91 commit 7730954

4 files changed

Lines changed: 165 additions & 60 deletions

File tree

‎heartbeat/docs/monitors/monitor-browser.asciidoc‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ Under `zip_url`, specify these options:
6565
located in the repository.
6666
*`username`*:: The username for authenticating with the zip endpoint. This setting is optional.
6767
*`password`*:: The password for authenticating with the zip endpoint. This setting is optional.
68+
*`ssl`*:: SSL options applied to downloading the zip, not the browser. See <<configuration-ssl>> for more details.
6869

6970
If `username` and `password` are provided, they will be sent as HTTP Basic Authentication
7071
headers to the remote zip endpoint.
@@ -83,9 +84,11 @@ Example configuration:
8384
folder: "examples/todos"
8485
username: ""
8586
password: ""
87+
# ssl options apply to downloading the zip, not the browser
88+
#ssl:
89+
# certificate_authorities: ['/etc/ca.crt']
8690
-------------------------------------------------------------------------------
8791

88-
8992
[float]
9093
[[monitor-source-local]]
9194
===== `Local directory`
@@ -198,7 +201,6 @@ Example configuration:
198201
*`tags`*:: run only journeys with the given tag(s), or globs
199202
*`match`*:: run only journeys with a name or tags that matches the configured glob
200203

201-
202204
[float]
203205
[[monitor-browser-synthetics-args]]
204206
==== `synthetics_args`

‎monitors.d/plaintodos.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
- name: Todos
2+
id: todos
3+
type: browser
4+
enabled: true
5+
schedule: "@every 3m"
6+
tags: todos-app
7+
params:
8+
url: "https://elastic.github.io/synthetics-demo/"
9+
source:
10+
zip_url:
11+
url: "https://github.com/elastic/synthetics-demo/archive/refs/heads/main.zip"
12+
folder: "todos/synthetics-tests"

‎x-pack/heartbeat/monitors/browser/source/zipurl.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ import (
1414
"path/filepath"
1515
"strings"
1616
"time"
17+
18+
"github.com/elastic/beats/v7/libbeat/common/transport/httpcommon"
1719
)
1820

1921
type ZipURLSource struct {
@@ -23,13 +25,25 @@ type ZipURLSource struct {
2325
Password string `config:"password" json:"password"`
2426
Retries int `config:"retries" default:"3" json:"retries"`
2527
BaseSource
26-
// Etag from last successful fetch
27-
etag string
2828
TargetDirectory string `config:"target_directory" json:"target_directory"`
29+
30+
// Etag from last successful fetch
31+
etag string
32+
33+
Transport httpcommon.HTTPTransportSettings `config:",inline" yaml:",inline"`
34+
35+
httpClient *http.Client
2936
}
3037

3138
var ErrNoEtag = fmt.Errorf("No ETag header in zip file response. Heartbeat requires an etag to efficiently cache downloaded code")
3239

40+
func (z *ZipURLSource) Validate() (err error) {
41+
if z.httpClient == nil {
42+
z.httpClient, _ = z.Transport.Client()
43+
}
44+
return err
45+
}
46+
3347
func (z *ZipURLSource) Fetch() error {
3448
changed, err := checkIfChanged(z)
3549
if err != nil {
@@ -181,14 +195,15 @@ func retryingZipRequest(method string, z *ZipURLSource) (resp *http.Response, er
181195
}
182196

183197
func zipRequest(method string, z *ZipURLSource) (*http.Response, error) {
198+
184199
req, err := http.NewRequest(method, z.URL, nil)
185200
if err != nil {
186201
return nil, fmt.Errorf("could not issue request to: %s %w", z.URL, err)
187202
}
188203
if z.Username != "" && z.Password != "" {
189204
req.SetBasicAuth(z.Username, z.Password)
190205
}
191-
return http.DefaultClient.Do(req)
206+
return z.httpClient.Do(req)
192207
}
193208

194209
func download(z *ZipURLSource, tf *os.File) (etag string, err error) {

‎x-pack/heartbeat/monitors/browser/source/zipurl_test.go‎

Lines changed: 131 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -5,69 +5,131 @@
55
package source
66

77
import (
8-
"context"
98
"fmt"
109
"net/http"
10+
"net/http/httptest"
1111
"os"
1212
"path"
1313
"path/filepath"
1414
"runtime"
1515
"testing"
1616

1717
"github.com/stretchr/testify/require"
18+
"gopkg.in/yaml.v2"
1819

20+
"github.com/elastic/beats/v7/libbeat/common"
1921
"github.com/elastic/beats/v7/x-pack/heartbeat/monitors/browser/source/fixtures"
2022
)
2123

22-
func TestZipUrlFetchNoAuth(t *testing.T) {
23-
address, teardown := setupTests()
24-
defer teardown()
25-
26-
zus := &ZipURLSource{
27-
URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address),
28-
Folder: "/",
29-
Retries: 3,
24+
func TestSimpleCases(t *testing.T) {
25+
type testCase struct {
26+
name string
27+
cfg common.MapStr
28+
tlsServer bool
29+
wantFetchErr bool
30+
}
31+
testCases := []testCase{
32+
{
33+
"basics",
34+
common.MapStr{
35+
"folder": "/",
36+
"retries": 3,
37+
},
38+
false,
39+
false,
40+
},
41+
{
42+
"targetdir",
43+
common.MapStr{
44+
"folder": "/",
45+
"retries": 3,
46+
"target_directory": "/tmp/synthetics/blah",
47+
},
48+
false,
49+
false,
50+
},
51+
{
52+
"auth success",
53+
common.MapStr{
54+
"folder": "/",
55+
"retries": 3,
56+
"username": "testuser",
57+
"password": "testpass",
58+
},
59+
false,
60+
false,
61+
},
62+
{
63+
"auth failure",
64+
common.MapStr{
65+
"folder": "/",
66+
"retries": 3,
67+
"username": "testuser",
68+
"password": "badpass",
69+
},
70+
false,
71+
true,
72+
},
73+
{
74+
"ssl ignore cert errors",
75+
common.MapStr{
76+
"folder": "/",
77+
"retries": 3,
78+
"ssl": common.MapStr{
79+
"enabled": "true",
80+
"verification_mode": "none",
81+
},
82+
},
83+
true,
84+
false,
85+
},
86+
{
87+
"bad ssl",
88+
common.MapStr{
89+
"folder": "/",
90+
"retries": 3,
91+
"ssl": common.MapStr{
92+
"enabled": "true",
93+
"certificate_authorities": []string{},
94+
},
95+
},
96+
true,
97+
true,
98+
},
3099
}
31-
fetchAndCheckDir(t, zus)
32-
}
33100

34-
func TestZipUrlFetchWithAuth(t *testing.T) {
35-
address, teardown := setupTests()
36-
defer teardown()
101+
for _, tc := range testCases {
102+
url, teardown := setupTests(tc.tlsServer)
103+
defer teardown()
104+
t.Run(tc.name, func(t *testing.T) {
105+
tc.cfg["url"] = fmt.Sprintf("%s/fixtures/todos.zip", url)
106+
zus, err := dummyZus(tc.cfg)
107+
require.NoError(t, err)
37108

38-
zus := &ZipURLSource{
39-
URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address),
40-
Folder: "/",
41-
Retries: 3,
42-
Username: "testuser",
43-
Password: "testpass",
44-
}
45-
fetchAndCheckDir(t, zus)
46-
}
109+
require.NotNil(t, zus.httpClient)
47110

48-
func TestZipUrlTargetDirectory(t *testing.T) {
49-
address, teardown := setupTests()
50-
defer teardown()
111+
if tc.wantFetchErr == true {
112+
err := zus.Fetch()
113+
require.Error(t, err)
114+
return
115+
}
51116

52-
zus := &ZipURLSource{
53-
URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address),
54-
Folder: "/",
55-
Retries: 3,
56-
TargetDirectory: "/tmp/synthetics/blah",
117+
fetchAndCheckDir(t, zus)
118+
})
57119
}
58-
fetchAndCheckDir(t, zus)
59120
}
60121

61122
func TestZipUrlWithSameEtag(t *testing.T) {
62-
address, teardown := setupTests()
123+
address, teardown := setupTests(false)
63124
defer teardown()
64125

65-
zus := ZipURLSource{
66-
URL: fmt.Sprintf("http://%s/fixtures/todos.zip", address),
67-
Folder: "/",
68-
Retries: 3,
69-
}
70-
err := zus.Fetch()
126+
zus, err := dummyZus(common.MapStr{
127+
"url": fmt.Sprintf("%s/fixtures/todos.zip", address),
128+
"folder": "/",
129+
"retries": 3,
130+
})
131+
require.NoError(t, err)
132+
err = zus.Fetch()
71133
defer zus.Close()
72134
require.NoError(t, err)
73135

@@ -80,32 +142,33 @@ func TestZipUrlWithSameEtag(t *testing.T) {
80142
}
81143

82144
func TestZipUrlWithBadUrl(t *testing.T) {
83-
_, teardown := setupTests()
145+
_, teardown := setupTests(false)
84146
defer teardown()
85147

86-
zus := ZipURLSource{
87-
URL: "http://notahost.notadomaintoehutoeuhn",
88-
Folder: "/",
89-
Retries: 2,
90-
}
91-
err := zus.Fetch()
148+
zus, err := dummyZus(common.MapStr{
149+
"url": "http://notahost.notadomaintoehutoeuhn",
150+
"folder": "/",
151+
"retries": 2,
152+
})
153+
require.NoError(t, err)
154+
err = zus.Fetch()
92155
defer zus.Close()
93156
require.Error(t, err)
94157
}
95158

96-
func setupTests() (addr string, teardown func()) {
159+
func setupTests(tls bool) (addr string, teardown func()) {
97160
// go offline, so we dont invoke npm install for unit tests
98161
GoOffline()
99162

100-
srv := createServer()
101-
address := srv.Addr
163+
srv := createServer(tls)
164+
address := srv.URL
102165
return address, func() {
103166
GoOnline()
104-
srv.Shutdown(context.Background())
167+
srv.Close()
105168
}
106169
}
107170

108-
func createServer() (addr *http.Server) {
171+
func createServer(tls bool) (addr *httptest.Server) {
109172
_, filename, _, _ := runtime.Caller(0)
110173
fixturesPath := path.Join(filepath.Dir(filename), "fixtures")
111174
fileServer := http.FileServer(http.Dir(fixturesPath))
@@ -121,10 +184,12 @@ func createServer() (addr *http.Server) {
121184
http.StripPrefix("/fixtures", fileServer).ServeHTTP(resp, req)
122185
})
123186

124-
srv := &http.Server{Addr: "localhost:1234", Handler: mux}
125-
go func() {
126-
srv.ListenAndServe()
127-
}()
187+
var srv *httptest.Server
188+
if tls {
189+
srv = httptest.NewTLSServer(mux)
190+
} else {
191+
srv = httptest.NewServer(mux)
192+
}
128193

129194
return srv
130195
}
@@ -140,3 +205,14 @@ func fetchAndCheckDir(t *testing.T, zip *ZipURLSource) {
140205
_, err = os.Stat(zip.TargetDirectory)
141206
require.True(t, os.IsNotExist(err), "TargetDirectory %s should have been deleted", zip.TargetDirectory)
142207
}
208+
209+
func dummyZus(conf map[string]interface{}) (*ZipURLSource, error) {
210+
zus := &ZipURLSource{}
211+
y, _ := yaml.Marshal(conf)
212+
c, err := common.NewConfigWithYAML(y, string(y))
213+
if err != nil {
214+
return nil, err
215+
}
216+
err = c.Unpack(zus)
217+
return zus, err
218+
}

0 commit comments

Comments
 (0)