Skip to content

Commit c3f9d13

Browse files
author
Christoph Wurm
committed
Fix make update
1 parent 8f49938 commit c3f9d13

8 files changed

Lines changed: 333 additions & 24 deletions

File tree

‎dev-tools/mage/check.go‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package mage
19+
20+
import (
21+
"bufio"
22+
"bytes"
23+
"fmt"
24+
"log"
25+
"os"
26+
"os/exec"
27+
"path/filepath"
28+
"runtime"
29+
"strings"
30+
31+
"github.com/magefile/mage/mg"
32+
"github.com/magefile/mage/sh"
33+
"github.com/pkg/errors"
34+
35+
"github.com/elastic/beats/libbeat/processors/dissect"
36+
)
37+
38+
// Check looks for created/modified/deleted/renamed files and returns an error
39+
// if it finds any modifications. If executed in in verbose mode it will write
40+
// the results of 'git diff' to stdout to indicate what changes have been made.
41+
//
42+
// It also checks the file permissions of nosetests test cases and YAML files.
43+
func Check() error {
44+
fmt.Println(">> check: Checking for modified files or incorrect permissions")
45+
46+
mg.Deps(CheckNosetestsNotExecutable, CheckYAMLNotExecutable)
47+
48+
changes, err := GitDiffIndex()
49+
if err != nil {
50+
return errors.Wrap(err, "failed to diff the git index")
51+
}
52+
53+
if len(changes) > 0 {
54+
if mg.Verbose() {
55+
GitDiff()
56+
}
57+
58+
return errors.Errorf("some files are not up-to-date. "+
59+
"Run 'mage fmt update' then review and commit the changes. "+
60+
"Modified: %v", changes)
61+
}
62+
return nil
63+
}
64+
65+
// GitDiffIndex returns a list of files that differ from what is committed.
66+
// These could file that were created, deleted, modified, or moved.
67+
func GitDiffIndex() ([]string, error) {
68+
// Ensure the index is updated so that diff-index gives accurate results.
69+
if err := sh.Run("git", "update-index", "-q", "--refresh"); err != nil {
70+
return nil, err
71+
}
72+
73+
// git diff-index provides a list of modified files.
74+
// https://www.git-scm.com/docs/git-diff-index
75+
out, err := sh.Output("git", "diff-index", "HEAD", "--", ".")
76+
if err != nil {
77+
return nil, err
78+
}
79+
80+
// Example formats.
81+
// :100644 100644 bcd1234... 0123456... M file0
82+
// :100644 100644 abcd123... 1234567... R86 file1 file3
83+
d, err := dissect.New(":%{src_mode} %{dst_mode} %{src_sha1} %{dst_sha1} %{status}\t%{paths}")
84+
if err != nil {
85+
return nil, err
86+
}
87+
88+
// Parse lines.
89+
var modified []string
90+
s := bufio.NewScanner(bytes.NewBufferString(out))
91+
for s.Scan() {
92+
m, err := d.Dissect(s.Text())
93+
if err != nil {
94+
return nil, errors.Wrap(err, "failed to dissect git diff-index output")
95+
}
96+
97+
paths := strings.Split(m["paths"], "\t")
98+
if len(paths) > 1 {
99+
modified = append(modified, paths[1])
100+
} else {
101+
modified = append(modified, paths[0])
102+
}
103+
}
104+
if err = s.Err(); err != nil {
105+
return nil, err
106+
}
107+
108+
return modified, nil
109+
}
110+
111+
// GitDiff runs 'git diff' and writes the output to stdout.
112+
func GitDiff() error {
113+
c := exec.Command("git", "--no-pager", "diff", "--minimal")
114+
c.Stdin = nil
115+
c.Stdout = os.Stdout
116+
c.Stderr = os.Stderr
117+
log.Println("exec:", strings.Join(c.Args, " "))
118+
err := c.Run()
119+
return err
120+
}
121+
122+
// CheckNosetestsNotExecutable checks that none of the nosetests files are
123+
// executable. Nosetests silently skips executable .py files and we don't want
124+
// this to happen.
125+
func CheckNosetestsNotExecutable() error {
126+
if runtime.GOOS == "windows" {
127+
// Skip windows because it doesn't have POSIX permissions.
128+
return nil
129+
}
130+
131+
tests, err := FindFiles(nosetestsTestFiles...)
132+
if err != nil {
133+
return err
134+
}
135+
136+
var executableTestFiles []string
137+
for _, file := range tests {
138+
info, err := os.Stat(file)
139+
if err != nil {
140+
return err
141+
}
142+
143+
if info.Mode().Perm()&0111 > 0 {
144+
executableTestFiles = append(executableTestFiles, file)
145+
}
146+
}
147+
148+
if len(executableTestFiles) > 0 {
149+
return errors.Errorf("nosetests files cannot be executable because "+
150+
"they will be skipped. Fix permissions of %v", executableTestFiles)
151+
}
152+
return nil
153+
}
154+
155+
// CheckYAMLNotExecutable checks that no .yml or .yaml files are executable.
156+
func CheckYAMLNotExecutable() error {
157+
if runtime.GOOS == "windows" {
158+
// Skip windows because it doesn't have POSIX permissions.
159+
return nil
160+
}
161+
162+
executableYAMLFiles, err := FindFilesRecursive(func(path string, info os.FileInfo) bool {
163+
switch filepath.Ext(path) {
164+
default:
165+
return false
166+
case ".yml", ".yaml":
167+
return info.Mode().Perm()&0111 > 0
168+
}
169+
})
170+
if err != nil {
171+
return errors.Wrap(err, "failed search for YAML files")
172+
}
173+
174+
if len(executableYAMLFiles) > 0 {
175+
return errors.Errorf("YAML files cannot be executable. Fix "+
176+
"permissions of %v", executableYAMLFiles)
177+
178+
}
179+
return nil
180+
}

‎dev-tools/mage/common.go‎

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,12 @@ func expandFile(src, dst string, args ...map[string]interface{}) error {
147147
}
148148

149149
// CWD return the current working directory.
150-
func CWD() string {
150+
func CWD(elem ...string) string {
151151
wd, err := os.Getwd()
152152
if err != nil {
153153
panic(errors.Wrap(err, "failed to get the CWD"))
154154
}
155-
return wd
155+
return filepath.Join(append([]string{wd}, elem...)...)
156156
}
157157

158158
// EnvOr returns the value of the specified environment variable if it is
@@ -215,6 +215,13 @@ func dockerInfo() (*DockerInfo, error) {
215215
return &info, nil
216216
}
217217

218+
// HaveDockerCompose returns an error if docker-compose is not found on the
219+
// PATH.
220+
func HaveDockerCompose() error {
221+
_, err := exec.LookPath("docker-compose")
222+
return errors.Wrap(err, "docker-compose was not found on the PATH")
223+
}
224+
218225
// FindReplace reads a file, performs a find/replace operation, then writes the
219226
// output to the same file path.
220227
func FindReplace(file string, re *regexp.Regexp, repl string) error {
@@ -513,6 +520,29 @@ func FindFiles(globs ...string) ([]string, error) {
513520
return configFiles, nil
514521
}
515522

523+
// FindFilesRecursive recursively traverses from the CWD and invokes the given
524+
// match function on each regular file to determine if the given path should be
525+
// returned as a match.
526+
func FindFilesRecursive(match func(path string, info os.FileInfo) bool) ([]string, error) {
527+
var matches []string
528+
err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
529+
if err != nil {
530+
return err
531+
}
532+
533+
if !info.Mode().IsRegular() {
534+
// continue
535+
return nil
536+
}
537+
538+
if match(filepath.ToSlash(path), info) {
539+
matches = append(matches, path)
540+
}
541+
return nil
542+
})
543+
return matches, err
544+
}
545+
516546
// FileConcat concatenates files and writes the output to out.
517547
func FileConcat(out string, perm os.FileMode, files ...string) error {
518548
f, err := os.OpenFile(createDir(out), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
@@ -646,23 +676,6 @@ func IsUpToDate(dst string, sources ...string) bool {
646676
return err == nil && !execute
647677
}
648678

649-
// OSSBeatDir returns the OSS beat directory. You can pass paths and they will
650-
// be joined and appended to the OSS beat dir.
651-
func OSSBeatDir(path ...string) string {
652-
ossDir := CWD()
653-
654-
// Check if we need to correct ossDir because it's in x-pack.
655-
if parentDir := filepath.Base(filepath.Dir(ossDir)); parentDir == "x-pack" {
656-
// If the OSS version of the beat exists.
657-
tmp := filepath.Join(ossDir, "../..", BeatName)
658-
if _, err := os.Stat(tmp); !os.IsNotExist(err) {
659-
ossDir = tmp
660-
}
661-
}
662-
663-
return filepath.Join(append([]string{ossDir}, path...)...)
664-
}
665-
666679
// LibbeatDir returns the libbeat directory. You can pass paths and
667680
// they will be joined and appended to the libbeat dir.
668681
func LibbeatDir(path ...string) string {

‎dev-tools/mage/dashboard.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Licensed to Elasticsearch B.V. under one or more contributor
2+
// license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright
4+
// ownership. Elasticsearch B.V. licenses this file to you under
5+
// the Apache License, Version 2.0 (the "License"); you may
6+
// not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package mage
19+
20+
import (
21+
"fmt"
22+
"path/filepath"
23+
24+
"github.com/magefile/mage/sh"
25+
)
26+
27+
// ExportDashboard exports a dashboard from Kibana and writes it into the given module.
28+
func ExportDashboard() error {
29+
module := EnvOr("MODULE", "")
30+
if module == "" {
31+
return fmt.Errorf("MODULE must be specified")
32+
}
33+
34+
id := EnvOr("ID", "")
35+
if id == "" {
36+
return fmt.Errorf("Dashboad ID must be specified")
37+
}
38+
39+
beatsDir, err := ElasticBeatsDir()
40+
if err != nil {
41+
return err
42+
}
43+
44+
// TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7
45+
file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json")
46+
47+
dashboardCmd := sh.RunCmd("go", "run",
48+
filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),
49+
"-output", file, "-dashboard", id,
50+
)
51+
52+
return dashboardCmd()
53+
}

‎dev-tools/mage/gotest.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,3 +325,16 @@ func (s *GoTestSummary) String() string {
325325

326326
return strings.TrimRight(b.String(), "\n")
327327
}
328+
329+
// BuildSystemTestBinary build a binary for testing that is instrumented for
330+
// testing and measuring code coverage. The binary is only instrumented for
331+
// coverage when TEST_COVERAGE=true (default is false).
332+
func BuildSystemTestBinary() error {
333+
args := []string{
334+
"test", "-c",
335+
}
336+
if TestCoverage {
337+
args = append(args, "-coverpkg", "./...")
338+
}
339+
return sh.RunV("go", args...)
340+
}

‎dev-tools/mage/pkgspecs.go‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package mage
1919

2020
import (
21+
"bytes"
2122
"io/ioutil"
2223
"log"
2324
"path/filepath"
@@ -88,10 +89,24 @@ func MustUsePackaging(specName, specFile string) {
8889
}
8990
}
9091

92+
// LoadLocalNamedSpec loads the named package spec from the packages.yml in the
93+
// current directory.
94+
func LoadLocalNamedSpec(name string) {
95+
beatsDir, err := ElasticBeatsDir()
96+
if err != nil {
97+
panic(err)
98+
}
99+
100+
err = LoadNamedSpec(name, filepath.Join(beatsDir, packageSpecFile), "packages.yml")
101+
if err != nil {
102+
panic(err)
103+
}
104+
}
105+
91106
// LoadNamedSpec loads a packaging specification with the given name from the
92107
// specified YAML file. name should be a sub-key of 'specs'.
93-
func LoadNamedSpec(name, file string) error {
94-
specs, err := LoadSpecs(file)
108+
func LoadNamedSpec(name string, files ...string) error {
109+
specs, err := LoadSpecs(files...)
95110
if err != nil {
96111
return errors.Wrap(err, "failed to load spec file")
97112
}
@@ -122,7 +137,7 @@ func LoadSpecs(files ...string) (map[string][]OSPackageArgs, error) {
122137
}
123138

124139
var packages PackageYAML
125-
if err = yaml.Unmarshal(data, &packages); err != nil {
140+
if err := yaml.Unmarshal(bytes.Join(data, []byte{'\n'}), &packages); err != nil {
126141
return nil, errors.Wrap(err, "failed to unmarshal spec data")
127142
}
128143

0 commit comments

Comments
 (0)