Skip to content

Commit 667b26e

Browse files
ruflinruflin
authored
Change package path from {packagename}-{version} to {packagename}/{version} (#300)
 The current package path is `/package/{packagename}-{version}`. EPM will need some info about which packages exist. To make it possible to populate some generic info about a package, the path is changed to `/package/{packagename}/{version}`. No additional info is added yet but it gives you the option to add it. This PR makes all changes in the code that are needed on the packaging side to make the above happen. Also all test files are updated. This is a breaking change for EPM.
1 parent 2a8ad9a commit 667b26e

77 files changed

Lines changed: 195 additions & 163 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ci/jenkins-go-agent/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
FROM golang:1.12
1+
FROM golang:1.13
22

33
RUN \
44
apt-get update \
55
&& apt-get install -y --no-install-recommends \
6-
zip \
6+
zip rsync\
77
&& rm -rf /var/lib/apt/lists/*
88

99
RUN go get -u github.com/magefile/mage \

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
99

1010
### Breaking changes
1111

12+
* Change package path from /package/{packagename}-{version} to /package/{packagename}/{version} [#](https://github.com/elastic/integrations-registry/pull/)
1213

1314
### Bugfixes
1415

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ FROM golang:${GO_VERSION:-1.13.4}
44
RUN \
55
apt-get update \
66
&& apt-get install -y --no-install-recommends \
7-
zip \
7+
zip rsync \
88
&& rm -rf /var/lib/apt/lists/*
99

1010
COPY ./ /go/src/github.com/elastic/package-registry

‎Jenkinsfile‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
@Library('apm@current') _
44

55
pipeline {
6-
agent { label 'docker && linux && immutable' }
6+
agent { label 'ubuntu && immutable' }
77
environment {
88
BASE_DIR="src/github.com/elastic/package-registry"
99
JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
@@ -27,8 +27,8 @@ pipeline {
2727
}
2828
stages {
2929
/**
30-
Checkout the code and stash it, to use it on other stages.
31-
*/
30+
Checkout the code and stash it, to use it on other stages.
31+
*/
3232
stage('Checkout') {
3333
steps {
3434
deleteDir()
@@ -37,8 +37,8 @@ pipeline {
3737
}
3838
}
3939
/**
40-
Checks formatting / linting.
41-
*/
40+
Checks formatting / linting.
41+
*/
4242
stage('Lint') {
4343
steps {
4444
deleteDir()
@@ -51,8 +51,8 @@ pipeline {
5151
}
5252
}
5353
/**
54-
Build the project from code..
55-
*/
54+
Build the project from code..
55+
*/
5656
stage('Build') {
5757
steps {
5858
deleteDir()
@@ -65,8 +65,8 @@ pipeline {
6565
}
6666
}
6767
/**
68-
Execute unit tests.
69-
*/
68+
Execute unit tests.
69+
*/
7070
stage('Test') {
7171
steps {
7272
deleteDir()
@@ -86,8 +86,8 @@ pipeline {
8686
}
8787
}
8888
/**
89-
Publish Docker images.
90-
*/
89+
Publish Docker images.
90+
*/
9191
stage('Publish Docker image'){
9292
environment {
9393
DOCKER_IMG_TAG = "${env.DOCKER_IMG}:${env.GIT_BASE_COMMIT}"
@@ -105,8 +105,8 @@ pipeline {
105105
script: "docker push ${env.DOCKER_IMG_TAG}")
106106
sh(label: 'Re-tag Docker image',
107107
script: "docker tag ${env.DOCKER_IMG_TAG} ${env.DOCKER_IMG_TAG_BRANCH}")
108-
sh(label: 'Push Docker image name',
109-
script: "docker push ${env.DOCKER_IMG_TAG_BRANCH}")
108+
sh(label: 'Push Docker image name',
109+
script: "docker push ${env.DOCKER_IMG_TAG_BRANCH}")
110110
}
111111
}
112112
}

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ Endpoints:
99
* `/`: Info about the registry
1010
* `/search`: Search for packages. By default returns all the most recent packages available.
1111
* `/categories`: List of the existing package categories and how many packages are in each category.
12-
* `/package/{name}-{version}`: Info about a package
13-
* `/package/{name}-{version}.tar.gz`: Download a package
12+
* `/package/{name}/{version}`: Info about a package
13+
* `/epr/{name}/{name}-{version}.tar.gz`: Download a package
1414

1515
Examples for each API endpoint can be found here: https://github.com/elastic/package-registry/tree/master/docs/api
1616

‎dev/generator/main.go‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"log"
1313
"os"
1414
"path/filepath"
15+
"strings"
1516

1617
"github.com/magefile/mage/sh"
1718

@@ -75,9 +76,9 @@ func Build(sourceDir, publicDir string) error {
7576

7677
// CopyPackage copies the files of a package to the public directory
7778
func CopyPackage(src, dst string) error {
78-
fmt.Println(">> Copy package: " + src)
79+
log.Println(">> Copy package: " + src)
7980
os.MkdirAll(dst, 0755)
80-
err := sh.RunV("cp", "-a", src, dst)
81+
err := sh.RunV("rsync", "-a", src, dst)
8182
if err != nil {
8283
return err
8384
}
@@ -100,14 +101,19 @@ func BuildPackages(sourceDir, packagesPath string) error {
100101
continue
101102
}
102103

104+
// Finds the last occurence of "-" and then below splits it up in 2 parts.
105+
dashIndex := strings.LastIndex(packageName, "-")
106+
dstDir := filepath.Join(packagesPath, packageName[0:dashIndex], packageName[dashIndex+1:])
107+
103108
if copy {
104-
err := CopyPackage(filepath.Join(sourceDir, packageName), packagesPath)
109+
// Trailing slash is to make sure content of package is copied
110+
err := CopyPackage(filepath.Join(sourceDir, packageName)+"/", dstDir)
105111
if err != nil {
106112
return err
107113
}
108114
}
109115

110-
p, err := util.NewPackage(filepath.Join(packagesPath, packageName))
116+
p, err := util.NewPackage(dstDir)
111117
if err != nil {
112118
return err
113119
}
@@ -200,15 +206,35 @@ func buildPackage(packagesBasePath string, p util.Package) error {
200206
}
201207

202208
if tarGz {
203-
err = os.MkdirAll(filepath.Join(packagesBasePath, "..", "epr", p.Name), 0755)
209+
tarGzDirPath := filepath.Join(packagesBasePath, "..", "epr", p.Name)
210+
err = os.MkdirAll(tarGzDirPath, 0755)
211+
if err != nil {
212+
return err
213+
}
214+
215+
tarGzName := p.Name + "-" + p.Version
216+
copiedPackagePath := filepath.Join(tarGzDirPath, tarGzName)
217+
218+
// As the package directories are now {packagename}/{version} when just running tar, the dir inside
219+
// the package had the wrong name. Using `-s` or `--transform` for some reason worked on the command line
220+
// but not when run through Golang. So the hack for now is to just copy over all files with the correct name
221+
// and then run tar on it.
222+
// This could become even useful in the future as things like images or videos should potentially not be part of
223+
// a tar.gz to keep it small.
224+
err := CopyPackage(packagesBasePath+"/"+p.Name+"/"+p.Version+"/", copiedPackagePath)
204225
if err != nil {
205226
return err
206227
}
207228

208-
err = sh.RunV("tar", "czf", filepath.Join(packagesBasePath, "..", "epr", p.Name, p.GetPath()+".tar.gz"), "-C", packagesBasePath+"/", filepath.Base(p.GetPath())+"/")
229+
err = sh.RunV("tar", "czf", filepath.Join(packagesBasePath, "..", "epr", p.Name, tarGzName+".tar.gz"), "-C", tarGzDirPath, tarGzName+"/")
209230
if err != nil {
210231
return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
211232
}
233+
234+
err = os.RemoveAll(copiedPackagePath)
235+
if err != nil {
236+
return err
237+
}
212238
}
213239

214240
return nil

‎docs/api/package.json‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -33,25 +33,25 @@
3333
}
3434
],
3535
"assets": [
36-
"/package/example-1.0.0/manifest.yml",
37-
"/package/example-1.0.0/docs/README.md",
38-
"/package/example-1.0.0/img/icon.png",
39-
"/package/example-1.0.0/img/kibana-envoyproxy.jpg",
40-
"/package/example-1.0.0/dataset/foo/manifest.yml",
41-
"/package/example-1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
42-
"/package/example-1.0.0/kibana/index-pattern/filebeat-*.json",
43-
"/package/example-1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
44-
"/package/example-1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
45-
"/package/example-1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
46-
"/package/example-1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
47-
"/package/example-1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
48-
"/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
49-
"/package/example-1.0.0/dataset/foo/fields/stream.yml",
50-
"/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
51-
"/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
52-
"/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
53-
"/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
54-
"/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
36+
"/package/example/1.0.0/manifest.yml",
37+
"/package/example/1.0.0/docs/README.md",
38+
"/package/example/1.0.0/img/icon.png",
39+
"/package/example/1.0.0/img/kibana-envoyproxy.jpg",
40+
"/package/example/1.0.0/dataset/foo/manifest.yml",
41+
"/package/example/1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
42+
"/package/example/1.0.0/kibana/index-pattern/filebeat-*.json",
43+
"/package/example/1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
44+
"/package/example/1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
45+
"/package/example/1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
46+
"/package/example/1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
47+
"/package/example/1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
48+
"/package/example/1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
49+
"/package/example/1.0.0/dataset/foo/fields/stream.yml",
50+
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
51+
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
52+
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
53+
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
54+
"/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
5555
],
5656
"datasets": [
5757
{
@@ -106,5 +106,5 @@
106106
}
107107
],
108108
"download": "/epr/example/example-1.0.0.tar.gz",
109-
"path": "/package/example-1.0.0"
110-
}
109+
"path": "/package/example/1.0.0"
110+
}

‎docs/api/search-category-logs.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"description": "Package with data sources",
44
"download": "/epr/datasources/datasources-1.0.0.tar.gz",
55
"name": "datasources",
6-
"path": "/package/datasources-1.0.0",
6+
"path": "/package/datasources/1.0.0",
77
"title": "Default datasource Integration",
88
"type": "integration",
99
"version": "1.0.0"
@@ -12,7 +12,7 @@
1212
"description": "Tests if no pipeline is set, it defaults to the default one",
1313
"download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
1414
"name": "default-pipeline",
15-
"path": "/package/default-pipeline-0.0.2",
15+
"path": "/package/default-pipeline/0.0.2",
1616
"title": "Default pipeline Integration",
1717
"type": "integration",
1818
"version": "0.0.2"
@@ -21,7 +21,7 @@
2121
"description": "This is the example integration",
2222
"download": "/epr/example/example-1.0.0.tar.gz",
2323
"name": "example",
24-
"path": "/package/example-1.0.0",
24+
"path": "/package/example/1.0.0",
2525
"title": "Example Integration",
2626
"type": "integration",
2727
"version": "1.0.0"

‎docs/api/search-category-metrics.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"description": "This is the example integration",
44
"download": "/epr/example/example-1.0.0.tar.gz",
55
"name": "example",
6-
"path": "/package/example-1.0.0",
6+
"path": "/package/example/1.0.0",
77
"title": "Example Integration",
88
"type": "integration",
99
"version": "1.0.0"
@@ -12,7 +12,7 @@
1212
"description": "This is the foo integration",
1313
"download": "/epr/foo/foo-1.0.0.tar.gz",
1414
"name": "foo",
15-
"path": "/package/foo-1.0.0",
15+
"path": "/package/foo/1.0.0",
1616
"title": "Foo",
1717
"type": "solution",
1818
"version": "1.0.0"

‎docs/api/search-kibana652.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
"description": "Package with data sources",
44
"download": "/epr/datasources/datasources-1.0.0.tar.gz",
55
"name": "datasources",
6-
"path": "/package/datasources-1.0.0",
6+
"path": "/package/datasources/1.0.0",
77
"title": "Default datasource Integration",
88
"type": "integration",
99
"version": "1.0.0"
@@ -12,7 +12,7 @@
1212
"description": "Tests if no pipeline is set, it defaults to the default one",
1313
"download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
1414
"name": "default-pipeline",
15-
"path": "/package/default-pipeline-0.0.2",
15+
"path": "/package/default-pipeline/0.0.2",
1616
"title": "Default pipeline Integration",
1717
"type": "integration",
1818
"version": "0.0.2"
@@ -21,7 +21,7 @@
2121
"description": "This is the example integration.",
2222
"download": "/epr/example/example-0.0.2.tar.gz",
2323
"name": "example",
24-
"path": "/package/example-0.0.2",
24+
"path": "/package/example/0.0.2",
2525
"title": "Example",
2626
"type": "integration",
2727
"version": "0.0.2"

0 commit comments

Comments
 (0)