Skip to content

Use UBI as base image #16599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kaisecheng merged 12 commits into from
Nov 8, 2024
Merged

Use UBI as base image #16599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
969a918
- replace ubuntu base image to wolfi
kaisecheng Oct 24, 2024
6c2e8c8
update docker tests
kaisecheng Oct 24, 2024
602fe68
add comments
kaisecheng Oct 25, 2024
806017a
add comments
kaisecheng Oct 28, 2024
17319d9
Merge branch 'main' of github.com:elastic/logstash into use_wolfi_bas…
kaisecheng Oct 31, 2024
790faa7
use UBI as base image
kaisecheng Oct 31, 2024
b47d7d2
remove ubi flavour
kaisecheng Oct 31, 2024
7e95415
Merge branch 'main' of github.com:elastic/logstash into use_wolfi_bas…
kaisecheng Nov 1, 2024
6e5d60d
use golang image for env2yaml in UBI
kaisecheng Nov 4, 2024
33a24db
use golang image for env2yaml in UBI
kaisecheng Nov 4, 2024
38e44de
split Dockerfile template to two
kaisecheng Nov 7, 2024
b19322f
comment
kaisecheng Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 1 addition & 17 deletions .buildkite/scripts/dra/build_docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,11 @@ case "$WORKFLOW_TYPE" in
rake artifact:docker_oss || error "artifact:docker_oss build failed."
rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
if [ "$ARCH" != "aarch64" ]; then
rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
fi
else
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
if [ "$ARCH" != "aarch64" ]; then
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
fi
# Qualifier is passed from CI as optional field and specify the version postfix
# in case of alpha or beta releases:
# e.g: 8.0.0-alpha1
Expand All @@ -41,17 +35,11 @@ case "$WORKFLOW_TYPE" in
RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
if [ "$ARCH" != "aarch64" ]; then
RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
fi
else
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
if [ "$ARCH" != "aarch64" ]; then
VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
fi
# Qualifier is passed from CI as optional field and specify the version postfix
# in case of alpha or beta releases:
# e.g: 8.0.0-alpha1
Expand All @@ -73,18 +61,14 @@ for file in build/logstash-*; do shasum $file;done
info "Uploading DRA artifacts in buildkite's artifact store ..."
# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh
images="logstash logstash-oss logstash-wolfi"
if [ "$ARCH" != "aarch64" ]; then
# No logstash-ubi8 for AARCH64
images="logstash logstash-oss logstash-wolfi logstash-ubi8"
fi
for image in ${images}; do
buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
done

# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
# overwritten when building aarch64 (or viceversa).
if [ "$ARCH" != "aarch64" ]; then
for image in logstash logstash-oss logstash-wolfi logstash-ubi8 logstash-ironbank; do
for image in logstash logstash-oss logstash-wolfi logstash-ironbank; do
buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
done
fi
Expand Down
4 changes: 0 additions & 4 deletions .buildkite/scripts/dra/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,6 @@ function save_docker_tarballs {
local arch="${1:?architecture required}"
local version="${2:?stack-version required}"
local images="logstash logstash-oss logstash-wolfi"
if [ "${arch}" != "aarch64" ]; then
# No logstash-ubi8 for AARCH64
images="logstash logstash-oss logstash-wolfi logstash-ubi8"
fi

for image in ${images}; do
tar_file="${image}-${version}-docker-image-${arch}.tar"
Expand Down
7 changes: 0 additions & 7 deletions .buildkite/scripts/dra/publish.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,6 @@ if [ "$RELEASE_VER" != "7.17" ]; then
:
fi

# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected
# by the release manager.
# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle
# for more details.
# TODO filter it out when uploading artifacts instead
rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz

info "Downloaded ARTIFACTS sha report"
for file in build/logstash-*; do shasum $file;done

Expand Down
11 changes: 0 additions & 11 deletions ci/docker_acceptance_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ fi
# Can run either a specific flavor, or all flavors -
# eg `ci/acceptance_tests.sh oss` will run tests for open source container
# `ci/acceptance_tests.sh full` will run tests for the default container
# `ci/acceptance_tests.sh ubi8` will run tests for the ubi8 based container
# `ci/acceptance_tests.sh wolfi` will run tests for the wolfi based container
# `ci/acceptance_tests.sh` will run tests for all containers
SELECTED_TEST_SUITE=$1
Expand Down Expand Up @@ -56,16 +55,6 @@ elif [[ $SELECTED_TEST_SUITE == "full" ]]; then

echo "--- Acceptance: Running the tests"
bundle exec rspec docker/spec/full/*_spec.rb
elif [[ $SELECTED_TEST_SUITE == "ubi8" ]]; then
echo "--- Building $SELECTED_TEST_SUITE docker images"
cd $LS_HOME
rake artifact:docker_ubi8
echo "--- Acceptance: Installing dependencies"
cd $QA_DIR
bundle install

echo "--- Acceptance: Running the tests"
bundle exec rspec docker/spec/ubi8/*_spec.rb
elif [[ $SELECTED_TEST_SUITE == "wolfi" ]]; then
echo "--- Building $SELECTED_TEST_SUITE docker images"
cd $LS_HOME
Expand Down
78 changes: 15 additions & 63 deletions docker/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ SHELL=/bin/bash
ELASTIC_REGISTRY ?= docker.elastic.co

# Determine the version to build.
ELASTIC_VERSION := $(shell ../vendor/jruby/bin/jruby bin/elastic-version)
ELASTIC_VERSION ?= $(shell ../vendor/jruby/bin/jruby bin/elastic-version)

ifdef STAGING_BUILD_NUM
VERSION_TAG := $(ELASTIC_VERSION)-$(STAGING_BUILD_NUM)
Expand All @@ -14,9 +14,13 @@ ifdef DOCKER_ARCHITECTURE
ARCHITECTURE := $(DOCKER_ARCHITECTURE)
else
ARCHITECTURE := $(shell uname -m)
# For MacOS
ifeq ($(ARCHITECTURE), arm64)
ARCHITECTURE := aarch64
endif
endif

IMAGE_FLAVORS ?= oss full ubi8 wolfi
IMAGE_FLAVORS ?= oss full wolfi
DEFAULT_IMAGE_FLAVOR ?= full

IMAGE_TAG := $(ELASTIC_REGISTRY)/logstash/logstash
Expand All @@ -26,7 +30,7 @@ all: build-from-local-artifacts build-from-local-oss-artifacts public-dockerfile

# Build from artifacts on the local filesystem, using an http server (running
# in a container) to provide the artifacts to the Dockerfile.
build-from-local-full-artifacts: dockerfile env2yaml
build-from-local-full-artifacts: dockerfile
docker run --rm -d --name=$(HTTPD) \
-p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
python:3 bash -c 'cd /mnt && python3 -m http.server'
Expand All @@ -36,7 +40,7 @@ build-from-local-full-artifacts: dockerfile env2yaml
docker tag $(IMAGE_TAG)-full:$(VERSION_TAG) $(IMAGE_TAG):$(VERSION_TAG);
docker kill $(HTTPD)

build-from-local-oss-artifacts: dockerfile env2yaml
build-from-local-oss-artifacts: dockerfile
docker run --rm -d --name=$(HTTPD) \
-p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
python:3 bash -c 'cd /mnt && python3 -m http.server'
Expand All @@ -45,15 +49,6 @@ build-from-local-oss-artifacts: dockerfile env2yaml
(docker kill $(HTTPD); false);
-docker kill $(HTTPD)

build-from-local-ubi8-artifacts: dockerfile env2yaml
docker run --rm -d --name=$(HTTPD) \
-p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
python:3 bash -c 'cd /mnt && python3 -m http.server'
timeout 120 bash -c 'until curl -s localhost:8000 > /dev/null; do sleep 1; done'
docker build --progress=plain --network=host -t $(IMAGE_TAG)-ubi8:$(VERSION_TAG) -f $(ARTIFACTS_DIR)/Dockerfile-ubi8 data/logstash || \
(docker kill $(HTTPD); false);
-docker kill $(HTTPD)

build-from-local-wolfi-artifacts: dockerfile
docker run --rm -d --name=$(HTTPD) \
-p 8000:8000 --expose=8000 -v $(ARTIFACTS_DIR):/mnt \
Expand All @@ -66,8 +61,6 @@ build-from-local-wolfi-artifacts: dockerfile
COPY_FILES := $(ARTIFACTS_DIR)/docker/config/pipelines.yml $(ARTIFACTS_DIR)/docker/config/logstash-oss.yml $(ARTIFACTS_DIR)/docker/config/logstash-full.yml
COPY_FILES += $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties $(ARTIFACTS_DIR)/docker/config/log4j2.properties
COPY_FILES += $(ARTIFACTS_DIR)/docker/pipeline/default.conf $(ARTIFACTS_DIR)/docker/bin/docker-entrypoint
COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64
COPY_FILES += $(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64

$(ARTIFACTS_DIR)/docker/config/pipelines.yml: data/logstash/config/pipelines.yml
$(ARTIFACTS_DIR)/docker/config/logstash-oss.yml: data/logstash/config/logstash-oss.yml
Expand All @@ -76,8 +69,6 @@ $(ARTIFACTS_DIR)/docker/config/log4j2.file.properties: data/logstash/config/log4
$(ARTIFACTS_DIR)/docker/config/log4j2.properties: data/logstash/config/log4j2.properties
$(ARTIFACTS_DIR)/docker/pipeline/default.conf: data/logstash/pipeline/default.conf
$(ARTIFACTS_DIR)/docker/bin/docker-entrypoint: data/logstash/bin/docker-entrypoint
$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-arm64: data/logstash/env2yaml/env2yaml-arm64
$(ARTIFACTS_DIR)/docker/env2yaml/env2yaml-amd64: data/logstash/env2yaml/env2yaml-amd64

$(ARTIFACTS_DIR)/docker/%:
cp -f $< $@
Expand All @@ -86,7 +77,6 @@ docker_paths:
mkdir -p $(ARTIFACTS_DIR)/docker/
mkdir -p $(ARTIFACTS_DIR)/docker/bin
mkdir -p $(ARTIFACTS_DIR)/docker/config
mkdir -p $(ARTIFACTS_DIR)/docker/env2yaml
mkdir -p $(ARTIFACTS_DIR)/docker/pipeline

COPY_IRONBANK_FILES := $(ARTIFACTS_DIR)/ironbank/scripts/config/pipelines.yml $(ARTIFACTS_DIR)/ironbank/scripts/config/logstash.yml
Expand Down Expand Up @@ -118,7 +108,7 @@ ironbank_docker_paths:
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/go/src/env2yaml/vendor
mkdir -p $(ARTIFACTS_DIR)/ironbank/scripts/pipeline

public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_ubi8 public-dockerfiles_wolfi public-dockerfiles_ironbank
public-dockerfiles: public-dockerfiles_oss public-dockerfiles_full public-dockerfiles_wolfi public-dockerfiles_ironbank

public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
Expand All @@ -132,7 +122,7 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES)
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \
tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline

public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
Expand All @@ -146,21 +136,7 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES)
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \
tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline

public-dockerfiles_ubi8: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
created_date="${BUILD_DATE}" \
elastic_version="${ELASTIC_VERSION}" \
arch="${ARCHITECTURE}" \
version_tag="${VERSION_TAG}" \
release="${RELEASE}" \
image_flavor="ubi8" \
local_artifacts="false" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ubi8" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-ubi8 Dockerfile && \
tar -zcf ../logstash-ubi8-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline

public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
Expand All @@ -174,9 +150,9 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES)
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \
cd $(ARTIFACTS_DIR)/docker && \
cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \
tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline
tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config pipeline

public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Dockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES)
../vendor/jruby/bin/jruby -S erb -T "-"\
elastic_version="${ELASTIC_VERSION}" \
templates/hardening_manifest.yaml.erb > $(ARTIFACTS_DIR)/ironbank/hardening_manifest.yaml && \
Expand All @@ -188,35 +164,11 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Doc
release="${RELEASE}" \
image_flavor="ironbank" \
local_artifacts="false" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \
cd $(ARTIFACTS_DIR)/ironbank && \
cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \
tar -zcf ../logstash-ironbank-$(VERSION_TAG)-docker-build-context.tar.gz scripts Dockerfile hardening_manifest.yaml LICENSE README.md

# Push the image to the dedicated push endpoint at "push.docker.elastic.co"
push:
$(foreach FLAVOR, $(IMAGE_FLAVORS), \
docker tag $(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG) push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \
docker push push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \
docker rmi push.$(IMAGE_TAG)-$(FLAVOR):$(VERSION_TAG); \
)
# Also push the default version, with no suffix like '-oss' or '-full'
docker tag $(IMAGE_TAG):$(VERSION_TAG) push.$(IMAGE_TAG):$(VERSION_TAG);
docker push push.$(IMAGE_TAG):$(VERSION_TAG);
docker rmi push.$(IMAGE_TAG):$(VERSION_TAG);

# Compile "env2yaml", the helper for configuring logstash.yml via environment
# variables.
env2yaml:
docker run --rm \
-v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-e GOARCH=arm64 -e GOOS=linux \
-w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-arm64
docker run --rm \
-v "$(PWD)/data/logstash/env2yaml:/usr/src/env2yaml" \
-e GOARCH=amd64 -e GOOS=linux \
-w /usr/src/env2yaml golang:1 go build -o /usr/src/env2yaml/env2yaml-amd64

# Generate the Dockerfiles from ERB templates.
dockerfile: templates/Dockerfile.erb
$(foreach FLAVOR, $(IMAGE_FLAVORS), \
Expand All @@ -226,7 +178,7 @@ dockerfile: templates/Dockerfile.erb
arch="${ARCHITECTURE}" \
version_tag="${VERSION_TAG}" \
image_flavor="${FLAVOR}" \
local_artifacts="true" \
local_artifacts="${LOCAL_ARTIFACTS}" \
templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-${FLAVOR}" ; \
)

Expand Down
Loading