-
Notifications
You must be signed in to change notification settings - Fork 4.4k
(aws-eks): Installing helm charts from e.g. public.ecr.aws in the OCI format #18001
Copy link
Copy link
Closed
Labels
@aws-cdk/aws-eksRelated to Amazon Elastic Kubernetes ServiceRelated to Amazon Elastic Kubernetes Serviceeffort/smallSmall work item – less than a day of effortSmall work item – less than a day of effortfeature-requestA feature should be added or improved.A feature should be added or improved.p1
Description
Description
I want to install Helm charts in the OCI format. For example oci://public.ecr.aws/aws-controllers-k8s/opensearchservice-chart.
cluster.addHelmChart('MyChart', {
chart: 'oci://public.ecr.aws/aws-controllers-k8s/opensearchservice-chart'
});Use Case
Because if it isn't supported these type of charts can't be installed with CDK.
Proposed Solution
- Set the Helm version to one that supports the
HELM_EXPERIMENTAL_OCIflag here, e.g.3.7.1 - Handle charts that start with
oci://in a special way here
import json
import logging
import os
import subprocess
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# these are coming from the kubectl layer
os.environ['PATH'] = '/opt/helm:/opt/awscli:' + os.environ['PATH']
outdir = os.environ.get('TEST_OUTDIR', '/tmp')
kubeconfig = os.path.join(outdir, 'kubeconfig')
def helm_handler(event, context):
logger.info(json.dumps(event))
request_type = event['RequestType']
props = event['ResourceProperties']
# resource properties
cluster_name = props['ClusterName']
role_arn = props['RoleArn']
release = props['Release']
chart = props['Chart']
version = props.get('Version', None)
wait = props.get('Wait', False)
timeout = props.get('Timeout', None)
namespace = props.get('Namespace', None)
create_namespace = props.get('CreateNamespace', None)
repository = props.get('Repository', None)
values_text = props.get('Values', None)
# "log in" to the cluster
subprocess.check_call([ 'aws', 'eks', 'update-kubeconfig',
'--role-arn', role_arn,
'--name', cluster_name,
'--kubeconfig', kubeconfig
])
if os.path.isfile(kubeconfig):
os.chmod(kubeconfig, 0o600)
# Write out the values to a file and include them with the install and upgrade
values_file = None
if not request_type == "Delete" and not values_text is None:
values = json.loads(values_text)
values_file = os.path.join(outdir, 'values.yaml')
with open(values_file, "w") as f:
f.write(json.dumps(values, indent=2))
if request_type == 'Create' or request_type == 'Update':
if chart.startswith('oci://'):
chart_dir = helm_pull(release, chart, version)
chart = chart_dir
helm('upgrade', release, chart, repository, values_file, namespace, version, wait, timeout, create_namespace)
elif request_type == "Delete":
try:
helm('uninstall', release, namespace=namespace, timeout=timeout)
except Exception as e:
logger.info("delete error: %s" % e)
def helm_pull(release, chart = None, version = None):
import subprocess
import tempfile
untardir = tempfile.mkdtemp()
cmnd = ['helm', 'pull', chart, '--untar', '--untardir', untardir]
if not version is None:
cmnd.extend(['--version', version])
maxAttempts = 3
retry = maxAttempts
while retry > 0:
try:
output = subprocess.check_output(['helm', 'version'], stderr=subprocess.STDOUT, cwd=outdir)
logger.info(output)
logger.info(cmnd)
env = os.environ.copy()
env['HELM_EXPERIMENTAL_OCI'] = '1'
output = subprocess.check_output(cmnd, stderr=subprocess.STDOUT, cwd=outdir, env=env)
logger.info(output)
return os.path.join(untardir, chart.split('/')[-1])
except subprocess.CalledProcessError as exc:
output = exc.output
if b'Broken pipe' in output:
retry = retry - 1
logger.info("Broken pipe, retries left: %s" % retry)
else:
raise Exception(output)
raise Exception(f'Operation failed after {maxAttempts} attempts: {output}')
def helm(verb, release, chart = None, repo = None, file = None, namespace = None, version = None, wait = False, timeout = None, create_namespace = None):
import subprocess
cmnd = ['helm', verb, release]
if not chart is None:
cmnd.append(chart)
if verb == 'upgrade':
cmnd.append('--install')
if create_namespace:
cmnd.append('--create-namespace')
if not repo is None:
cmnd.extend(['--repo', repo])
if not file is None:
cmnd.extend(['--values', file])
if not version is None:
cmnd.extend(['--version', version])
if not namespace is None:
cmnd.extend(['--namespace', namespace])
if wait:
cmnd.append('--wait')
if not timeout is None:
cmnd.extend(['--timeout', timeout])
cmnd.extend(['--kubeconfig', kubeconfig])
maxAttempts = 3
retry = maxAttempts
while retry > 0:
try:
logger.info(cmnd)
env = os.environ.copy()
env['HELM_EXPERIMENTAL_OCI'] = '1'
output = subprocess.check_output(cmnd, stderr=subprocess.STDOUT, cwd=outdir, env=env)
logger.info(output)
return
except subprocess.CalledProcessError as exc:
output = exc.output
if b'Broken pipe' in output:
retry = retry - 1
logger.info("Broken pipe, retries left: %s" % retry)
else:
raise Exception(output)
raise Exception(f'Operation failed after {maxAttempts} attempts: {output}')Other information
Unfortunately this is really hard to test.
Currently I copy over my patched file in a postinstall hook and I provide my own layer with Cluster({kubectlLayer: myLayer})
Other issues exist as well, but the state of those is a bit confusing:
- [aws-eks] Install custom helm chart from local artifact or from ecr #10421
- [aws-eks] Support S3 private bucket repository from helm deployment #9273
Acknowledge
- I may be able to implement this feature request
- This feature might incur a breaking change
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
@aws-cdk/aws-eksRelated to Amazon Elastic Kubernetes ServiceRelated to Amazon Elastic Kubernetes Serviceeffort/smallSmall work item – less than a day of effortSmall work item – less than a day of effortfeature-requestA feature should be added or improved.A feature should be added or improved.p1