|
| 1 | +#!/bin/bash |
| 2 | +set -e |
| 3 | + |
| 4 | +#source <(curl -s https://raw.githubusercontent.com/keptn/keptn/0.8.5/test/utils.sh) |
| 5 | + |
| 6 | +function print_headline() { |
| 7 | + HEADLINE=$1 |
| 8 | + |
| 9 | + echo "" |
| 10 | + echo "---------------------------------------------------------------------" |
| 11 | + echo $HEADLINE |
| 12 | + echo "---------------------------------------------------------------------" |
| 13 | + echo "" |
| 14 | +} |
| 15 | + |
| 16 | +# wait for a deployment to be up and running |
| 17 | +function wait_for_deployment_in_namespace() { |
| 18 | + DEPLOYMENT=$1; NAMESPACE=$2; |
| 19 | + RETRY=0; RETRY_MAX=10; |
| 20 | + |
| 21 | + while [[ $RETRY -lt $RETRY_MAX ]]; do |
| 22 | + DEPLOYMENT_LIST=$(eval "kubectl get deployments -n ${NAMESPACE} | awk '/$DEPLOYMENT /'" | awk '{print $1}') # list of multiple deployments when starting with the same name |
| 23 | + if [[ -z "$DEPLOYMENT_LIST" ]]; then |
| 24 | + RETRY=$((RETRY+1)) |
| 25 | + echo "Retry: ${RETRY}/${RETRY_MAX} - Deployment not found - waiting 5s for deployment ${DEPLOYMENT} in namespace ${NAMESPACE}" |
| 26 | + sleep 5 |
| 27 | + else |
| 28 | + READY_REPLICAS=$(eval kubectl get deployments "$DEPLOYMENT" -n "$NAMESPACE" -o=jsonpath='{$.status.availableReplicas}') |
| 29 | + WANTED_REPLICAS=$(eval kubectl get deployments "$DEPLOYMENT" -n "$NAMESPACE" -o=jsonpath='{$.spec.replicas}') |
| 30 | + UNAVAILABLE_REPLICAS=$(eval kubectl get deployments "$DEPLOYMENT" -n "$NAMESPACE" -o=jsonpath='{$.status.unavailableReplicas}') |
| 31 | + if [[ "$READY_REPLICAS" = "$WANTED_REPLICAS" && "$UNAVAILABLE_REPLICAS" = "" ]]; then |
| 32 | + echo "Found deployment ${DEPLOYMENT} in namespace ${NAMESPACE}: ${DEPLOYMENT_LIST}" |
| 33 | + break |
| 34 | + else |
| 35 | + RETRY=$((RETRY+1)) |
| 36 | + echo "Retry: ${RETRY}/${RETRY_MAX} - Unsufficient replicas for deployment - waiting 5s for deployment ${DEPLOYMENT} in namespace ${NAMESPACE}" |
| 37 | + sleep 5 |
| 38 | + fi |
| 39 | + fi |
| 40 | + done |
| 41 | + |
| 42 | + if [[ $RETRY == "$RETRY_MAX" ]]; then |
| 43 | + print_error "Could not find deployment ${DEPLOYMENT} in namespace ${NAMESPACE}" |
| 44 | + exit 1 |
| 45 | + fi |
| 46 | +} |
| 47 | + |
| 48 | +PROJECT="podtatohead" |
| 49 | +SERVICE="helloservice" |
| 50 | + |
| 51 | +print_headline "Preparation of Auto-remediation in Production" |
| 52 | + |
| 53 | +echo "Adding SLIs for Prometheus" |
| 54 | +keptn add-resource --project=$PROJECT --stage=production --service=$SERVICE --resource=./demo/prometheus/sli.yaml --resourceUri=prometheus/sli.yaml |
| 55 | +echo "Adding SLO definition file for the quality gate" |
| 56 | +keptn add-resource --project=$PROJECT --stage=production --service=$SERVICE --resource=./demo/slo.yaml --resourceUri=slo.yaml |
| 57 | +echo "Adding Remediation Configuration" |
| 58 | +keptn add-resource --project=$PROJECT --stage=production --service=$SERVICE --resource=./demo/remediation.yaml --resourceUri=remediation.yaml |
| 59 | + |
| 60 | +print_headline "Deploy Job Executor" |
| 61 | + |
| 62 | +kubectl apply -f ./demo/job/job-executor.yaml |
| 63 | +keptn add-resource --project=$PROJECT --service=$SERVICE --stage=production --resource=./demo/job/config.yaml --resourceUri=job/config.yaml |
| 64 | + |
| 65 | +wait_for_deployment_in_namespace "job-executor-service" "keptn" |
| 66 | + |
| 67 | +print_headline "Simulate Alert (Problem)" |
| 68 | +echo -e "{\"type\": \"sh.keptn.event.production.remediation.triggered\",\"specversion\":\"1.0\",\"source\":\"https:\/\/github.com\/keptn\/keptn\/prometheus-service\",\"id\": \"f2b878d3-03c0-4e8f-bc3f-454bc1b3d79d\", \"time\": \"2019-06-07T07:02:15.64489Z\", \"contenttype\": \"application\/json\", \"data\": {\"project\": \"podtatohead\",\"stage\": \"production\",\"service\": \"helloservice\",\"problem\": { \"problemTitle\": \"out_of_memory\",\"rootCause\": \"Response time degradation\"}}}" > remediation_trigger.json | keptn send event -f remediation_trigger.json |
| 69 | + |
| 70 | +print_headline "Have a look at the Keptn Bridge and explore the demo project" |
| 71 | + |
| 72 | +echo "You can simulate a new problem any time by executing the following command:" |
| 73 | +echo "keptn send event -f remediation_trigger.json" |
0 commit comments