3장 실습 - Amazon EKS FinOps - KubeCost와 OpenCost 구성하기

cd cnaee_class_tf/ch3

export TF_VAR_KeyName=[각자 ssh keypair] export TF_VAR_MyIamUserAccessKeyID=[각자 iam 사용자의 access key id] export TF_VAR_MyIamUserSecretAccessKey=[각자 iam 사용자의 secret access key] export TF_VAR_SgIngressSshCidr=$(curl -s ipinfo.io/ip)/32

terraform init terraform plan

nohup sh -c "terraform apply -auto-approve" > create.log 2>&1 &

aws eks update-kubeconfig \ --region $AWS_DEFAULT_REGION \ --name $CLUSTER_NAME

kubens default

echo $AWS_DEFAULT_REGION echo $CLUSTER_NAME echo $VPCID echo $PublicSubnet1,$PublicSubnet2,$PublicSubnet3 echo $PrivateSubnet1,$PrivateSubnet2,$PrivateSubnet3

eksctl get cluster

eksctl get nodegroup \ --cluster $CLUSTER_NAME \ --name ${CLUSTER_NAME}-node-group

kubectl get node -owide

PublicN1=$(kubectl get node --label-columns=topology.kubernetes.io/zone --selector=topology.kubernetes.io/zone=ap-northeast-2a -o jsonpath={.items[0].status.addresses[0].address}) PublicN2=$(kubectl get node --label-columns=topology.kubernetes.io/zone --selector=topology.kubernetes.io/zone=ap-northeast-2b -o jsonpath={.items[0].status.addresses[0].address}) PublicN3=$(kubectl get node --label-columns=topology.kubernetes.io/zone --selector=topology.kubernetes.io/zone=ap-northeast-2c -o jsonpath={.items[0].status.addresses[0].address}) echo "export PublicN1=$PublicN1" >> /etc/profile echo "export PublicN2=$PublicN2" >> /etc/profile echo "export PublicN3=$PublicN3" >> /etc/profile echo $PublicN1, $PublicN2, $PublicN3

for node in $PublicN1 $PublicN2 $PublicN3; \ do \ ssh -i ~/.ssh/kp_node.pem -o StrictHostKeyChecking=no ec2-user@$node hostname; \ done

export MyDomain=[각자 도메인] echo "export MyDomain=$MyDomain" >> /etc/profile; echo $MyDomain export NICKNAME=[각자의 닉네임] echo "export NICKNAME=$NICKNAME" >> /etc/profile; echo $NICKNAME export OIDC_ARN=$(aws iam list-open-id-connect-providers --query 'OpenIDConnectProviderList[*].Arn' --output text) echo "export OIDC_ARN=$OIDC_ARN" >> /etc/profile; echo $OIDC_ARN export OIDC_URL=${OIDC_ARN#*oidc-provider/} echo "export OIDC_URL=$OIDC_URL" >> /etc/profile; echo $OIDC_URL

cat <<EOT | kubectl apply -f - kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: gp3 allowVolumeExpansion: true provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer parameters: type: gp3 allowAutoIOPSPerGBIncrease: 'true' encrypted: 'true' EOT

kubectl create ns monitoring kubectl create ns kubecost kubectl create ns opencost

helm repo add grafana https://grafana.github.io/helm-charts helm repo update

aws s3api create-bucket \ --bucket mimir-${NICKNAME} \ --region $AWS_DEFAULT_REGION \ --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION aws s3 ls

export MIMIR_BUCKET_NAME="mimir-${NICKNAME}" echo "export MIMIR_BUCKET_NAME=$MIMIR_BUCKET_NAME" >> /etc/profile; echo $MIMIR_BUCKET_NAME

cat >grafana-mimir-s3-policy.json <<EOF { "Version": "2012-10-17", "Statement": [ { "Sid": "MimirStorage", "Effect": "Allow", "Action": [ "s3:ListBucket", "s3:PutObject", "s3:GetObject", "s3:DeleteObject" ], "Resource": [ "arn:aws:s3:::${MIMIR_BUCKET_NAME}", "arn:aws:s3:::${MIMIR_BUCKET_NAME}/*" ] } ] } EOF cat grafana-mimir-s3-policy.json

aws iam create-policy \ --policy-name aws-mimir-s3 \ --policy-document file://grafana-mimir-s3-policy.json

cat >trust-rs-mimir.json <<EOF { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Federated": "${OIDC_ARN}" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { "${OIDC_URL}:sub": "system:serviceaccount:monitoring:mimir", "${OIDC_URL}:aud": "sts.amazonaws.com" } } } ] } EOF cat trust-rs-mimir.json

aws iam create-role \ --role-name AWS-Mimir-Role \ --assume-role-policy-document file://trust-rs-mimir.json

aws iam attach-role-policy \ --role-name AWS-Mimir-Role \ --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/aws-mimir-s3

export MIMIR_ROLE_ARN=arn:aws:iam::${ACCOUNT_ID}:role/AWS-Mimir-Role echo "export MIMIR_ROLE_ARN=$MIMIR_ROLE_ARN" >> /etc/profile; echo $MIMIR_ROLE_ARN

cat >mimir-values.yaml <<EOF image: repository: grafana/mimir tag: 2.10.3 pullPolicy: IfNotPresent mimir: structuredConfig: limits: max_label_names_per_series: 60 compactor_blocks_retention_period: 30d blocks_storage: backend: s3 s3: bucket_name: ${MIMIR_BUCKET_NAME} endpoint: s3.${AWS_DEFAULT_REGION}.amazonaws.com region: ${AWS_DEFAULT_REGION} tsdb: retention_period: 13h bucket_store: ignore_blocks_within: 10h querier: query_store_after: 12h ingester: ring: replication_factor: 3 serviceAccount: create: true name: "mimir" annotations: "eks.amazonaws.com/role-arn": "${MIMIR_ROLE_ARN}" minio: enabled: false alertmanager: enabled: false ruler: enabled: false compactor: persistentVolume: enabled: true annotations: {} accessModes: - ReadWriteOnce size: 5Gi storageClass: gp3 ingester: zoneAwareReplication: enabled: false persistentVolume: enabled: true annotations: {} accessModes: - ReadWriteOnce size: 5Gi storageClass: gp3 store_gateway: zoneAwareReplication: enabled: false persistentVolume: enabled: true annotations: {} accessModes: - ReadWriteOnce size: 5Gi storageClass: gp3 EOF cat mimir-values.yaml

watch kubectl get pod,pv,pvc,cm -n monitoring

helm install mimir grafana/mimir-distributed \ -n monitoring \ -f mimir-values.yaml \ --version 5.4.0

export MIMIR_ENDPOINT_PUSH="http://mimir-nginx.monitoring/api/v1/push"

cat <<EOF | kind: ConfigMap metadata: name: grafana-agent apiVersion: v1 data: agent.yaml: | metrics: wal_directory: /var/lib/agent/wal global: scrape_interval: 60s external_labels: cluster: ${CLUSTER_NAME} configs: - name: integrations remote_write: - headers: X-Scope-OrgID: kubecost_mimir url: ${MIMIR_ENDPOINT_PUSH} - url: ${MIMIR_ENDPOINT_PUSH} scrape_configs: - job_name: kubecost honor_labels: true scrape_interval: 1m scrape_timeout: 10s metrics_path: /metrics scheme: http dns_sd_configs: - names: - kubecost-cost-analyzer.kubecost type: 'A' port: 9003 - job_name: kubecost-networking kubernetes_sd_configs: - role: pod relabel_configs: # Scrape only the the targets matching the following metadata - source_labels: [__meta_kubernetes_pod_label_app] action: keep regex: 'kubecost-network-costs' - job_name: prometheus static_configs: - targets: - localhost:9090 - job_name: 'kubernetes-nodes-cadvisor' scheme: https tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/\$\$1/proxy/metrics/cadvisor metric_relabel_configs: - source_labels: [ __name__ ] regex: (container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_network_receive_errors_total|container_network_transmit_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_packets_dropped_total|container_memory_usage_bytes|container_cpu_cfs_throttled_periods_total|container_cpu_cfs_periods_total|container_fs_usage_bytes|container_fs_limit_bytes|container_cpu_cfs_periods_total|container_fs_inodes_free|container_fs_inodes_total|container_fs_usage_bytes|container_fs_limit_bytes|container_cpu_cfs_throttled_periods_total|container_cpu_cfs_periods_total|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_fs_inodes_free|container_fs_inodes_total|container_fs_usage_bytes|container_fs_limit_bytes|container_spec_cpu_shares|container_spec_memory_limit_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total|container_fs_reads_bytes_total|container_network_receive_bytes_total|container_fs_writes_bytes_total|container_fs_reads_bytes_total|cadvisor_version_info|kubecost_pv_info|kubelet_volume_stats_used_bytes|kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_available_bytes|kubelet_volume_stats_inodes|kubelet_volume_stats_inodes_free|kubelet_volume_stats_inodes_used) action: keep - source_labels: [ container ] target_label: container_name regex: (.+) action: replace - source_labels: [ pod ] target_label: pod_name regex: (.+) action: replace - job_name: 'kubernetes-nodes' tls_config: ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecure_skip_verify: true bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.+) - target_label: __address__ replacement: kubernetes.default.svc:443 - source_labels: [__meta_kubernetes_node_name] regex: (.+) target_label: __metrics_path__ replacement: /api/v1/nodes/\$\$1/proxy/metrics metric_relabel_configs: - source_labels: [ __name__ ] regex: (kubelet_volume_stats_used_bytes) # this metric is in alpha action: keep - job_name: 'kubernetes-service-endpoints' kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_endpoints_name] action: keep regex: (.*kube-state-metrics|.*prometheus-node-exporter|kubecost-network-costs) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ regex: ([^:]+)(?::\d+)?;(\d+) replacement: \$\$1:\$\$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: kubernetes_name - source_labels: [__meta_kubernetes_pod_node_name] action: replace target_label: kubernetes_node metric_relabel_configs: - source_labels: [ __name__ ] regex: (container_cpu_allocation|container_cpu_usage_seconds_total|container_fs_limit_bytes|container_fs_writes_bytes_total|container_gpu_allocation|container_memory_allocation_bytes|container_memory_usage_bytes|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_transmit_bytes_total|DCGM_FI_DEV_GPU_UTIL|deployment_match_labels|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_ready|kube_deployment_spec_replicas|kube_deployment_status_replicas|kube_deployment_status_replicas_available|kube_job_status_failed|kube_namespace_annotations|kube_namespace_labels|kube_node_info|kube_node_labels|kube_node_status_allocatable|kube_node_status_allocatable_cpu_cores|kube_node_status_allocatable_memory_bytes|kube_node_status_capacity|kube_node_status_capacity_cpu_cores|kube_node_status_capacity_memory_bytes|kube_node_status_condition|kube_persistentvolume_capacity_bytes|kube_persistentvolume_status_phase|kube_persistentvolumeclaim_info|kube_persistentvolumeclaim_resource_requests_storage_bytes|kube_pod_container_info|kube_pod_container_resource_limits|kube_pod_container_resource_limits_cpu_cores|kube_pod_container_resource_limits_memory_bytes|kube_pod_container_resource_requests|kube_pod_container_resource_requests_cpu_cores|kube_pod_container_resource_requests_memory_bytes|kube_pod_container_status_restarts_total|kube_pod_container_status_running|kube_pod_container_status_terminated_reason|kube_pod_labels|kube_pod_owner|kube_pod_status_phase|kube_replicaset_owner|kube_statefulset_replicas|kube_statefulset_status_replicas|kubecost_cluster_info|kubecost_cluster_management_cost|kubecost_cluster_memory_working_set_bytes|kubecost_load_balancer_cost|kubecost_network_internet_egress_cost|kubecost_network_region_egress_cost|kubecost_network_zone_egress_cost|kubecost_node_is_spot|kubecost_pod_network_egress_bytes_total|node_cpu_hourly_cost|node_cpu_seconds_total|node_disk_reads_completed|node_disk_reads_completed_total|node_disk_writes_completed|node_disk_writes_completed_total|node_filesystem_device_error|node_gpu_count|node_gpu_hourly_cost|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_MemAvailable_bytes|node_memory_MemFree_bytes|node_memory_MemTotal_bytes|node_network_transmit_bytes_total|node_ram_hourly_cost|node_total_hourly_cost|pod_pvc_allocation|pv_hourly_cost|service_selector_labels|statefulSet_match_labels|kubecost_pv_info|up) action: keep - job_name: 'kubernetes-service-endpoints-slow' scrape_interval: 5m scrape_timeout: 30s kubernetes_sd_configs: - role: endpoints relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow] action: keep regex: true - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] action: replace target_label: __scheme__ regex: (https?) - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.+) - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] action: replace target_label: __address__ regex: ([^:]+)(?::\d+)?;(\d+) replacement: \$\$1:\$\$2 - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] action: replace target_label: kubernetes_name - source_labels: [__meta_kubernetes_pod_node_name] action: replace target_label: kubernetes_node - job_name: 'prometheus-pushgateway' honor_labels: true kubernetes_sd_configs: - role: service relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] action: keep regex: pushgateway - job_name: 'kubernetes-services' metrics_path: /probe params: module: [http_2xx] kubernetes_sd_configs: - role: service relabel_configs: - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe] action: keep regex: true - source_labels: [__address__] target_label: __param_target - target_label: __address__ replacement: blackbox - source_labels: [__param_target] target_label: instance - action: labelmap regex: __meta_kubernetes_service_label_(.+) - source_labels: [__meta_kubernetes_namespace] target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_service_name] target_label: kubernetes_name EOF (export NAMESPACE=kubecost && kubectl apply -n $NAMESPACE -f -)

curl -O https://raw.githubusercontent.com/cloudneta/cnaeelab/master/_data/agent-bare.yaml kubectl apply -f agent-bare.yaml kubectl get all -n kubecost

kubectl -n kubecost logs grafana-agent-0

MIMIR_ENDPOINT="mimir-nginx.monitoring" echo "export MIMIR_ENDPOINT=$MIMIR_ENDPOINT" >> /etc/profile; echo $MIMIR_ENDPOINT MIMIR_ORG_ID="0" echo "export MIMIR_ORG_ID=$MIMIR_ORG_ID" >> /etc/profile; echo $MIMIR_ORG_ID

cat >kubecost-values.yaml <<EOF global: mimirProxy: enabled: true mimirEndpoint: http://${MIMIR_ENDPOINT} orgIdentifier: ${MIMIR_ORG_ID} prometheus: enabled: false fqdn: http://${MIMIR_ENDPOINT}/prometheus persistentVolume: enabled: true storageClass: gp3 kubecostProductConfigs: clusterName: ${CLUSTER_NAME} ingress: enabled: true annotations: kubernetes.io/ingress.class: "alb" alb.ingress.kubernetes.io/scheme: "internet-facing" alb.ingress.kubernetes.io/target-type: "ip" alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80}]' hosts: - "kubecost.${MyDomain}" paths: - "/*" pathType: "ImplementationSpecific" EOF cat kubecost-values.yaml

helm repo add kubecost https://kubecost.github.io/cost-analyzer/ helm repo update

watch kubectl get all -n kubecost

helm upgrade -i kubecost kubecost/cost-analyzer \ -n kubecost \ --version 2.6.3 \ --values kubecost-values.yaml

kubectl run curl --image=appropriate/curl --restart=Never --rm -it -- sh

KUBECOST_EP="http://kubecost-cost-analyzer.kubecost.svc.cluster.local:9090" apk add --no-cache jq

curl $KUBECOST_EP/model/allocation \ -d window=3d \ -d aggregate=namespace \ -d accumulate=false \ -d shareIdle=false \ -d format=json \ -G | jq

curl $KUBECOST_EP/model/assets \ -d window=today \ -d aggregate=type \ -d accumulate=true \ -d disableAdjustments=true \ -d format=json \ -G | jq

os=$(uname | tr '[:upper:]' '[:lower:]') && \ arch=$(uname -m | tr '[:upper:]' '[:lower:]' | sed -e s/x86_64/amd64/) && \ curl -s -L https://github.com/kubecost/kubectl-cost/releases/latest/download/kubectl-cost-$os-$arch.tar.gz | tar xz -C /tmp && \ chmod +x /tmp/kubectl-cost && \ sudo mv /tmp/kubectl-cost /usr/local/bin/kubectl-cost

kubectl cost namespace

kubectl cost deployment --show-cpu

kubectl cost pod --show-pv

kubectl cost node --window 7d --show-cpu --show-memory

kubectl cost tui

helm uninstall kubecost -n kubecost

cat >opencost.yaml <<EOF --- apiVersion: v1 kind: ServiceAccount metadata: name: opencost namespace: opencost --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: opencost rules: - apiGroups: - '' resources: - configmaps - deployments - nodes - pods - services - resourcequotas - replicationcontrollers - limitranges - persistentvolumeclaims - persistentvolumes - namespaces - endpoints verbs: - get - list - watch - apiGroups: - extensions resources: - daemonsets - deployments - replicasets verbs: - get - list - watch - apiGroups: - apps resources: - statefulsets - deployments - daemonsets - replicasets verbs: - list - watch - apiGroups: - batch resources: - cronjobs - jobs verbs: - get - list - watch - apiGroups: - autoscaling resources: - horizontalpodautoscalers verbs: - get - list - watch - apiGroups: - policy resources: - poddisruptionbudgets verbs: - get - list - watch - apiGroups: - storage.k8s.io resources: - storageclasses verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: opencost roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: opencost subjects: - kind: ServiceAccount name: opencost namespace: opencost --- apiVersion: apps/v1 kind: Deployment metadata: name: opencost namespace: opencost labels: app: opencost spec: replicas: 1 selector: matchLabels: app: opencost strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: metadata: labels: app: opencost spec: restartPolicy: Always serviceAccountName: opencost containers: - image: ghcr.io/opencost/opencost:latest name: opencost resources: requests: cpu: "10m" memory: "55M" limits: cpu: "999m" memory: "1G" env: - name: PROMETHEUS_SERVER_ENDPOINT value: "http://${MIMIR_ENDPOINT}.svc/prometheus" - name: CLUSTER_ID value: "${CLUSTER_NAME}" imagePullPolicy: Always securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL privileged: false readOnlyRootFilesystem: true runAsUser: 1001 - image: ghcr.io/opencost/opencost-ui:latest name: opencost-ui resources: requests: cpu: "10m" memory: "55M" limits: cpu: "999m" memory: "1G" imagePullPolicy: Always --- kind: Service apiVersion: v1 metadata: name: opencost namespace: opencost spec: selector: app: opencost type: ClusterIP ports: - name: opencost port: 9003 targetPort: 9003 - name: opencost-ui port: 9090 targetPort: 9090 --- apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: opencost namespace: opencost annotations: nginx.ingress.kubernetes.io/rewrite-target: / kubernetes.io/ingress.class: alb alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/target-type: ip spec: ingressClassName: alb rules: - host: "opencost.${MyDomain}" http: paths: - path: / pathType: Prefix backend: service: name: opencost port: number: 9090 EOF cat opencost.yaml

watch kubectl get all -n opencost

kubectl create -f opencost.yaml

kubectl cost \ --service-port 9003 \ --service-name opencost \ --kubecost-namespace opencost \ --allocation-path /allocation/compute \ namespace \ --show-efficiency=true

kubectl cost \ --service-port 9003 \ --service-name opencost \ --kubecost-namespace opencost \ --allocation-path /allocation/compute \ tui

kubectl delete -f opencost.yaml kubectl delete -f agent-bare.yaml kubectl delete pvc --all -n kubecost

helm uninstall mimir -n monitoring kubectl delete pvc --all -n monitoring

aws iam detach-role-policy --role-name AWS-Mimir-Role --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/aws-mimir-s3 aws iam delete-policy --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/aws-mimir-s3 aws iam delete-role --role-name AWS-Mimir-Role

aws s3 rm s3://$MIMIR_BUCKET_NAME --recursive aws s3api delete-bucket --bucket $MIMIR_BUCKET_NAME --region $AWS_DEFAULT_REGION aws s3 ls

nohup sh -c "terraform destroy -auto-approve" > delete.log 2>&1 &