4장 실습 - Amazon EKS 로깅 구성하기

aws eks describe-cluster \ --region $AWS_DEFAULT_REGION \ --name $CLUSTER_NAME \ --query "cluster.logging.clusterLogging" \ --output json | jq

aws eks update-cluster-config \ --region $AWS_DEFAULT_REGION \ --name $CLUSTER_NAME \ --logging '{"clusterLogging":[{"types":["api","audit","authenticator","controllerManager","scheduler"],"enabled":true}]}'

aws logs describe-log-groups | jq

aws logs describe-log-streams \ --log-group-name /aws/eks/${CLUSTER_NAME}/cluster | grep logStreamName

aws logs tail /aws/eks/$CLUSTER_NAME/cluster | more

aws logs tail /aws/eks/$CLUSTER_NAME/cluster --follow

aws logs tail /aws/eks/$CLUSTER_NAME/cluster --since 1h

aws logs tail /aws/eks/$CLUSTER_NAME/cluster --since 30m --format short

aws logs tail /aws/eks/$CLUSTER_NAME/cluster \ --log-stream-name-prefix kube-controller-manager \ --follow

kubectl scale deployment \ -n kube-system coredns \ --replicas=1

kubectl scale deployment \ -n kube-system coredns \ --replicas=2

fields userAgent, requestURI, @timestamp, @message | filter @logStream ~= "kube-apiserver-audit" | stats count(userAgent) as count by userAgent | sort count desc

fields @timestamp, @message | filter @logStream ~= "kube-apiserver" | sort @timestamp desc

fields @timestamp, @message | filter @logStream ~= "authenticator" | sort @timestamp asc

fields @timestamp, @message | filter @logStream ~= "kube-controller-manager" | sort @timestamp desc

eksctl utils update-cluster-logging \ --cluster $CLUSTER_NAME \ --region $AWS_DEFAULT_REGION \ --disable-types all \ --approve

aws eks describe-cluster \ --region $AWS_DEFAULT_REGION \ --name $CLUSTER_NAME \ --query "cluster.logging.clusterLogging" \ --output json | jq

aws logs delete-log-group \ --log-group-name /aws/eks/$CLUSTER_NAME/cluster aws logs describe-log-groups | jq

kubectl get nodes -o name \ | grep ip-192-168-1- \ | xargs -I {} \ kubectl label {} role=nginx-node --overwrite

helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update

cat <<EOT > nginx-values.yaml service: type: NodePort networkPolicy: enabled: false resourcesPreset: "nano" nodeSelector: role: nginx-node ingress: enabled: true ingressClassName: alb hostname: nginx.$MyDomain path: /* annotations: alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/target-type: ip alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}, {"HTTP":80}]' alb.ingress.kubernetes.io/certificate-arn: $CERT_ARN alb.ingress.kubernetes.io/success-codes: 200-399 alb.ingress.kubernetes.io/load-balancer-name: $CLUSTER_NAME-ingress-alb alb.ingress.kubernetes.io/group.name: study alb.ingress.kubernetes.io/ssl-redirect: '443' EOT cat nginx-values.yaml | yh

helm install nginx bitnami/nginx \ --version 20.0.0 \ -f nginx-values.yaml

kubectl get ingress,deploy,svc,ep nginx

kubectl get targetgroupbindings

curl -s https://nginx.$MyDomain

curl -s http://nginx.$MyDomain

curl -vs http://nginx.$MyDomain

curl -L http://nginx.$MyDomain

MyDomain=<자신의 도메인> while true; do curl -s https://nginx.$MyDomain -I | head -n 1; date; sleep 1; done

kubectl logs deploy/nginx -c nginx -f

kubectl exec -it deploy/nginx -c nginx -- ls -l /opt/bitnami/nginx/logs/

ssh ec2-user@$N1 \ sudo tree -C /var/log/containers

ssh ec2-user@$N1 \ sudo tree -C /var/log/pods

ssh ec2-user@$N1 sudo tail -f <XXXXXXXXX> ssh ec2-user@$N1 sudo head <XXXXXXXXX>

ssh ec2-user@$N1 \ journalctl -u kubelet | grep FLAG | grep container-log-max

for node in $N1 $N2 $N3; \ do \ echo ">>>>> $node <<<<<"; \ ssh ec2-user@$node sudo tree /var/log/containers; \ echo; done

for node in $N1 $N2 $N3; \ do \ echo ">>>>> $node <<<<<"; \ ssh ec2-user@$node sudo tree /var/log/pods; \ echo; done

for node in $N1 $N2 $N3; \ do \ echo ">>>>> $node <<<<<"; \ ssh ec2-user@$node sudo tree /var/log/ -L 1; \ echo; done

for log in dmesg secure messages; \ do \ echo ">>>>> Node1: /var/log/$log <<<<<"; \ ssh ec2-user@$N1 sudo tail /var/log/$log; \ echo; done

for node in $N1 $N2 $N3; \ do \ echo ">>>>> $node <<<<<"; \ ssh ec2-user@$node sudo tree /var/log/journal -L 1; \ echo; done

ssh ec2-user@$N1 \ sudo journalctl -u kubelet -x -n 200

ssh ec2-user@$N1 \ sudo journalctl -u kubelet -f

kubectl create ns amazon-cloudwatch kubectl label ns amazon-cloudwatch name=amazon-cloudwatch

kubectl get ns

FluentBitHttpServer='On' FluentBitHttpPort='2020' FluentBitReadFromHead='Off' FluentBitReadFromTail='On'

kubectl create configmap fluent-bit-cluster-info \ --from-literal=cluster.name=${CLUSTER_NAME} \ --from-literal=http.server=${FluentBitHttpServer} \ --from-literal=http.port=${FluentBitHttpPort} \ --from-literal=read.head=${FluentBitReadFromHead} \ --from-literal=read.tail=${FluentBitReadFromTail} \ --from-literal=logs.region=${AWS_DEFAULT_REGION} \ -n amazon-cloudwatch

kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml

kubectl get ds,pod,cm,sa -n amazon-cloudwatch

kubectl get cm fluent-bit-cluster-info -n amazon-cloudwatch -o yaml | yh

kubectl describe cm fluent-bit-config -n amazon-cloudwatch

kubectl describe ds fluent-bit -n amazon-cloudwatch

ssh ec2-user@$N1 sudo tree /var/log

kubectl logs deploy/nginx -f

yum install -y httpd

ab -c 500 -n 30000 https://nginx.$MyDomain/

ab -c 500 -n 30000 https://nginx.$MyDomain/notfound

kubectl exec deploy/nginx -c nginx -- kill -s SIGINT 1

kubectl delete ds fluent-bit -n amazon-cloudwatch \ && kubectl delete cm fluent-bit-cluster-info -n amazon-cloudwatch \ && kubectl delete cm fluent-bit-config -n amazon-cloudwatch \ && kubectl delete sa fluent-bit -n amazon-cloudwatch

helm uninstall nginx

aws logs delete-log-group \ --log-group-name /aws/containerinsights/$CLUSTER_NAME/application aws logs delete-log-group \ --log-group-name /aws/containerinsights/$CLUSTER_NAME/dataplane aws logs delete-log-group \ --log-group-name /aws/containerinsights/$CLUSTER_NAME/host