Prometheus Metrics and AWS CloudWatch

Introduction

Example

$ git clone https://github.com/sudhindra-hub/metrics-demo.git
$ cd prometheus/java
$ make package
$ make docker
docker build -t prometheus/metrics-demo:latest . -f Dockerfile
Sending build context to Docker daemon 3.866MB
Step 1/5 : FROM java:8-jdk-alpine
---> 3fd9dd82815c
Step 2/5 : MAINTAINER sudhip1234@yahoo.com
---> Using cache
---> 00bf9f90704d
Step 3/5 : COPY target/metrics-demo-1.0-SNAPSHOT.jar .
---> Using cache
---> bce6744988b4
Step 4/5 : WORKDIR /
---> Using cache
---> 2739b1f19267
Step 5/5 : ENTRYPOINT ["java", "-jar", "metrics-demo-1.0-SNAPSHOT.jar"]
---> Using cache
---> 241f44d5b55e
Successfully built 241f44d5b55e
Successfully tagged prometheus/metrics-demo:latest
$ docker images | grep metrics-demo
prometheus/metrics-demo latest 241f44d5b55e 5 minutes ago 149MB
$ cd ../kubernetes$ kubectl create namespace metrics-demo                                                                      
namespace/metrics-demo created
$ kubectl apply -f pod_metrics.yaml
pod/metrics-pod created
$ kubectl get pods -n metrics-demo
NAME READY STATUS RESTARTS AGE
metrics-pod 1/1 Running 0 11s
$ kubectl --namespace metrics-demo port-forward metrics-pod 8082                                             
Forwarding from 127.0.0.1:8082 -> 8082
Forwarding from [::1]:8082 -> 8082
$ cat prometheus-k8s.yaml# create amazon-cloudwatch namespace
apiVersion: v1
kind: Namespace
metadata:
name: amazon-cloudwatch
labels:
name: amazon-cloudwatch
---
# create amazon-cloudwatch namespace
apiVersion: v1
kind: Namespace
metadata:
name: amazon-cloudwatch
labels:
name: amazon-cloudwatch
---
# create configmap for prometheus cwagent config
apiVersion: v1
data:
# cwagent json config
cwagentconfig.json: |
{
"agent": {
"region": "us-west-2"
},
"logs": {
"metrics_collected": {
"prometheus": {
"cluster_name": "monitoring",
"prometheus_config_path": "/etc/prometheusconfig/prometheus.yaml",
"emf_processor": {
"metric_declaration_dedup": true,
"metric_declaration": [
{
"source_labels": ["job"],
"label_matcher": "^metrics-demo$",
"dimensions": [["ClusterName","Namespace"]],
"metric_selectors": [
"^num_cars$",
"^speed_cars$"
]
}
]
}
}
},
"force_flush_interval": 5
}
}
kind: ConfigMap
metadata:
name: prometheus-cwagentconfig
namespace: amazon-cloudwatch
---
# create configmap for prometheus cwagent config
apiVersion: v1
data:
# cwagent json config
credentials: |
[AmazonCloudWatchAgent]
aws_access_key_id=****
aws_secret_access_key=****
region = us-west-2
kind: ConfigMap
metadata:
name: aws-credentials
namespace: amazon-cloudwatch
---
# create configmap for prometheus scrape config
apiVersion: v1
data:
# prometheus config
prometheus.yaml: |
global:
scrape_interval: 1m
scrape_timeout: 10s
scrape_configs:
- job_name: 'metrics-demo'
sample_limit: 10000
metrics_path: /metrics
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__address__]
action: keep
regex: '.*:8082$'
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: Namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: pod_name
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container_name
- action: replace
source_labels:
- __meta_kubernetes_pod_controller_name
target_label: pod_controller_name
- action: replace
source_labels:
- __meta_kubernetes_pod_controller_kind
target_label: pod_controller_kind
- action: replace
source_labels:
- __meta_kubernetes_pod_phase
target_label: pod_phase
kind: ConfigMap
metadata:
name: prometheus-config
namespace: amazon-cloudwatch
---
# create cwagent service account and role binding
apiVersion: v1
kind: ServiceAccount
metadata:
name: cwagent-prometheus
namespace: amazon-cloudwatch
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cwagent-prometheus-role
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups:
- extensions
resources:
- ingresses
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cwagent-prometheus-role-binding
subjects:
- kind: ServiceAccount
name: cwagent-prometheus
namespace: amazon-cloudwatch
roleRef:
kind: ClusterRole
name: cwagent-prometheus-role
apiGroup: rbac.authorization.k8s.io
---
# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
name: cwagent-prometheus
namespace: amazon-cloudwatch
spec:
replicas: 1
selector:
matchLabels:
app: cwagent-prometheus
template:
metadata:
labels:
app: cwagent-prometheus
spec:
containers:
- name: cloudwatch-agent
image: amazon/cloudwatch-agent:1.243835.0-prometheus-beta
imagePullPolicy: Always
resources:
limits:
cpu: 1000m
memory: 1000Mi
requests:
cpu: 200m
memory: 200Mi
# Please don't change below envs
env:
- name: CI_VERSION
value: "k8s/1.1.0-prometheus-beta"
# Please don't change the mountPath
volumeMounts:
- name: prometheus-cwagentconfig
mountPath: /etc/cwagentconfig
- name: prometheus-config
mountPath: /etc/prometheusconfig
- name: aws-credentials
mountPath: /root/.aws
volumes:
- name: prometheus-cwagentconfig
configMap:
name: prometheus-cwagentconfig
- name: prometheus-config
configMap:
name: prometheus-config
- name: aws-credentials
configMap:
name: aws-credentials
terminationGracePeriodSeconds: 60
serviceAccountName: cwagent-prometheus
$ kubectl apply -f prometheus-k8s.yaml                                                                       
namespace/amazon-cloudwatch created
configmap/prometheus-cwagentconfig created
configmap/aws-credentials created
configmap/prometheus-config created
serviceaccount/cwagent-prometheus created
clusterrole.rbac.authorization.k8s.io/cwagent-prometheus-role created
clusterrolebinding.rbac.authorization.k8s.io/cwagent-prometheus-role-binding created
deployment.apps/cwagent-prometheus created
$ kubectl get pod -l "app=cwagent-prometheus" -n amazon-cloudwatch                                                                        
NAME READY STATUS RESTARTS AGE
cwagent-prometheus-5bd976c894-4dpch 1/1 Running 0 27s
$ kubectl logs -l "app=cwagent-prometheus" -n amazon-cloudwatch --tail=-1
...
2020/07/20 02:33:19 Reading json config file path: /opt/aws/amazon-cloudwatch-agent/bin/default_linux_config.json ...
/opt/aws/amazon-cloudwatch-agent/bin/default_linux_config.json does not exist or cannot read. Skipping it.
2020/07/20 02:33:19 Reading json config file path: /etc/cwagentconfig/..2020_07_20_02_33_10.801334158/cwagentconfig.json ...
2020/07/20 02:33:19 Find symbolic link /etc/cwagentconfig/..data
2020/07/20 02:33:19 Find symbolic link /etc/cwagentconfig/cwagentconfig.json
2020/07/20 02:33:19 Reading json config file path: /etc/cwagentconfig/cwagentconfig.json ...
Valid Json input schema.
Got Home directory: /root
No csm configuration found.
No metric configuration found.
Configuration validation first phase succeeded
2020/07/20 02:33:19 I! Config has been translated into TOML /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.toml
2020/07/20 02:33:19 I! AmazonCloudWatchAgent Version 1.243835.0.
...
{
"source_labels": ["job"],
"label_matcher": "^metrics-demo$",
"dimensions": [["ClusterName","Namespace"]],
"metric_selectors": [
"^num_cars$",
"^speed_cars$"
]
}
"prometheus": {
"cluster_name": "monitoring",
scrape_configs:
- job_name: 'metrics-demo'
- source_labels: [__address__]
action: keep
regex: '.*:8082$'

CloudWatch

Dashboards

$ wget https://raw.githubusercontent.com/sudhindra-hub/metrics-demo/master/cloudwatch/dashboard.json$ content=$(cat dashboard.json)$ aws cloudwatch put-dashboard --dashboard-name MetricDemoApp --dashboard-body $content
$ while :; do; curl -s "http://localhost:8082/cars"; sleep 5; done

Conclusion

Passionate about building products/technologies that delight customers and solving business problems. Focus on excellence of delivered solutions.