Deploying Prometheus + Grafana with Docker

Resource Inventory##

Host	IP
Prometheus + Grafana	10.0.0.1

Software	Version
docker	20.10.12
docker-compose	1.23.1
Prometheus	2.18.1
Grafana	7.2.1
Pushgateway	1.4.2
Alertmanager	0.11.0
node-exporter	1.3.1

I. Installing `Docker`##

1. Use a domestic `yum` repo##

1
# yum install -y yum-utils device-mapper-persistent-data lvm2
2
# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

2. Remove old versions of `docker`##

1
## If docker is already installed on the host and it's not the desired version, remove it first.
2
# yum remove -y docker \
3
              docker-client \
4
              docker-client-latest \
5
              docker-common \
6
              docker-latest \
7
              docker-latest-logrotate \
8
              docker-logrotate \
9
              docker-selinux \
10
              docker-engine-selinux \
11
              docker-engine \
12
              container*

3. Install `Docker 20.10`##

1
# yum -y install docker-ce-20.10.12-3.el7 docker-ce-cli-20.10.12-3.el7

4. Configure registry mirror##

1
# mkdir /etc/docker
2
# vi /etc/docker/daemon.json
3

4
{
5
  "registry-mirrors": ["https://xxxxxxxxx.mirror.aliyuncs.com"]
6
}

5. Start `docker`##

1
# systemctl start docker
2
# systemctl enable docker
3
# systemctl status docker

II. Installing `Docker-compose`##

1. Install `Docker-compose`##

1
## github.com may time out; you can use the following URL, download it locally, then upload it to the server
2
# curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
3

4
# curl -k "https://dl.cactifans.com/zabbix_docker/docker-compose" -o /usr/bin/docker-compose
5

6
# chmod a+x /usr/bin/docker-compose

2. Check `docker-compose` version##

1
# docker-compose version

III. Install `prometheus + grafana + pushgateway`##

1. Get `docker-compose.yaml`##

1
## If you use the open source project below, you can skip this step and just use the docker-compose.yaml file provided there
2
# cd /opt/
3
# git clone https://github.com/evnsio/prom-stack.git
4
# cd prom-stack

2. Edit the `docker-compose.yml` file##

1
version: "3"
2
services:
3

4
  pushgateway:
5
    #image: prom/pushgateway:v0.4.0
6
    image: prom/pushgateway:v1.4.2
7
    container_name: pushgateway
8
    command: --persistence.file=/pushgateway/pushgateway.data
9
    ports:
10
      - 9091:9091
11
    volumes:
12
      - /data/pushgateway:/pushgateway
13

14
  prometheus:
15
    #image: prom/prometheus:v2.0.0
16
    image: prom/prometheus:v2.18.1
17
    container_name: prometheus
18
    command: --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/prometheus --storage.tsdb.retention=30d
19
    user: 0:0
20
    ports:
21
      - 9090:9090
22
    volumes:
23
      - /data/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
24
      - /data/prometheus/alerts/:/etc/prometheus/rules.d/
25
      - /data/prometheus/data/:/prometheus
26

27
  grafana:
28
    #image: grafana/grafana:4.6.2
29
    image: grafana/grafana:7.2.1
30
    container_name: grafana
31
    ports:
32
      - 3000:3000
33
    volumes:
34
      - /data/grafana:/var/lib/grafana
35
    environment:
36
      - GF_SECURITY_ADMIN_PASSWORD=admin
37

38
  alertmanager:
39
    image: prom/alertmanager:v0.11.0
40
    container_name: alertmanager
41
    command: --config.file=/etc/alertmanager/alertmanager.yml
42
    ports:
43
      - 9093:9093
44
    volumes:
45
      - /data/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
46

47
###############node-exporter###############
48
  node-exporter:
49
    image: "prom/node-exporter:v1.3.1"
50
    #hostname: node-exporter
51
    container_name: node-exporter
52
    ports:
53
      - '9100:9100'
54
    volumes:
55
      - /etc/localtime:/etc/localtime:ro
56
      - /proc:/host/proc:ro
57
      - /sys:/host/sys:ro
58
      - /:/rootfs:ro
59
      - /etc/hostname:/etc/hostname:ro
60
    restart: always
61
    network_mode: host
62
    command:
63
      - '--path.procfs=/host/proc'
64
      - '--path.sysfs=/host/sys'
65
      - '--path.rootfs=/rootfs'

3. Create mount directories for services##

1
# mkdir -pv /data/{pushgateway,prometheus,alertmanager,grafana}
2

3
# mkdir /data/prometheus/{alerts,data}
4

5
## Set file permissions for Grafana persistent storage
6
# chown 472:472 -R /data/grafana/

4. Edit configuration files##

a | `prometheus.yml`##

1
# vim /data/prometheus/prometheus.yml
2

3
# my global config
4
global:
5
  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
6
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
7
  # scrape_timeout is set to the global default (10s).
8

9
  # Attach these labels to any time series or alerts when communicating with
10
  # external systems (federation, remote storage, Alertmanager).
11
  external_labels:
12
      monitor: 'edocyun-prom-stack'
13

14
# Alertmanager configuration
15
alerting:
16
  alertmanagers:
17
  - static_configs:
18
    - targets:
19
      - alertmanager:9093
20

21
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
22
rule_files:
23
  # - "first_rules.yml"
24
  # - "second_rules.yml"
25

26
scrape_configs:
27
  - job_name: 'prom-stack'
28
    static_configs:
29
      - targets:
30
        - prometheus:9090
31
        - pushgateway:9091
32
        - alertmanager:9093
33
        - grafana:3000
34

35
  - job_name: 'node-exporter'
36
    static_configs:
37
      - targets:
38
        - 10.0.0.1:9100  ## The host needs to have the node-exporter service deployed

b | `alertmanager.yml`##

1
# vim  /data/alertmanager/alertmanager.yml
2

3
global:
4
  smtp_smarthost: 'localhost:25'
5
  smtp_from: '[email protected]'
6
  smtp_auth_username: 'alertmanager'
7
  smtp_auth_password: 'password'
8

9
route:
10
  group_by: ['cluster']
11
  receiver: team-a
12

13
receivers:
14
- name: 'team-a'
15
  email_configs:
16
  - to: '[email protected]'

5. Start the services##

1
# docker-compose -f docker-compose.yml pull
2
# docker-compose -f docker-compose.yml up -d

Log in to Grafana with the default credentials: username admin, password admin.

a | Check service status##

1
# docker-compse ps
2

3
# docker ps -a

6. Use `curl` to generate test data##

You can use $RANDOM to generate random numbers

1
# while true; do
2
    echo "mymetric $RANDOM" | curl --data-binary @- http://localhost:9091/metrics/job/my-push-job
3
    sleep 1
4
done

“mymetric” is the metric being monitored, and pushing this metric to Pushgateway is a simple and intuitive way to test.

Open Pushgateway in your browser: “http://10.0.0.1:9091/”

You should see that the test data has already been sent into Pushgateway.

Deploying Prometheus + Grafana with Docker

Resource Inventory##

I. Installing Docker##

1. Use a domestic yum repo##

2. Remove old versions of docker##

3. Install Docker 20.10##

4. Configure registry mirror##

5. Start docker##

II. Installing Docker-compose##

1. Install Docker-compose##

2. Check docker-compose version##

III. Install prometheus + grafana + pushgateway##

1. Get docker-compose.yaml##

2. Edit the docker-compose.yml file##

3. Create mount directories for services##

4. Edit configuration files##

a | prometheus.yml##

b | alertmanager.yml##

5. Start the services##

a | Check service status##

6. Use curl to generate test data##

I. Installing `Docker`##

1. Use a domestic `yum` repo##

2. Remove old versions of `docker`##

3. Install `Docker 20.10`##

5. Start `docker`##

II. Installing `Docker-compose`##

1. Install `Docker-compose`##

2. Check `docker-compose` version##

III. Install `prometheus + grafana + pushgateway`##

1. Get `docker-compose.yaml`##

2. Edit the `docker-compose.yml` file##

a | `prometheus.yml`##

b | `alertmanager.yml`##

6. Use `curl` to generate test data##