loki: add log aggregation on prom + promtail daemonset in home-cluster

- new ansible loki role installs loki 3.5.5 on prometheus_servers as a
  systemd unit (single-binary, filesystem store, 30d retention)
- open tcp/3100 on prom.w5isp.com to 10.0.0.0/16
- grafana: provision loki datasource at http://127.0.0.1:3100
- k8s: promtail daemonset + rbac in monitoring namespace, ships pod logs
  to http://10.0.15.31:3100 with cluster=home-cluster label
- argocd: Application points at home/cluster/promtail

ansible/host_vars/prom.w5isp.com.yml

@@ -33,3 +33,7 @@ firewall_allow_rules:
     proto: tcp
     from_ip: 10.0.0.0/16
     comment: Grafana UI
+  - port: 3100
+    proto: tcp
+    from_ip: 10.0.0.0/16
+    comment: Loki HTTP (push + query)

ansible/playbook.yml

@@ -257,5 +257,7 @@
       tags: alertmanager
     - role: prometheus
       tags: prometheus
+    - role: loki
+      tags: loki
     - role: grafana
       tags: grafana

ansible/roles/grafana/defaults/main.yml

@@ -32,3 +32,7 @@ grafana_datasources:
     url: "http://127.0.0.1:9093"
     jsonData:
       implementation: prometheus
+  - name: Loki
+    type: loki
+    access: proxy
+    url: "http://127.0.0.1:3100"

ansible/roles/loki/defaults/main.yml

@@ -0,0 +1,20 @@
+---
+loki_version: "3.5.5"
+loki_arch: amd64
+
+loki_user: loki
+loki_group: loki
+
+loki_install_dir: /usr/local/bin
+loki_config_dir: /etc/loki
+loki_data_dir: /var/lib/loki
+
+loki_http_listen_port: 3100
+loki_grpc_listen_port: 9095
+
+# Logs older than this are dropped at ingest time, and old chunks are deleted
+# by the compactor.
+loki_retention_period: "720h"  # 30 days
+
+# Alertmanager target for the ruler (same host by default)
+loki_alertmanager_url: "http://127.0.0.1:9093"

ansible/roles/loki/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart loki
+  ansible.builtin.systemd:
+    name: loki
+    state: restarted
+    daemon_reload: true

ansible/roles/loki/tasks/main.yml

@@ -0,0 +1,106 @@
+---
+# Installs Loki from the upstream zip release and runs it under systemd in
+# single-binary mode with filesystem storage.
+
+- name: Install unzip (required to extract loki release)
+  ansible.builtin.apt:
+    name: unzip
+    state: present
+    update_cache: true
+  when: ansible_os_family == "Debian"
+
+- name: Create loki group
+  ansible.builtin.group:
+    name: "{{ loki_group }}"
+    system: true
+    state: present
+
+- name: Create loki user
+  ansible.builtin.user:
+    name: "{{ loki_user }}"
+    group: "{{ loki_group }}"
+    system: true
+    shell: /usr/sbin/nologin
+    home: "{{ loki_data_dir }}"
+    create_home: false
+    state: present
+
+- name: Create loki directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ loki_user }}"
+    group: "{{ loki_group }}"
+    mode: "0755"
+  loop:
+    - "{{ loki_config_dir }}"
+    - "{{ loki_data_dir }}"
+    - "{{ loki_data_dir }}/chunks"
+    - "{{ loki_data_dir }}/rules"
+    - "{{ loki_data_dir }}/compactor"
+
+- name: Check installed loki version
+  ansible.builtin.command: "{{ loki_install_dir }}/loki -version"
+  register: loki_installed
+  changed_when: false
+  failed_when: false
+
+- name: Decide whether loki needs (re)install
+  ansible.builtin.set_fact:
+    loki_needs_install: >-
+      {{
+        loki_installed.rc != 0
+        or loki_version not in (loki_installed.stdout | default(''))
+        and loki_version not in (loki_installed.stderr | default(''))
+      }}
+
+- name: Download loki zip
+  ansible.builtin.get_url:
+    url: "https://github.com/grafana/loki/releases/download/v{{ loki_version }}/loki-linux-{{ loki_arch }}.zip"
+    dest: "/tmp/loki-{{ loki_version }}-linux-{{ loki_arch }}.zip"
+    mode: "0644"
+  when: loki_needs_install
+
+- name: Extract loki
+  ansible.builtin.unarchive:
+    src: "/tmp/loki-{{ loki_version }}-linux-{{ loki_arch }}.zip"
+    dest: /tmp/
+    remote_src: true
+    creates: "/tmp/loki-linux-{{ loki_arch }}"
+  when: loki_needs_install
+
+- name: Install loki binary
+  ansible.builtin.copy:
+    src: "/tmp/loki-linux-{{ loki_arch }}"
+    dest: "{{ loki_install_dir }}/loki"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+  when: loki_needs_install
+  notify: Restart loki
+
+- name: Render loki.yml
+  ansible.builtin.template:
+    src: loki.yml.j2
+    dest: "{{ loki_config_dir }}/loki.yml"
+    owner: "{{ loki_user }}"
+    group: "{{ loki_group }}"
+    mode: "0640"
+  notify: Restart loki
+
+- name: Install loki systemd unit
+  ansible.builtin.template:
+    src: loki.service.j2
+    dest: /etc/systemd/system/loki.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart loki
+
+- name: Enable and start loki
+  ansible.builtin.systemd:
+    name: loki
+    enabled: true
+    state: started
+    daemon_reload: true

ansible/roles/loki/templates/loki.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Loki log aggregation server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ loki_user }}
+Group={{ loki_group }}
+ExecStart={{ loki_install_dir }}/loki \
+  -config.file={{ loki_config_dir }}/loki.yml
+
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths={{ loki_data_dir }}
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/loki/templates/loki.yml.j2

@@ -0,0 +1,58 @@
+{{ ansible_managed | comment }}
+# Single-binary Loki with filesystem storage. Suitable for a single-node
+# log aggregation target (no clustering, no object storage).
+
+auth_enabled: false
+
+server:
+  http_listen_address: 0.0.0.0
+  http_listen_port: {{ loki_http_listen_port }}
+  grpc_listen_address: 127.0.0.1
+  grpc_listen_port: {{ loki_grpc_listen_port }}
+  log_level: info
+
+common:
+  path_prefix: {{ loki_data_dir }}
+  storage:
+    filesystem:
+      chunks_directory: {{ loki_data_dir }}/chunks
+      rules_directory: {{ loki_data_dir }}/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+  instance_addr: 127.0.0.1
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: {{ loki_retention_period }}
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: true
+  volume_enabled: true
+
+compactor:
+  working_directory: {{ loki_data_dir }}/compactor
+  retention_enabled: true
+  retention_delete_delay: 2h
+  delete_request_store: filesystem
+
+ruler:
+  alertmanager_url: {{ loki_alertmanager_url }}
+  storage:
+    type: local
+    local:
+      directory: {{ loki_data_dir }}/rules
+  rule_path: {{ loki_data_dir }}/rules-tmp
+
+analytics:
+  reporting_enabled: false

home/cluster/argocd/apps/promtail.yaml

@@ -0,0 +1,23 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: promtail
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://codeberg.org/gmcintire/infra.git
+    targetRevision: main
+    path: home/cluster/promtail
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

home/cluster/promtail/configmap.yaml

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: promtail
+  namespace: monitoring
+data:
+  promtail.yaml: |
+    server:
+      http_listen_port: 3101
+      grpc_listen_port: 0
+      log_level: info
+
+    positions:
+      filename: /run/promtail/positions.yaml
+
+    clients:
+      - url: http://10.0.15.31:3100/loki/api/v1/push
+        backoff_config:
+          min_period: 500ms
+          max_period: 5m
+          max_retries: 10
+        external_labels:
+          cluster: home-cluster
+
+    scrape_configs:
+      # Discover and scrape logs for every pod via the kubelet log directory.
+      - job_name: kubernetes-pods
+        kubernetes_sd_configs:
+          - role: pod
+        pipeline_stages:
+          - cri: {}
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: __host__
+          - source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - source_labels: [__meta_kubernetes_pod_container_name]
+            target_label: container
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+            target_label: app
+          - source_labels: [__meta_kubernetes_pod_label_app]
+            target_label: app
+            regex: (.+)
+          - action: replace
+            replacement: /var/log/pods/*$1/*.log
+            separator: /
+            source_labels:
+              - __meta_kubernetes_pod_uid
+              - __meta_kubernetes_pod_container_name
+            target_label: __path__
+          # Drop logs from this promtail daemonset itself
+          - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app]
+            regex: monitoring;promtail
+            action: drop

home/cluster/promtail/daemonset.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: promtail
+  namespace: monitoring
+  labels:
+    app: promtail
+spec:
+  selector:
+    matchLabels:
+      app: promtail
+  template:
+    metadata:
+      labels:
+        app: promtail
+    spec:
+      serviceAccountName: promtail
+      tolerations:
+        - operator: Exists
+      containers:
+        - name: promtail
+          image: grafana/promtail:3.5.5
+          args:
+            - -config.file=/etc/promtail/promtail.yaml
+          ports:
+            - name: http-metrics
+              containerPort: 3101
+          env:
+            - name: HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 0
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          volumeMounts:
+            - name: config
+              mountPath: /etc/promtail
+            - name: run
+              mountPath: /run/promtail
+            - name: varlog
+              mountPath: /var/log
+              readOnly: true
+            - name: containers
+              mountPath: /var/lib/containerd
+              readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: promtail
+        - name: run
+          hostPath:
+            path: /run/promtail
+            type: DirectoryOrCreate
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: containers
+          hostPath:
+            path: /var/lib/containerd

home/cluster/promtail/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - rbac.yaml
+  - configmap.yaml
+  - daemonset.yaml

home/cluster/promtail/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

home/cluster/promtail/rbac.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: promtail
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: promtail
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - nodes/proxy
+      - services
+      - endpoints
+      - pods
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: promtail
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: promtail
+subjects:
+  - kind: ServiceAccount
+    name: promtail
+    namespace: monitoring