diff --git a/ansible/host_vars/prom.w5isp.com.yml b/ansible/host_vars/prom.w5isp.com.yml index da72880..119bbb1 100644 --- a/ansible/host_vars/prom.w5isp.com.yml +++ b/ansible/host_vars/prom.w5isp.com.yml @@ -33,3 +33,7 @@ firewall_allow_rules: proto: tcp from_ip: 10.0.0.0/16 comment: Grafana UI + - port: 3100 + proto: tcp + from_ip: 10.0.0.0/16 + comment: Loki HTTP (push + query) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index b33082b..b8f6a98 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -257,5 +257,7 @@ tags: alertmanager - role: prometheus tags: prometheus + - role: loki + tags: loki - role: grafana tags: grafana diff --git a/ansible/roles/grafana/defaults/main.yml b/ansible/roles/grafana/defaults/main.yml index 3fc315b..27368ec 100644 --- a/ansible/roles/grafana/defaults/main.yml +++ b/ansible/roles/grafana/defaults/main.yml @@ -32,3 +32,7 @@ grafana_datasources: url: "http://127.0.0.1:9093" jsonData: implementation: prometheus + - name: Loki + type: loki + access: proxy + url: "http://127.0.0.1:3100" diff --git a/ansible/roles/loki/defaults/main.yml b/ansible/roles/loki/defaults/main.yml new file mode 100644 index 0000000..b1052aa --- /dev/null +++ b/ansible/roles/loki/defaults/main.yml @@ -0,0 +1,20 @@ +--- +loki_version: "3.5.5" +loki_arch: amd64 + +loki_user: loki +loki_group: loki + +loki_install_dir: /usr/local/bin +loki_config_dir: /etc/loki +loki_data_dir: /var/lib/loki + +loki_http_listen_port: 3100 +loki_grpc_listen_port: 9095 + +# Logs older than this are dropped at ingest time, and old chunks are deleted +# by the compactor. +loki_retention_period: "720h" # 30 days + +# Alertmanager target for the ruler (same host by default) +loki_alertmanager_url: "http://127.0.0.1:9093" diff --git a/ansible/roles/loki/handlers/main.yml b/ansible/roles/loki/handlers/main.yml new file mode 100644 index 0000000..57de0c9 --- /dev/null +++ b/ansible/roles/loki/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart loki + ansible.builtin.systemd: + name: loki + state: restarted + daemon_reload: true diff --git a/ansible/roles/loki/tasks/main.yml b/ansible/roles/loki/tasks/main.yml new file mode 100644 index 0000000..38df20f --- /dev/null +++ b/ansible/roles/loki/tasks/main.yml @@ -0,0 +1,106 @@ +--- +# Installs Loki from the upstream zip release and runs it under systemd in +# single-binary mode with filesystem storage. + +- name: Install unzip (required to extract loki release) + ansible.builtin.apt: + name: unzip + state: present + update_cache: true + when: ansible_os_family == "Debian" + +- name: Create loki group + ansible.builtin.group: + name: "{{ loki_group }}" + system: true + state: present + +- name: Create loki user + ansible.builtin.user: + name: "{{ loki_user }}" + group: "{{ loki_group }}" + system: true + shell: /usr/sbin/nologin + home: "{{ loki_data_dir }}" + create_home: false + state: present + +- name: Create loki directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: "{{ loki_user }}" + group: "{{ loki_group }}" + mode: "0755" + loop: + - "{{ loki_config_dir }}" + - "{{ loki_data_dir }}" + - "{{ loki_data_dir }}/chunks" + - "{{ loki_data_dir }}/rules" + - "{{ loki_data_dir }}/compactor" + +- name: Check installed loki version + ansible.builtin.command: "{{ loki_install_dir }}/loki -version" + register: loki_installed + changed_when: false + failed_when: false + +- name: Decide whether loki needs (re)install + ansible.builtin.set_fact: + loki_needs_install: >- + {{ + loki_installed.rc != 0 + or loki_version not in (loki_installed.stdout | default('')) + and loki_version not in (loki_installed.stderr | default('')) + }} + +- name: Download loki zip + ansible.builtin.get_url: + url: "https://github.com/grafana/loki/releases/download/v{{ loki_version }}/loki-linux-{{ loki_arch }}.zip" + dest: "/tmp/loki-{{ loki_version }}-linux-{{ loki_arch }}.zip" + mode: "0644" + when: loki_needs_install + +- name: Extract loki + ansible.builtin.unarchive: + src: "/tmp/loki-{{ loki_version }}-linux-{{ loki_arch }}.zip" + dest: /tmp/ + remote_src: true + creates: "/tmp/loki-linux-{{ loki_arch }}" + when: loki_needs_install + +- name: Install loki binary + ansible.builtin.copy: + src: "/tmp/loki-linux-{{ loki_arch }}" + dest: "{{ loki_install_dir }}/loki" + remote_src: true + owner: root + group: root + mode: "0755" + when: loki_needs_install + notify: Restart loki + +- name: Render loki.yml + ansible.builtin.template: + src: loki.yml.j2 + dest: "{{ loki_config_dir }}/loki.yml" + owner: "{{ loki_user }}" + group: "{{ loki_group }}" + mode: "0640" + notify: Restart loki + +- name: Install loki systemd unit + ansible.builtin.template: + src: loki.service.j2 + dest: /etc/systemd/system/loki.service + owner: root + group: root + mode: "0644" + notify: Restart loki + +- name: Enable and start loki + ansible.builtin.systemd: + name: loki + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/loki/templates/loki.service.j2 b/ansible/roles/loki/templates/loki.service.j2 new file mode 100644 index 0000000..e491ea3 --- /dev/null +++ b/ansible/roles/loki/templates/loki.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=Loki log aggregation server +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User={{ loki_user }} +Group={{ loki_group }} +ExecStart={{ loki_install_dir }}/loki \ + -config.file={{ loki_config_dir }}/loki.yml + +Restart=on-failure +RestartSec=5 +NoNewPrivileges=true +ProtectHome=true +ProtectSystem=full +ReadWritePaths={{ loki_data_dir }} + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/loki/templates/loki.yml.j2 b/ansible/roles/loki/templates/loki.yml.j2 new file mode 100644 index 0000000..05133cd --- /dev/null +++ b/ansible/roles/loki/templates/loki.yml.j2 @@ -0,0 +1,58 @@ +{{ ansible_managed | comment }} +# Single-binary Loki with filesystem storage. Suitable for a single-node +# log aggregation target (no clustering, no object storage). + +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: {{ loki_http_listen_port }} + grpc_listen_address: 127.0.0.1 + grpc_listen_port: {{ loki_grpc_listen_port }} + log_level: info + +common: + path_prefix: {{ loki_data_dir }} + storage: + filesystem: + chunks_directory: {{ loki_data_dir }}/chunks + rules_directory: {{ loki_data_dir }}/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + instance_addr: 127.0.0.1 + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: {{ loki_retention_period }} + reject_old_samples: true + reject_old_samples_max_age: 168h + allow_structured_metadata: true + volume_enabled: true + +compactor: + working_directory: {{ loki_data_dir }}/compactor + retention_enabled: true + retention_delete_delay: 2h + delete_request_store: filesystem + +ruler: + alertmanager_url: {{ loki_alertmanager_url }} + storage: + type: local + local: + directory: {{ loki_data_dir }}/rules + rule_path: {{ loki_data_dir }}/rules-tmp + +analytics: + reporting_enabled: false diff --git a/home/cluster/argocd/apps/promtail.yaml b/home/cluster/argocd/apps/promtail.yaml new file mode 100644 index 0000000..cd05a7b --- /dev/null +++ b/home/cluster/argocd/apps/promtail.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: promtail + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: default + source: + repoURL: https://codeberg.org/gmcintire/infra.git + targetRevision: main + path: home/cluster/promtail + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/home/cluster/promtail/configmap.yaml b/home/cluster/promtail/configmap.yaml new file mode 100644 index 0000000..e504135 --- /dev/null +++ b/home/cluster/promtail/configmap.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: promtail + namespace: monitoring +data: + promtail.yaml: | + server: + http_listen_port: 3101 + grpc_listen_port: 0 + log_level: info + + positions: + filename: /run/promtail/positions.yaml + + clients: + - url: http://10.0.15.31:3100/loki/api/v1/push + backoff_config: + min_period: 500ms + max_period: 5m + max_retries: 10 + external_labels: + cluster: home-cluster + + scrape_configs: + # Discover and scrape logs for every pod via the kubelet log directory. + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + pipeline_stages: + - cri: {} + relabel_configs: + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: __host__ + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + target_label: app + - source_labels: [__meta_kubernetes_pod_label_app] + target_label: app + regex: (.+) + - action: replace + replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + # Drop logs from this promtail daemonset itself + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app] + regex: monitoring;promtail + action: drop diff --git a/home/cluster/promtail/daemonset.yaml b/home/cluster/promtail/daemonset.yaml new file mode 100644 index 0000000..77cd5b0 --- /dev/null +++ b/home/cluster/promtail/daemonset.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: promtail + namespace: monitoring + labels: + app: promtail +spec: + selector: + matchLabels: + app: promtail + template: + metadata: + labels: + app: promtail + spec: + serviceAccountName: promtail + tolerations: + - operator: Exists + containers: + - name: promtail + image: grafana/promtail:3.5.5 + args: + - -config.file=/etc/promtail/promtail.yaml + ports: + - name: http-metrics + containerPort: 3101 + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + readOnlyRootFilesystem: true + runAsUser: 0 + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + volumeMounts: + - name: config + mountPath: /etc/promtail + - name: run + mountPath: /run/promtail + - name: varlog + mountPath: /var/log + readOnly: true + - name: containers + mountPath: /var/lib/containerd + readOnly: true + volumes: + - name: config + configMap: + name: promtail + - name: run + hostPath: + path: /run/promtail + type: DirectoryOrCreate + - name: varlog + hostPath: + path: /var/log + - name: containers + hostPath: + path: /var/lib/containerd diff --git a/home/cluster/promtail/kustomization.yaml b/home/cluster/promtail/kustomization.yaml new file mode 100644 index 0000000..17cd3c4 --- /dev/null +++ b/home/cluster/promtail/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - rbac.yaml + - configmap.yaml + - daemonset.yaml diff --git a/home/cluster/promtail/namespace.yaml b/home/cluster/promtail/namespace.yaml new file mode 100644 index 0000000..d325236 --- /dev/null +++ b/home/cluster/promtail/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring diff --git a/home/cluster/promtail/rbac.yaml b/home/cluster/promtail/rbac.yaml new file mode 100644 index 0000000..7aaf800 --- /dev/null +++ b/home/cluster/promtail/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: promtail + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: promtail +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: promtail +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: promtail +subjects: + - kind: ServiceAccount + name: promtail + namespace: monitoring