diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 000000000..592659d75 --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.24.5-alpine AS builder + +WORKDIR /src + +COPY go.mod ./ +RUN go mod download + +COPY . . + +RUN CGO_ENABLED=0 go build \ + -trimpath \ + -ldflags="-s -w" \ + -o /quicknotes + +FROM gcr.io/distroless/static:nonroot + +COPY --from=builder /quicknotes /quicknotes +COPY --from=builder /src/seed.json /seed.json + +EXPOSE 8080 + +USER 65532:65532 + +ENTRYPOINT ["/quicknotes"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 000000000..b6352d463 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,51 @@ +services: + quicknotes: + image: quicknotes:lab6 + build: + context: ./app + user: "0:0" + + ports: + - "8080:8080" + + environment: + ADDR: ":8080" + DATA_PATH: "/data/notes.json" + SEED_PATH: "/seed.json" + + volumes: + - quicknotes-data:/data + + restart: unless-stopped + + + prometheus: + image: prom/prometheus:v3.3.1 + + ports: + - "9090:9090" + + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + + depends_on: + - quicknotes + grafana: + image: grafana/grafana:11.6.16 + + ports: + - "3000:3000" + + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: supersecret123 + + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards + + depends_on: + - prometheus + +volumes: + quicknotes-data: \ No newline at end of file diff --git a/docs/runbook/high-error-rate.md b/docs/runbook/high-error-rate.md new file mode 100644 index 000000000..a62e29642 --- /dev/null +++ b/docs/runbook/high-error-rate.md @@ -0,0 +1,16 @@ +# High Error Rate Alert + +## What this alert means +More than 5% of requests to QuickNotes are failing with HTTP 4xx or 5xx statuses over the last 5 minutes. + +## Triage steps +1. Check the Grafana dashboard → Errors panel to confirm the error ratio and see which endpoints are affected. +2. Look at the QuickNotes logs (`docker logs quicknotes`) for stack traces or specific error messages. +3. Verify the health of dependent services (database, external APIs) by checking their connectivity and response times. + +## Mitigations +- **Rollback** the latest deployment if a code change was recently pushed (revert to the previous image tag). +- **Increase logging** and enable debug mode to capture more details, then restart the service to clear any transient state. + +## Post-incident +After resolving, create a postmortem following the [template](https://github.com/your-repo/docs/postmortem-template.md) to document the root cause, impact, and action items. \ No newline at end of file diff --git a/monitoring/grafana/provisioning/dashboards/dashboard.yml b/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 000000000..5c705b1ee --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: default + folder: "" + type: file + + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/datasource.yml b/monitoring/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 000000000..86fd3465e --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 000000000..f83efe3db --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,9 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: quicknotes + + static_configs: + - targets: + - quicknotes:8080 diff --git a/submissions/image-1.png b/submissions/image-1.png new file mode 100644 index 000000000..7bbb02e2c Binary files /dev/null and b/submissions/image-1.png differ diff --git a/submissions/image.png b/submissions/image.png new file mode 100644 index 000000000..3475f88ae Binary files /dev/null and b/submissions/image.png differ diff --git a/submissions/lab8.md b/submissions/lab8.md new file mode 100644 index 000000000..5cf32082a --- /dev/null +++ b/submissions/lab8.md @@ -0,0 +1,552 @@ +# Lab 8 submission + +`dashboard.yml` + +``` +apiVersion: 1 + +providers: + - name: default + folder: "" + type: file + + options: + path: /var/lib/grafana/dashboards +``` + +`datasource.yml` + +``` +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true +``` + +`prometeus.yml` + +``` +global: + scrape_interval: 15s + +scrape_configs: + - job_name: quicknotes + + static_configs: + - targets: + - quicknotes:8080 +``` + +`golden-signals` + +``` +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.16", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(quicknotes_http_requests_total{instance=\"quicknotes:8080\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.16", + "targets": [ + { + "editorMode": "code", + "expr": "rate(quicknotes_http_requests_total[$__rate_interval])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.16", + "targets": [ + { + "editorMode": "code", + "expr": "quicknotes_notes_total", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Saturation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.16", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(quicknotes_http_responses_by_code_total{code=~\"[45]..\"}[$__rate_interval])) / sum(rate(quicknotes_http_requests_total[$__rate_interval]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Errors", + "type": "timeseries" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "New dashboard", + "uid": "", + "version": 0 +} +``` +![alt text](image.png) + +``` +long1tail@JustSomeLaptop ~/DevOps-Intro (feature/lab8)> curl http://localhost:9090/api/v1/targets | jq '.data.activeTargets[].health' + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed +100 612 100 612 0 0 544k 0 --:--:-- --:--:-- --:--:-- 597k +"up" +``` + +- a. Pull means Prometheus must be able to reach QuickNotes (the target must be network-reachable from Prometheus). If Prometheus can't reach QuickNotes, the scrape fails, up becomes 0, and you lose metrics until connectivity is restored. + +- b. Setting 5s creates excessive load on both systems and increases cardinality, while making queries slower; setting 5m reduces resolution, causing you to miss short-lived error spikes and making rate() calculations inaccurate. + +- c. rate() is the right choice for the Traffic panel because it computes the per‑second average increase over the chosen time window, smoothing out bursts and providing a stable view of request flow, unlike irate() which reacts too sharply to single spikes, and delta() which returns total count rather than a rate. + +- d. Provisioning from files ensures dashboards are version‑controlled, reproducible, and automatically applied on every fresh stack start, eliminating manual, error‑prone UI steps and keeping the setup consistent across environments. + + +``` +apiVersion: 1 +groups: + - orgId: 1 + name: main + folder: folder + interval: 1m + rules: + - uid: efqqj408ng0zke + title: Errors + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + expr: (sum(rate(quicknotes_http_requests_total{code=~"4..|5.."}[5m])) / sum(rate(quicknotes_http_requests_total[5m]))) > 0.05 + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.05 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 1m + annotations: {} + labels: {} + isPaused: false + notification_settings: + receiver: grafana-default-email +``` + +![alt text](image-1.png) + +- e. A 5‑minute sustain period prevents false positives caused by transient glitches (e.g., a single misbehaving client, a momentary network hiccup). It confirms that the error condition is persistent and actually affecting users, not just a one‑off spike. + +- f. A cause alert might be: "Database connection pool usage > 90%". +It's worse because high pool usage doesn't necessarily mean users are seeing errors—the app might be handling the load gracefully. On‑call engineers would then have to correlate this metric with user‑visible symptoms. Symptom alerts (like error ratio) directly measure what users experience, so they are more actionable and reduce cognitive overhead during incidents. + +- g. A reasonable rule of thumb: if your alert pages the on‑call team more than 5% of the time when the user was not actually affected (i.e., false‑positive rate > 5%), it's too noisy. In practice, a healthy false‑positive rate should be < 1% to keep on‑call attention sharp and prevent pager fatigue. \ No newline at end of file