From e3b48757ac3e02415b86797b7639d82e9d24108c Mon Sep 17 00:00:00 2001 From: Magik <30931287+unseenmagik@users.noreply.github.com> Date: Wed, 29 Apr 2026 20:28:35 +0100 Subject: [PATCH 1/4] Add overpass user_agent config to identify importer traffic Overpass enforces a User-Agent on incoming requests; missing/generic agents get rate limited or rejected. Plumb a configurable user_agent through overpass.Config into the client so requests carry a contact string. Co-Authored-By: Claude Opus 4.7 (1M context) --- app_config/config.go | 6 ++++-- bin/fletchling-osm-importer/fletchling-osm-importer.go | 2 +- configs/fletchling.toml.example | 3 +++ overpass/client.go | 7 ++++++- overpass/config.go | 3 ++- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/app_config/config.go b/app_config/config.go index 2d1c22e..5ddfa3d 100644 --- a/app_config/config.go +++ b/app_config/config.go @@ -28,7 +28,8 @@ import ( ) const ( - DEFAULT_OVERPASS_URL = "https://overpass-api.de/api/interpreter" + DEFAULT_OVERPASS_URL = "https://overpass-api.de/api/interpreter" + DEFAULT_OVERPASS_USER_AGENT = "Fletchling MAGI importer contact: your-email@example.com" DEFAULT_NEST_NAME = "Unknown Nest" ) @@ -151,7 +152,8 @@ func GetDefaultConfig() Config { Processor: processor.GetDefaultConfig(), Overpass: overpass.Config{ - Url: DEFAULT_OVERPASS_URL, + Url: DEFAULT_OVERPASS_URL, + UserAgent: DEFAULT_OVERPASS_USER_AGENT, }, Importer: importer.Config{ diff --git a/bin/fletchling-osm-importer/fletchling-osm-importer.go b/bin/fletchling-osm-importer/fletchling-osm-importer.go index 7984948..44d401b 100644 --- a/bin/fletchling-osm-importer/fletchling-osm-importer.go +++ b/bin/fletchling-osm-importer/fletchling-osm-importer.go @@ -247,7 +247,7 @@ func main() { continue } - overpassCli, err := overpass.NewClient(logger, cfg.Overpass.Url) + overpassCli, err := overpass.NewClient(logger, cfg.Overpass.Url, cfg.Overpass.UserAgent) if err != nil { logger.Fatalf("failed to create overpass client for area %s: %v", areaName, err) } diff --git a/configs/fletchling.toml.example b/configs/fletchling.toml.example index 11f8283..4ad6a9c 100644 --- a/configs/fletchling.toml.example +++ b/configs/fletchling.toml.example @@ -145,3 +145,6 @@ no_nesting_pokemon_age_hours = 12 # In case there's a need to change this: [overpass] #url = "https://overpass-api.de/api/interpreter" +## User-Agent header sent with overpass requests. Replace the email with your own +## contact address so the overpass operators can reach you if there are issues. +user_agent = "Fletchling USER importer contact: overpass@mymail.fyi" diff --git a/overpass/client.go b/overpass/client.go index 57f5078..1570ae3 100644 --- a/overpass/client.go +++ b/overpass/client.go @@ -32,6 +32,7 @@ var ( type Client struct { logger *logrus.Logger apiUrl string + userAgent string httpClient *http.Client } @@ -42,6 +43,9 @@ func (cli *Client) doSingleQuery(ctx context.Context, v url.Values) (*osm.OSM, e } req = req.WithContext(ctx) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + if cli.userAgent != "" { + req.Header.Set("User-Agent", cli.userAgent) + } resp, err := cli.httpClient.Do(req) if err != nil { @@ -120,7 +124,7 @@ func (cli *Client) GetPossibleNestLocations(ctx context.Context, bound orb.Bound } } -func NewClient(logger *logrus.Logger, apiUrl string) (*Client, error) { +func NewClient(logger *logrus.Logger, apiUrl string, userAgent string) (*Client, error) { if logger == nil { return nil, errors.New("No logger given") } @@ -130,6 +134,7 @@ func NewClient(logger *logrus.Logger, apiUrl string) (*Client, error) { return &Client{ logger: logger, apiUrl: apiUrl, + userAgent: userAgent, httpClient: &http.Client{}, }, nil } diff --git a/overpass/config.go b/overpass/config.go index 2e15e7f..6372eec 100644 --- a/overpass/config.go +++ b/overpass/config.go @@ -3,7 +3,8 @@ package overpass import "errors" type Config struct { - Url string `koanf:"url" json:"url"` + Url string `koanf:"url"` + UserAgent string `koanf:"user_agent"` } func (cfg *Config) Validate() error { From c3604c38e4b8950e1c8afec826fe6d15c05e22c0 Mon Sep 17 00:00:00 2001 From: Magik <30931287+unseenmagik@users.noreply.github.com> Date: Wed, 29 Apr 2026 21:11:57 +0100 Subject: [PATCH 2/4] Add Grafana dashboard and Prometheus setup docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship an importable Grafana dashboard covering all exposed Fletchling metrics (webhook counters, gin HTTP, Go runtime, process), and document how to wire up Prometheus scraping via the Zapdos stack — including the required [prometheus] enabled = true opt-in. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 34 ++ grafana/fletchling-dashboard.json | 721 ++++++++++++++++++++++++++++++ 2 files changed, 755 insertions(+) create mode 100644 grafana/fletchling-dashboard.json diff --git a/README.md b/README.md index 0ea9164..75a0180 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,40 @@ You will need to have at least golang 1.22.1 installed. It is rather new as of t 2. Check the logs in logs/ (or the log_dir that you configured in fletchling.toml) for errors. * Every minute, a log message will appear saying how many pokemon were processed. If this is 0, it means that Fletchling is not getting any webhooks. Check your Golbat webhooks configuration. Check the address Fletchling is listening on (http section in config). +# Prometheus / Grafana + +Fletchling exposes Prometheus metrics (webhook processing counters, HTTP stats, Go runtime, and process metrics) on the same listen address as the API, at `/metrics`. The endpoint is **disabled by default** — you must opt in. + +## 1. Enable the metrics endpoint in Fletchling + +Edit `configs/fletchling.toml` and uncomment the `enabled` line: + +```toml +[prometheus] +enabled = true +``` + +Restart Fletchling. Verify it works: `curl http://127.0.0.1:9042/metrics` should return a long list of `fletchling_*` metrics. If you get `404 page not found`, the endpoint isn't enabled — recheck your config. + +## 2. Scrape it with Zapdos + +[Zapdos](https://github.com/UnownHash/Zapdos) is the UnownHash metrics stack (VmAgent + VictoriaMetrics + Grafana). Follow its docs to get the stack running, then add a scrape job for Fletchling in your `prometheus.yml`: + +```yml +scrape_configs: + - job_name: 'fletchling' + static_configs: + - targets: ['fletchling:9042'] # docker; for pm2 use 127.0.0.1:9042 + labels: + instance: 'fletchling' +``` + +Reload VmAgent so the new target is picked up. + +## 3. Import the dashboard + +A Grafana dashboard covering all exposed metrics is included at [`grafana/fletchling-dashboard.json`](grafana/fletchling-dashboard.json). In Grafana: **Dashboards → New → Import**, upload the file, and select your VictoriaMetrics/Prometheus datasource. + # Migrating from other nest processors ## nestcollector to Fletching using existing Golbat DB for nests (SIMPLEST) diff --git a/grafana/fletchling-dashboard.json b/grafana/fletchling-dashboard.json new file mode 100644 index 0000000..5b042b8 --- /dev/null +++ b/grafana/fletchling-dashboard.json @@ -0,0 +1,721 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus datasource scraping Fletchling's /metrics endpoint", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "All Prometheus metrics exposed by Fletchling: webhook processing counters, HTTP request stats (gin), Go runtime, and process metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": ["fletchling", "pokemon", "nests"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "fletchling", + "value": "fletchling" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(fletchling_pokemon_processed, job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(fletchling_pokemon_processed, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(fletchling_pokemon_processed{job=\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(fletchling_pokemon_processed{job=\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Fletchling", + "uid": "fletchling-overview", + "version": 1, + "weekStart": "", + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "panels": [], + "title": "Webhook Processing", + "type": "row" + }, + { + "type": "stat", + "id": 1, + "title": "Pokemon Processed (total)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "unit": "short" + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(fletchling_pokemon_processed{job=\"$job\", instance=~\"$instance\"})", + "legendFormat": "processed", + "refId": "A" + } + ] + }, + { + "type": "stat", + "id": 2, + "title": "Pokemon Matched (total)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(fletchling_pokemon_matched{job=\"$job\", instance=~\"$instance\"})", + "legendFormat": "matched", + "refId": "A" + } + ] + }, + { + "type": "stat", + "id": 3, + "title": "Nests Matched (total)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": null }] }, + "unit": "short" + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(fletchling_nests_matched{job=\"$job\", instance=~\"$instance\"})", + "legendFormat": "nests matched", + "refId": "A" + } + ] + }, + { + "type": "stat", + "id": 4, + "title": "Match Ratio (5m)", + "description": "Pokemon matched / pokemon processed over the last 5 minutes.", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.05 }, + { "color": "green", "value": 0.2 } + ] + }, + "unit": "percentunit", + "max": 1, + "min": 0 + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(rate(fletchling_pokemon_matched{job=\"$job\", instance=~\"$instance\"}[5m])) / clamp_min(sum(rate(fletchling_pokemon_processed{job=\"$job\", instance=~\"$instance\"}[5m])), 1)", + "legendFormat": "ratio", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "id": 5, + "title": "Pokemon Processing Rate", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (instance) (rate(fletchling_pokemon_processed{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "processed - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (instance) (rate(fletchling_pokemon_matched{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "matched - {{instance}}", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "id": 6, + "title": "Nests Matched Rate", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "spanNulls": false }, + "unit": "ops" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (instance) (rate(fletchling_nests_matched{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "id": 101, + "panels": [], + "title": "HTTP (gin)", + "type": "row" + }, + { + "type": "timeseries", + "id": 10, + "title": "HTTP Request Rate by Status Code", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never", "stacking": { "mode": "normal", "group": "A" } }, + "unit": "reqps" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (code) (rate(fletchling_gin_requests_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{code}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "id": 11, + "title": "HTTP Request Rate by Path", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "reqps" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum by (path, method) (rate(fletchling_gin_requests_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "id": 12, + "title": "HTTP Request Duration (p50 / p95 / p99)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(fletchling_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(fletchling_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(fletchling_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "id": 13, + "title": "HTTP Request / Response Size (avg)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "bytes" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(rate(fletchling_gin_request_size_bytes_sum{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(fletchling_gin_request_size_bytes_count{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])), 1)", + "legendFormat": "request avg", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "sum(rate(fletchling_gin_response_size_bytes_sum{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(fletchling_gin_response_size_bytes_count{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])), 1)", + "legendFormat": "response avg", + "refId": "B" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "id": 102, + "panels": [], + "title": "Process", + "type": "row" + }, + { + "type": "timeseries", + "id": 20, + "title": "CPU Usage", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "percentunit" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "rate(fletchling_process_cpu_seconds_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "id": 21, + "title": "Resident Memory", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "bytes" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_process_resident_memory_bytes{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "RSS - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_process_virtual_memory_bytes{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "VSZ - {{instance}}", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "id": 22, + "title": "File Descriptors", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_process_open_fds{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "open - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_process_max_fds{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "max - {{instance}}", + "refId": "B" + } + ] + }, + { + "type": "stat", + "id": 23, + "title": "Process Uptime", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 39 }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "unit": "s" + } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "time() - fletchling_process_start_time_seconds{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 47 }, + "id": 103, + "panels": [], + "title": "Go Runtime", + "type": "row" + }, + { + "type": "timeseries", + "id": 30, + "title": "Goroutines", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "short" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_go_goroutines{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "type": "timeseries", + "id": 31, + "title": "Go Heap Memory", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "bytes" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_go_memory_classes_heap_objects_bytes{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "heap objects - {{instance}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_go_memory_classes_heap_free_bytes{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "heap free - {{instance}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "fletchling_go_memory_classes_total_bytes{job=\"$job\", instance=~\"$instance\"}", + "legendFormat": "total - {{instance}}", + "refId": "C" + } + ] + }, + { + "type": "timeseries", + "id": 32, + "title": "GC Pause (p95 / p99)", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "s" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(fletchling_go_gc_pauses_seconds_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(fletchling_go_gc_pauses_seconds_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "B" + } + ] + }, + { + "type": "timeseries", + "id": 33, + "title": "GC Cycles", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, + "fieldConfig": { + "defaults": { + "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" }, + "unit": "ops" + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "expr": "rate(fletchling_go_gc_cycles_total_gc_cycles_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + } + ] +} From af44f14ef24e5a73c652878d26e87c52ce75d102 Mon Sep 17 00:00:00 2001 From: Magik <30931287+unseenmagik@users.noreply.github.com> Date: Wed, 29 Apr 2026 21:19:31 +0100 Subject: [PATCH 3/4] Fix dashboard metric names for Go runtime and gin sections NewGoCollector and ginprom don't pick up the fletchling namespace, so runtime metrics are exposed as go_* (not fletchling_go_*) and ginprom v1.8.1 emits gin_gin_* (its default namespace is "gin", combined with the configured Subsystem("gin")). Update all dashboard queries to match. Co-Authored-By: Claude Opus 4.7 (1M context) --- grafana/fletchling-dashboard.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/grafana/fletchling-dashboard.json b/grafana/fletchling-dashboard.json index 5b042b8..9a2859a 100644 --- a/grafana/fletchling-dashboard.json +++ b/grafana/fletchling-dashboard.json @@ -367,7 +367,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (code) (rate(fletchling_gin_requests_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum by (code) (rate(gin_gin_requests_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{code}}", "refId": "A" } @@ -392,7 +392,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (path, method) (rate(fletchling_gin_requests_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum by (path, method) (rate(gin_gin_requests_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "legendFormat": "{{method}} {{path}}", "refId": "A" } @@ -417,19 +417,19 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le) (rate(fletchling_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le) (rate(gin_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(fletchling_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(gin_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(fletchling_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(gin_gin_request_duration_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "C" } @@ -454,13 +454,13 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(fletchling_gin_request_size_bytes_sum{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(fletchling_gin_request_size_bytes_count{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])), 1)", + "expr": "sum(rate(gin_gin_request_size_bytes_sum{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(gin_gin_request_size_bytes_count{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])), 1)", "legendFormat": "request avg", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(rate(fletchling_gin_response_size_bytes_sum{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(fletchling_gin_response_size_bytes_count{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])), 1)", + "expr": "sum(rate(gin_gin_response_size_bytes_sum{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(gin_gin_response_size_bytes_count{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])), 1)", "legendFormat": "response avg", "refId": "B" } @@ -618,7 +618,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "fletchling_go_goroutines{job=\"$job\", instance=~\"$instance\"}", + "expr": "go_goroutines{job=\"$job\", instance=~\"$instance\"}", "legendFormat": "{{instance}}", "refId": "A" } @@ -643,19 +643,19 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "fletchling_go_memory_classes_heap_objects_bytes{job=\"$job\", instance=~\"$instance\"}", + "expr": "go_memory_classes_heap_objects_bytes{job=\"$job\", instance=~\"$instance\"}", "legendFormat": "heap objects - {{instance}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "fletchling_go_memory_classes_heap_free_bytes{job=\"$job\", instance=~\"$instance\"}", + "expr": "go_memory_classes_heap_free_bytes{job=\"$job\", instance=~\"$instance\"}", "legendFormat": "heap free - {{instance}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "fletchling_go_memory_classes_total_bytes{job=\"$job\", instance=~\"$instance\"}", + "expr": "go_memory_classes_total_bytes{job=\"$job\", instance=~\"$instance\"}", "legendFormat": "total - {{instance}}", "refId": "C" } @@ -680,13 +680,13 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le) (rate(fletchling_go_gc_pauses_seconds_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.95, sum by (le) (rate(go_gc_pauses_seconds_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p95", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le) (rate(fletchling_go_gc_pauses_seconds_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(go_gc_pauses_seconds_bucket{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "legendFormat": "p99", "refId": "B" } @@ -711,7 +711,7 @@ "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(fletchling_go_gc_cycles_total_gc_cycles_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(go_gc_cycles_total_gc_cycles_total{job=\"$job\", instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "{{instance}}", "refId": "A" } From 6c9265c904f1f271a89b7410f8ad0c5cd399e310 Mon Sep 17 00:00:00 2001 From: Magik <30931287+unseenmagik@users.noreply.github.com> Date: Wed, 29 Apr 2026 23:00:06 +0100 Subject: [PATCH 4/4] Update default overpass user agent string Co-Authored-By: Claude Opus 4.7 (1M context) --- app_config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app_config/config.go b/app_config/config.go index 5ddfa3d..2f86be6 100644 --- a/app_config/config.go +++ b/app_config/config.go @@ -29,7 +29,7 @@ import ( const ( DEFAULT_OVERPASS_URL = "https://overpass-api.de/api/interpreter" - DEFAULT_OVERPASS_USER_AGENT = "Fletchling MAGI importer contact: your-email@example.com" + DEFAULT_OVERPASS_USER_AGENT = "Fletchling-USR-OSM-Importer/1.0 contact: overpass@mymail.fiy" DEFAULT_NEST_NAME = "Unknown Nest" )