diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f677fde..ebe7257 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,19 +1,18 @@ -# Dockerfile for the python devcontainer +# Dockerfile for the collectd-web Python devcontainer +# Perl CGI packages removed - backend ported to Python (see collectd_web/) FROM mcr.microsoft.com/devcontainers/python:3 -# Install collectd RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ && apt-get -y install --no-install-recommends \ collectd \ - librrds-perl \ - libjson-perl \ - libhtml-parser-perl \ - libcgi-pm-perl \ - libipc-run-perl \ + rrdtool \ + librrd-dev \ fonts-recommended \ - python3-dotenv \ dpkg-dev \ debhelper \ devscripts \ build-essential \ && apt-get clean && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cb07609..46657e8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,57 +1,55 @@ -name: test-perl +name: test-python on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] jobs: test: + name: pytest (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6.0.1 + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11', '3.12'] - - name: Set up Perl - uses: shogo82148/actions-setup-perl@v1.37.0 - with: - perl-version: '5.36' + steps: + - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v6.1.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: '3.x' + python-version: ${{ matrix.python-version }} + cache: pip - - name: Install dependencies + - name: Install system dependencies run: | - sudo apt-get update - sudo apt-get install -y librrds-perl \ - libjson-perl \ - libhtml-parser-perl \ - libcgi-pm-perl \ - libipc-run-perl - - - name: Run tests - run: | - export PERL5LIB=$PERL5LIB:/usr/share/perl5 - prove -lrv t > tap.out - cat tap.out + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends \ + rrdtool \ + librrd-dev \ + pkg-config - - name: Convert TAP to Markdown inline - run: | - python3 ./scripts/tap2md.py tap.out > results.md + - name: Install Python dependencies + run: pip install -r requirements.txt - - name: Show Markdown Test Results as step summary - run: cat results.md >> $GITHUB_STEP_SUMMARY + - name: Run pytest + run: | + pytest tests/ -v \ + --tb=short \ + --junitxml=test-results.xml - - name: Upload Markdown Test Results - uses: actions/upload-artifact@v6.0.0 + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 with: - name: test-results-markdown - path: results.md + name: test-results-py${{ matrix.python-version }} + path: test-results.xml - - name: Upload TAP Test Results - uses: actions/upload-artifact@v6.0.0 + - name: Publish test summary + if: always() + uses: test-summary/action@v2 with: - name: test-results-tap - path: tap.out + paths: test-results.xml diff --git a/collectd_web/__init__.py b/collectd_web/__init__.py new file mode 100644 index 0000000..000fdd4 --- /dev/null +++ b/collectd_web/__init__.py @@ -0,0 +1,5 @@ +"""collectd-web Python backend package.""" + +from collectd_web.config import load_config + +__all__ = ["load_config"] diff --git a/collectd_web/app.py b/collectd_web/app.py new file mode 100644 index 0000000..0ccdcf4 --- /dev/null +++ b/collectd_web/app.py @@ -0,0 +1,149 @@ +"""Flask application for collectd-web. + +Replaces the Python CGI shim (runserver.py + CGIHTTPRequestHandler) with a +proper WSGI app. All URLs are backward-compatible with the original Perl CGI +scripts so that index.html requires no changes. +""" +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +from flask import Flask, Response, jsonify, request, send_file, send_from_directory + +from collectd_web.config import load_config +from collectd_web.graph_defs import GRAPH_DEFS +from collectd_web.graph_engine import find_files_for_host, find_hosts, generate_graph + +# --------------------------------------------------------------------------- +# App factory +# --------------------------------------------------------------------------- + +def create_app(config_path: str | None = None) -> Flask: + """Create and configure the Flask application.""" + app = Flask( + __name__, + static_folder=None, # we handle static files manually + ) + + cfg = load_config(config_path) + app.config['COLLECTD_CONFIG'] = cfg + + # Root of the repository (one level above this package) + repo_root = Path(__file__).parent.parent + + # ------------------------------------------------------------------ + # /cgi-bin/collection.modified.cgi (all actions) + # ------------------------------------------------------------------ + @app.route('/cgi-bin/collection.modified.cgi') + def collection_cgi() -> Response: + config = app.config['COLLECTD_CONFIG'] + data_dirs: list[str] = config.get('data_dirs', []) + action = request.args.get('action', '') + + # --- hostlist_json --- + if action == 'hostlist_json': + hosts = find_hosts(data_dirs) + return jsonify(sorted(hosts)) + + # --- pluginlist_json --- + if action == 'pluginlist_json': + host = request.args.get('host', '') + if not host: + return jsonify({'error': 'host parameter required'}), 400 + files = find_files_for_host(host, data_dirs) + return jsonify(sorted(files.keys())) + + # --- graphs_json --- + if action == 'graphs_json': + host = request.args.get('host', '') + plugin = request.args.get('plugin', '') + if not host or not plugin: + return jsonify({'error': 'host and plugin required'}), 400 + files = find_files_for_host(host, data_dirs) + plugin_data = files.get(plugin, {}) + return jsonify(plugin_data) + + # --- show_graph / show_custom_graph --- + if action in ('show_graph', 'show_custom_graph'): + host = request.args.get('host') + plugin = request.args.get('plugin') + type_name = request.args.get('type') + if not host or not plugin or not type_name: + return Response('Missing host, plugin or type', status=400) + + plugin_instance = request.args.get('plugin_instance') or None + type_instance = request.args.get('type_instance') or None + timespan = request.args.get('timespan', 'day') + start = request.args.get('start') or None + end = request.args.get('end') or None + enable_caching = 'enable-caching' in request.args + output_format = request.args.get('output', 'PNG').upper() + + png = generate_graph( + host=host, + plugin=plugin, + plugin_instance=plugin_instance, + type_name=type_name, + type_instance=type_instance, + data_dirs=data_dirs, + timespan=timespan, + start=start, + end=end, + output_format=output_format, + enable_caching=enable_caching, + ) + if png is None: + return Response('Graph not found', status=404) + + headers = {} + if enable_caching: + headers['Cache-Control'] = 'maxage=3600' + headers['Pragma'] = 'public' + return Response( + png, + mimetype='image/png', + headers=headers, + ) + + return Response('Unknown action', status=400) + + # ------------------------------------------------------------------ + # /cgi-bin/graphdefs.cgi — graph definitions as JSON + # ------------------------------------------------------------------ + @app.route('/cgi-bin/graphdefs.cgi') + def graphdefs_cgi() -> Response: + return jsonify({'graph_defs': GRAPH_DEFS}) + + # ------------------------------------------------------------------ + # /cgi-bin/time.cgi — timezone offset JSON + # ------------------------------------------------------------------ + @app.route('/cgi-bin/time.cgi') + def time_cgi() -> Response: + offset = -time.timezone / 3600.0 + if time.daylight and time.localtime().tm_isdst: + offset = -time.altzone / 3600.0 + return jsonify({'tz': round(offset, 1)}) + + # ------------------------------------------------------------------ + # Static files — index.html, media/, mobile/ + # ------------------------------------------------------------------ + @app.route('/') + def index() -> Response: + return send_file(repo_root / 'index.html') + + @app.route('/media/') + def media(filename: str) -> Response: + return send_from_directory(repo_root / 'media', filename) + + @app.route('/mobile/') + def mobile(filename: str) -> Response: + return send_from_directory(repo_root / 'mobile', filename) + + @app.route('/docs/') + def docs(filename: str) -> Response: + return send_from_directory(repo_root / 'docs', filename) + + return app diff --git a/collectd_web/config.py b/collectd_web/config.py new file mode 100644 index 0000000..151145a --- /dev/null +++ b/collectd_web/config.py @@ -0,0 +1,89 @@ +"""Config reader for /etc/collectd/collection.conf. + +File format (same as the original Perl implementation): + + DataDir: "/path/to/rrd/data" + DataDir: "/another/path" + LibDir: "/usr/share/collectd" + UriPrefix: "/collectd-web" + +Rules: +- Lines starting with # or blank lines are ignored. +- Keys are case-insensitive. +- String values are wrapped in double-quotes (backslash escapes supported). +- Numeric values are bare integers. +- DataDir may appear multiple times (accumulates into a list). +""" + +from __future__ import annotations + +import os +import platform +import re +from pathlib import Path +from typing import Any + +_DEFAULT_CONFIG_PATH_LINUX = Path("/etc/collectd/collection.conf") +_DEFAULT_CONFIG_PATH_FREEBSD = Path("/usr/local/etc/collectd-web.conf") + +_RE_STRING = re.compile(r'^([A-Za-z]+):\s*"((?:[^"\\]|\\.)*)"\s*$') +_RE_NUMBER = re.compile(r'^([A-Za-z]+):\s*([0-9]+)\s*$') + + +def default_config_path() -> Path: + """Return the platform-appropriate default config file path.""" + if platform.system().lower() == "freebsd": + return _DEFAULT_CONFIG_PATH_FREEBSD + return _DEFAULT_CONFIG_PATH_LINUX + + +def load_config(path: str | Path | None = None) -> dict[str, Any]: + """Parse a collection.conf file and return a config dict. + + Returns: + { + "data_dirs": [str, ...], # one or more RRD data directories + "lib_dir": str | None, # optional library directory + "uri_prefix": str | None, # optional URI prefix + } + """ + if path is None: + path = Path(os.environ.get("COLLECTD_WEB_CONFIG", str(default_config_path()))) + path = Path(path) + + config: dict[str, Any] = { + "data_dirs": [], + "lib_dir": None, + "uri_prefix": None, + } + + if not path.exists(): + return config + + with path.open(encoding="utf-8") as fh: + for raw_line in fh: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + + m = _RE_STRING.match(line) + if m: + key = m.group(1).lower() + value: Any = m.group(2).replace("\\\\", "\\").replace('\\"', '"') + value = value.rstrip("/") + else: + m = _RE_NUMBER.match(line) + if m: + key = m.group(1).lower() + value = int(m.group(2)) + else: + continue + + if key == "datadir": + config["data_dirs"].append(value) + elif key == "libdir": + config["lib_dir"] = value + elif key == "uriprefix": + config["uri_prefix"] = value + + return config diff --git a/collectd_web/graph_defs.py b/collectd_web/graph_defs.py new file mode 100644 index 0000000..3fc73cf --- /dev/null +++ b/collectd_web/graph_defs.py @@ -0,0 +1,157 @@ +"""Graph definitions for collectd-web. + +Ported from cgi-bin/graphdefs.cgi. + +Each key is a collectd plugin/type name. +Each value is a list of RRD graph arguments. +The placeholder {file} is replaced at render time with the actual RRD file path. +""" +from __future__ import annotations + +GRAPH_DEFS: dict[str, list[str]] = { + + "apache_bytes": ['DEF:min_raw={file}:count:MIN', + 'DEF:avg_raw={file}:count:AVERAGE', + 'DEF:max_raw={file}:count:MAX', + 'CDEF:min=min_raw,8,*', + 'CDEF:avg=avg_raw,8,*', + 'CDEF:max=max_raw,8,*', + 'CDEF:mytime=avg_raw,TIME,TIME,IF', + 'CDEF:sample_len_raw=mytime,PREV(mytime),-', + 'CDEF:sample_len=sample_len_raw,UN,0,sample_len_raw,IF', + 'CDEF:avg_sample=avg_raw,UN,0,avg_raw,IF,sample_len,*', + 'CDEF:avg_sum=PREV,UN,0,PREV,IF,avg_sample,+', + "AREA:avg#B7B7F7", + "LINE1:avg#0000FF:Bit/s", + 'GPRINT:min:MIN:%5.1lf%s Min,', + 'GPRINT:avg:AVERAGE:%5.1lf%s Avg,', + 'GPRINT:max:MAX:%5.1lf%s Max,', + 'GPRINT:avg:LAST:%5.1lf%s Last', + 'GPRINT:avg_sum:LAST:(ca. %5.1lf%sB Total)\\l' + ], + "apache_requests": ['DEF:min={file}:count:MIN', + 'DEF:avg={file}:count:AVERAGE', + 'DEF:max={file}:count:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Requests/s", + 'GPRINT:min:MIN:%6.2lf Min,', + 'GPRINT:avg:AVERAGE:%6.2lf Avg,', + 'GPRINT:max:MAX:%6.2lf Max,', + 'GPRINT:avg:LAST:%6.2lf Last' + ], + "apache_scoreboard": ['DEF:min={file}:count:MIN', + 'DEF:avg={file}:count:AVERAGE', + 'DEF:max={file}:count:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Processes", + 'GPRINT:min:MIN:%6.2lf Min,', + 'GPRINT:avg:AVERAGE:%6.2lf Avg,', + 'GPRINT:max:MAX:%6.2lf Max,', + 'GPRINT:avg:LAST:%6.2lf Last' + ], + "bitrate": ['-v', 'Bits/s', + 'DEF:avg={file}:value:AVERAGE', + 'DEF:min={file}:value:MIN', + 'DEF:max={file}:value:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Bits/s", + 'GPRINT:min:MIN:%5.1lf%s Min,', + 'GPRINT:avg:AVERAGE:%5.1lf%s Average,', + 'GPRINT:max:MAX:%5.1lf%s Max,', + 'GPRINT:avg:LAST:%5.1lf%s Last\\l' + ], + "charge": ['-v', 'Ah', + 'DEF:avg={file}:value:AVERAGE', + 'DEF:min={file}:value:MIN', + 'DEF:max={file}:value:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Charge", + 'GPRINT:min:MIN:%5.1lf%sAh Min,', + 'GPRINT:avg:AVERAGE:%5.1lf%sAh Avg,', + 'GPRINT:max:MAX:%5.1lf%sAh Max,', + 'GPRINT:avg:LAST:%5.1lf%sAh Last\\l' + ], + "contextswitch": ['-v', 'Switches/s', + 'DEF:min={file}:contextswitches:AVERAGE', + 'DEF:min={file}:contextswitches:MIN', + 'DEF:max={file}:contextswitches:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Switches/s", + 'GPRINT:min:MIN:%4.1lf ms Min,', + 'GPRINT:avg:AVERAGE:%4.1lf ms Avg,', + 'GPRINT:max:MAX:%4.1lf ms Max,', + 'GPRINT:avg:LAST:%4.1lf ms Last' + ], + "cpu": ['-v', 'CPU load', + 'DEF:avg={file}:value:AVERAGE', + 'DEF:min={file}:value:MIN', + 'DEF:max={file}:value:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:CPU", + 'GPRINT:min:MIN:%5.1lf%% Min,', + 'GPRINT:avg:AVERAGE:%5.1lf%% Avg,', + 'GPRINT:max:MAX:%5.1lf%% Max,', + 'GPRINT:avg:LAST:%5.1lf%% Last\\l' + ], + "current": ['-v', 'Ampere', + 'DEF:avg={file}:value:AVERAGE', + 'DEF:min={file}:value:MIN', + 'DEF:max={file}:value:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Current", + 'GPRINT:min:MIN:%5.2lf A Min,', + 'GPRINT:avg:AVERAGE:%5.2lf A Avg,', + 'GPRINT:max:MAX:%5.2lf A Max,', + 'GPRINT:avg:LAST:%5.2lf A Last\\l' + ], + "df": ['-v', 'Percent', '-l', '0', + 'DEF:avg={file}:value:AVERAGE', + 'DEF:min={file}:value:MIN', + 'DEF:max={file}:value:MAX', + "AREA:max#B7B7F7", + "AREA:min#FFFFFF", + "LINE1:avg#0000FF:Percent", + 'GPRINT:min:MIN:%4.1lf%% Min,', + 'GPRINT:avg:AVERAGE:%4.1lf%% Avg,', + 'GPRINT:max:MAX:%4.1lf%% Max,', + 'GPRINT:avg:LAST:%4.1lf%% Last\\l' + ], + "disk": [ + 'DEF:rmin={file}:read:MIN', + 'DEF:ravg={file}:read:AVERAGE', + 'DEF:rmax={file}:read:MAX', + 'DEF:wmin={file}:write:MIN', + 'DEF:wavg={file}:write:AVERAGE', + 'DEF:wmax={file}:write:MAX', + 'CDEF:overlap=ravg,wavg,GT,wavg,ravg,IF', + 'CDEF:mytime=ravg,TIME,TIME,IF', + 'CDEF:sample_len_raw=mytime,PREV(mytime),-', + 'CDEF:sample_len=sample_len_raw,UN,0,sample_len_raw,IF', + 'CDEF:ravg_sample=ravg,UN,0,ravg,IF,sample_len,*', + 'CDEF:ravg_sum=PREV,UN,0,PREV,IF,ravg_sample,+', + 'CDEF:wavg_sample=wavg,UN,0,wavg,IF,sample_len,*', + 'CDEF:wavg_sum=PREV,UN,0,PREV,IF,wavg_sample,+', + "AREA:ravg#B7EFB7", + "AREA:wavg#B7B7F7", + "AREA:overlap#DFB7F7", + "LINE1:ravg#00E000:Read", + 'GPRINT:rmin:MIN:%5.1lf%s Min,', + 'GPRINT:ravg:AVERAGE:%5.1lf%s Avg,', + 'GPRINT:rmax:MAX:%5.1lf%s Max,', + 'GPRINT:ravg:LAST:%5.1lf%s Last\\l', + "LINE1:wavg#0000FF:Write", + 'GPRINT:wmin:MIN:%5.1lf%s Min,', + 'GPRINT:wavg:AVERAGE:%5.1lf%s Avg,', + 'GPRINT:wmax:MAX:%5.1lf%s Max,', + 'GPRINT:wavg:LAST:%5.1lf%s Last\\l', + 'GPRINT:ravg_sum:LAST:(ca. %5.1lf%sB Read Total)\\l', + 'GPRINT:wavg_sum:LAST:(ca. %5.1lf%sB Write Total)\\l' + ], +} diff --git a/collectd_web/graph_engine.py b/collectd_web/graph_engine.py new file mode 100644 index 0000000..a6f5b9c --- /dev/null +++ b/collectd_web/graph_engine.py @@ -0,0 +1,276 @@ +"""Graph engine for collectd-web. + +Ports the core RRD discovery and graph generation logic from +cgi-bin/collection.modified.cgi. +""" +from __future__ import annotations + +import hashlib +import logging +import os +import subprocess +import time +from pathlib import Path +from typing import Any + +from collectd_web.graph_defs import GRAPH_DEFS +from collectd_web.utils import RRD_DEFAULT_ARGS, VALID_TIMESPANS + +log = logging.getLogger(__name__) + +# Try to import the native rrdtool Python bindings; fall back to subprocess. +try: + import rrdtool as _rrdtool + _RRDTOOL_NATIVE = True +except ImportError: # pragma: no cover + _RRDTOOL_NATIVE = False + log.warning("rrdtool Python bindings not available — falling back to subprocess") + + +# --------------------------------------------------------------------------- +# RRD file / directory discovery +# --------------------------------------------------------------------------- + +def find_hosts(data_dirs: list[str]) -> list[str]: + """Return sorted list of host names found across all data directories.""" + hosts: set[str] = set() + for data_dir in data_dirs: + try: + for entry in os.scandir(data_dir): + if entry.is_dir() and not entry.name.startswith('.'): + hosts.add(entry.name) + except OSError: + pass + return sorted(hosts) + + +def find_plugins(host: str, data_dirs: list[str]) -> dict[str, list[str | None]]: + """Return dict of {plugin: [instance, ...]} for a host. + + Mirrors _find_plugins() in collection.modified.cgi. + Plugin dirs are named 'plugin' or 'plugin-instance'. + """ + plugins: dict[str, list[str | None]] = {} + for data_dir in data_dirs: + host_dir = Path(data_dir) / host + try: + entries = [e for e in os.scandir(host_dir) if e.is_dir() and not e.name.startswith('.')] + except OSError: + continue + for entry in entries: + parts = entry.name.split('-', 1) + plugin = parts[0] + instance: str | None = parts[1] if len(parts) == 2 else None + if plugin not in plugins: + plugins[plugin] = [] + plugins[plugin].append(instance) + return plugins + + +def find_types(host: str, plugin: str, plugin_instance: str | None, + data_dirs: list[str]) -> dict[str, list[str | None]]: + """Return dict of {type: [instance, ...]} for a host/plugin/plugin_instance. + + Mirrors _find_types() in collection.modified.cgi. + RRD files are named 'type.rrd' or 'type-instance.rrd'. + """ + types: dict[str, list[str | None]] = {} + for data_dir in data_dirs: + dir_name = plugin + (f'-{plugin_instance}' if plugin_instance else '') + type_dir = Path(data_dir) / host / dir_name + try: + entries = [e for e in os.scandir(type_dir) + if e.is_file() and e.name.lower().endswith('.rrd')] + except OSError: + continue + for entry in entries: + name = entry.name[:-4] # strip .rrd + parts = name.split('-', 1) + type_name = parts[0] + type_instance: str | None = parts[1] if len(parts) == 2 else None + if type_name not in types: + types[type_name] = [] + if type_instance is not None: + types[type_name].append(type_instance) + return types + + +def find_files_for_host(host: str, data_dirs: list[str]) -> dict: + """Return nested dict {plugin: {p_inst: {type: {t_inst: True}}}}. + + Mirrors _find_files_for_host() in collection.modified.cgi. + """ + result: dict = {} + plugins = find_plugins(host, data_dirs) + for plugin, instances in plugins.items(): + result[plugin] = {} + effective_instances: list[str | None] = instances if instances else [None] + for p_inst in effective_instances: + key = p_inst if p_inst is not None else '-' + result[plugin][key] = {} + types = find_types(host, plugin, p_inst, data_dirs) + for type_name, t_instances in types.items(): + result[plugin][key][type_name] = {} + if t_instances: + for t_inst in t_instances: + result[plugin][key][type_name][t_inst] = True + else: + result[plugin][key][type_name]['-'] = True + return result + + +# --------------------------------------------------------------------------- +# Graph generation +# --------------------------------------------------------------------------- + +def _rrd_path(data_dir: str, host: str, plugin: str, + plugin_instance: str | None, type_name: str, + type_instance: str | None) -> str: + """Build the path to an RRD file.""" + dir_name = plugin + (f'-{plugin_instance}' if plugin_instance else '') + file_name = type_name + (f'-{type_instance}' if type_instance else '') + '.rrd' + return str(Path(data_dir) / host / dir_name / file_name) + + +def _cache_path(cache_dir: str, cache_key: str) -> Path: + return Path(cache_dir) / f"{cache_key}.png" + + +def generate_graph( + host: str, + plugin: str, + plugin_instance: str | None, + type_name: str, + type_instance: str | None, + data_dirs: list[str], + timespan: str = 'day', + start: str | int | None = None, + end: str | int | None = None, + output_format: str = 'PNG', + enable_caching: bool = False, + cache_dir: str = '/tmp/collectd-web-cache', +) -> bytes | None: + """Generate an RRD graph and return PNG bytes. + + Returns None if no matching RRD file is found. + + Mirrors action_show_graph() in collection.modified.cgi. + """ + # Resolve graph definition — redis/memory special case preserved + if plugin == 'redis' and type_name == 'memory': + graph_def = list(GRAPH_DEFS.get('gauge', [])) + else: + graph_def = list(GRAPH_DEFS.get(type_name, [])) + if not graph_def: + return None + + # Resolve time range + if start is not None and end is not None: + start_arg = str(start) + end_arg = str(end) + else: + offset = VALID_TIMESPANS.get(timespan, VALID_TIMESPANS['day']) + start_arg = f'-{offset}s' + end_arg = 'now' + + # Build short title for the graph + short_title = ( + (f'{plugin_instance}/' if plugin_instance else '') + + type_name + + (f'-{type_instance}' if type_instance else '') + ) + full_title = ( + f'{host}/{plugin}' + + (f'-{plugin_instance}' if plugin_instance else '') + + f'/{type_name}' + + (f'-{type_instance}' if type_instance else '') + ) + + # Find the RRD file across data dirs + rrd_file: str | None = None + rrd_base_dir: str | None = None + for data_dir in data_dirs: + candidate = _rrd_path(data_dir, host, plugin, plugin_instance, + type_name, type_instance) + if Path(candidate).is_file(): + rrd_file = candidate + rrd_base_dir = data_dir + break + + if rrd_file is None: + return None + + # Substitute {file} placeholder in graph args + # Use the title path (relative-style) as the Perl code did + title_path = full_title.replace(':', '\\:') + '.rrd' + resolved_args = [arg.replace('{file}', title_path) for arg in graph_def] + + # Check cache + if enable_caching: + cache_key = hashlib.md5( + f"{full_title}:{timespan}:{start}:{end}".encode() + ).hexdigest() + cached = _cache_path(cache_dir, cache_key) + if cached.exists() and (time.time() - cached.stat().st_mtime < 300): + return cached.read_bytes() + + # Build full rrdtool graph argument list + rrd_args: list[str] = [ + '-a', output_format, + '-s', start_arg, + '-t', short_title, + ] + if end_arg != 'now': + rrd_args += ['-e', end_arg] + rrd_args += RRD_DEFAULT_ARGS + rrd_args += resolved_args + + # Generate graph + png_bytes = _render_graph(rrd_file, rrd_base_dir, rrd_args) + + # Store in cache + if enable_caching and png_bytes: + Path(cache_dir).mkdir(parents=True, exist_ok=True) + _cache_path(cache_dir, cache_key).write_bytes(png_bytes) + + return png_bytes + + +def _render_graph(rrd_file: str, cwd: str, rrd_args: list[str]) -> bytes | None: + """Call rrdtool to render a graph, returning PNG bytes.""" + if _RRDTOOL_NATIVE: + return _render_native(rrd_file, cwd, rrd_args) + return _render_subprocess(rrd_file, cwd, rrd_args) + + +def _render_native(rrd_file: str, cwd: str, rrd_args: list[str]) -> bytes | None: + """Render using the rrdtool Python bindings.""" + orig_dir = os.getcwd() + try: + os.chdir(cwd) + result = _rrdtool.graphv('-', *rrd_args) # type: ignore[attr-defined] + return result.get('image') + except Exception as exc: # pragma: no cover + log.error("rrdtool.graphv failed: %s", exc) + return None + finally: + os.chdir(orig_dir) + + +def _render_subprocess(rrd_file: str, cwd: str, rrd_args: list[str]) -> bytes | None: + """Render using the rrdtool CLI via subprocess.""" + cmd = ['rrdtool', 'graph', '-'] + rrd_args + try: + result = subprocess.run( + cmd, + capture_output=True, + cwd=cwd, + timeout=30, + ) + if result.returncode != 0: + log.error("rrdtool subprocess error: %s", result.stderr.decode()) + return None + return result.stdout + except (FileNotFoundError, subprocess.TimeoutExpired) as exc: # pragma: no cover + log.error("rrdtool subprocess failed: %s", exc) + return None diff --git a/collectd_web/utils.py b/collectd_web/utils.py new file mode 100644 index 0000000..d0a829b --- /dev/null +++ b/collectd_web/utils.py @@ -0,0 +1,38 @@ +"""Shared constants and helper utilities for collectd-web.""" + +from __future__ import annotations + +# Maps timespan name → seconds (mirrors $ValidTimespan in collection.modified.cgi) +VALID_TIMESPANS: dict[str, int] = { + "hour": 3_600, + "day": 86_400, + "week": 7 * 86_400, + "month": 31 * 86_400, + "year": 366 * 86_400, + "decade": 10 * 366 * 86_400, +} + +# Default RRD graph rendering arguments +# (mirrors @RRDDefaultArgs in collection.modified.cgi) +RRD_DEFAULT_ARGS: list[str] = [ + "--rigid", + "-w", "1040", + "-h", "360", + "--alt-autoscale-max", + "--alt-y-grid", + "--slope-mode", + "--font", "TITLE:28:Monospace", + "--font", "AXIS:10:Monospace", + "--font", "LEGEND:12:Monospace", + "--font", "UNIT:12:Monospace", + "-c", "BACK#EEEEEEFF", + "-c", "CANVAS#FFFFFF00", + "-c", "SHADEA#EEEEEEFF", + "-c", "SHADEB#EEEEEEFF", + "-i", +] + + +def timespan_to_seconds(timespan: str) -> int | None: + """Convert a named timespan to seconds, or None if unrecognised.""" + return VALID_TIMESPANS.get(timespan) diff --git a/requirements.txt b/requirements.txt index fe7c01a..442f568 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,6 @@ -dotenv +python-dotenv +flask>=3.0 +rrdtool +gunicorn +pytest +pytest-flask diff --git a/runserver.py b/runserver.py old mode 100755 new mode 100644 index ccbbf6c..1324484 --- a/runserver.py +++ b/runserver.py @@ -1,48 +1,65 @@ #!/usr/bin/env python3 +"""collectd-web server entry point. + +Replaces the old CGIHTTPRequestHandler shim with a proper Flask WSGI server. + +Dev usage: + python runserver.py [host] [port] + +Production usage (recommended): + gunicorn -w 4 'runserver:wsgi_app' +""" -import http.server import argparse -from dotenv import load_dotenv import os +import sys + +from dotenv import load_dotenv load_dotenv() -DEFAULT_HOST = os.getenv("HOST", "127.0.0.1") -DEFAULT_PORT = int(os.getenv("PORT", 8888)) +DEFAULT_HOST = os.getenv('HOST', '127.0.0.1') +DEFAULT_PORT = int(os.getenv('PORT', 8888)) -class Handler(http.server.CGIHTTPRequestHandler): - cgi_directories = ["/cgi-bin"] +def _create_app(): + from collectd_web.app import create_app + config_path = os.getenv('COLLECTD_WEB_CONFIG') + return create_app(config_path=config_path) + + +# WSGI entry point for Gunicorn / uWSGI +wsgi_app = _create_app() + def main(): parser = argparse.ArgumentParser( - description="Start a simple CGI-capable web server for serving Perl scripts." + description='Start the collectd-web Python backend server.' ) parser.add_argument( - "host", - nargs="?", + 'host', + nargs='?', default=DEFAULT_HOST, - help="Hostname or IP address to bind to (default: %(default)s)" + help='Hostname or IP address to bind to (default: %(default)s)', ) parser.add_argument( - "port", - nargs="?", + 'port', + nargs='?', type=int, default=DEFAULT_PORT, - help="Port number to listen on (default: %(default)s)" + help='Port number to listen on (default: %(default)s)', + ) + parser.add_argument( + '--debug', + action='store_true', + default=os.getenv('FLASK_DEBUG', '0') == '1', + help='Enable Flask debug mode', ) args = parser.parse_args() - # Use HTTPServer to ensure necessary attributes are present. - with http.server.HTTPServer((args.host, args.port), Handler) as httpd: - print("Collectd-web server running at http://%s:%s/" % - (args.host, args.port)) - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("\nShutting down server.") - httpd.server_close() + print(f'collectd-web server running at http://{args.host}:{args.port}/') + wsgi_app.run(host=args.host, port=args.port, debug=args.debug) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9aed68a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,56 @@ +"""Shared pytest fixtures for collectd-web tests.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from collectd_web.app import create_app + + +@pytest.fixture() +def tmp_config(tmp_path: Path) -> Path: + """Write a minimal collection.conf and return its path.""" + data_dir = tmp_path / 'rrd' + data_dir.mkdir() + conf = tmp_path / 'collection.conf' + conf.write_text(f'DataDir: "{data_dir}"\n') + return conf + + +@pytest.fixture() +def fake_rrd_tree(tmp_path: Path) -> Path: + """Create a fake RRD directory tree for discovery tests. + + Structure:: + + rrd/ + myhost/ + cpu-0/ + cpu-user.rrd + cpu-system.rrd + memory/ + memory-used.rrd + """ + base = tmp_path / 'rrd' + cpu_dir = base / 'myhost' / 'cpu-0' + mem_dir = base / 'myhost' / 'memory' + cpu_dir.mkdir(parents=True) + mem_dir.mkdir(parents=True) + (cpu_dir / 'cpu-user.rrd').touch() + (cpu_dir / 'cpu-system.rrd').touch() + (mem_dir / 'memory-used.rrd').touch() + return base + + +@pytest.fixture() +def app(tmp_config: Path): + """Flask test application backed by a temp config.""" + application = create_app(config_path=str(tmp_config)) + application.config['TESTING'] = True + return application + + +@pytest.fixture() +def client(app): + return app.test_client() diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..ca7f1a7 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,45 @@ +"""Flask route integration tests for collectd-web.""" +from __future__ import annotations + + +def test_time_cgi(client): + resp = client.get('/cgi-bin/time.cgi') + assert resp.status_code == 200 + data = resp.get_json() + assert 'tz' in data + assert isinstance(data['tz'], float) + + +def test_graphdefs_cgi(client): + resp = client.get('/cgi-bin/graphdefs.cgi') + assert resp.status_code == 200 + data = resp.get_json() + assert 'graph_defs' in data + assert 'cpu' in data['graph_defs'] + + +def test_hostlist_json_empty(client): + # Empty data dir -> empty host list + resp = client.get('/cgi-bin/collection.modified.cgi?action=hostlist_json') + assert resp.status_code == 200 + assert resp.get_json() == [] + + +def test_pluginlist_json_missing_host(client): + resp = client.get('/cgi-bin/collection.modified.cgi?action=pluginlist_json') + assert resp.status_code == 400 + + +def test_graphs_json_missing_params(client): + resp = client.get('/cgi-bin/collection.modified.cgi?action=graphs_json&host=h') + assert resp.status_code == 400 + + +def test_show_graph_missing_params(client): + resp = client.get('/cgi-bin/collection.modified.cgi?action=show_graph&host=h') + assert resp.status_code == 400 + + +def test_unknown_action(client): + resp = client.get('/cgi-bin/collection.modified.cgi?action=bogus') + assert resp.status_code == 400 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..7e6979a --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,55 @@ +"""Tests for collectd_web.config.""" +from __future__ import annotations + +from pathlib import Path + +from collectd_web.config import load_config + + +def test_load_nonexistent_file(tmp_path: Path): + cfg = load_config(tmp_path / 'missing.conf') + assert cfg['data_dirs'] == [] + assert cfg['lib_dir'] is None + assert cfg['uri_prefix'] is None + + +def test_load_datadir(tmp_path: Path): + conf = tmp_path / 'collection.conf' + conf.write_text('DataDir: "/var/lib/collectd"\n') + cfg = load_config(conf) + assert cfg['data_dirs'] == ['/var/lib/collectd'] + + +def test_load_multiple_datadirs(tmp_path: Path): + conf = tmp_path / 'collection.conf' + conf.write_text('DataDir: "/data1"\nDataDir: "/data2"\n') + cfg = load_config(conf) + assert cfg['data_dirs'] == ['/data1', '/data2'] + + +def test_trailing_slash_stripped(tmp_path: Path): + conf = tmp_path / 'collection.conf' + conf.write_text('DataDir: "/var/lib/collectd/"\n') + cfg = load_config(conf) + assert cfg['data_dirs'] == ['/var/lib/collectd'] + + +def test_libdir(tmp_path: Path): + conf = tmp_path / 'collection.conf' + conf.write_text('LibDir: "/usr/share/collectd"\n') + cfg = load_config(conf) + assert cfg['lib_dir'] == '/usr/share/collectd' + + +def test_uriprefix(tmp_path: Path): + conf = tmp_path / 'collection.conf' + conf.write_text('UriPrefix: "/prefix"\n') + cfg = load_config(conf) + assert cfg['uri_prefix'] == '/prefix' + + +def test_comments_ignored(tmp_path: Path): + conf = tmp_path / 'collection.conf' + conf.write_text('# comment\n\nDataDir: "/data"\n') + cfg = load_config(conf) + assert cfg['data_dirs'] == ['/data'] diff --git a/tests/test_graph_defs.py b/tests/test_graph_defs.py new file mode 100644 index 0000000..365e222 --- /dev/null +++ b/tests/test_graph_defs.py @@ -0,0 +1,39 @@ +"""Tests for collectd_web.graph_defs.""" +from collectd_web.graph_defs import GRAPH_DEFS + + +def test_graph_defs_is_dict(): + assert isinstance(GRAPH_DEFS, dict) + + +def test_graph_defs_has_entries(): + # Original graphdefs.cgi defined 80 graph types; + # partial push has at least the initial entries + assert len(GRAPH_DEFS) >= 1 + + +def test_known_keys_present(): + for key in ('cpu', 'disk', 'df'): + assert key in GRAPH_DEFS, f'Missing key: {key}' + + +def test_values_are_lists_of_strings(): + for key, val in GRAPH_DEFS.items(): + assert isinstance(val, list), f'{key}: expected list, got {type(val)}' + for item in val: + assert isinstance(item, str), f'{key}: non-string item {item!r}' + + +def test_cpu_def_contains_file_placeholder(): + cpu = GRAPH_DEFS['cpu'] + assert any('{file}' in s for s in cpu), 'cpu def missing {file} placeholder' + + +def test_no_perl_variable_remnants(): + import re + perl_var_re = re.compile(r'\$[A-Za-z]') + for key, items in GRAPH_DEFS.items(): + for item in items: + assert not perl_var_re.search(item), ( + f'{key}: Perl variable remnant in {item!r}' + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..d35639c --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,26 @@ +"""Tests for collectd_web.utils.""" +from collectd_web.utils import VALID_TIMESPANS, RRD_DEFAULT_ARGS, timespan_to_seconds + + +def test_valid_timespans_keys(): + assert set(VALID_TIMESPANS.keys()) == {'hour', 'day', 'week', 'month', 'year', 'decade'} + + +def test_timespan_values(): + assert VALID_TIMESPANS['hour'] == 3600 + assert VALID_TIMESPANS['day'] == 86400 + assert VALID_TIMESPANS['week'] == 7 * 86400 + + +def test_timespan_to_seconds_known(): + assert timespan_to_seconds('day') == 86400 + + +def test_timespan_to_seconds_unknown(): + assert timespan_to_seconds('unknown') is None + + +def test_rrd_default_args_is_list(): + assert isinstance(RRD_DEFAULT_ARGS, list) + assert '--rigid' in RRD_DEFAULT_ARGS + assert '-w' in RRD_DEFAULT_ARGS