Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions components/status/ComponentStatus.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* component's status with methods for status management.
*/

import { type ComponentStatusLevel, COMPONENT_STATUS_LEVELS } from './types.ts';
import { type ComponentStatusLevel, type ComponentStatusSource, COMPONENT_STATUS_LEVELS } from './types.ts';

/**
* Component status information class
Expand All @@ -19,12 +19,23 @@ export class ComponentStatus {
public message?: string;
/** Error information if status is 'error' */
public error?: Error | string;
/** How the status was set */
public source?: ComponentStatusSource;
/** Epoch ms when this status should auto-clear */
public expiresAt?: number;
/** Number of times this status has been reported */
public occurrenceCount: number;
/** When this status was first reported */
public firstOccurrence: Date;

constructor(status: ComponentStatusLevel, message?: string, error?: Error | string) {
constructor(status: ComponentStatusLevel, message?: string, error?: Error | string, source?: ComponentStatusSource) {
this.lastChecked = new Date();
this.status = status;
this.message = message;
this.error = error;
this.source = source;
this.occurrenceCount = 1;
this.firstOccurrence = this.lastChecked;
}

/**
Expand Down
15 changes: 13 additions & 2 deletions components/status/ComponentStatusRegistry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import { ComponentStatus } from './ComponentStatus.ts';
import {
type ComponentStatusLevel,
type ComponentStatusSource,
COMPONENT_STATUS_LEVELS,
type AggregatedComponentStatus,
type ComponentApplicationStatus,
Expand Down Expand Up @@ -43,7 +44,8 @@ export class ComponentStatusRegistry {
componentName: string,
status: ComponentStatusLevel,
message?: string,
error?: Error | string
error?: Error | string,
source?: ComponentStatusSource
): void {
if (!componentName || typeof componentName !== 'string') {
throw new ComponentStatusOperationError(
Expand All @@ -61,7 +63,16 @@ export class ComponentStatusRegistry {
);
}

this.statusMap.set(componentName, new ComponentStatus(status, message, error));
const existing = this.statusMap.get(componentName);
if (existing && existing.status === status) {
existing.lastChecked = new Date();
existing.message = message;
if (error !== undefined) existing.error = error;
if (source !== undefined) existing.source = source;
existing.occurrenceCount++;
} else {
this.statusMap.set(componentName, new ComponentStatus(status, message, error, source));
}
}

/**
Expand Down
16 changes: 9 additions & 7 deletions components/status/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,19 @@

import { componentStatusRegistry } from './registry.ts';
import { ComponentStatus } from './ComponentStatus.ts';
import { COMPONENT_STATUS_LEVELS } from './types.ts';
import { COMPONENT_STATUS_LEVELS, type ComponentStatusSource } from './types.ts';

/**
* Component Status Builder
* Provides a fluent interface for reporting component status
*/
export class ComponentStatusBuilder {
private componentName: string;
private source: ComponentStatusSource;

constructor(componentName: string) {
constructor(componentName: string, source: ComponentStatusSource = 'explicit') {
this.componentName = componentName;
this.source = source;
}

/**
Expand All @@ -26,7 +28,7 @@ export class ComponentStatusBuilder {
* @returns this for chaining
*/
healthy(message?: string): this {
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.HEALTHY, message);
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.HEALTHY, message, undefined, this.source);
return this;
}

Expand All @@ -36,7 +38,7 @@ export class ComponentStatusBuilder {
* @returns this for chaining
*/
warning(message: string): this {
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.WARNING, message);
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.WARNING, message, undefined, this.source);
return this;
}

Expand All @@ -47,7 +49,7 @@ export class ComponentStatusBuilder {
* @returns this for chaining
*/
error(message: string, error?: Error): this {
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.ERROR, message, error);
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.ERROR, message, error, this.source);
return this;
}

Expand All @@ -57,7 +59,7 @@ export class ComponentStatusBuilder {
* @returns this for chaining
*/
loading(message?: string): this {
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.LOADING, message || 'Loading...');
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.LOADING, message || 'Loading...', undefined, this.source);
return this;
}

Expand All @@ -67,7 +69,7 @@ export class ComponentStatusBuilder {
* @returns this for chaining
*/
unknown(message?: string): this {
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.UNKNOWN, message);
componentStatusRegistry.setStatus(this.componentName, COMPONENT_STATUS_LEVELS.UNKNOWN, message, undefined, this.source);
return this;
}

Expand Down
8 changes: 8 additions & 0 deletions components/status/crossThread.ts
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,8 @@ export class StatusAggregator {
let mostRecentCheckTime = 0;
let latestMessage: string | undefined;
let error: Error | string | undefined;
let source: string | undefined;
let totalOccurrenceCount = 0;
const statusCounts = new Map<ComponentStatusLevel, number>();
const abnormalities = new Map<string, ComponentStatusAbnormality>();

Expand Down Expand Up @@ -340,6 +342,10 @@ export class StatusAggregator {
if (status.error && !error) {
error = status.error;
}

// Track source and occurrence count
if (status.source) source = status.source;
if (status.occurrenceCount) totalOccurrenceCount += status.occurrenceCount;
}

// Determine overall status (priority: error > warning > loading > unknown > healthy)
Expand Down Expand Up @@ -368,6 +374,8 @@ export class StatusAggregator {
lastChecked: lastCheckedTimes,
latestMessage,
error,
source: source as any,
occurrenceCount: totalOccurrenceCount || undefined,
};

// Only add abnormalities if there are any
Expand Down
127 changes: 127 additions & 0 deletions components/status/healthChecks.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/**
* System Health Checks
*
* Periodic monitoring of system resources (disk, memory, CPU) that reports
* status via the component status system. Health check statuses self-heal:
* when the metric returns to normal, status reverts to healthy.
*
* Runs on the main thread only.
*/

import { componentStatusRegistry } from './registry.ts';
import { COMPONENT_STATUS_LEVELS } from './types.ts';

interface ThresholdConfig {
warning: number;
error: number;
}

interface HealthCheckConfig {
enabled?: boolean;
intervalSeconds?: number;
thresholds?: {
disk?: ThresholdConfig;
memory?: ThresholdConfig;
cpu?: ThresholdConfig;
};
}

const DEFAULT_THRESHOLDS: Record<string, ThresholdConfig> = {
disk: { warning: 80, error: 95 },
memory: { warning: 80, error: 95 },
cpu: { warning: 85, error: 95 },
};

const DEFAULT_INTERVAL_SECONDS = 60;

function setHealthStatus(name: string, percent: number, thresholds: ThresholdConfig, label: string) {
const key = `system.${name}`;
if (percent >= thresholds.error) {
componentStatusRegistry.setStatus(
key, COMPONENT_STATUS_LEVELS.ERROR,
`${label} at ${percent.toFixed(1)}% utilization`, undefined, 'health-check'
);
} else if (percent >= thresholds.warning) {
componentStatusRegistry.setStatus(
key, COMPONENT_STATUS_LEVELS.WARNING,
`${label} at ${percent.toFixed(1)}% utilization`, undefined, 'health-check'
);
} else {
componentStatusRegistry.setStatus(
key, COMPONENT_STATUS_LEVELS.HEALTHY,
`${label} at ${percent.toFixed(1)}% utilization`, undefined, 'health-check'
);
}
}

async function checkDisk(thresholds: ThresholdConfig) {
try {
const si = await import('systeminformation');
const fsSizes = await si.fsSize();
// Check the filesystem with the highest usage
let worstPercent = 0;
let worstMount = '';
for (const fs of fsSizes) {
if (fs.use > worstPercent) {
worstPercent = fs.use;
worstMount = fs.mount;
}
}
if (worstPercent > 0) {
setHealthStatus('disk', worstPercent, thresholds, `Disk (${worstMount})`);
}
} catch {
// systeminformation may not be available in all environments
}
}

async function checkMemory(thresholds: ThresholdConfig) {
try {
const si = await import('systeminformation');
const mem = await si.mem();
if (mem.total > 0) {
const usedPercent = ((mem.total - mem.available) / mem.total) * 100;
setHealthStatus('memory', usedPercent, thresholds, 'Memory');
}
} catch {
// systeminformation may not be available in all environments
}
}

async function checkCPU(thresholds: ThresholdConfig) {
try {
const si = await import('systeminformation');
const load = await si.currentLoad();
if (load.currentLoad !== undefined) {
setHealthStatus('cpu', load.currentLoad, thresholds, 'CPU');
}
} catch {
// systeminformation may not be available in all environments
}
}

async function runChecks(thresholds: Record<string, ThresholdConfig>) {
await Promise.all([
checkDisk(thresholds.disk),
checkMemory(thresholds.memory),
checkCPU(thresholds.cpu),
]);
}

export function startHealthChecks(config?: HealthCheckConfig) {
if (config?.enabled === false) return;

const thresholds = {
disk: config?.thresholds?.disk || DEFAULT_THRESHOLDS.disk,
memory: config?.thresholds?.memory || DEFAULT_THRESHOLDS.memory,
cpu: config?.thresholds?.cpu || DEFAULT_THRESHOLDS.cpu,
};
const intervalMs = (config?.intervalSeconds || DEFAULT_INTERVAL_SECONDS) * 1000;

// Run immediately
runChecks(thresholds);

// Then periodically
const timer = setInterval(() => runChecks(thresholds), intervalMs);
timer.unref();
}
105 changes: 105 additions & 0 deletions components/status/hierarchy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/**
* Status Hierarchy Builder
*
* Builds a tree-structured view of component status from the flat status map.
* Component names are split on '.' to create parent-child relationships.
* Parent status rolls up from children (worst status wins).
*/

import {
type ComponentStatusLevel,
type ComponentStatusSource,
type AggregatedComponentStatus,
COMPONENT_STATUS_LEVELS,
} from './types.ts';

export interface StatusNode {
status: ComponentStatusLevel;
message?: string;
source?: ComponentStatusSource;
lastChecked?: { main?: number; workers: Record<number, number> };
occurrenceCount?: number;
error?: Error | string;
children?: Record<string, StatusNode>;
}

const STATUS_PRIORITY: Record<ComponentStatusLevel, number> = {
[COMPONENT_STATUS_LEVELS.ERROR]: 4,
[COMPONENT_STATUS_LEVELS.WARNING]: 3,
[COMPONENT_STATUS_LEVELS.LOADING]: 2,
[COMPONENT_STATUS_LEVELS.UNKNOWN]: 1,
[COMPONENT_STATUS_LEVELS.HEALTHY]: 0,
};

/**
* Build a hierarchical status tree from a flat map of aggregated statuses.
* Names are split on '.' to create parent/child structure.
*
* Example:
* 'system.disk' -> { system: { children: { disk: { status: 'warning', ... } } } }
* 'replication' -> { replication: { status: 'error', ... } }
*/
export function buildHierarchy(
statuses: Map<string, AggregatedComponentStatus>
): Record<string, StatusNode> {
const root: Record<string, StatusNode> = {};

for (const [name, status] of statuses) {
const parts = name.split('.');
let currentLevel = root;

for (let i = 0; i < parts.length; i++) {
const part = parts[i];
if (!currentLevel[part]) {
currentLevel[part] = {
status: COMPONENT_STATUS_LEVELS.HEALTHY,
};
}
const node = currentLevel[part];

if (i === parts.length - 1) {
// Leaf node -- set actual status from aggregated data
node.status = status.status;
node.message = status.latestMessage;
node.source = status.source;
node.lastChecked = status.lastChecked;
node.occurrenceCount = status.occurrenceCount;
if (status.error) node.error = status.error;
} else {
// Intermediate node -- ensure children map exists
if (!node.children) node.children = {};
currentLevel = node.children;
}
}
}

rollUpStatus(root);
return root;
}

/**
* Roll up status from children to parents.
* A parent's status is the worst (highest priority) status among its children.
*/
function rollUpStatus(nodes: Record<string, StatusNode>): ComponentStatusLevel {
let worstStatus: ComponentStatusLevel = COMPONENT_STATUS_LEVELS.HEALTHY;

for (const node of Object.values(nodes)) {
let nodeStatus = node.status;

if (node.children) {
const childWorst = rollUpStatus(node.children);
// Parent status = worst of own status and children's worst
if (STATUS_PRIORITY[childWorst] > STATUS_PRIORITY[nodeStatus]) {
nodeStatus = childWorst;
node.status = nodeStatus;
}
}

if (STATUS_PRIORITY[nodeStatus] > STATUS_PRIORITY[worstStatus]) {
worstStatus = nodeStatus;
}
}

return worstStatus;
}
Loading
Loading