diff --git a/docs/examples/cryptocurrency-quickstart.ipynb b/docs/examples/cryptocurrency-quickstart.ipynb index e54bf0e..4d4a39a 100644 --- a/docs/examples/cryptocurrency-quickstart.ipynb +++ b/docs/examples/cryptocurrency-quickstart.ipynb @@ -63,7 +63,7 @@ "outputs": [], "source": [ "files = os.listdir(path)\n", - "files = [path + \"/\" + x for x in files]" + "files = [path+'/'+x for x in files]" ] }, { @@ -198,18 +198,18 @@ "# Read all filez and set them up to the readable structure for timecopilot\n", "for file in files:\n", " temp_df = pd.read_csv(file)\n", - " temp_df = temp_df[[\"Symbol\", \"Date\", \"Close\"]]\n", - " temp_df.columns = [\"unique_id\", \"ds\", \"y\"]\n", - " big_df = pd.concat([big_df, temp_df])\n", + " temp_df = temp_df[['Symbol','Date','Close']]\n", + " temp_df.columns = ['unique_id','ds','y']\n", + " big_df = pd.concat([big_df,temp_df])\n", "\n", "big_df = big_df.reset_index(drop=True)\n", "big_df[\"ds\"] = pd.to_datetime(big_df[\"ds\"], dayfirst=True, errors=\"coerce\")\n", "\n", - "# This line will be kept for execution time sanity, feel free to remove it if you want to stress timing a little further.\n", + "# This line will be kept for execution time sanity, feel free to remove it if you want to stress timing a little further. \n", "# big_df = big_df[big_df.ds >= \"2021-01-01\"]\n", - "cryptos = [\"MIOTA\", \"XEM\", \"ETH\", \"LTC\", \"DOGE\", \"CRO\", \"USDC\", \"ADA\"]\n", - "big_df = big_df[big_df.unique_id.isin(cryptos)]\n", - "big_df = big_df.reset_index(drop=True)\n", + "cryptos=['MIOTA','XEM','ETH','LTC','DOGE','CRO','USDC','ADA']\n", + "big_df=big_df[big_df.unique_id.isin(cryptos)]\n", + "big_df=big_df.reset_index(drop=True)\n", "big_df" ] }, @@ -341,7 +341,6 @@ " df_out.loc[idx, col] = np.nan\n", " return df_out\n", "\n", - "\n", "df_missing = add_missing(big_df, col=\"y\", frac=0.03, seed=42)\n", "df_missing = df_missing.sample(frac=1, random_state=42).reset_index(drop=True)\n", "print(df_missing)" @@ -710,14 +709,12 @@ } ], "source": [ - "anomaly_summary_xlm = anomalies_df[\n", + "anomaly_summary_xlm=anomalies_df[\n", " # (anomalies_df.unique_id=='SOL') & \\\n", - " (\n", - " (anomalies_df[\"Chronos-anomaly\"] == True)\n", - " | (anomalies_df[\"SeasonalNaive-anomaly\"] == True)\n", - " | (anomalies_df[\"Theta-anomaly\"] == True)\n", - " )\n", - "].reset_index(drop=True)\n", + " ((anomalies_df['Chronos-anomaly']==True) | \\\n", + " (anomalies_df['SeasonalNaive-anomaly']==True) |\n", + " (anomalies_df['Theta-anomaly']==True)\n", + " )].reset_index(drop=True)\n", "anomaly_summary_xlm" ] }, @@ -957,14 +954,12 @@ } ], "source": [ - "anomaly_summary_xlm = anomalies_df[\n", - " (anomalies_df.unique_id == \"ADA\")\n", - " & (\n", - " (anomalies_df[\"Chronos-anomaly\"] == True)\n", - " | (anomalies_df[\"SeasonalNaive-anomaly\"] == True)\n", - " | (anomalies_df[\"Theta-anomaly\"] == True)\n", - " )\n", - "].reset_index(drop=True)\n", + "anomaly_summary_xlm=anomalies_df[\n", + " (anomalies_df.unique_id=='ADA') & \\\n", + " ((anomalies_df['Chronos-anomaly']==True) | \\\n", + " (anomalies_df['SeasonalNaive-anomaly']==True) |\n", + " (anomalies_df['Theta-anomaly']==True)\n", + " )].reset_index(drop=True)\n", "anomaly_summary_xlm" ] }, @@ -1204,14 +1199,12 @@ } ], "source": [ - "anomaly_summary_xlm = anomalies_df[\n", - " (anomalies_df.unique_id == \"ADA\")\n", - " & (\n", - " (anomalies_df[\"Chronos-anomaly\"] == True)\n", - " & (anomalies_df[\"SeasonalNaive-anomaly\"] == True)\n", - " # (anomalies_df['Theta-anomaly']==True)\n", - " )\n", - "].reset_index(drop=True)\n", + "anomaly_summary_xlm=anomalies_df[\n", + " (anomalies_df.unique_id=='ADA') & \\\n", + " ((anomalies_df['Chronos-anomaly']==True) & \\\n", + " (anomalies_df['SeasonalNaive-anomaly']==True) \\\n", + " # (anomalies_df['Theta-anomaly']==True)\n", + " )].reset_index(drop=True)\n", "anomaly_summary_xlm" ] }, @@ -1248,12 +1241,12 @@ "source": [ "tcf1 = TimeCopilotForecaster(\n", " models=[\n", - " AutoARIMA(),\n", + " AutoARIMA(), \n", " Chronos(repo_id=\"amazon/chronos-bolt-mini\"),\n", " Theta(),\n", - " AutoETS(),\n", - " Moirai(),\n", - " Prophet(),\n", + " AutoETS(), \n", + " Moirai(), \n", + " Prophet(), \n", " SeasonalNaive(),\n", " ]\n", ")" @@ -1266,7 +1259,7 @@ "metadata": {}, "outputs": [], "source": [ - "fcst_df = tcf1.forecast(df=big_df, h=30, level=[80, 90])" + "fcst_df = tcf1.forecast(df=big_df, h=30, level=[80,90])" ] }, { @@ -1310,9 +1303,9 @@ "metadata": {}, "outputs": [], "source": [ - "eth_fcst_normal = fcst_df[(fcst_df.unique_id == \"ETH\")][\n", - " [\"unique_id\", \"ds\", \"Chronos\", \"Chronos-lo-80\"]\n", - "].reset_index(drop=True)" + "eth_fcst_normal=fcst_df[(fcst_df.unique_id=='ETH')]\\\n", + " [['unique_id','ds','Chronos','Chronos-lo-80']]\\\n", + " .reset_index(drop=True)" ] }, { @@ -1352,9 +1345,9 @@ "metadata": {}, "outputs": [], "source": [ - "eth_fcst_missing = fcst_df[(fcst_df.unique_id == \"ETH\")][\n", - " [\"unique_id\", \"ds\", \"Chronos\", \"Chronos-lo-80\"]\n", - "].reset_index(drop=True)" + "eth_fcst_missing=fcst_df[(fcst_df.unique_id=='ETH')]\\\n", + " [['unique_id','ds','Chronos','Chronos-lo-80']]\\\n", + " .reset_index(drop=True)" ] }, { @@ -1522,9 +1515,9 @@ } ], "source": [ - "compare = eth_fcst_normal.merge(eth_fcst_missing, on=[\"ds\", \"unique_id\"])\n", - "compare[\"dif\"] = abs(compare[\"Chronos_x\"] - compare[\"Chronos_y\"])\n", - "print(compare[\"dif\"].sum())" + "compare=eth_fcst_normal.merge(eth_fcst_missing,on=['ds','unique_id'])\n", + "compare['dif']=abs(compare['Chronos_x']-compare['Chronos_y'])\n", + "print(compare['dif'].sum())" ] }, { diff --git a/mkdocs.yml b/mkdocs.yml index a40294e..f2fb26c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,12 +22,12 @@ nav: - examples/agent-quickstart.ipynb - examples/llm-providers.ipynb - examples/aws-bedrock.ipynb - - examples/google-llms.ipynb - examples/forecaster-quickstart.ipynb - examples/anomaly-detection-forecaster-quickstart.ipynb - examples/ts-foundation-models-comparison-quickstart.ipynb - examples/gift-eval.ipynb - examples/chronos-family.ipynb + - examples/crytpocurrency-quickstart.ipynb - examples/finetuning.ipynb - examples/cryptocurrency-quickstart.ipynb - examples/sktime.ipynb @@ -47,7 +47,6 @@ nav: - api/models/ml.md - api/models/neural.md - api/models/ensembles.md - - api/models/adapters/adapters.md - api/models/utils/forecaster.md - api/gift-eval/gift-eval.md - Changelogs: @@ -76,7 +75,6 @@ nav: theme: name: "material" - custom_dir: docs/overrides logo: https://timecopilot.s3.amazonaws.com/public/logos/logo-white.svg favicon: https://timecopilot.s3.amazonaws.com/public/logos/favicon-white.svg palette: diff --git a/timecopilot/agent.py b/timecopilot/agent.py index f24e053..cf1440f 100644 --- a/timecopilot/agent.py +++ b/timecopilot/agent.py @@ -146,7 +146,6 @@ def prettify( """Pretty print the forecast results using rich formatting.""" console = console or Console() - # Create header with title and overview header = Panel( f"[bold cyan]{self.selected_model}[/bold cyan] forecast analysis\n" f"[{'green' if self.is_better_than_seasonal_naive else 'red'}]" @@ -157,7 +156,6 @@ def prettify( style="blue", ) - # Time Series Analysis Section - check if features_df is available ts_features = Table( title="Time Series Features", show_header=True, @@ -167,13 +165,11 @@ def prettify( ts_features.add_column("Feature", style="cyan") ts_features.add_column("Value", style="magenta") - # Use features_df if available (attached after forecast run) if features_df is not None: for feature_name, feature_value in features_df.iloc[0].items(): if pd.notna(feature_value): ts_features.add_row(feature_name, f"{float(feature_value):.3f}") else: - # Fallback: show a note that detailed features are not available ts_features.add_row("Features", "Available in analysis text below") ts_analysis = Panel( @@ -182,7 +178,6 @@ def prettify( style="blue", ) - # Model Selection Section model_details = Panel( f"[bold]Technical Details[/bold]\n{self.model_details}\n\n" f"[bold]Selection Rationale[/bold]\n{self.reason_for_selection}", @@ -190,27 +185,39 @@ def prettify( style="green", ) - # Model Comparison Table - check if eval_df is available model_scores = Table( title="Model Performance", show_header=True, title_style="bold yellow" ) model_scores.add_column("Model", style="yellow") model_scores.add_column("MASE", style="cyan", justify="right") - # Use eval_df if available (attached after forecast run) + has_std = False + if eval_df is not None: + has_std = any(col.endswith("_std") for col in eval_df.columns) + + if has_std: + model_scores.add_column("STD", style="magenta", justify="right") + if eval_df is not None: - # Get the MASE scores from eval_df model_scores_data = [] for col in eval_df.columns: - if col != "metric" and pd.notna(eval_df[col].iloc[0]): - model_scores_data.append((col, float(eval_df[col].iloc[0]))) + if col.endswith("_std") or col == "metric": + continue + + mean = float(eval_df[col].iloc[0]) + std = float(eval_df.get(f"{col}_std", pd.Series([0])).iloc[0]) + + if pd.notna(mean): + model_scores_data.append((col, mean, std)) - # Sort by score (lower MASE is better) model_scores_data.sort(key=lambda x: x[1]) - for model, score in model_scores_data: - model_scores.add_row(model, f"{score:.3f}") + + for model, mean, std in model_scores_data: + if has_std: + model_scores.add_row(model, f"{mean:.3f}", f"{std:.3f}") + else: + model_scores.add_row(model, f"{mean:.3f}") else: - # Fallback: show a note that detailed scores are not available model_scores.add_row("Scores", "Available in analysis text below") model_analysis = Panel( @@ -219,16 +226,13 @@ def prettify( style="yellow", ) - # Forecast Results Section - check if fcst_df is available forecast_table = Table( title="Forecast Values", show_header=True, title_style="bold magenta" ) forecast_table.add_column("Period", style="magenta") forecast_table.add_column("Value", style="cyan", justify="right") - # Use fcst_df if available (attached after forecast run) if fcst_df is not None: - # Show forecast values from fcst_df fcst_data = fcst_df.copy() if "ds" in fcst_data.columns and self.selected_model in fcst_data.columns: for _, row in fcst_data.iterrows(): @@ -240,7 +244,6 @@ def prettify( value = row[self.selected_model] forecast_table.add_row(period, f"{value:.2f}") - # Add note about number of periods if many if len(fcst_data) > 12: forecast_table.caption = ( f"[dim]Showing all {len(fcst_data)} forecasted periods. " @@ -249,7 +252,6 @@ def prettify( else: forecast_table.add_row("Forecast", "Available in analysis text below") else: - # Fallback: show a note that detailed forecast is not available forecast_table.add_row("Forecast", "Available in analysis text below") forecast_analysis = Panel( @@ -258,14 +260,12 @@ def prettify( style="magenta", ) - # Anomaly Detection Section anomaly_analysis = Panel( self.anomaly_analysis, title="[bold red]Anomaly Detection[/bold red]", style="red", ) - # Optional user response section user_response = None if self.user_query_response: user_response = Panel( @@ -274,7 +274,6 @@ def prettify( style="cyan", ) - # Print all sections with clear separation console.print("\n") console.print(header) @@ -300,7 +299,6 @@ def prettify( console.print("\n") - def _transform_time_series_to_text(df: pd.DataFrame) -> str: df_agg = df.groupby("unique_id").agg(list) output = ( @@ -323,11 +321,22 @@ def _transform_features_to_text(features_df: pd.DataFrame) -> str: ) return output +def _transform_eval_to_text(eval_df, models): + parts = [] -def _transform_eval_to_text(eval_df: pd.DataFrame, models: list[str]) -> str: - output = ", ".join([f"{model}: {eval_df[model].iloc[0]}" for model in models]) - return output + for model in models: + mean = eval_df[model].iloc[0] + + # Filter for cross-validation involving only one fold + std = eval_df.get(f"{model}_std") + stability = "N/A" if std is None or std.iloc[0] == 0 \ + else f"{std.iloc[0]:.3f}" + parts.append( + f"{model}: mean MASE={mean:.3f}, stability={stability}" + ) + + return ", ".join(parts) def _transform_fcst_to_text(fcst_df: pd.DataFrame) -> str: df_agg = fcst_df.groupby("unique_id").agg(list) @@ -995,15 +1004,26 @@ async def cross_validation_tool( df=ctx.deps.df, h=ctx.deps.h, freq=ctx.deps.freq, + cv_mode=ctx.deps.cv_mode, ) + n_folds = fcst_cv["cutoff"].nunique() eval_df = ctx.deps.evaluate_forecast_df( forecast_df=fcst_cv, models=[model.alias for model in callable_models], ) - eval_df = eval_df.groupby( - ["metric"], - as_index=False, - ).mean(numeric_only=True) + eval_mean = eval_df.groupby(["metric"], as_index=False)\ + .mean(numeric_only=True) + eval_std = eval_df.groupby(["metric"], as_index=False)\ + .std(numeric_only=True) + eval_df = eval_mean.copy() + + if n_folds > 1: + eval_std = eval_df.groupby(["metric"], as_index=False)\ + .std(numeric_only=True) + for col in eval_mean.columns: + if col != "metric": + eval_df[f"{col}_std"] = eval_std[col] + self.eval_df = eval_df self.eval_forecasters = models return _transform_eval_to_text(eval_df, models) @@ -1216,6 +1236,7 @@ def analyze( freq: str | None = None, seasonality: int | None = None, query: str | None = None, + cv_mode: str | None = None, ) -> AgentRunResult[ForecastAgentOutput]: """Generate forecast and anomaly analysis. @@ -1257,6 +1278,7 @@ def analyze( seasonality, query, ) + self.dataset.cv_mode = cv_mode query = query if query else "complete system prompt" result = self.forecasting_agent.run_sync( user_prompt=query, @@ -1427,6 +1449,7 @@ async def analyze( freq: str | None = None, seasonality: int | None = None, query: str | None = None, + cv_mode: str | None = None, ) -> AgentRunResult[ForecastAgentOutput]: """ Asynchronously analyze time series data with forecasting, anomaly detection, diff --git a/timecopilot/forecaster.py b/timecopilot/forecaster.py index a27ac5f..a7120aa 100644 --- a/timecopilot/forecaster.py +++ b/timecopilot/forecaster.py @@ -96,11 +96,9 @@ def _call_models( res_df_model = fn(**known_kwargs, **kwargs) res_df_model = res_df_model.rename( columns={ - col: ( - col.replace(self.fallback_model.alias, model.alias) - if col.startswith(self.fallback_model.alias) - else col - ) + col: col.replace(self.fallback_model.alias, model.alias) + if col.startswith(self.fallback_model.alias) + else col for col in res_df_model.columns } ) @@ -192,61 +190,54 @@ def cross_validation( step_size: int | None = None, level: list[int | float] | None = None, quantiles: list[float] | None = None, + cv_mode: str | None = None, + verbose: bool = True, ) -> pd.DataFrame: """ - This method splits the time series into multiple training and testing - windows and generates forecasts for each window. It enables evaluating - forecast accuracy over different historical periods. Supports point - forecasts and, optionally, prediction intervals or quantile forecasts. + Cross-validation with optional preset modes. - Args: - df (pd.DataFrame): - DataFrame containing the time series to forecast. It must - include as columns: + This method performs rolling-origin cross-validation by splitting the + time series into multiple training and testing windows and generating + forecasts for each window. It enables evaluating forecast accuracy over + different historical periods. - - "unique_id": an ID column to distinguish multiple series. - - "ds": a time column indicating timestamps or periods. - - "y": a target column with the observed values. + Backward compatibility is preserved: if no additional parameters are + provided, the method behaves exactly as before with `n_windows=1`. - h (int): - Forecast horizon specifying how many future steps to predict in - each window. - freq (str, optional): - Frequency of the time series (e.g. "D" for daily, "M" for - monthly). See [Pandas frequency aliases](https://pandas.pydata. - org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases) - for valid values. If not provided, the frequency will be inferred - from the data. - n_windows (int, optional): - Number of cross-validation windows to generate. Defaults to 1. - step_size (int, optional): - Step size between the start of consecutive windows. If None, it - defaults to `h`. - level (list[int | float], optional): - Confidence levels for prediction intervals, expressed as - percentages (e.g. [80, 95]). When specified, the output - DataFrame includes lower and upper interval columns for each - level. - quantiles (list[float], optional): - Quantiles to forecast, expressed as floats between 0 and 1. - Should not be used simultaneously with `level`. If provided, - additional columns named "model-q-{percentile}" will appear in - the output, where {percentile} is 100 × quantile value. + New optional parameter: + cv_mode (str, optional): + Preset cross-validation configurations for convenience. - Returns: - pd.DataFrame: - DataFrame containing the forecasts for each cross-validation - window. The output includes: + - "quick": 1 window (fastest, equivalent to previous behavior) + - "std": 3 windows (balanced accuracy vs runtime) + - "robust": 5 windows (more stable model evaluation) - - "unique_id" column to indicate the series. - - "ds" column to indicate the timestamp. - - "y" column to indicate the target. - - "cutoff" column to indicate which window each forecast - belongs to. - - point forecasts for each timestamp, series and model. - - prediction intervals if `level` is specified. - - quantile forecasts if `quantiles` is specified. + If `cv_mode` is provided and `n_windows` is left at its default + value (1), the number of windows will be overridden by the + selected preset. + + If `n_windows` is explicitly provided by the user, it always + takes precedence over `cv_mode`. + + This design allows existing code to run unchanged while enabling users + and the agent to opt into more robust cross-validation with a simple + parameter. """ + + # Backward compatibility: explicit n_windows always wins + if cv_mode is not None: + if cv_mode == "quick": + n_windows = 1 + elif cv_mode == "std": + n_windows = 3 + elif cv_mode == "robust": + n_windows = 5 + else: + raise ValueError("cv_mode must be one of: quick, std, robust") + + if verbose: + print(f"[TimeCopilot] Starting cross-validation with {n_windows} window(s)") + return self._call_models( "cross_validation", merge_on=["unique_id", "ds", "cutoff"], @@ -259,6 +250,9 @@ def cross_validation( quantiles=quantiles, ) + if verbose: + print(f"[TimeCopilot] Starting cross-validation with {n_windows} window(s)") + def detect_anomalies( self, df: pd.DataFrame, diff --git a/timecopilot/utils/experiment_handler.py b/timecopilot/utils/experiment_handler.py index 5e9bc4c..d760dc5 100644 --- a/timecopilot/utils/experiment_handler.py +++ b/timecopilot/utils/experiment_handler.py @@ -177,6 +177,7 @@ class ExperimentDataset: freq: str h: int seasonality: int + cv_mode: str | None = None def evaluate_forecast_df( self,