diff --git a/.gitignore b/.gitignore index 85e6e08..6af63bd 100644 --- a/.gitignore +++ b/.gitignore @@ -213,4 +213,5 @@ log.txt *.broken test_*coverage*.py test_remaining*.py -test_final*.py \ No newline at end of file +test_final*.py +src/health.db diff --git a/pyproject.toml b/pyproject.toml index 518cb90..60016a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools>=61.0", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" [project] name = "garmy" diff --git a/src/garmy/core/utils.py b/src/garmy/core/utils.py index 9df46ae..b3fd754 100644 --- a/src/garmy/core/utils.py +++ b/src/garmy/core/utils.py @@ -248,19 +248,22 @@ class TimestampMixin: @staticmethod def timestamp_to_datetime(timestamp: int) -> datetime: - """Convert Unix timestamp (milliseconds) to datetime object. + """Convert Unix timestamp (milliseconds) to a timezone-aware UTC datetime object. Args: timestamp: Unix timestamp in milliseconds. Returns: - Corresponding datetime object. + Corresponding timezone-aware UTC datetime object. Example: - >>> TimestampMixin.timestamp_to_datetime(1640995200000) - datetime.datetime(2022, 1, 1, 0, 0) + >>> from datetime import timezone + >>> ts = TimestampMixin.timestamp_to_datetime(1640995200000) + >>> ts.year == 2022 and ts.tzinfo == timezone.utc + True """ - return datetime.fromtimestamp(timestamp / 1000) + from datetime import timezone + return datetime.fromtimestamp(timestamp / 1000, tz=timezone.utc) @staticmethod def iso_to_datetime(iso_string: Optional[str]) -> Optional[datetime]: diff --git a/src/garmy/localdb/activities_iterator.py b/src/garmy/localdb/activities_iterator.py index 7f88a9c..d0c081d 100644 --- a/src/garmy/localdb/activities_iterator.py +++ b/src/garmy/localdb/activities_iterator.py @@ -33,31 +33,33 @@ def initialize(self): self._advance_to_next_activity() def _load_next_batch(self) -> bool: - """Load next batch of activities from API.""" + """Load next batch of activities from API in ascending order (oldest first).""" if not self.has_more_data: return False - + try: batch_size = self.sync_config.activities_batch_size + # Request activities in ascending order (oldest first) to match sync order activities_batch = self.api_client.metrics.get('activities').list( limit=batch_size, - start=self.batch_offset + start=self.batch_offset, + sort_order='asc' ) - + if not activities_batch or len(activities_batch) == 0: self.has_more_data = False return False - + # Append to cache and update offset self.activities_cache.extend(activities_batch) self.batch_offset += len(activities_batch) - + # Check if we got less than requested (indicates end of data) if len(activities_batch) < batch_size: self.has_more_data = False - + return True - + except Exception as e: self.progress.warning(f"Failed to load activities batch at offset {self.batch_offset}: {e}") self.has_more_data = False @@ -112,37 +114,40 @@ def _extract_activity_date(self, activity) -> Optional[date]: return None def get_activities_for_date(self, target_date: date) -> List[Any]: - """Get all activities for a specific date.""" + """Get all activities for a specific date. + + Note: Activities are loaded in ascending order (oldest first) from the API. + """ activities = [] - + # Ensure we have a current activity if self.current_activity is None: if not self._advance_to_next_activity(): return activities - - # Process activities while they match or are newer than target_date + + # Process activities while they match or are older than target_date while self.current_activity is not None: if self.current_activity_date is None: # Skip activities without dates if not self._advance_to_next_activity(): break continue - - if self.current_activity_date > target_date: - # Activity is newer than target - skip it + + if self.current_activity_date < target_date: + # Activity is older than target - skip it if not self._advance_to_next_activity(): break continue - + elif self.current_activity_date == target_date: # Activity matches target date - collect it activities.append(self.current_activity) if not self._advance_to_next_activity(): break continue - - else: # self.current_activity_date < target_date - # Activity is older than target - we're done for this date + + else: # self.current_activity_date > target_date + # Activity is newer than target - we're done for this date break - + return activities \ No newline at end of file diff --git a/src/garmy/localdb/cli.py b/src/garmy/localdb/cli.py index 185e692..339d19b 100644 --- a/src/garmy/localdb/cli.py +++ b/src/garmy/localdb/cli.py @@ -76,13 +76,10 @@ def cmd_sync(args) -> int: start_date = end_date - timedelta(days=6) print(f"Syncing data from {start_date} to {end_date}") - - # Get credentials - email, password = get_credentials() - + # Setup progress reporter progress_reporter = ProgressReporter(use_tqdm=args.progress == 'tqdm') - + # Initialize sync manager config = LocalDBConfig() manager = SyncManager( @@ -90,10 +87,16 @@ def cmd_sync(args) -> int: config=config, progress_reporter=progress_reporter ) - - # Initialize with credentials + + # Try to initialize with saved tokens first print("Connecting to Garmin Connect...") - manager.initialize(email, password) + try: + manager.initialize() + print("Using saved authentication tokens") + except RuntimeError: + # No valid tokens, prompt for credentials + email, password = get_credentials() + manager.initialize(email, password) # Parse metrics metrics = parse_metrics(args.metrics) if args.metrics else list(MetricType) diff --git a/src/garmy/localdb/extractors.py b/src/garmy/localdb/extractors.py index c867863..c0982c2 100644 --- a/src/garmy/localdb/extractors.py +++ b/src/garmy/localdb/extractors.py @@ -208,7 +208,8 @@ def extract_timeseries_data(self, data: Any, metric_type: MetricType) -> List[Tu for reading in data.heart_rate_values_array: if isinstance(reading, (list, tuple)) and len(reading) >= 2: timestamp, heart_rate = reading[0], reading[1] - timeseries_data.append((timestamp, heart_rate, {})) + if heart_rate is not None: + timeseries_data.append((timestamp, heart_rate, {})) elif metric_type == MetricType.RESPIRATION: # Respiration might have different format - check if it has readings @@ -219,11 +220,16 @@ def extract_timeseries_data(self, data: Any, metric_type: MetricType) -> List[Tu return timeseries_data def _extract_steps_data(self, data: Any) -> Dict[str, Any]: - """Extract steps data.""" - return { - 'total_steps': getattr(data, 'total_steps', None), - 'step_goal': getattr(data, 'step_goal', None) - } + """Extract steps data for the most recent day in the data object.""" + if hasattr(data, 'daily_steps') and data.daily_steps: + # The 'data' object can contain multiple daily summaries. + # We assume the last one corresponds to the single 'sync_date' used for the sync operation. + latest_day = data.daily_steps[-1] + return { + 'total_steps': getattr(latest_day, 'total_steps', None), + 'step_goal': getattr(latest_day, 'step_goal', None) + } + return {} def _extract_calories_data(self, data: Any) -> Dict[str, Any]: """Extract calories data.""" diff --git a/src/garmy/localdb/progress.py b/src/garmy/localdb/progress.py index 208530c..8bb9633 100644 --- a/src/garmy/localdb/progress.py +++ b/src/garmy/localdb/progress.py @@ -60,6 +60,10 @@ def info(self, message: str): def error(self, message: str): """Log error message.""" self.logger.error(message) + + def warning(self, message: str): + """Log warning message.""" + self.logger.warning(message) def end_sync(self): """End sync progress tracking.""" diff --git a/src/garmy/localdb/sync.py b/src/garmy/localdb/sync.py index e71bbe6..c6d7603 100644 --- a/src/garmy/localdb/sync.py +++ b/src/garmy/localdb/sync.py @@ -37,13 +37,34 @@ def __init__(self, self.api_client = None self.activities_iterator = None - def initialize(self, email: str, password: str): - """Initialize with Garmin credentials.""" + def initialize(self, email: Optional[str] = None, password: Optional[str] = None): + """Initialize with Garmin credentials or saved tokens. + + Args: + email: Garmin account email (optional if tokens are saved) + password: Garmin account password (optional if tokens are saved) + """ try: from garmy import AuthClient, APIClient auth_client = AuthClient() - auth_client.login(email, password) + + # Check if already authenticated with saved tokens + if not auth_client.is_authenticated: + if auth_client.needs_refresh: + self.progress.info("Refreshing authentication tokens...") + auth_client.refresh_tokens() + elif email and password: + auth_client.login( + email, + password, + prompt_mfa=lambda: input("MFA code: "), + ) + else: + raise RuntimeError( + "No valid saved tokens found. Please provide email and password." + ) + self.api_client = APIClient(auth_client=auth_client) self.activities_iterator = ActivitiesIterator( @@ -187,6 +208,8 @@ def _sync_activities_for_date(self, user_id: int, sync_date: date, stats: Dict[s self.db.store_activity(user_id, activity_data) stats['completed'] += 1 + # Update sync status for the entire ACTIVITIES metric for this date + self.db.update_sync_status(user_id, sync_date, MetricType.ACTIVITIES, 'completed') self.progress.task_complete("activities", sync_date) except Exception as e: @@ -250,4 +273,4 @@ def query_timeseries(self, user_id: int, metric_type: MetricType, 'timestamp': ts, 'value': value, 'metadata': metadata - } for ts, value, metadata in data] + } for ts, value, metadata in data] \ No newline at end of file diff --git a/src/garmy/metrics/activities.py b/src/garmy/metrics/activities.py index a2e92fb..1447fd5 100644 --- a/src/garmy/metrics/activities.py +++ b/src/garmy/metrics/activities.py @@ -250,9 +250,15 @@ def __init__(self, api_client: Any) -> None: self.api_client = api_client self.parse_func = parse_activities_data - def raw(self, limit: int = 20, start: int = 0) -> Any: - """Get raw API response for activities data.""" - endpoint = f"/activitylist-service/activities/search/activities?limit={limit}&start={start}" + def raw(self, limit: int = 20, start: int = 0, sort_order: str = "desc") -> Any: + """Get raw API response for activities data. + + Args: + limit: Maximum number of activities to return + start: Starting offset for pagination + sort_order: Sort order - "asc" for oldest first, "desc" for newest first (default) + """ + endpoint = f"/activitylist-service/activities/search/activities?limit={limit}&start={start}&sortOrder={sort_order}" try: return self.api_client.connectapi(endpoint) except (SystemExit, KeyboardInterrupt, GeneratorExit): @@ -262,17 +268,18 @@ def raw(self, limit: int = 20, start: int = 0) -> Any: return handle_api_exception(e, "fetching Activities", endpoint, []) - def list(self, limit: int = 20, start: int = 0) -> List[ActivitySummary]: + def list(self, limit: int = 20, start: int = 0, sort_order: str = "desc") -> List[ActivitySummary]: """Get parsed activities data. Args: limit: Maximum number of activities to return (default: 20) start: Starting offset for pagination (default: 0) + sort_order: Sort order - "asc" for oldest first, "desc" for newest first (default) Returns: List of ActivitySummary objects """ - raw_data = self.raw(limit, start) + raw_data = self.raw(limit, start, sort_order) if not raw_data: return [] result = self.parse_func(raw_data) diff --git a/tests/test_core_utils.py b/tests/test_core_utils.py index 07f80c4..80f1077 100644 --- a/tests/test_core_utils.py +++ b/tests/test_core_utils.py @@ -440,6 +440,7 @@ class TestTimestampMixin: def test_timestamp_to_datetime(self): """Test timestamp_to_datetime method.""" + from datetime import timezone # Test Unix timestamp in milliseconds timestamp = 1640995200000 # 2022-01-01 00:00:00 UTC result = TimestampMixin.timestamp_to_datetime(timestamp) @@ -448,6 +449,7 @@ def test_timestamp_to_datetime(self): assert result.year == 2022 assert result.month == 1 assert result.day == 1 + assert result.tzinfo == timezone.utc def test_timestamp_to_datetime_different_values(self): """Test timestamp_to_datetime with different values.""" diff --git a/tests/test_localdb_db.py b/tests/test_localdb_db.py new file mode 100644 index 0000000..6efc621 --- /dev/null +++ b/tests/test_localdb_db.py @@ -0,0 +1,236 @@ + +from datetime import date, datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from garmy.localdb.db import HealthDB +from garmy.localdb.models import Base, TimeSeries, Activity, DailyHealthMetric, SyncStatus, MetricType + + +@pytest.fixture +def db_path(tmp_path: Path) -> Path: + """Fixture for temporary database path.""" + return tmp_path / "test_health.db" + + +@pytest.fixture +def health_db(db_path: Path) -> HealthDB: + """Fixture for HealthDB instance.""" + db = HealthDB(db_path) + yield db + db.engine.dispose() + + +def test_initialization(health_db: HealthDB, db_path: Path): + """Test database initialization.""" + assert db_path.exists() + assert health_db.engine is not None + assert health_db.SessionLocal is not None + +def test_get_session(health_db: HealthDB): + """Test getting a database session.""" + with health_db.get_session() as session: + assert session is not None + +def test_get_schema_info(health_db: HealthDB, db_path: Path): + """Test getting schema information.""" + schema_info = health_db.get_schema_info() + assert set(schema_info["tables"]) == {"timeseries", "activities", "daily_health_metrics", "sync_status"} + assert schema_info["db_path"] == str(db_path) + +def test_validate_schema(health_db: HealthDB): + """Test schema validation.""" + assert health_db.validate_schema() + +def test_store_timeseries_batch(health_db: HealthDB): + """Test storing a batch of timeseries data.""" + user_id = 1 + metric_type = MetricType.HEART_RATE + data = [ + (datetime(2023, 1, 1, 12, 0, 0), 80, {}), + (datetime(2023, 1, 1, 12, 1, 0), 82, {}), + ] + health_db.store_timeseries_batch(user_id, metric_type, data) + with health_db.get_session() as session: + timeseries = session.query(TimeSeries).all() + assert len(timeseries) == 2 + assert timeseries[0].user_id == user_id + assert timeseries[0].metric_type == metric_type.value + +def test_store_activity(health_db: HealthDB): + """Test storing activity data.""" + user_id = 1 + activity_data = { + "activity_id": "12345", + "activity_date": date(2023, 1, 1), + "activity_name": "Running", + } + health_db.store_activity(user_id, activity_data) + with health_db.get_session() as session: + activity = session.query(Activity).first() + assert activity is not None + assert activity.user_id == user_id + assert activity.activity_id == "12345" + +def test_store_health_metric(health_db: HealthDB): + """Test storing daily health metric data.""" + user_id = 1 + metric_date = date(2023, 1, 1) + health_db.store_health_metric(user_id, metric_date, total_steps=1000) + with health_db.get_session() as session: + metric = session.query(DailyHealthMetric).first() + assert metric is not None + assert metric.user_id == user_id + assert metric.metric_date == metric_date + assert metric.total_steps == 1000 + +def test_create_and_get_sync_status(health_db: HealthDB): + """Test creating and getting sync status.""" + user_id = 1 + sync_date = date(2023, 1, 1) + metric_type = MetricType.HEART_RATE + health_db.create_sync_status(user_id, sync_date, metric_type) + status = health_db.get_sync_status(user_id, sync_date, metric_type) + assert status == "pending" + +def test_update_sync_status(health_db: HealthDB): + """Test updating sync status.""" + user_id = 1 + sync_date = date(2023, 1, 1) + metric_type = MetricType.HEART_RATE + health_db.create_sync_status(user_id, sync_date, metric_type) + health_db.update_sync_status( + user_id, sync_date, metric_type, "success" + ) + status = health_db.get_sync_status(user_id, sync_date, metric_type) + assert status == "success" + +def test_get_pending_metrics(health_db: HealthDB): + """Test getting pending metrics.""" + user_id = 1 + sync_date = date(2023, 1, 1) + health_db.create_sync_status( + user_id, sync_date, MetricType.HEART_RATE, "pending" + ) + health_db.create_sync_status( + user_id, sync_date, MetricType.STEPS, "success" + ) + pending_metrics = health_db.get_pending_metrics(user_id, sync_date) + assert len(pending_metrics) == 1 + assert pending_metrics[0] == MetricType.HEART_RATE.value + +def test_existence_checks(health_db: HealthDB): + """Test existence check methods.""" + user_id = 1 + activity_id = "12345" + metric_date = date(2023, 1, 1) + sync_date = date(2023, 1, 1) + metric_type = MetricType.HEART_RATE + + assert not health_db.activity_exists(user_id, activity_id) + assert not health_db.health_metric_exists(user_id, metric_date) + assert not health_db.sync_status_exists(user_id, sync_date, metric_type) + + health_db.store_activity(user_id, {"activity_id": activity_id, "activity_date": metric_date}) + health_db.store_health_metric(user_id, metric_date) + health_db.create_sync_status(user_id, sync_date, metric_type) + + assert health_db.activity_exists(user_id, activity_id) + assert health_db.health_metric_exists(user_id, metric_date) + assert health_db.sync_status_exists(user_id, sync_date, metric_type) + +def test_get_health_metrics(health_db: HealthDB): + """Test querying health metrics.""" + user_id = 1 + start_date = date(2023, 1, 1) + end_date = date(2023, 1, 2) + health_db.store_health_metric(user_id, start_date, total_steps=1000) + health_db.store_health_metric(user_id, end_date, total_steps=2000) + + metrics = health_db.get_health_metrics(user_id, start_date, end_date) + assert len(metrics) == 2 + assert metrics[0]["total_steps"] == 1000 + assert metrics[1]["total_steps"] == 2000 + +def test_get_activities(health_db: HealthDB): + """Test querying activities.""" + user_id = 1 + start_date = date(2023, 1, 1) + end_date = date(2023, 1, 2) + health_db.store_activity( + user_id, {"activity_id": "1", "activity_date": start_date, "activity_name": "Running"} + ) + health_db.store_activity( + user_id, {"activity_id": "2", "activity_date": end_date, "activity_name": "Cycling"} + ) + + activities = health_db.get_activities(user_id, start_date, end_date) + assert len(activities) == 2 + assert activities[0]["activity_name"] == "Running" + assert activities[1]["activity_name"] == "Cycling" + +def test_get_timeseries(health_db: HealthDB): + """Test querying timeseries data.""" + user_id = 1 + metric_type = MetricType.HEART_RATE + start_timestamp = int(datetime(2023, 1, 1, 12, 0, 0).timestamp()) + end_timestamp = int(datetime(2023, 1, 1, 12, 1, 0).timestamp()) + data = [ + (start_timestamp, 80, {}), + (end_timestamp, 82, {}), + ] + health_db.store_timeseries_batch(user_id, metric_type, data) + + timeseries = health_db.get_timeseries( + user_id, metric_type, start_timestamp, end_timestamp + ) + assert len(timeseries) == 2 + assert timeseries[0][1] == 80 + assert timeseries[1][1] == 82 + +@patch('garmy.AuthClient') +@patch('garmy.APIClient') +@patch('garmy.localdb.sync.ActivitiesIterator') +def test_sync_activities_updates_status(mock_activities_iterator, mock_api_client, mock_auth_client, health_db: HealthDB): + """Test that syncing activities updates their sync_status to 'completed'.""" + from garmy.localdb.sync import SyncManager + from garmy.localdb.config import LocalDBConfig + + user_id = 1 + sync_date = date(2023, 1, 1) + + # Mock APIClient and ActivitiesIterator behavior + mock_api_client_instance = mock_api_client.return_value + mock_api_client_instance.metrics.get.return_value.get.return_value = MagicMock() # For other metrics if any + + mock_activities_iterator_instance = mock_activities_iterator.return_value + mock_activities_iterator_instance.get_activities_for_date.return_value = [ + MagicMock(activity_id="123", activity_date=sync_date, activity_name="Running"), + MagicMock(activity_id="456", activity_date=sync_date, activity_name="Walking") + ] + mock_activities_iterator_instance.initialize.return_value = None + + # Mock DataExtractor to return some data + with patch('garmy.localdb.sync.DataExtractor') as MockDataExtractor: + mock_extractor_instance = MockDataExtractor.return_value + mock_extractor_instance.extract_metric_data.side_effect = [ + {"activity_id": "123", "activity_date": sync_date, "activity_name": "Running"}, + {"activity_id": "456", "activity_date": sync_date, "activity_name": "Walking"} + ] + + # Initialize SyncManager + sync_manager = SyncManager(db_path=health_db.db_path, config=LocalDBConfig()) + sync_manager.api_client = mock_api_client_instance + sync_manager.activities_iterator = mock_activities_iterator_instance + + # Call sync_range for activities + sync_manager.sync_range(user_id, sync_date, sync_date, metrics=[MetricType.ACTIVITIES]) + + # Assert that the sync status for ACTIVITIES is 'completed' + status = health_db.get_sync_status(user_id, sync_date, MetricType.ACTIVITIES) + assert status == 'completed' + diff --git a/tests/test_metrics_comprehensive.py b/tests/test_metrics_comprehensive.py index 6454c8c..b5de261 100644 --- a/tests/test_metrics_comprehensive.py +++ b/tests/test_metrics_comprehensive.py @@ -984,7 +984,7 @@ def test_activities_accessor_list(self): result = accessor.list(limit=20) assert isinstance(result, list) - mock_raw.assert_called_once_with(20, 0) + mock_raw.assert_called_once_with(20, 0, 'desc') mock_parse.assert_called_once() def test_activities_accessor_list_empty_data(self): diff --git a/tests/test_metrics_remaining.py b/tests/test_metrics_remaining.py index f87876f..7b75d8b 100644 --- a/tests/test_metrics_remaining.py +++ b/tests/test_metrics_remaining.py @@ -665,6 +665,7 @@ def test_calculated_properties_edge_cases(self): def test_timestamp_property_edge_cases(self): """Test timestamp-related properties with edge cases.""" from garmy.metrics.body_battery import BodyBatteryReading + from datetime import timezone # Test with very old timestamp reading = BodyBatteryReading( @@ -677,6 +678,7 @@ def test_timestamp_property_edge_cases(self): dt = reading.datetime assert isinstance(dt, datetime) assert dt.year == 1970 + assert dt.tzinfo == timezone.utc def test_list_processing_edge_cases(self): """Test list processing with various edge cases."""