HDXDSYS-2612 Make HDX Python API Path aware (#102)

* Make path aware

Author: mcarans

pyproject.toml

@@ -37,8 +37,8 @@ dependencies = [
     "ckanapi>=4.8",
     "defopt>=7.0.0",
     "email_validator",
-    "hdx-python-country>=4.0.0",
-    "hdx-python-utilities>=4.0.0",
+    "hdx-python-country>=4.0.1",
+    "hdx-python-utilities>=4.0.2",
     "libhxl>=5.2.2",
     "makefun",
     "quantulum3",

requirements.txt

@@ -34,9 +34,9 @@ et-xmlfile==2.0.0
     # via openpyxl
 frictionless==5.18.1
     # via hdx-python-utilities
-hdx-python-country==4.0.0
+hdx-python-country==4.0.1
     # via hdx-python-api (pyproject.toml)
-hdx-python-utilities==4.0.0
+hdx-python-utilities==4.0.2
     # via
     #   hdx-python-api (pyproject.toml)
     #   hdx-python-country

src/hdx/api/configuration.py

@@ -4,7 +4,8 @@
 import os
 from base64 import b64decode
 from collections import UserDict
-from os.path import expanduser, isfile, join
+from os.path import expanduser
+from pathlib import Path
 from typing import Any, Optional
 
 import ckanapi
@@ -38,22 +39,22 @@ class Configuration(UserDict):
         hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
         hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
         hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
-        hdx_config_json (str): Path to JSON HDX configuration OR
-        hdx_config_yaml (str): Path to YAML HDX configuration
+        hdx_config_json (Path | str): Path to JSON HDX configuration OR
+        hdx_config_yaml (Path | str): Path to YAML HDX configuration
         project_config_dict (dict): Project configuration dictionary OR
-        project_config_json (str): Path to JSON Project configuration OR
-        project_config_yaml (str): Path to YAML Project configuration
+        project_config_json (Path | str): Path to JSON Project configuration OR
+        project_config_yaml (Path | str): Path to YAML Project configuration
         hdx_base_config_dict (dict): HDX base configuration dictionary OR
-        hdx_base_config_json (str): Path to JSON HDX base configuration OR
-        hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
+        hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
+        hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
     """
 
     _configuration = None
     home_folder = expanduser("~")
     default_hdx_base_config_yaml = script_dir_plus_file(
         "hdx_base_configuration.yaml", ConfigurationError
     )
-    default_hdx_config_yaml = join(home_folder, ".hdx_configuration.yaml")
+    default_hdx_config_yaml = Path(home_folder, ".hdx_configuration.yaml")
 
     prefix = f"HDXPythonLibrary/{__version__}"
 
@@ -111,10 +112,10 @@ def __init__(self, **kwargs: Any) -> None:
                 raise ConfigurationError("More than one HDX configuration given!")
         else:
             if not hdx_config_yaml:
-                hdx_config_yaml = Configuration.default_hdx_config_yaml
-                if not isfile(hdx_config_yaml):
-                    hdx_config_yaml = hdx_config_yaml.replace(".yaml", ".yml")
-                if isfile(hdx_config_yaml):
+                hdx_config_yaml = Path(Configuration.default_hdx_config_yaml)
+                if not hdx_config_yaml.is_file():
+                    hdx_config_yaml = hdx_config_yaml.with_suffix(".yml")
+                if hdx_config_yaml.is_file():
                     logger.info(
                         f"No HDX configuration parameter. Using default configuration file: {hdx_config_yaml}."
                     )
@@ -364,7 +365,7 @@ def create_session_user_agent(
         cls,
         session: requests.Session = None,
         user_agent: str | None = None,
-        user_agent_config_yaml: str | None = None,
+        user_agent_config_yaml: Path | str | None = None,
         user_agent_lookup: str | None = None,
         use_env: bool = False,
         **kwargs: Any,
@@ -509,14 +510,14 @@ def setup(
             hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
             hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
             hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
-            hdx_config_json (str): Path to JSON HDX configuration OR
-            hdx_config_yaml (str): Path to YAML HDX configuration
+            hdx_config_json (Path | str): Path to JSON HDX configuration OR
+            hdx_config_yaml (Path | str): Path to YAML HDX configuration
             project_config_dict (dict): Project configuration dictionary OR
-            project_config_json (str): Path to JSON Project configuration OR
-            project_config_yaml (str): Path to YAML Project configuration
+            project_config_json (Path | str): Path to JSON Project configuration OR
+            project_config_yaml (Path | str): Path to YAML Project configuration
             hdx_base_config_dict (dict): HDX base configuration dictionary OR
-            hdx_base_config_json (str): Path to JSON HDX base configuration OR
-            hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
+            hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
+            hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
 
         Returns:
             None
@@ -549,14 +550,14 @@ def _create(
             hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
             hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
             hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
-            hdx_config_json (str): Path to JSON HDX configuration OR
-            hdx_config_yaml (str): Path to YAML HDX configuration
+            hdx_config_json (Path | str): Path to JSON HDX configuration OR
+            hdx_config_yaml (Path | str): Path to YAML HDX configuration
             project_config_dict (dict): Project configuration dictionary OR
-            project_config_json (str): Path to JSON Project configuration OR
-            project_config_yaml (str): Path to YAML Project configuration
+            project_config_json (Path | str): Path to JSON Project configuration OR
+            project_config_yaml (Path | str): Path to YAML Project configuration
             hdx_base_config_dict (dict): HDX base configuration dictionary OR
-            hdx_base_config_json (str): Path to JSON HDX base configuration OR
-            hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
+            hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
+            hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
 
         Returns:
             HDX site url
@@ -588,14 +589,14 @@ def create(
             hdx_read_only (bool): Whether to access HDX in read only mode. Defaults to False.
             hdx_key (str): Your HDX key. Ignored if hdx_read_only = True.
             hdx_config_dict (dict): HDX configuration dictionary to use instead of above 3 parameters OR
-            hdx_config_json (str): Path to JSON HDX configuration OR
-            hdx_config_yaml (str): Path to YAML HDX configuration
+            hdx_config_json (Path | str): Path to JSON HDX configuration OR
+            hdx_config_yaml (Path | str): Path to YAML HDX configuration
             project_config_dict (dict): Project configuration dictionary OR
-            project_config_json (str): Path to JSON Project configuration OR
-            project_config_yaml (str): Path to YAML Project configuration
+            project_config_json (Path | str): Path to JSON Project configuration OR
+            project_config_yaml (Path | str): Path to YAML Project configuration
             hdx_base_config_dict (dict): HDX base configuration dictionary OR
-            hdx_base_config_json (str): Path to JSON HDX base configuration OR
-            hdx_base_config_yaml (str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
+            hdx_base_config_json (Path | str): Path to JSON HDX base configuration OR
+            hdx_base_config_yaml (Path | str): Path to YAML HDX base configuration. Defaults to library's internal hdx_base_configuration.yaml.
 
         Returns:
             HDX site url

src/hdx/api/utilities/hdx_state.py

@@ -2,7 +2,7 @@
 
 import logging
 from collections.abc import Callable
-from os.path import join
+from pathlib import Path
 from typing import Any
 
 from hdx.utilities.loader import load_text
@@ -32,7 +32,7 @@ class HDXState(State):
     def __init__(
         self,
         dataset_name_or_id: str,
-        path: str,
+        path: Path | str,
         read_fn: Callable[[str], Any] = lambda x: x,
         write_fn: Callable[[Any], str] = lambda x: x,
         configuration: Configuration | None = None,
@@ -65,7 +65,7 @@ def write(self) -> None:
         """
         logger.info(f"State written to {self._dataset_name_or_id} = {self.state}")
         filename = self._resource["name"]
-        file_to_upload = join(self.path, filename)
+        file_to_upload = self.path / filename
         save_text(self.write_fn(self.state), file_to_upload)
         self._resource.set_file_to_upload(file_to_upload)
         self._resource.update_in_hdx()

src/hdx/data/dataset.py

@@ -7,7 +7,8 @@
 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
 from copy import deepcopy
 from datetime import datetime
-from os.path import isfile, join
+from os.path import isfile
+from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -198,7 +199,7 @@ def get_dataset_dict(self) -> dict:
             package["resources"] = self._convert_hdxobjects(self._resources)
         return package
 
-    def save_to_json(self, path: str, follow_urls: bool = False):
+    def save_to_json(self, path: Path | str, follow_urls: bool = False):
         """Save dataset to JSON. If follow_urls is True, resource urls that point to
         datasets, HXL proxy urls etc. are followed to retrieve final urls.
 
@@ -219,7 +220,7 @@ def save_to_json(self, path: str, follow_urls: bool = False):
         save_json(dataset_dict, path)
 
     @staticmethod
-    def load_from_json(path: str) -> Optional["Dataset"]:
+    def load_from_json(path: Path | str) -> Optional["Dataset"]:
         """Load dataset from JSON
 
         Args:
@@ -457,7 +458,7 @@ def move_resource(
         return resource
 
     def update_from_yaml(
-        self, path: str = join("config", "hdx_dataset_static.yaml")
+        self, path: Path | str = Path("config", "hdx_dataset_static.yaml")
     ) -> None:
         """Update dataset metadata with static metadata from YAML file
 
@@ -471,7 +472,7 @@ def update_from_yaml(
         self.separate_resources()
 
     def update_from_json(
-        self, path: str = join("config", "hdx_dataset_static.json")
+        self, path: Path | str = Path("config", "hdx_dataset_static.json")
     ) -> None:
         """Update dataset metadata with static metadata from JSON file
 
@@ -2330,7 +2331,7 @@ def _create_preview_resourceview(self) -> None:
     def _generate_resource_view(
         self,
         resource: Union["Resource", dict, str, int] = 0,
-        path: str | None = None,
+        path: Path | str | None = None,
         bites_disabled: Sequence[bool] | None = None,
         indicators: Sequence[dict] | None = None,
         findreplace: dict | None = None,
@@ -2374,9 +2375,9 @@ def _generate_resource_view(
         resourceview = resource_view.ResourceView(resourceview_data)
         if path is None:
             if indicators is None:
-                path = join("config", "hdx_resource_view_static.yaml")
+                path = Path("config", "hdx_resource_view_static.yaml")
                 if not isfile(path):
-                    path = path.replace(".yaml", ".yml")
+                    path = path.with_suffix(".yml")
             else:
                 path = script_dir_plus_file(
                     "indicator_resource_view_template.yaml",
@@ -2488,7 +2489,7 @@ def replace_indicator(qc_config, index):
     def generate_quickcharts(
         self,
         resource: Union["Resource", dict, str, int] = 0,
-        path: str | None = None,
+        path: Path | str | None = None,
         bites_disabled: Sequence[bool] | None = None,
         indicators: Sequence[dict] | None = None,
         findreplace: dict | None = None,
@@ -2607,7 +2608,7 @@ def remove_dates_from_title(
 
     def generate_resource(
         self,
-        folder: str,
+        folder: Path | str,
         filename: str,
         rows: Iterable[Sequence | Mapping],
         resourcedata: dict,
@@ -2703,7 +2704,7 @@ def process_row(row: Sequence | Mapping) -> Sequence | Mapping | None:
                     dates[1] = enddate
             return row
 
-        filepath = join(folder, filename)
+        filepath = Path(folder) / filename
         rows = save_iterable(
             filepath,
             rows,
@@ -2739,7 +2740,7 @@ def process_row(row: Sequence | Mapping) -> Sequence | Mapping | None:
 
     def generate_resource_from_rows(
         self,
-        folder: str,
+        folder: Path | str,
         filename: str,
         rows: Iterable[Sequence | Mapping],
         resourcedata: dict,
@@ -2776,7 +2777,7 @@ def generate_resource_from_iterable(
         headers: Sequence[str],
         iterable: Iterable[Sequence | dict],
         hxltags: dict[str, str],
-        folder: str,
+        folder: Path | str,
         filename: str,
         resourcedata: dict,
         datecol: int | str | None = None,
@@ -2988,7 +2989,7 @@ def generate_resource_from_iterator(
         headers: Sequence[str],
         iterator: Iterator[Sequence | dict],
         hxltags: dict[str, str],
-        folder: str,
+        folder: Path | str,
         filename: str,
         resourcedata: dict,
         datecol: int | str | None = None,
@@ -3019,7 +3020,7 @@ def download_generate_resource(
         self,
         downloader: BaseDownload,
         url: str,
-        folder: str,
+        folder: Path | str,
         filename: str,
         resourcedata: dict,
         header_insertions: Sequence[tuple[int, str]] | None = None,
@@ -3122,7 +3123,7 @@ def download_and_generate_resource(
         downloader: BaseDownload,
         url: str,
         hxltags: dict[str, str],
-        folder: str,
+        folder: Path | str,
         filename: str,
         resourcedata: dict,
         header_insertions: Sequence[tuple[int, str]] | None = None,

src/hdx/data/hdxobject.py

@@ -8,6 +8,7 @@
 from collections import UserDict
 from collections.abc import Sequence
 from os.path import isfile
+from pathlib import Path
 from typing import Any, Optional, Union
 
 from ckanapi.errors import NotFound
@@ -62,7 +63,7 @@ def get_old_data_dict(self) -> dict:
         """
         return self._old_data
 
-    def update_from_yaml(self, path: str) -> None:
+    def update_from_yaml(self, path: Path | str) -> None:
         """Update metadata with static metadata from YAML file
 
         Args:
@@ -72,10 +73,10 @@ def update_from_yaml(self, path: str) -> None:
             None
         """
         if not isfile(path):
-            path = path.replace(".yaml", ".yml")
-        self.data = load_yaml_into_existing_dict(self.data, path)
+            path = Path(path).with_suffix(".yml")
+        load_yaml_into_existing_dict(self.data, path)
 
-    def update_from_json(self, path: str) -> None:
+    def update_from_json(self, path: Path | str) -> None:
         """Update metadata with static metadata from JSON file
 
         Args:
@@ -84,7 +85,7 @@ def update_from_json(self, path: str) -> None:
         Returns:
             None
         """
-        self.data = load_json_into_existing_dict(self.data, path)
+        load_json_into_existing_dict(self.data, path)
 
     def _read_from_hdx(
         self,

src/hdx/data/organization.py

@@ -2,7 +2,7 @@
 
 import logging
 from collections.abc import Sequence
-from os.path import join
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Union
 
 import hdx.data.dataset
@@ -50,7 +50,7 @@ def actions() -> dict[str, str]:
         }
 
     def update_from_yaml(
-        self, path: str = join("config", "hdx_organization_static.yaml")
+        self, path: Path | str = Path("config", "hdx_organization_static.yaml")
     ) -> None:
         """Update organization metadata with static metadata from YAML file
 
@@ -63,7 +63,7 @@ def update_from_yaml(
         super().update_from_yaml(path)
 
     def update_from_json(
-        self, path: str = join("config", "hdx_organization_static.json")
+        self, path: Path | str = Path("config", "hdx_organization_static.json")
     ) -> None:
         """Update organization metadata with static metadata from JSON file