Add CDSE endpoint for data download

.gitignore

@@ -1,3 +1,6 @@
+# Sandbox (local testing, not part of the repository)
+sandbox/
+
 # OS Files
 **/.DS_Store
 **/.DS_Store?

CHANGELOG.md

@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.3]
+
+### Added
+* Support for downloading Sentinel-1 SLC granules from the Copernicus Data Space Ecosystem (CDSE) as an alternative to ASF. Metadata retrieval and bounding box checks continue to use ASF. Download from CDSE uses the compressed `.zip` archive directly, leveraging ISCE2's virtual file access to reduce download traffic.
+
 ## [1.0.2]
 
 ### Changed

README.md

@@ -61,6 +61,20 @@ python -m ipykernel install --user --name topsapp_env
     ```
     The `username`/`password` pair are the appropriate Earthdata Login credentials that are used to access NASA data. This file is necessary for downloading the Sentinel-1 files, and auxiliary data. Additionally, the [`requests`](https://docs.python-requests.org/en/latest/) library automatically uses credentials stored in the `~/.netrc` for authentification when none are supplied.
 
+2. (Optional) To download Sentinel-1 SLC granules from the [Copernicus Data Space Ecosystem (CDSE)](https://dataspace.copernicus.eu/) instead of ASF, you will need CDSE credentials. These can be provided in one of two ways:
+    - Add an entry to your `~/.netrc` file:
+        ```
+        machine dataspace.copernicus.eu
+            login <cdse-username>
+            password <cdse-password>
+        ```
+    - Or set environment variables:
+        ```
+        export CDSE_USERNAME=<cdse-username>
+        export CDSE_PASSWORD=<cdse-password>
+        ```
+    A free CDSE account can be registered at [dataspace.copernicus.eu](https://dataspace.copernicus.eu/).
+
 ## Generate an ARIA-S1-GUNW
 
 Make sure you have `~/.netrc` as described above. Run the following command:
@@ -77,6 +91,19 @@ This is reflected in the [`sample_run.sh`](sample_run.sh).
 
 To be even more explicit, you can use [`tee`](https://en.wikipedia.org/wiki/Tee_(command)) to record output to both including `> >(tee -a topsapp_img.out) 2> >(tee -a topsapp_img.err >&2)`.
 
+## Generate an ARIA-S1-GUNW using CDSE for Sentinel-1 Download
+
+By default, Sentinel-1 SLC granules are downloaded from the [Alaska Satellite Facility (ASF)](https://asf.alaska.edu/). As an alternative, particularly suited for European users or those operating within the European cloud ecosystem, granules can be downloaded from the [Copernicus Data Space Ecosystem (CDSE)](https://dataspace.copernicus.eu/) by passing `--download-source CDSE`. Metadata lookups and bounding box checks still use ASF, so a valid `~/.netrc` entry for `urs.earthdata.nasa.gov` remains required. Additionally, CDSE credentials must be configured as described in the *Additional setup* section above.
+
+```
+isce2_topsapp --reference-scenes S1A_IW_SLC__1SDV_20220212T222803_20220212T222830_041886_04FCA3_2B3E \
+                                 S1A_IW_SLC__1SDV_20220212T222828_20220212T222855_041886_04FCA3_A3E2  \
+              --secondary-scenes S1A_IW_SLC__1SDV_20220131T222803_20220131T222830_041711_04F690_8F5F \
+                                 S1A_IW_SLC__1SDV_20220131T222828_20220131T222855_041711_04F690_28D7  \
+              --frame-id 25502 \
+              --download-source CDSE
+```
+
 ## What makes an ARIA-S1-GUNW Product *standard*?
 
 Each ARIA-S1-GUNW at the ASF that ensures that down-stream analysis by [ARIA-Tools](https://github.com/aria-tools/ARIA-tools) and [Mintpy](https://github.com/insarlab/MintPy) is done consistently and reproducibly. There are a number of exposed parameters in this plugin that we require to be set in a certain manner for a product to be considered "standard". We now discuss the standard parameters with respect to this plugin.

isce2_topsapp/__main__.py

@@ -44,6 +44,7 @@ def localize_data(
     dry_run: bool = False,
     water_mask_flag: bool = True,
     geocode_resolution: int = 90,
+    download_source: str = "ASF",
 ) -> dict:
     """The dry-run prevents gets necessary metadata from SLCs and orbits.
 
@@ -58,6 +59,7 @@ def localize_data(
         frame_id=frame_id,
         min_frame_coverage=min_frame_coverage,
         dry_run=dry_run,
+        download_source=download_source,
     )
 
     out_orbits = download_orbits(reference_scenes, secondary_scenes)
@@ -204,6 +206,18 @@ def get_slc_parser():
         default=0.5,
         help="The power applied to the patch FFT of the phase filter",
     )
+    parser.add_argument(
+        "--download-source",
+        type=str,
+        choices=["ASF", "CDSE"],
+        default="ASF",
+        help=(
+            "Source for downloading Sentinel-1 SLC granules. "
+            "'ASF' uses Alaska Satellite Facility (requires Earthdata credentials). "
+            "'CDSE' uses Copernicus Data Space Ecosystem (requires CDSE credentials "
+            "via CDSE_USERNAME/CDSE_PASSWORD env vars or ~/.netrc)."
+        ),
+    )
     return parser
 
 
@@ -245,10 +259,22 @@ def gunw_slc():
 
     # Validation
     ensure_earthdata_credentials(args.username, args.password)
+    # Also validate CDSE credentials if using CDSE for download
+    if args.download_source == "CDSE":
+        from isce2_topsapp.localize_slc_cdse import ensure_cdse_credentials
+
+        ensure_cdse_credentials()
     cli_params = vars(args).copy()
     [
         cli_params.pop(key)
-        for key in ["username", "password", "bucket", "bucket_prefix", "dry_run"]
+        for key in [
+            "username",
+            "password",
+            "bucket",
+            "bucket_prefix",
+            "dry_run",
+            "download_source",
+        ]
     ]
     topsapp_params_obj = topsappParams(**cli_params)
 
@@ -268,6 +294,7 @@ def gunw_slc():
         frame_id=args.frame_id,
         min_frame_coverage=args.min_frame_coverage,
         water_mask_flag=args.estimate_ionosphere_delay,
+        download_source=args.download_source,
     )
     loc_data["frame_id"] = args.frame_id
     loc_data["cmd_line_str"] = cmd_line_str
@@ -460,12 +487,24 @@ def coseis_sar():
     args = parser.parse_args()
     args = update_slc_namespace(args)
 
-    # Validation
+    # Validation - always need Earthdata creds for ASF metadata lookup
     ensure_earthdata_credentials(args.username, args.password)
+    # Also validate CDSE credentials if using CDSE for download
+    if args.download_source == "CDSE":
+        from isce2_topsapp.localize_slc_cdse import ensure_cdse_credentials
+
+        ensure_cdse_credentials()
     cli_params = vars(args).copy()
     [
         cli_params.pop(key)
-        for key in ["username", "password", "bucket", "bucket_prefix", "dry_run"]
+        for key in [
+            "username",
+            "password",
+            "bucket",
+            "bucket_prefix",
+            "dry_run",
+            "download_source",
+        ]
     ]
     topsapp_params_obj = topsappParams(**cli_params)
 
@@ -484,6 +523,7 @@ def coseis_sar():
         geocode_resolution=args.output_resolution,
         frame_id=args.frame_id,
         water_mask_flag=True,  # Download water mask regardless of iono computation. It is needed for viz masking
+        download_source=args.download_source,
     )
     loc_data["frame_id"] = args.frame_id
     loc_data["cmd_line_str"] = cmd_line_str

isce2_topsapp/localize_slc.py

@@ -170,6 +170,7 @@ def download_slcs(
     min_frame_coverage: float = MIN_FRAME_COVERAGE_DEFAULT,
     max_workers_for_download: int = 5,
     dry_run: bool = False,
+    download_source: str = "ASF",
 ) -> dict:
     reference_obs = get_asf_slc_objects(reference_ids)
     secondary_obs = get_asf_slc_objects(secondary_ids)
@@ -229,23 +230,39 @@ def download_slcs(
             category=RuntimeWarning,
         )
 
-    def download_one(resp):
-        session = get_session()
-        file_name = resp.properties["fileName"]
-        if not dry_run:
-            resp.download(path=".", session=session)
-        return file_name
-
     processing_geo = ifg_geo
     if frame_id != -1:
         processing_geo = _get_frame_by_id(frame_id).geometry
 
-    all_obs = reference_obs + secondary_obs
-    n = len(all_obs)
-    with ThreadPoolExecutor(max_workers=max_workers_for_download) as executor:
-        results = list(
-            tqdm(executor.map(download_one, all_obs), total=n, desc="Downloading SLCs")
+    if download_source == "CDSE":
+        from isce2_topsapp.localize_slc_cdse import download_slcs_from_cdse
+
+        all_ids = reference_ids + secondary_ids
+        results = download_slcs_from_cdse(
+            all_ids,
+            output_dir=".",
+            max_workers=max_workers_for_download,
+            dry_run=dry_run,
         )
+    else:
+
+        def download_one(resp):
+            session = get_session()
+            file_name = resp.properties["fileName"]
+            if not dry_run:
+                resp.download(path=".", session=session)
+            return file_name
+
+        all_obs = reference_obs + secondary_obs
+        n = len(all_obs)
+        with ThreadPoolExecutor(max_workers=max_workers_for_download) as executor:
+            results = list(
+                tqdm(
+                    executor.map(download_one, all_obs),
+                    total=n,
+                    desc="Downloading SLCs",
+                )
+            )
 
     n0 = len(reference_obs)
     return {