feat: cop_ghsl - add GHS_UCDB product types

Author: jlahovnik

eodag/plugins/search/cop_ghsl.py

@@ -163,11 +163,15 @@ def _check_input_parameters_valid(self, product_type: str, params: Any):
         Check if all required parameters are given and if the values are valid
         raises a ValidationError if this is not the case
         """
-        constraints_values = self._fetch_constraints(product_type)["constraints"]
+        constraints_data = self._fetch_constraints(product_type)
+        constraints_values = constraints_data["constraints"]
         # get available values - will raise error if wrong parameters or wrong parameter values in request
+        grouped_by = params.pop("grouped_by", None)
         available_values = _get_available_values_from_constraints(
             constraints_values, params, product_type
         )
+        if grouped_by and grouped_by not in params:
+            params[grouped_by] = available_values[grouped_by]
         missing_params = set(available_values.keys()) - set(params.keys())
         if missing_params:
             raise ValidationError(
@@ -182,6 +186,50 @@ def _check_input_parameters_valid(self, product_type: str, params: Any):
                 set(available_values["month"])
             )
 
+    def _get_start_and_end_from_properties(
+        self, properties: dict[str, Any]
+    ) -> dict[str, str]:
+        """get the start and end time from year/month in the properties or missionStart/EndDate"""
+        if "month" in properties:
+            start_date = datetime.datetime(
+                year=int(properties["year"]),
+                month=int(properties["month"]),
+                day=1,
+                hour=0,
+                minute=0,
+                second=0,
+            )
+            end_day = monthrange(int(properties["year"]), int(properties["month"]))[1]
+            end_date = datetime.datetime(
+                year=int(properties["year"]),
+                month=int(properties["month"]),
+                day=end_day,
+                hour=23,
+                minute=59,
+                second=59,
+            )
+        elif "year" in properties:
+            start_date = datetime.datetime(
+                year=int(properties["year"]), month=1, day=1, hour=0, minute=0, second=0
+            )
+            end_date = datetime.datetime(
+                year=int(properties["year"]),
+                month=12,
+                day=31,
+                hour=23,
+                minute=59,
+                second=59,
+            )
+        else:
+            start_date = self.get_product_type_cfg_value("missionStartDate")
+            end_date = self.get_product_type_cfg_value("missionEndDate")
+            return {"start_date": start_date, "end_date": end_date}
+
+        result = {}
+        result["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
+        result["end_date"] = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
+        return result
+
     def _create_products_from_tiles(
         self,
         tiles: dict[str, list[dict[str, Any]]],
@@ -287,77 +335,63 @@ def _create_products_without_tiles(
         default_geometry = getattr(self.config, "metadata_mapping")["defaultGeometry"]
         properties = {}
         properties["geometry"] = default_geometry[1]
-        download_link = (
-            self.config.products.get(product_type, {})
-            .get("metadata_mapping", {})
-            .get("downloadLink", None)
+        product_type_config = self.config.products.get(product_type, {})
+        download_link = product_type_config.get("metadata_mapping", {}).get(
+            "downloadLink", None
         )
         if not download_link:
             raise MisconfiguredError(
                 f"Download link configuration missing for product type {product_type}"
             )
-        properties["downloadLink"] = download_link
+
         # product type with assets mapping
         assets_mapping = filters.pop("assets_mapping", None)
         products = []
         start_index = prep.items_per_page * (prep.page - 1)
         end_index = start_index + prep.items_per_page - 1
-        if assets_mapping:
+        grouped_by = filters.pop("grouped_by", None)
+        if grouped_by:  # dataset with several files differentiated by one parameter
             format_params = {k: str(v) for k, v in filters.items() if v}
             format_params.pop("metadata_mapping", None)
-            months = filters.get("month", "12")
-            num_products = len(months)
-            if isinstance(months, str) or isinstance(months, int):
-                months = [months]
-            for i, month in enumerate(months):
+            grouped_by_values = filters[grouped_by]
+            if isinstance(grouped_by_values, str) or isinstance(grouped_by_values, int):
+                grouped_by_values = [grouped_by_values]
+            num_products = len(grouped_by_values)
+            for i, value in enumerate(grouped_by_values):
                 if i < start_index:
                     continue
-                filters["month"] = format_params["month"] = str(month)
+                filters[grouped_by] = format_params[grouped_by] = str(value)
                 product_id = product_type + "__" + "_".join(format_params.values())
                 properties["id"] = properties["title"] = product_id
-                end_day = monthrange(int(filters["year"]), int(month))[1]
-                start_time = datetime.datetime(
-                    year=int(filters["year"]),
-                    month=int(month),
-                    day=1,
-                    hour=0,
-                    minute=0,
-                    second=0,
-                )
-                end_time = datetime.datetime(
-                    year=int(filters["year"]),
-                    month=int(month),
-                    day=end_day,
-                    hour=23,
-                    minute=59,
-                    second=59,
-                )
-                properties["startTimeFromAscendingNode"] = start_time.strftime(
-                    "%Y-%m-%dT%H:%M:%SZ"
-                )
-                properties["completionTimeFromAscendingNode"] = end_time.strftime(
-                    "%Y-%m-%dT%H:%M:%SZ"
-                )
+                properties["downloadLink"] = download_link.format(**format_params)
+                datetimes = self._get_start_and_end_from_properties(format_params)
+                properties["startTimeFromAscendingNode"] = datetimes["start_date"]
+                properties["completionTimeFromAscendingNode"] = datetimes["end_date"]
                 product = EOProduct(
                     provider="cop_ghsl", properties=properties, productType=product_type
                 )
-                assets = AssetsDict(product=product)
-                for key, mapping in assets_mapping.items():
-                    download_link = mapping["href"].format(**filters)
-                    assets[key] = Asset(
-                        product=product,
-                        key=key,
-                        href=download_link,
-                        title=mapping["title"],
-                        type=mapping["type"],
-                    )
-                product.assets = assets
+                if assets_mapping:  # item with several assets
+                    assets = AssetsDict(product=product)
+                    for key, mapping in assets_mapping.items():
+                        download_link = mapping["href"].format(**filters)
+                        assets[key] = Asset(
+                            product=product,
+                            key=key,
+                            href=download_link,
+                            title=mapping["title"],
+                            type=mapping["type"],
+                        )
+                    product.assets = assets
                 products.append(product)
                 if i == end_index:
                     break
         else:  # product type with only one file to download
             product_id = f"{product_type}_ALL"
             properties["id"] = properties["title"] = product_id
+            datetimes = self._get_start_and_end_from_properties(properties)
+            properties["startTimeFromAscendingNode"] = datetimes["start_date"]
+            properties["completionTimeFromAscendingNode"] = datetimes["end_date"]
+            properties["downloadLink"] = download_link
             product = EOProduct(
                 provider="cop_ghsl", properties=properties, productType=product_type
             )
@@ -423,10 +457,13 @@ def _get_tiles_for_filters(
         filter_params.pop("assets_mapping", None)
 
         self._check_input_parameters_valid(product_type, filter_params)
-        if "year" in filter_params:
-            params["year"] = filter_params["year"]
+        # update parameters based on changes during validation
+        params.update(filter_params)
 
         # fetch available tiles based on filters
+        if "year" not in filter_params:
+            logger.warning(f"no tiles available for {product_type}")
+            return None
         if isinstance(filter_params["year"], int) or isinstance(
             filter_params["year"], str
         ):

eodag/resources/product_types.yml

@@ -6713,6 +6713,40 @@ GHS_ENACT_POP:
   missionStartDate: "2011-01-01T00:00:00Z"
   missionEndDate: "2011-12-31T00:00:00Z"
 
+GHS_UCDB_DOMAIN:
+  abstract: |
+       The GHS Urban Centre Database (GHS-UCDB) describes spatial entities called “Urban Centres” described through a set of multi-temporal thematic
+       attributes from the GHSL data combined with other free and open data sets. The Urban Centres are defined by specific cut-off values on
+       resident population and built-up surface share in a 1x1 km uniform global grid. The input data is generated by the GHSL, and the
+       operating parameters are set in the frame of the “Degree of Urbanisation” (DEGURBA) methodology. The data is grouped by thematic domain.
+  instrument:
+  platform:
+  platformSerialIdentifier:
+  processingLevel:
+  keywords: GHS,Settlement,UCDB,Urbanisation,thematic,domain,DEGURBA
+  sensorType: multi
+  license: other
+  title: Stats in the City - the GHS Urban Centre Database UCDB R2024A by Thematic Domain
+  missionStartDate: "1975-01-01T00:00:00Z"
+  missionEndDate: "2030-12-31T00:00:00Z"
+
+GHS_UCDB_REGION:
+  abstract: |
+       The GHS Urban Centre Database (GHS-UCDB) describes spatial entities called “Urban Centres” described through a set of multi-temporal thematic
+       attributes from the GHSL data combined with other free and open data sets. The Urban Centres are defined by specific cut-off values on
+       resident population and built-up surface share in a 1x1 km uniform global grid. The input data is generated by the GHSL, and the
+       operating parameters are set in the frame of the “Degree of Urbanisation” (DEGURBA) methodology. The data is grouped by region.
+  instrument:
+  platform:
+  platformSerialIdentifier:
+  processingLevel:
+  keywords: GHS,Settlement,UCDB,Urbanisation,region,DEGURBA
+  sensorType: multi
+  license: other
+  title: Stats in the City - the GHS Urban Centre Database UCDB R2024A by Region
+  missionStartDate: "1975-01-01T00:00:00Z"
+  missionEndDate: "2030-12-31T00:00:00Z"
+
 # MARK: GENERIC ------------------------------------------------------------------------
 GENERIC_PRODUCT_TYPE:
   abstract:

eodag/resources/providers.yml

@@ -6064,6 +6064,7 @@
     # products with several files
     GHS_ENACT_POP:
       productType: ENACT_POP
+      grouped_by: month
       year: 2011
       metadata_mapping:
         downloadLink: https://jeodpp.jrc.ec.europa.eu
@@ -6076,6 +6077,16 @@
           href: 'https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/ENACT/ENACT_POP_2011_EU28_R2020A/ENACT_POP_N{month}2011_EU28_R2020A_{coord_system}_{tile_size}/V1-0/ENACT_POP_N{month}2011_EU28_R2020A_{coord_system}_{tile_size}_V1_0.zip'
           type: "application/zip"
           title: Nightime grid
+    GHS_UCDB_DOMAIN:
+      productType: UCDB
+      grouped_by: thematic_domain
+      metadata_mapping:
+        downloadLink: https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_UCDB_GLOBE_R2024A/GHS_UCDB_THEME_GLOBE_R2024A/GHS_UCDB_THEME_{thematic_domain}_GLOBE_R2024A/V1-1/GHS_UCDB_THEME_{thematic_domain}_GLOBE_R2024A_V1_1.zip
+    GHS_UCDB_REGION:
+      productType: UCDB
+      grouped_by: region
+      metadata_mapping:
+        downloadLink: https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/GHSL/GHS_UCDB_GLOBE_R2024A/GHS_UCDB_REGION_GLOBE_R2024A/GHS_UCDB_REGION_{region}_R2024A/V1-1/GHS_UCDB_REGION_{region}_R2024A_V1_1.zip
   download: !plugin
     type: HTTPDownload
     ssl_verify: true