Feature/grid file etl

Dockerfile.temp_transfer → Dockerfile.create_ranked_grid

@@ -1,5 +1,5 @@
-# Dockerfile supporting temperature transfer from zarr datasets to existing DSS documents for SST model runs
-FROM osgeo/gdal:ubuntu-small-3.5.1 as prod
+# Dockerfile supporting plugin for HMS GRID file generation
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.5.1 as prod
 RUN apt-get update && \
     apt-get install -y python3-pip && \
     pip3 install rasterio --no-binary rasterio
@@ -25,6 +25,25 @@ RUN cp -a heclib/heclib.a pydsstools/pydsstools/src/external/dss/linux64/heclib.
 # install
 RUN ( cd pydsstools && python3 -m pip install . )
 
-# copy over script and install packages used in temperature transfer
-COPY stormcloud/etl/temp_transfer/. .
+# install etl packages
+COPY stormcloud/etl/create_ranked_grid/requirements.txt .
 RUN pip3 install -r requirements.txt
+
+# copy scripts called by GRID generation script
+COPY stormcloud/common/ common/.
+
+# copy script to write AORC zarr to DSS
+COPY stormcloud/write_aorc_zarr_to_dss.py .
+
+# copy script called by metadata standardization process
+COPY stormcloud/construct_meta.py .
+
+# copy scripts to retrieve meilisearch documents
+COPY stormcloud/ms/storm_query.py ms/storm_query.py
+COPY stormcloud/ms/constants.py ms/constants.py
+
+# copy hms grid creation script
+COPY stormcloud/create_hms_grid.py .
+
+# copy etl scripts
+COPY stormcloud/etl/create_ranked_grid/create_ranked_grid.py .

Dockerfile.hms_grid

@@ -1,5 +1,5 @@
 # Dockerfile supporting plugin for HMS GRID file generation
-FROM osgeo/gdal:ubuntu-small-3.5.1 as prod
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.5.1 as prod
 RUN apt-get update && \
     apt-get install -y python3-pip && \
     pip3 install rasterio --no-binary rasterio

Dockerfile.sst

@@ -1,5 +1,5 @@
 # Dockerfile supporting plugin for SST modeling runs
-FROM osgeo/gdal:ubuntu-small-3.5.1 as prod
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.5.1 as prod
 RUN apt-get update && \
     apt-get install -y python3-pip && \
     pip3 install rasterio --no-binary rasterio

Dockerfile.standardize_meta

@@ -1,5 +1,5 @@
 # Dockerfile supporting plugin for metadata standardization used in GRID file generation
-FROM osgeo/gdal:ubuntu-small-3.5.1 as prod
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.5.1 as prod
 RUN apt-get update && \
     apt-get install -y python3-pip && \
     pip3 install rasterio --no-binary rasterio

Dockerfile.temp_precip

@@ -1,5 +1,5 @@
 # Dockerfile supporting plugin for temperature and precipitation extraction and conversion to DSS format
-FROM osgeo/gdal:ubuntu-small-3.5.1 as prod
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.5.1 as prod
 RUN apt-get update && \
     apt-get install -y python3-pip && \
     pip3 install rasterio --no-binary rasterio

Dockerfile.transpose_geom

@@ -1,5 +1,5 @@
 # Dockerfile supporting plugin for SST modeling runs
-FROM osgeo/gdal:ubuntu-small-3.5.1 as prod
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.5.1 as prod
 RUN apt-get update && \
     apt-get install -y python3-pip && \
     pip3 install rasterio --no-binary rasterio

README.md

@@ -14,18 +14,19 @@ Creates datasets for hydrologic modeling in the cloud.
 
 - Requires use of docker container as specified in [Dockerfile.temp_precip](Dockerfile.temp_precip)
 
-## Identifying storms of interest
-
-[Identify top ranked storms tracked in meilisearch database, rerun zarr to DSS conversion and save to local path](stormcloud/etl/top_storms/extract_top_storms_dss.py)
-
-- Requires use of docker container as specified in [Dockerfile.top_storms_dss](Dockerfile.top_storms_dss)
 
 ## Getting valid transposition geometry
 
-[Get the geometry defining all valid transposes within a transposition region for a given watershed and saves the geometry as a simple geojson polygon](stormcloud/etl/transpose_geom/get_valid_transpose_geom.py)
+[Get the geometry defining all valid transposes within a transposition region for a given watershed and saves the geometry as a simple geojson polygon](stormcloud/etl/transpose_geom/README.md)
 
 - Requires use of docker container as specified in [Dockerfile.transpose_geom](Dockerfile.transpose_geom)
 
+## Creating a zipped GRID package using top n storm events per year
+
+[Use DSS files created during SST runs for a specified watershed and transposition region to create a zipped package with an HMS .grid file](stormcloud/etl/create_ranked_grid/README.md)
+
+- Requires use of docker container as specified in [Dockerfile.create_ranked_grid](Dockerfile.create_ranked_grid)
+
 ## Plugins -- Process API support
 
 Process API plugins exist which support:
@@ -40,15 +41,15 @@ Meilisearch acts as a back end for the [StormViewer site](https://storms.dewberr
     - create an HMS grid package for a subsection of events from s3 data located using SST metadata
     - update the meilisearch index with ranked documents generated by the [ranked document plugin](stormcloud/plugins/doc_rank/README.md)
 
-[SST plugin directory](stormcloud/plugins/sst/)
+[SST plugin](stormcloud/plugins/sst/README.md)
 
-[Temperature and precipitation extraction plugin directory](stormcloud/plugins/temp_precip/)
+[Temperature and precipitation extraction plugin directory](stormcloud/plugins/temp_precip/README.md)
 
-[Metadata standardization plugin directory](stormcloud/plugins/standardize_meta/)
+[Metadata standardization plugin directory](stormcloud/plugins/standardize_meta/README.md)
 
-[HMS GRID package generation plugin directory](stormcloud/plugins/hms_grid/)
+[HMS GRID package generation plugin directory](stormcloud/plugins/hms_grid/README.md)
 
-[Ranked document plugin directory](stormcloud/plugins/doc_rank/)
+[Ranked document plugin directory](stormcloud/plugins/doc_rank/README.md)
 
 ---
 

docker-compose.yml

@@ -26,26 +26,19 @@ services:
     env_file:
       - ./stormcloud/plugins/doc_rank/.env
 
-  # stormcloud-etl-temp-transfer:
-  #   container_name: stormcloud-temp-transfer
+  # stormcloud-etl-transpose-geom:
+  #   container_name: stormcloud-transpose-geom
   #   build:
-  #     dockerfile: Dockerfile.temp_transfer
+  #     dockerfile: Dockerfile.transpose_geom
   #   env_file:
-  #     - ./stormcloud/etl/temp_transfer/.env
+  #     - ./stormcloud/etl/transpose_geom/.env
   #   tty: true
 
-  # stormcloud-etl-top-storms-dss:
-  #   container_name: stormcloud-top-storms-dss
+  # stormcloud-etl-create-ranked-grid:
+  #   container_name: stormcloud-create-ranked-grid
   #   build:
-  #     dockerfile: Dockerfile.top_storms_dss
+  #     context: .
+  #     dockerfile: Dockerfile.create_ranked_grid
   #   env_file:
-  #     - ./stormcloud/etl/top_storms/.env
+  #     - ./stormcloud/etl/create_ranked_grid/.env
   #   tty: true
-
-  # stormcloud-etl-transpose-geom:
-  #   container_name: stormcloud-transpose-geom
-  #   build:
-  #     dockerfile: Dockerfile.transpose_geom
-  #   env_file:
-  #     - ./stormcloud/etl/transpose_geom/.env
-  #   tty: true

stormcloud/common/cloud.py

@@ -10,6 +10,14 @@
 from shapely.geometry import MultiPolygon, Polygon, shape
 
 
+def check_if_exists(s3, bucket: str, key: str) -> bool:
+    try:
+        s3.head_object(Bucket=bucket, Key=key)
+    except ClientError:
+        return False
+    return True
+
+
 def create_presigned_url(bucket_name, object_name, expiration=604800):
     """Generate a presigned URL to share an S3 object
 

stormcloud/common/zarr.py

@@ -1,4 +1,5 @@
 """ Script to isolate time periods of storms using meilisearch data and search NOAA s3 zarr data for data matching these time periods to save to DSS format """
+
 import datetime
 import logging
 from typing import Iterator, List, Tuple, Union
@@ -104,6 +105,7 @@ def extract_period_zarr(
                 )
 
             zarr_key = f"{zarr_key_prefix}/{current_dt.year}/{current_dt.strftime('%Y%m%d%H')}.zarr"
+            logging.debug(f"loading data from zarr s3 path s3://{zarr_bucket}/{zarr_key}")
             hour_ds = load_zarr(zarr_bucket, zarr_key, access_key_id, secret_access_key)
             if hour_ds:
                 hour_ds.rio.write_crs("epsg:4326", inplace=True)

stormcloud/etl/create_ranked_grid/README.md

@@ -0,0 +1,20 @@
+# create ranked grid
+
+## Summary
+
+ETL which accesses the meilisearch database backing the StormViewer application and queries for top n storms per year, then formats this data and uses it to download and package precipitation DSS files generated from an SST run into a zipped package containing an HMS GRID file.
+
+## Usage
+
+This ETL has an associated [docker image](../../../Dockerfile.create_ranked_grid) which can be used to build an image useable to launch this ETL. There is also a service listed in the [docker compose file](../../../docker-compose.yml) but which is commented out. To use docker compose to build and run a container for this ETL, uncomment the service called something like "stormcloud-etl-create-ranked-grid"
+
+It is necessary to have a .env file in this directory in order for the docker compose run definition to work. This .env file should follow the template of [.env.sample](./.env.sample)
+
+### Parameters
+
+For help on parameters used in this etl, you can use the `-h` flag after the python command (example below). This will print what the parameters are and a short description.
+
+```shell
+python create_ranked_grid.py -h
+```
+