{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# 10: Dask and rioxarray demo\n", "\n", "UW Geospatial Data Analysis \n", "CEE467/CEWA567 \n", "David Shean \n", "\n", "These are incomplete, some were not working - ran out of time to test, preserving here for the future" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "\n", "\n", "\n", "
\n", "

Client

\n", "\n", "		\n", "

Cluster

\n", "
    \n", "
  • Workers: 4
    • \n", "
  • Cores: 4
    • \n", "
  • Memory: 8.59 GB
    • \n", "
\n", "
" ], "text/plain": [ "" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from dask.distributed import Client\n", "\n", "client = Client(\"tcp://127.0.0.1:36567\")\n", "client" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 79 ms, sys: 51.3 ms, total: 130 ms\n", "Wall time: 129 ms\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 't2m' ()>\n",
       "array(8.996196, dtype=float32)
" ], "text/plain": [ "\n", "array(8.996196, dtype=float32)" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%%time\n", "wa_merge['t2m'].mean()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", "\n", "\n", "\n", "\n", "
\n", "

Client

\n", "\n", "		\n", "

Cluster

\n", "
    \n", "
  • Workers: 2
    • \n", "
  • Cores: 4
    • \n", "
  • Memory: 2.00 GB
    • \n", "
\n", "
" ], "text/plain": [ "" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from dask.distributed import Client\n", "\n", "#client = Client(\"tcp://127.0.0.1:35733\")\n", "client = Client(n_workers=2, threads_per_worker=2, memory_limit='1GB')\n", "client" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import hvplot.xarray" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#This was pulled from the part 2 notebook, originally separate variables\n", "wa_t.mean(dim=(\"latitude\", \"longitude\")).hvplot()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "wa_t.mean(dim=\"time\").hvplot(aspect=2)" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "dem_fn = '../07_Raster2_DEMs_Warp_Clip_Sample/WA_SRTMGL3_utm_gdalwarp_lzw.tif'\n", "dem_fn = '/home/jovyan/gda_course_2021_solutions/modules/07_Raster2_DEMs_Warp_Clip_Sample/WA_SRTMGL3_utm_gdalwarp_lzw.tif'" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/jovyan/gda_course_2021_solutions/modules/07_Raster2_DEMs_Warp_Clip_Sample/WA_SRTMGL3_utm_gdalwarp_lzw.tif\n" ] } ], "source": [ "ls $dem_fn" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [], "source": [ "import rioxarray" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "https://corteva.github.io/rioxarray/stable/getting_started/getting_started.html" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [], "source": [ "chunks={'x': 1024, 'y': 1024}\n", "#chunks='auto'\n", "overview_level=0" ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'SRTM' (band: 1, y: 5851, x: 8877)>\n",
       "dask.array<open_rasterio-1fd172fd27c59599ee677e71323da560<this-array>, shape=(1, 5851, 8877), dtype=float64, chunksize=(1, 1024, 1024), chunktype=numpy.ndarray>\n",
       "Coordinates:\n",
       "  * band         (band) int64 1\n",
       "  * y            (y) float64 5.446e+06 5.446e+06 ... 5.043e+06 5.043e+06\n",
       "  * x            (x) float64 3.647e+05 3.648e+05 ... 9.748e+05 9.749e+05\n",
       "    spatial_ref  int64 0\n",
       "Attributes:\n",
       "    scale_factor:  1.0\n",
       "    add_offset:    0.0\n",
       "    grid_mapping:  spatial_ref
" ], "text/plain": [ "\n", "dask.array, shape=(1, 5851, 8877), dtype=float64, chunksize=(1, 1024, 1024), chunktype=numpy.ndarray>\n", "Coordinates:\n", " * band (band) int64 1\n", " * y (y) float64 5.446e+06 5.446e+06 ... 5.043e+06 5.043e+06\n", " * x (x) float64 3.647e+05 3.648e+05 ... 9.748e+05 9.749e+05\n", " spatial_ref int64 0\n", "Attributes:\n", " scale_factor: 1.0\n", " add_offset: 0.0\n", " grid_mapping: spatial_ref" ] }, "execution_count": 62, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dem = rioxarray.open_rasterio(dem_fn, masked=True, default_name=\"SRTM\", chunks=chunks)\n", "dem" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Affine(68.74846185120099, 0.0, 364652.9635476386,\n", " 0.0, -68.74846185120099, 5445635.966252405)" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dem.rio.transform()" ] }, { "cell_type": "code", "execution_count": 63, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "nan" ] }, "execution_count": 63, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dem.rio.nodata" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'SRTM' (band: 1, y: 5851, x: 8877)>\n",
       "array([[[nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        ...,\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan]]])\n",
       "Coordinates:\n",
       "  * band         (band) int64 1\n",
       "  * y            (y) float64 5.446e+06 5.446e+06 ... 5.043e+06 5.043e+06\n",
       "  * x            (x) float64 3.647e+05 3.648e+05 ... 9.748e+05 9.749e+05\n",
       "    spatial_ref  int64 0\n",
       "Attributes:\n",
       "    scale_factor:  1.0\n",
       "    add_offset:    0.0\n",
       "    grid_mapping:  spatial_ref
" ], "text/plain": [ "\n", "array([[[nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " ...,\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan]]])\n", "Coordinates:\n", " * band (band) int64 1\n", " * y (y) float64 5.446e+06 5.446e+06 ... 5.043e+06 5.043e+06\n", " * x (x) float64 3.647e+05 3.648e+05 ... 9.748e+05 9.749e+05\n", " spatial_ref int64 0\n", "Attributes:\n", " scale_factor: 1.0\n", " add_offset: 0.0\n", " grid_mapping: spatial_ref" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" }, { "name": "stderr", "output_type": "stream", "text": [ "distributed.client - ERROR - Failed to reconnect to scheduler after 10.00 seconds, closing client\n", "_GatheringFuture exception was never retrieved\n", "future: <_GatheringFuture finished exception=CancelledError()>\n", "asyncio.exceptions.CancelledError\n" ] } ], "source": [ "dem.load()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'SRTM' (band: 1, y: 5851, x: 8877)>\n",
       "array([[[nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        ...,\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan],\n",
       "        [nan, nan, nan, ..., nan, nan, nan]]])\n",
       "Coordinates:\n",
       "  * band         (band) int64 1\n",
       "  * y            (y) float64 5.446e+06 5.446e+06 ... 5.043e+06 5.043e+06\n",
       "  * x            (x) float64 3.647e+05 3.648e+05 ... 9.748e+05 9.749e+05\n",
       "    spatial_ref  int64 0\n",
       "Attributes:\n",
       "    scale_factor:  1.0\n",
       "    add_offset:    0.0\n",
       "    grid_mapping:  spatial_ref
" ], "text/plain": [ "\n", "array([[[nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " ...,\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan],\n", " [nan, nan, nan, ..., nan, nan, nan]]])\n", "Coordinates:\n", " * band (band) int64 1\n", " * y (y) float64 5.446e+06 5.446e+06 ... 5.043e+06 5.043e+06\n", " * x (x) float64 3.647e+05 3.648e+05 ... 9.748e+05 9.749e+05\n", " spatial_ref int64 0\n", "Attributes:\n", " scale_factor: 1.0\n", " add_offset: 0.0\n", " grid_mapping: spatial_ref" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dem.compute()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "dem.plot()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "data": {}, "metadata": {}, "output_type": "display_data" } ], "source": [ "dem.hvplot(x='x', y='y')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.8" } }, "nbformat": 4, "nbformat_minor": 4 }