feat: pixel driller API (#221)

* feat: pixel driller API

The pixel driller API converts many single-band raster files into one gigantic multi-band raster file with each band tagged with its initial file name.
The API will return the value for any given pixel on the `/{x}/{y}` endpoint.

The first time it is run, or anytime the source rasters are updated, the ingestion process will need to be rerun by sending a POST request to the `/reimport` endpoint with `{"code": str}` matching the `ADMIN_CODE` envvar.

* fix: volumes must be a list

* fix: docker-compose setup

* Drop reimport endpoint

* Format with black

* Rewrite pixel_driller with xarray and zarr

* Include metadata in response; change to columnar format

* Test metadata join

* Simplify reshaping

* Expand docs; dict equality

* Check query is within bounds of grids

* Tidy metadata handling

* Small fixes
- make sure `libgdal-dev` is installed during the Docker build.
- update VSCode settings.

* feat: pixel driller frontend (#251)

First pass at a frontend for the pixel-driller API. Clicking anywhere on the map opens a righthand sidebar panel with tables of hazard information for that point.

---------

Co-authored-by: Tom Russell <tom.russell@ouce.ox.ac.uk>
Co-authored-by: Fred Thomas <fred.thomas@eci.ox.ac.uk>
Co-authored-by: Jim O'Donnell <james.odonnell@dtc.ox.ac.uk>

.gitignore

@@ -2,6 +2,7 @@
 node_modules/
 tileserver/raster/data
 tileserver/vector/data
+tileserver/stacks
 incoming_data/
 intermediate_data/
 *.zip
@@ -20,6 +21,8 @@ public/*.csv
 public/**/*.csv
 .snakemake
 
+notebooks
+
 # local config
 envs
 .env*

.vscode/settings.json

@@ -9,6 +9,6 @@
     "[typescript]": {
         "editor.tabSize": 2
     },
-    "editor.formatOnSave": true,
-    "editor.formatOnPaste": true
-}
+    "editor.formatOnPaste": true,
+    "editor.formatOnSave": true
+}

AUTHORS.txt

@@ -2,5 +2,8 @@ Authors
 
 - Tom Russell
 - Maciej Ziarkowski
+- Jim O'Donnell
+- Fred Thomas
+- Matt Jaquiery
 - Roald Schoenmakers
 - Raghav Pant

docker-compose.dev.yml

@@ -87,10 +87,30 @@ services:
       - "traefik.http.middlewares.raster-tileserver-stripprefix.stripprefix.prefixes=/raster"
       - "traefik.http.services.raster-tileserver.loadbalancer.server.port=5000"
 
+  pixel-driller:
+    build: ./pixel_driller
+    volumes:
+      - ./tileserver/stacks:/data
+      - ./pixel_driller:/code
+      - ./etl/hazard_layers.csv:/data/hazard_layers.csv
+    ports:
+      - 5080:80
+    environment:
+      - PIXEL_STACK_DATA_DIR=/data
+      - LAYER_METADATA_PATH=/data/hazard_layers.csv
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.pixel-driller.rule=Host(`localhost`) && PathPrefix(`/pixel`)"
+      - "traefik.http.routers.pixel-driller.entrypoints=web"
+      - "traefik.http.routers.pixel-driller.middlewares=pixel-driller-stripprefix"
+      - "traefik.http.middlewares.pixel-driller-stripprefix.stripprefix.prefixes=/pixel"
+      - "traefik.http.services.pixel-driller.loadbalancer.server.port=80"
+
   db:
     image: kartoza/postgis:14-3.1
     volumes:
       - postgis-data:/var/lib/postgresql
+    shm_size: 1g
     environment:
       - POSTGRES_DB=jamaicadev
       - POSTGRES_USER=docker

frontend/dev-proxy/proxy-table.dev.json

@@ -19,5 +19,12 @@
     "pathRewrite": {
       "^/api": "/"
     }
+  },
+  "/pixel": {
+    "target": "http://localhost",
+    "changeOrigin": true,
+    "pathRewrite": {
+      "^/pixel": "/pixel"
+    }
   }
 }

frontend/dev-proxy/proxy-table.docker.json

@@ -19,5 +19,12 @@
     "pathRewrite": {
       "^/api": "/api"
     }
+  },
+  "/pixel": {
+    "target": "http://localhost",
+    "changeOrigin": true,
+    "pathRewrite": {
+      "^/pixel": "/pixel"
+    }
   }
 }

frontend/package.json

@@ -12,6 +12,7 @@
     "@mui/icons-material": "^6.1.0",
     "@mui/lab": "^6.0.0-beta.9",
     "@mui/material": "^6.1.0",
+    "@mui/x-data-grid": "^7.24.1",
     "@mui/x-tree-view": "^7.5.0",
     "@react-hook/debounce": "^4.0.0",
     "@recoiljs/refine": "^0.1.1",

frontend/src/details/DetailsSidebar.tsx

@@ -7,6 +7,7 @@ import { AdaptationsSidebar } from './adaptations/AdaptationsSidebar';
 import { FeatureSidebar } from './features/FeatureSidebar';
 import { RegionDetails } from './regions/RegionDetails';
 import { SolutionsSidebar } from './solutions/SolutionsSidebar';
+import { PixelData } from './pixel-data/PixelData';
 
 export const showAdaptationsTableState = selector<boolean>({
   key: 'showAdaptationsTable',
@@ -18,6 +19,9 @@ export const DetailsSidebar = () => {
   const showAdaptationsTable = useRecoilValue(showAdaptationsTableState);
   return (
     <>
+      <Box mb={2}>
+        <PixelData />
+      </Box>
       <Box mb={2}>
         <SolutionsSidebar />
       </Box>

frontend/src/details/pixel-data/PixelData.stories.tsx

@@ -0,0 +1,50 @@
+import { StoryObj, Meta } from '@storybook/react';
+import { expect, within } from '@storybook/test';
+import { http, HttpResponse } from 'msw';
+
+import mockPixelData from 'mocks/details/pixel-data/mockPixelData.json';
+import { PixelData } from './PixelData';
+
+function FixedWidthDecorator(Story) {
+  return (
+    <div style={{ width: '60ch' }}>
+      <Story />
+    </div>
+  );
+}
+
+const meta = {
+  title: 'Details/PixelData',
+  component: PixelData,
+  decorators: [FixedWidthDecorator],
+} as Meta;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  parameters: {
+    msw: {
+      handlers: [
+        http.get('/pixel/0.000/0.000', () => {
+          return HttpResponse.json(mockPixelData);
+        }),
+      ],
+    },
+  },
+  play: async ({ canvasElement }) => {
+    const canvas = within(canvasElement);
+    expect(await canvas.findByText('Cyclones: speed (m s-1)')).toBeTruthy();
+    expect(await canvas.findByText('River flooding: depth (m)')).toBeTruthy();
+    expect(await canvas.findByText('Surface flooding: depth (m)')).toBeTruthy();
+    const grids = await canvas.findAllByRole('grid');
+    expect(grids).toHaveLength(6);
+    grids.forEach((grid, i) => {
+      const rowGroup = within(grid).getByRole('rowgroup');
+      expect(rowGroup).toBeTruthy();
+      const rows = within(rowGroup).getAllByRole('row');
+      expect(rows.length).toBeGreaterThan(0);
+    });
+  },
+};

frontend/src/details/pixel-data/PixelData.tsx

@@ -0,0 +1,52 @@
+import { useRecoilValue, useSetRecoilState } from 'recoil';
+
+import { Box, IconButton } from '@mui/material';
+import { SidePanel } from 'details/SidePanel';
+import { ErrorBoundary } from 'lib/react/ErrorBoundary';
+import { MobileTabContentWatcher } from 'lib/map/layouts/tab-has-content';
+import {
+  pixelDrillerDataHeaders,
+  pixelDrillerDataState,
+  pixelSelectionState,
+} from 'lib/state/pixel-driller';
+import { PixelDataGrid } from './PixelDataGrid';
+import { Close } from '@mui/icons-material';
+
+/**
+ * Display detailed information about a selected pixel (lat/lon point.)
+ */
+export const PixelData = () => {
+  const { data: selectedData } = useRecoilValue(pixelDrillerDataState);
+  const headers = useRecoilValue(pixelDrillerDataHeaders);
+  const setPixelSelection = useSetRecoilState(pixelSelectionState);
+
+  function clearSelectedLocation() {
+    setPixelSelection(null);
+  }
+
+  if (!selectedData) {
+    return null;
+  }
+  if (!headers.length) {
+    return null;
+  }
+  const hazards = [...new Set(selectedData.hazard)];
+
+  return (
+    <SidePanel position="relative">
+      <MobileTabContentWatcher tabId="details" />
+      <ErrorBoundary message="There was a problem displaying these details.">
+        <Box position="absolute" top={0} right={0} p={2}>
+          <IconButton onClick={clearSelectedLocation} title={'Close'}>
+            <Close />
+          </IconButton>
+        </Box>
+        {hazards.map((hazard) => (
+          <Box key={hazard} mt={2}>
+            <PixelDataGrid hazard={hazard} />
+          </Box>
+        ))}
+      </ErrorBoundary>
+    </SidePanel>
+  );
+};

frontend/src/details/pixel-data/PixelDataGrid.tsx

@@ -0,0 +1,52 @@
+import { DataGrid } from '@mui/x-data-grid';
+
+import {
+  pixelDrillerDataHeaders,
+  pixelDrillerDataRows,
+  pixelDrillerDataRPs,
+} from 'lib/state/pixel-driller';
+import { useRecoilValue } from 'recoil';
+
+import './hazard-table.css';
+import { Typography } from '@mui/material';
+
+const headings = {
+  cyclone: 'Cyclones',
+  fluvial: 'River flooding',
+  surface: 'Surface flooding',
+  coastal_mangrove: 'Coastal (mangrove)',
+  coastal_nomangrove: 'Coastal (no mangrove)',
+  coastal_nomangrove_minus_mangrove: 'Coastal (no mangrove - mangrove)',
+};
+
+const displayReturnPeriods = new Set([10, 20, 50, 100, 200, 500]);
+
+export const PixelDataGrid = ({ hazard }) => {
+  const headers = useRecoilValue(pixelDrillerDataHeaders);
+  const rows = useRecoilValue(pixelDrillerDataRows(hazard));
+  const dataReturnPeriods = useRecoilValue(pixelDrillerDataRPs(hazard));
+  const columns = [
+    { field: 'epoch', headerName: 'Epoch' },
+    { field: 'rcp', headerName: 'RCP' },
+  ];
+  const returnPeriods = displayReturnPeriods.intersection(dataReturnPeriods);
+
+  returnPeriods.forEach((rp) => {
+    columns.push({ field: `rp-${rp}`, headerName: `RP ${rp}` });
+  });
+  if (!headers.length) {
+    return null;
+  }
+  const variable = rows[0].variable;
+  const unit = rows[0].unit;
+
+  return (
+    <>
+      <Typography variant="subtitle2" component="h3">
+        {headings[hazard]}: {variable} ({unit})
+      </Typography>
+
+      <DataGrid columns={columns} rows={rows} density="compact" />
+    </>
+  );
+};

frontend/src/details/pixel-data/hazard-table.css

@@ -0,0 +1,3 @@
+.MuiDataGrid-footerContainer {
+  display: none;
+}

frontend/src/lib/state/interactions/use-interactions.ts

@@ -21,6 +21,7 @@ import {
 } from './interaction-state';
 import { RecoilStateFamily } from 'lib/recoil/types';
 import { PickingInfo } from 'deck.gl/typed';
+import { pixelSelectionState } from '../pixel-driller';
 
 function processRasterTarget(info: any): RasterTarget {
   const { bitmap, sourceLayer } = info;
@@ -118,6 +119,7 @@ export function useInteractions(
 
   const setInteractionGroupHover = useSetInteractionGroupState(hoverState);
   const setInteractionGroupSelection = useSetInteractionGroupState(selectionState);
+  const setPixelSelection = useSetRecoilState(pixelSelectionState);
 
   const [primaryGroup] = [...interactionGroups.keys()];
   const primaryGroupPickingRadius = interactionGroups.get(primaryGroup).pickingRadius;
@@ -176,6 +178,8 @@ export function useInteractions(
         setInteractionGroupSelection(groupName, selectionTarget);
       }
     }
+    const [lon, lat] = info.coordinate;
+    setPixelSelection({ lon, lat });
   };
 
   /**