From c4f81d4c353810eeacba8a1ad1c0258d02bbab6e Mon Sep 17 00:00:00 2001 From: matt rice Date: Sun, 12 Jul 2026 16:53:10 -0700 Subject: [PATCH 1/6] Add a water layer --- process.sh | 4 +++- sql/macros.sql | 24 +++++++++++++++++++ sql/water.sql | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 sql/water.sql diff --git a/process.sh b/process.sh index 27082c0..c6d267f 100755 --- a/process.sh +++ b/process.sh @@ -41,6 +41,7 @@ for arg in "$@"; do --parks) PARKS=1; ALL=0 ;; --pois) POIS=1; ALL=0 ;; --addresses) ADDRESSES=1; ALL=0 ;; + --water) WATER=1; ALL=0 ;; --osmium-index-type=*) OSMIUM_INDEX_TYPE="${arg#*=}" ;; --duckdb-memory-limit=*) DUCKDB_MEMORY_LIMIT="${arg#*=}" ;; *) echo "Unknown argument: $arg" >&2; exit 1 ;; @@ -48,7 +49,7 @@ for arg in "$@"; do done if [ "$ALL" = "1" ]; then - BUILDINGS=1; HIGHWAYS=1; BOUNDARIES=1; SETTLEMENTS=1; PARKS=1; POIS=1; ADDRESSES=1 + BUILDINGS=1; HIGHWAYS=1; BOUNDARIES=1; SETTLEMENTS=1; PARKS=1; POIS=1; ADDRESSES=1; WATER=1 fi mkdir -p "$OUTPUT_DIR" @@ -76,5 +77,6 @@ run_layer() { [ "$PARKS" = "1" ] && run_layer parks [ "$POIS" = "1" ] && run_layer pois [ "$ADDRESSES" = "1" ] && run_layer addresses +[ "$WATER" = "1" ] && run_layer water echo "Done" diff --git a/sql/macros.sql b/sql/macros.sql index 01f9021..850d391 100644 --- a/sql/macros.sql +++ b/sql/macros.sql @@ -32,3 +32,27 @@ CREATE OR REPLACE MACRO prefix_map_split(pfx, t) AS ( ) ) ); + +-- If a map is empty, returns NULL, otherwise return the map. +CREATE OR REPLACE MACRO try_map(m1) AS ( + CASE + -- A little tricky to do without triggering the type checker, + -- This is why we route through `len(map_keys(..))` instead of calling `cardinality()` + WHEN len(map_keys(m1)) = 0 THEN NULL + ELSE m1 + END +); + +-- Given a map of tags, and a list of tag names, yields a map with those named values. +-- e.g. given map_from_tags(tags, ['boat, 'canoe']) on {id: 1, canoe: 'yes', boat: 'no', ...} +-- yields {canoe: 'yes', boat: 'no'}. +CREATE OR REPLACE MACRO map_from_tag_list(tags, tag_list) AS ( + try_map( + MAP_FROM_ENTRIES( + LIST_FILTER( + MAP_ENTRIES(tags), + lambda kv: kv.key IN tag_list AND kv.value IS NOT NULL + ) + ) + ) +); diff --git a/sql/water.sql b/sql/water.sql new file mode 100644 index 0000000..7cb7b23 --- /dev/null +++ b/sql/water.sql @@ -0,0 +1,65 @@ +COPY ( + WITH raw AS ( + SELECT type, id, tags, geometry + FROM '{{INPUT}}' + WHERE kind = 'line' + AND tags['waterway'] IN ('river', 'stream', 'canal', 'ditch', 'drain', 'flowline', 'fairway', 'link', + -- Reason: Routing obstructions. + 'dam', 'weir', 'waterfall' + ) + ) + SELECT + type, + id, + tags['waterway'] AS waterway, + tags['name'] AS name, + tags['ref'] AS ref, + tags['intermittent'] AS intermittent, + tags['layer'] AS layer, + tags['bridge'] AS bridge, + tags['tunnel'] AS tunnel, + tags['access'] AS access, + + -- waterway-relevant tags i've added or changed... + -- Reason: general + tags['usage'] AS usage, + tags['natural'] AS natural, + split_multi(tags['seasonal']) AS seasonal, + tags['tidal'] AS tidal, + + -- Reason: dimensions. + tags['width'] AS width, + tags['depth'] AS depth, + tags['depth:'] AS 'depth:', + tags['height'] AS height, -- For waterfalls? + + -- Reason: navigation direction and obstruction bypass, + prefix_map('oneway:', tags) AS 'oneway:', + tags['oneway'] AS oneway, + tags['canoe_pass'] AS canoe_pass, + tags['lock'] AS lock, + + -- Boat access + map_from_tag_list(tags, ['motorboat', 'ship', 'sailboat', 'boat', 'canoe']) as vessel_access, + + -- Reason: Hazards, + tags['open_water'] AS open_water, + prefix_map('whitewater:', tags) AS whitewater, + tags['narrow'] AS narrow, + tags['rapids'] AS rapids, + tags['rapids:'] AS rapids, + tags['hazard'] AS hazard, + + -- Reason: Fish navigation? + tags['fish_pass'] AS fish_pass, + -- Reason: hydrology + tags['order:strahler'] AS 'order:strahler', + { + xmin: ST_XMin(geometry)::FLOAT, + ymin: ST_YMin(geometry)::FLOAT, + xmax: ST_XMax(geometry)::FLOAT, + ymax: ST_YMax(geometry)::FLOAT + } AS bbox, + geometry + FROM raw +) TO '{{OUTPUT}}' WITH (FORMAT PARQUET, COMPRESSION ZSTD); From 273a60ab289e002a11f4ab80bcfa6cf1c206e978 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 29 Jul 2026 22:12:19 -0700 Subject: [PATCH 2/6] Remove some tags after review --- sql/macros.sql | 24 ------------------------ sql/water.sql | 42 +++++++++++++++--------------------------- 2 files changed, 15 insertions(+), 51 deletions(-) diff --git a/sql/macros.sql b/sql/macros.sql index 850d391..01f9021 100644 --- a/sql/macros.sql +++ b/sql/macros.sql @@ -32,27 +32,3 @@ CREATE OR REPLACE MACRO prefix_map_split(pfx, t) AS ( ) ) ); - --- If a map is empty, returns NULL, otherwise return the map. -CREATE OR REPLACE MACRO try_map(m1) AS ( - CASE - -- A little tricky to do without triggering the type checker, - -- This is why we route through `len(map_keys(..))` instead of calling `cardinality()` - WHEN len(map_keys(m1)) = 0 THEN NULL - ELSE m1 - END -); - --- Given a map of tags, and a list of tag names, yields a map with those named values. --- e.g. given map_from_tags(tags, ['boat, 'canoe']) on {id: 1, canoe: 'yes', boat: 'no', ...} --- yields {canoe: 'yes', boat: 'no'}. -CREATE OR REPLACE MACRO map_from_tag_list(tags, tag_list) AS ( - try_map( - MAP_FROM_ENTRIES( - LIST_FILTER( - MAP_ENTRIES(tags), - lambda kv: kv.key IN tag_list AND kv.value IS NOT NULL - ) - ) - ) -); diff --git a/sql/water.sql b/sql/water.sql index 7cb7b23..3bc98fa 100644 --- a/sql/water.sql +++ b/sql/water.sql @@ -3,57 +3,45 @@ COPY ( SELECT type, id, tags, geometry FROM '{{INPUT}}' WHERE kind = 'line' - AND tags['waterway'] IN ('river', 'stream', 'canal', 'ditch', 'drain', 'flowline', 'fairway', 'link', - -- Reason: Routing obstructions. - 'dam', 'weir', 'waterfall' - ) + AND tags['waterway'] IN ('river', 'stream', 'canal', 'ditch', 'drain', 'flowline', 'fairway', 'link') ) SELECT type, id, tags['waterway'] AS waterway, - tags['name'] AS name, + split_multi(tags['name']) AS name, + prefix_map_split('name:', tags) AS names, tags['ref'] AS ref, tags['intermittent'] AS intermittent, tags['layer'] AS layer, tags['bridge'] AS bridge, tags['tunnel'] AS tunnel, tags['access'] AS access, - - -- waterway-relevant tags i've added or changed... - -- Reason: general + -- General tags['usage'] AS usage, - tags['natural'] AS natural, split_multi(tags['seasonal']) AS seasonal, tags['tidal'] AS tidal, - - -- Reason: dimensions. + -- Dimensions. tags['width'] AS width, tags['depth'] AS depth, - tags['depth:'] AS 'depth:', - tags['height'] AS height, -- For waterfalls? - - -- Reason: navigation direction and obstruction bypass, - prefix_map('oneway:', tags) AS 'oneway:', + -- Navigation direction and obstruction bypass, tags['oneway'] AS oneway, - tags['canoe_pass'] AS canoe_pass, tags['lock'] AS lock, - + tags['lock_name'] AS lock, + tags['lock_ref'] AS lock, -- Boat access - map_from_tag_list(tags, ['motorboat', 'ship', 'sailboat', 'boat', 'canoe']) as vessel_access, - + tags['motorboat'] AS motorboat, + tags['ship'] AS ship, + tags['sailboat'] AS sailboat, + tags['boat'] AS boat, + tags['canoe'] AS canoe, -- Reason: Hazards, tags['open_water'] AS open_water, - prefix_map('whitewater:', tags) AS whitewater, + prefix_map('whitewater:', tags) AS whitewaters, tags['narrow'] AS narrow, tags['rapids'] AS rapids, - tags['rapids:'] AS rapids, + tags['rapids:name'] AS 'rapids:name', tags['hazard'] AS hazard, - - -- Reason: Fish navigation? - tags['fish_pass'] AS fish_pass, - -- Reason: hydrology - tags['order:strahler'] AS 'order:strahler', { xmin: ST_XMin(geometry)::FLOAT, ymin: ST_YMin(geometry)::FLOAT, From 867d9cba520b85cbab0245faaaeccd25d11b7c7e Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 29 Jul 2026 22:12:41 -0700 Subject: [PATCH 3/6] remove extra indentation --- sql/water.sql | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/sql/water.sql b/sql/water.sql index 3bc98fa..ca5787e 100644 --- a/sql/water.sql +++ b/sql/water.sql @@ -18,30 +18,30 @@ COPY ( tags['tunnel'] AS tunnel, tags['access'] AS access, -- General - tags['usage'] AS usage, - split_multi(tags['seasonal']) AS seasonal, - tags['tidal'] AS tidal, - -- Dimensions. - tags['width'] AS width, - tags['depth'] AS depth, - -- Navigation direction and obstruction bypass, - tags['oneway'] AS oneway, - tags['lock'] AS lock, - tags['lock_name'] AS lock, - tags['lock_ref'] AS lock, - -- Boat access - tags['motorboat'] AS motorboat, - tags['ship'] AS ship, - tags['sailboat'] AS sailboat, - tags['boat'] AS boat, - tags['canoe'] AS canoe, - -- Reason: Hazards, - tags['open_water'] AS open_water, - prefix_map('whitewater:', tags) AS whitewaters, - tags['narrow'] AS narrow, - tags['rapids'] AS rapids, - tags['rapids:name'] AS 'rapids:name', - tags['hazard'] AS hazard, + tags['usage'] AS usage, + split_multi(tags['seasonal']) AS seasonal, + tags['tidal'] AS tidal, + -- Dimensions. + tags['width'] AS width, + tags['depth'] AS depth, + -- Navigation direction and obstruction bypass, + tags['oneway'] AS oneway, + tags['lock'] AS lock, + tags['lock_name'] AS lock, + tags['lock_ref'] AS lock, + -- Boat access + tags['motorboat'] AS motorboat, + tags['ship'] AS ship, + tags['sailboat'] AS sailboat, + tags['boat'] AS boat, + tags['canoe'] AS canoe, + -- Reason: Hazards, + tags['open_water'] AS open_water, + prefix_map('whitewater:', tags) AS whitewaters, + tags['narrow'] AS narrow, + tags['rapids'] AS rapids, + tags['rapids:name'] AS 'rapids:name', + tags['hazard'] AS hazard, { xmin: ST_XMin(geometry)::FLOAT, ymin: ST_YMin(geometry)::FLOAT, From a9509bd992eefd1dba67a03c5527985a920b6a43 Mon Sep 17 00:00:00 2001 From: Jake Low Date: Thu, 13 Aug 2026 12:48:46 -0700 Subject: [PATCH 4/6] remove whitewater tags for now (unclear which are well established) --- sql/water.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/water.sql b/sql/water.sql index ca5787e..d48cffa 100644 --- a/sql/water.sql +++ b/sql/water.sql @@ -37,7 +37,6 @@ COPY ( tags['canoe'] AS canoe, -- Reason: Hazards, tags['open_water'] AS open_water, - prefix_map('whitewater:', tags) AS whitewaters, tags['narrow'] AS narrow, tags['rapids'] AS rapids, tags['rapids:name'] AS 'rapids:name', From d33e342b0af88ac6df625586f0bd5158d2c73931 Mon Sep 17 00:00:00 2001 From: Jake Low Date: Thu, 13 Aug 2026 12:51:55 -0700 Subject: [PATCH 5/6] format code --- sql/water.sql | 64 +++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/sql/water.sql b/sql/water.sql index d48cffa..2a7b4d5 100644 --- a/sql/water.sql +++ b/sql/water.sql @@ -8,39 +8,39 @@ COPY ( SELECT type, id, - tags['waterway'] AS waterway, - split_multi(tags['name']) AS name, - prefix_map_split('name:', tags) AS names, - tags['ref'] AS ref, - tags['intermittent'] AS intermittent, - tags['layer'] AS layer, - tags['bridge'] AS bridge, - tags['tunnel'] AS tunnel, - tags['access'] AS access, + tags['waterway'] AS waterway, + split_multi(tags['name']) AS name, + prefix_map_split('name:', tags) AS names, + tags['ref'] AS ref, + tags['intermittent'] AS intermittent, + tags['layer'] AS layer, + tags['bridge'] AS bridge, + tags['tunnel'] AS tunnel, + tags['access'] AS access, -- General - tags['usage'] AS usage, - split_multi(tags['seasonal']) AS seasonal, - tags['tidal'] AS tidal, - -- Dimensions. - tags['width'] AS width, - tags['depth'] AS depth, - -- Navigation direction and obstruction bypass, - tags['oneway'] AS oneway, - tags['lock'] AS lock, - tags['lock_name'] AS lock, - tags['lock_ref'] AS lock, - -- Boat access - tags['motorboat'] AS motorboat, - tags['ship'] AS ship, - tags['sailboat'] AS sailboat, - tags['boat'] AS boat, - tags['canoe'] AS canoe, - -- Reason: Hazards, - tags['open_water'] AS open_water, - tags['narrow'] AS narrow, - tags['rapids'] AS rapids, - tags['rapids:name'] AS 'rapids:name', - tags['hazard'] AS hazard, + tags['usage'] AS usage, + split_multi(tags['seasonal']) AS seasonal, + tags['tidal'] AS tidal, + -- Dimensions + tags['width'] AS width, + tags['depth'] AS depth, + -- Navigation and obstructions + tags['oneway'] AS oneway, + tags['lock'] AS lock, + tags['lock_name'] AS lock, + tags['lock_ref'] AS lock, + -- Access + tags['motorboat'] AS motorboat, + tags['ship'] AS ship, + tags['sailboat'] AS sailboat, + tags['boat'] AS boat, + tags['canoe'] AS canoe, + -- Hazards + tags['open_water'] AS open_water, + tags['narrow'] AS narrow, + tags['rapids'] AS rapids, + tags['rapids:name'] AS 'rapids:name', + tags['hazard'] AS hazard, { xmin: ST_XMin(geometry)::FLOAT, ymin: ST_YMin(geometry)::FLOAT, From ed1ded790de8e1e0f47f9657d634dffac87f4af8 Mon Sep 17 00:00:00 2001 From: Jake Low Date: Thu, 13 Aug 2026 12:53:25 -0700 Subject: [PATCH 6/6] rename water to waterways --- process.sh | 6 +++--- sql/{water.sql => waterways.sql} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename sql/{water.sql => waterways.sql} (100%) diff --git a/process.sh b/process.sh index c6d267f..3feb9af 100755 --- a/process.sh +++ b/process.sh @@ -41,7 +41,7 @@ for arg in "$@"; do --parks) PARKS=1; ALL=0 ;; --pois) POIS=1; ALL=0 ;; --addresses) ADDRESSES=1; ALL=0 ;; - --water) WATER=1; ALL=0 ;; + --waterways) WATERWAYS=1; ALL=0 ;; --osmium-index-type=*) OSMIUM_INDEX_TYPE="${arg#*=}" ;; --duckdb-memory-limit=*) DUCKDB_MEMORY_LIMIT="${arg#*=}" ;; *) echo "Unknown argument: $arg" >&2; exit 1 ;; @@ -49,7 +49,7 @@ for arg in "$@"; do done if [ "$ALL" = "1" ]; then - BUILDINGS=1; HIGHWAYS=1; BOUNDARIES=1; SETTLEMENTS=1; PARKS=1; POIS=1; ADDRESSES=1; WATER=1 + BUILDINGS=1; HIGHWAYS=1; BOUNDARIES=1; SETTLEMENTS=1; PARKS=1; POIS=1; ADDRESSES=1; WATERWAYS=1 fi mkdir -p "$OUTPUT_DIR" @@ -77,6 +77,6 @@ run_layer() { [ "$PARKS" = "1" ] && run_layer parks [ "$POIS" = "1" ] && run_layer pois [ "$ADDRESSES" = "1" ] && run_layer addresses -[ "$WATER" = "1" ] && run_layer water +[ "$WATERWAYS" = "1" ] && run_layer waterways echo "Done" diff --git a/sql/water.sql b/sql/waterways.sql similarity index 100% rename from sql/water.sql rename to sql/waterways.sql