Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# SWRData/Boundaries

Ready-to-use, timestamped, tiled boundary data for Germany.
Ready-to-use, timestamped boundary data for Germany.

[![deploy demo](https://github.com/SWRdata/boundaries/actions/workflows/deploy-demo.yaml/badge.svg)](https://github.com/SWRdata/boundaries/actions/workflows/deploy-demo.yaml) [![deploy pipeline](https://github.com/SWRdata/boundaries/actions/workflows/deploy-pipeline.yaml/badge.svg)](https://github.com/SWRdata/boundaries/actions/workflows/deploy-pipeline.yaml) [![ty](https://github.com/SWRdata/boundaries/actions/workflows/ty.yaml/badge.svg)](https://github.com/SWRdata/boundaries/actions/workflows/ty.yaml)

Expand All @@ -14,15 +14,17 @@ See [demo](https://static.datenhub.net/apps/boundaries/main/index.html) for code

### Tilesets

| Name | Description | Source | License |
| ------------------ | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- |
| `admin_boundaries` | Staat, Länder, Kreise, Gemeinden 1:250,000 | [BKG VG250](https://gdz.bkg.bund.de/index.php/default/digitale-geodaten/verwaltungsgebiete/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html) | [DNN 2.0](https://sgx.geodatenzentrum.de/web_public/gdz/lizenz/deu/nutzungsbedingungen_vg250.pdf) |
| `admin_labels` | Label points for all features in `admin_boundaries` | [BKG VG250](https://gdz.bkg.bund.de/index.php/default/digitale-geodaten/verwaltungsgebiete/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html) | [DNN 2.0](https://sgx.geodatenzentrum.de/web_public/gdz/lizenz/deu/nutzungsbedingungen_vg250.pdf) |
| Name | Description | Sources | License |
| ------------------ | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `admin_boundaries` | Staat, Länder, Kreise, Gemeinden 1:250,000 | [BKG VG250](https://gdz.bkg.bund.de/index.php/default/digitale-geodaten/verwaltungsgebiete/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html), [Natural Earth 10m Cultural Vectors](https://www.naturalearthdata.com/downloads/10m-cultural-vectors/) | [DNN 2.0](https://sgx.geodatenzentrum.de/web_public/gdz/lizenz/deu/nutzungsbedingungen_vg250.pdf), [Public Domain](https://www.naturalearthdata.com/about/terms-of-use/) |
| `admin_labels` | Label points for all features in `admin_boundaries` | [BKG VG250](https://gdz.bkg.bund.de/index.php/default/digitale-geodaten/verwaltungsgebiete/verwaltungsgebiete-1-250-000-stand-01-01-vg250-01-01.html) | [DNN 2.0](https://sgx.geodatenzentrum.de/web_public/gdz/lizenz/deu/nutzungsbedingungen_vg250.pdf) |

### Timestamps

<!-- BEGIN TIMESTAMPS -->

`2024-01-01`, `2025-01-01`

<!-- END TIMESTAMPS -->

### Fields
Expand Down
2 changes: 1 addition & 1 deletion demo/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
"sync": "svelte-kit sync"
},
"devDependencies": {
"@sveltejs/kit": "^2.61.1",
"@sveltejs/adapter-auto": "^7.0.1",
"@sveltejs/adapter-static": "^3.0.10",
"@sveltejs/kit": "^2.61.1",
"@sveltejs/vite-plugin-svelte": "^7.1.2",
"dotenv": "^17.4.2",
"prettier": "^3.8.3",
Expand Down
64 changes: 53 additions & 11 deletions demo/src/lib/App.svelte
Original file line number Diff line number Diff line change
@@ -1,28 +1,46 @@
<script>
import { dev } from "$app/environment";

import { Map, SWRDataLabLight, VectorTileSource, VectorLayer, NavigationControl, Tooltip, tokens, AttributionControl } from "@swr-data-lab/components";
import {
Map,
SWRDataLabLight,
VectorTileSource,
VectorLayer,
NavigationControl,
Tooltip,
tokens,
AttributionControl,
} from "@swr-data-lab/components";

import Sidebar from "./Sidebar.svelte";

const { timestamps } = $props();
const levels = [2, 4, 6, 8];
const labels = ["Default", "None"];

let filter = $state(levels[1]);
let filter = $state(levels[0]);
let showLabels = $state(labels[0]);
let currentDate = $state(0);
let date = $derived(timestamps[currentDate] ?? null);

let tileUrls = $derived(
false
dev
? [
["boundaries", `http://localhost:8080/tiles/admin_boundaries/tiles.json`],
[
"boundaries",
`http://localhost:8080/tiles/admin_boundaries/tiles.json`,
],
["labels", `http://localhost:8080/tiles/admin_labels/tiles.json`],
]
: [
["boundaries", `https://static.datenhub.net/data/boundaries/admin_boundaries_${date}.versatiles?{z}/{x}/{y}`],
["labels", `https://static.datenhub.net/data/boundaries/admin_labels_${date}.versatiles?{z}/{x}/{y}`],
[
"boundaries",
`https://static.datenhub.net/data/boundaries/admin_boundaries_${date}.versatiles?{z}/{x}/{y}`,
],
[
"labels",
`https://static.datenhub.net/data/boundaries/admin_labels_${date}.versatiles?{z}/{x}/{y}`,
],
],
);

Expand All @@ -32,7 +50,9 @@

let fills = [];
for (let i = 0; i <= 16; i++) {
const key = Object.keys(tokens.shades)[i % Object.keys(tokens.shades).length];
const key = Object.keys(tokens.shades)[
i % Object.keys(tokens.shades).length
];
fills.push(tokens.shades[key].base);
}

Expand All @@ -46,7 +66,14 @@
</script>

<main class="container">
<Sidebar {labels} {levels} {timestamps} bind:date bind:filter bind:showLabels />
<Sidebar
{labels}
{levels}
{timestamps}
bind:date
bind:filter
bind:showLabels
/>
<Map
initialBounds={[5.86625, 47.270124, 15.041816, 55.058778]}
initialLocation={{ zoom: 5.9 }}
Expand All @@ -63,7 +90,12 @@
{#if url.includes("tiles.json")}
<VectorTileSource {id} {url} />
{:else}
<VectorTileSource {id} tiles={[url]} maxZoom={8} attribution={`© BKG ${date.replace("-01-01", "")} dl-de/by-2-0`} />
<VectorTileSource
{id}
tiles={[url]}
maxZoom={8}
attribution={`© BKG ${date.replace("-01-01", "")} dl-de/by-2-0`}
/>
{/if}
{/each}

Expand Down Expand Up @@ -125,7 +157,12 @@
type="line"
filter={["==", "admin_level", filter]}
paint={{
"line-width": ["case", ["any", ["boolean", ["feature-state", "hovered"], false]], 1.75, 0],
"line-width": [
"case",
["any", ["boolean", ["feature-state", "hovered"], false]],
1.75,
0,
],
"line-color": "black",
"line-opacity": 1,
}}
Expand Down Expand Up @@ -155,7 +192,12 @@
{/if}

{#if hovered}
<Tooltip position={tooltipCoordinates} mouseEvents={false} showCloseButton={false} closeOnClick={false}>
<Tooltip
position={tooltipCoordinates}
mouseEvents={false}
showCloseButton={false}
closeOnClick={false}
>
<table class="tooltip-body">
<tbody>
{#each Object.entries(hovered.properties) as [k, v]}
Expand Down
2 changes: 1 addition & 1 deletion tasks/make_boundaries/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,4 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
uv sync --frozen --no-dev --no-editable

CMD [".venv/bin/python", "src/main.py"]
CMD [".venv/bin/python", "src/main.py", "--mode=production", "--min_year=2024"]
143 changes: 143 additions & 0 deletions tasks/make_boundaries/exploration/country_snapping.ipynb

Large diffs are not rendered by default.

12 changes: 5 additions & 7 deletions tasks/make_boundaries/exploration/labels.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,14 @@
"id": "d83ab3d6",
"metadata": {},
"source": [
"# `boundaries`: Exploration\n",
"\n",
"This notebook is a place to work out processing steps before we move them into the production pipeline.\n",
"# `boundaries`: Label points exploration\n",
"\n",
"max.koehler@swr.de"
]
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "e20f696c",
"metadata": {},
"outputs": [],
Expand All @@ -26,7 +24,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"id": "612b0753",
"metadata": {},
"outputs": [],
Expand All @@ -35,7 +33,7 @@
"for k in [\"STA\", \"LAN\", \"KRS\"]:\n",
" data[k] = gp.read_file(f\"zip://../tmp/raw/2025-01-01_vg250_01-01.utm32s.shape.ebenen.zip!vg250_01-01.utm32s.shape.ebenen/vg250_ebenen_0101/VG250_{k}.shp\")\n",
"\n",
"data[\"subs\"] = gp.read_file(\"../label_substitutions.geojson\")\n"
"data[\"subs\"] = gp.read_file(\"../label_substitutions.geojson\")"
]
},
{
Expand Down Expand Up @@ -182,7 +180,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.10"
"version": "3.12.2"
}
},
"nbformat": 4,
Expand Down
5 changes: 4 additions & 1 deletion tasks/make_boundaries/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[project]
name = "fetch-geometry"
name = "make_boundaries"
version = "0.1.0"
description = ""
readme = "README.md"
Expand All @@ -9,8 +9,11 @@ dependencies = [
"dotenv>=0.9.9",
"geopandas>=1.1.2",
"google-cloud-storage>=3.9.0",
"ipykernel>=7.3.0",
"matplotlib>=3.10.9",
"pandas>=3.0.3",
"requests>=2.34.2",
"shapely>=2.1.2",
"tqdm>=4.68.2",
"typed-argument-parser>=1.12.0",
]
12 changes: 12 additions & 0 deletions tasks/make_boundaries/src/globals.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,15 @@
"Dillingen a.d.Donau": "Dillingen an der Donau",
"Mühldorf a.Inn": "Mühldorf am Inn",
}

NEIGHBOURS = (
"Austria",
"Belgium",
"Czechia",
"Denmark",
"France",
"Luxembourg",
"Netherlands",
"Poland",
"Switzerland",
)
89 changes: 57 additions & 32 deletions tasks/make_boundaries/src/main.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import datetime
import os
from typing import Dict
from typing import Dict, Literal

from dotenv import load_dotenv
from google.cloud import storage
from tap import Tap

from entities.Tileset import Tileset
from usecases.fetch_bkg_years import fetch_bkg_years
Expand All @@ -12,35 +13,51 @@
from usecases.make_admin_labels import make_admin_labels
from usecases.upload_blob import upload_blob

gcs_project = "swr-data-1"
gcs_bucket = "datenhub-net-static"
gcs_path = "data/boundaries/"
min_year = 2024

raw_dir = "./tmp/raw/"
processed_dir = "./tmp/processed/"
manifest_path = os.path.join(processed_dir, "manifest.csv")
class ArgParser(Tap):
mode: Literal["production", "dev"] = "dev"
"""dev mode disables reading/writing to GCS and some data fetches"""
min_year: int = 2024
gcs_project: str = "swr-data-1"
gcs_bucket: str = "datenhub-net-static"
gcs_path: str = "data/boundaries/"
raw_dir: str = "./tmp/raw/"
processed_dir: str = "./tmp/processed/"


tilesets: Dict[str, Tileset] = {}
def main(args: ArgParser):
tilesets: Dict[str, Tileset] = {}


def run():
load_dotenv()
os.makedirs(os.path.dirname(args.raw_dir), exist_ok=True)
os.makedirs(os.path.dirname(args.processed_dir), exist_ok=True)
manifest_path = os.path.join(args.processed_dir, "manifest.csv")

print(f"running in {args.mode} mode")

os.makedirs(os.path.dirname(raw_dir), exist_ok=True)
os.makedirs(os.path.dirname(processed_dir), exist_ok=True)
if args.mode == "production":
print(f"{args.get_reproducibility_info()}")

storage_client = storage.Client(project=gcs_project)
if args.mode == "production":
storage_client = storage.Client(project=args.gcs_project)

print("Fetching BKG files... ", end="")
available_years = fetch_bkg_years()
# available_years = [2025]
# Fetch available years on the BKG wesite
if args.mode == "production":
print("fetching available bkg files... ", end="")
available_years = fetch_bkg_years()
else:
print("skipping fetching bkg files in dev mode... ", end="")
available_years = [2025]
print(f"found {len(available_years)}\n")

print("Fetching existing files... ", end="")
existing_files = fetch_existing(storage_client, gcs_bucket, gcs_path)
# existing_files = []
# Fetch existing tilesets
if args.mode == "production":
print("fetching existing files... ", end="")
existing_files = fetch_existing(storage_client, args.gcs_bucket, args.gcs_path)
else:
print("skipping fetching existing files in dev mode... ", end="")
existing_files = []

if len(existing_files) > 0:
print(f"found {len(existing_files)}:\n- {'\n- '.join(existing_files)}")
else:
Expand All @@ -49,25 +66,25 @@ def run():
new_files: list[str] = []
failed_files: list[str] = []

for y in [y for y in available_years if y >= min_year]:
for y in [y for y in available_years if y >= args.min_year]:
date = datetime.date(y, 1, 1)

tilesets[f"admin_boundaries_{date.strftime('%Y-%m-%d')}"] = Tileset(
name=f"Administrative Boundaries {y}",
make_fn=make_admin,
make_args={
"cache_dir": raw_dir,
"output_dir": processed_dir,
"cache_dir": args.raw_dir,
"output_dir": args.processed_dir,
"date": date,
},
)

continue
tilesets[f"admin_labels_{date.strftime('%Y-%m-%d')}"] = Tileset(
name=f"Administrative Labels {date}",
make_fn=make_admin_labels,
make_args={
"cache_dir": raw_dir,
"output_dir": processed_dir,
"cache_dir": args.raw_dir,
"output_dir": args.processed_dir,
"date": date,
},
)
Expand Down Expand Up @@ -96,12 +113,20 @@ def run():
f.write(f"name\n{'\n'.join([*existing_files, *new_files])}")
print(f"Wrote manifest to {manifest_path}")

for f in [*new_files, manifest_path]:
upload_blob(
storage_client, f, gcs_bucket, os.path.join(gcs_path, os.path.basename(f))
if args.mode == "production":
for f in [*new_files, manifest_path]:
upload_blob(
storage_client,
f,
args.gcs_bucket,
os.path.join(args.gcs_path, os.path.basename(f)),
)
print(f"Uploaded {len(new_files)} new files ({len(failed_files)} failed)")
else:
print(
f"skipping updloading {len(new_files)} new files in dev mode ({len(failed_files)} failed)"
)

print(f"Uploaded {len(new_files)} new files, {len(failed_files)} failed")


run()
args = ArgParser().parse_args()
main(args)
Loading