diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 9f7bde5f..b554fe2a 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -18,7 +18,7 @@ jobs: timeout-minutes: 60 strategy: matrix: - python-version: ['3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10','3.11','3.12','3.13'] steps: - uses: actions/checkout@v6 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8054cceb..84b829d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: 24.8.0 hooks: - id: black -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.3.0 +- repo: https://github.com/pycqa/flake8 + rev: 7.1.2 hooks: - id: flake8 \ No newline at end of file diff --git a/Makefile b/Makefile index 22037acb..e32196d6 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ SQLDIFF := $(shell command -v sqldiff 2> /dev/null) UNAME := $(shell uname) ifeq ($(UNAME),Darwin) - SPATIAL := $(shell ls /usr/local/lib/*spatialite* 2> /dev/null) + # /opt/homebrew/lib on Apple Silicon, /usr/local/lib on Intel Macs + SPATIAL := $(shell ls /opt/homebrew/lib/*spatialite* 2> /dev/null || ls /usr/local/lib/*spatialite* 2> /dev/null) else SPATIAL := $(shell ls /usr/lib/x86_64-linux-gnu/*spatialite* 2> /dev/null) endif diff --git a/digital_land/collection.py b/digital_land/collection.py index fed8b152..62903918 100644 --- a/digital_land/collection.py +++ b/digital_land/collection.py @@ -639,8 +639,13 @@ def retire_endpoints_and_sources(collection, collection_df_to_retire) -> None: source_csv_path = os.path.join(collection.dir, "source.csv") # Read endpoint and source CSV files - endpoint_csv_df = pd.read_csv(endpoint_csv_path) - source_csv_df = pd.read_csv(source_csv_path) + # end-date is read as object (not left to dtype inference) so an + # all-blank column doesn't get inferred as float64 -- assigning a + # date string into a float64 column is an error under pandas 3.0 + endpoint_csv_df = pd.read_csv( + endpoint_csv_path, dtype={"end-date": "object"} + ) + source_csv_df = pd.read_csv(source_csv_path, dtype={"end-date": "object"}) # Get today's date in the format YYYY-MM-DD today_date = datetime.now().strftime("%Y-%m-%d") diff --git a/digital_land/expectations/checkpoints/dataset.py b/digital_land/expectations/checkpoints/dataset.py index bb8a6838..16dfb837 100644 --- a/digital_land/expectations/checkpoints/dataset.py +++ b/digital_land/expectations/checkpoints/dataset.py @@ -181,7 +181,7 @@ def run(self, prefetch_resources=False): org_name = org.get("organisation", "") if org else "" label = f"{expectation['operation'].__name__}({org_name})" logger.warning( - f"[expectations] {i+1}/{len(self.expectations)} {label} — {'PASSED' if passed else 'FAILED'}" + f"[expectations] {i + 1}/{len(self.expectations)} {label} — {'PASSED' if passed else 'FAILED'}" ) self.log.add( diff --git a/digital_land/phase/convert.py b/digital_land/phase/convert.py index ebb5c93e..b73989f7 100644 --- a/digital_land/phase/convert.py +++ b/digital_land/phase/convert.py @@ -1,5 +1,5 @@ import csv -from cchardet import UniversalDetector +from charset_normalizer import from_bytes, from_path import logging import json_stream import os @@ -21,20 +21,26 @@ class ConversionError(Exception): pass +def _best_encoding(best): + if not best: + return None + # charset-normalizer reports the base codec even when a BOM is present + # (best.bom=True); Python needs the "-sig" variant to strip it, or the + # BOM decodes as a leading U+FEFF character in the content. + return best.encoding + "-sig" if best.bom else best.encoding + + def detect_file_encoding(path): - with open(path, "rb") as f: - return detect_encoding(f) + if not os.path.getsize(path): + return None + return _best_encoding(from_path(path).best()) def detect_encoding(f): - detector = UniversalDetector() - detector.reset() - for line in f: - detector.feed(line) - if detector.done: - break - detector.close() - return detector.result["encoding"] + data = f.read() + if not data: + return None + return _best_encoding(from_bytes(data).best()) def load_csv(path, encoding="UTF-8", log=None): diff --git a/pyproject.toml b/pyproject.toml index ada25530..5228e4dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ name = "digital-land" dynamic = ["version", "readme"] description = "Data pipeline tools to collect data and process it into a dataset" +requires-python = ">=3.8" authors = [ { name = "MHCLG Planning Data Team", email = "DigitalLand@communities.gov.uk" }, ] @@ -18,16 +19,17 @@ dependencies = [ "datasette", "canonicaljson", "click", - "cchardet", + "charset-normalizer", "esridump", - "pandas", + "pandas==2.0.3; python_version < '3.11'", + "pandas==3.0.5; python_version >= '3.11'", "pyproj", "requests", "validators", "xlrd==1.2.0", "openpyxl", "numpy<2", - "Shapely==2.0.2", + "Shapely==2.0.7", "SPARQLWrapper", "geojson", "spatialite", @@ -52,14 +54,18 @@ classifiers = [ "Intended Audience :: Developers", "Topic :: Database", "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] [project.optional-dependencies] test = [ "coverage", - "flake8", + "flake8==7.1.2", "pytest", "coveralls", "twine", diff --git a/tests/unit/plugins/test_wfs.py b/tests/unit/plugins/test_wfs.py index 5bafc2d8..c45a3e32 100644 --- a/tests/unit/plugins/test_wfs.py +++ b/tests/unit/plugins/test_wfs.py @@ -27,6 +27,21 @@ def get(self, url, log, plugin): ) +def test_get_does_not_crash_on_empty_content(): + class FakeCollector: + def get(self, url, log, plugin): + log["status"] = "200" + return log, None + + log, content = wfs_get( + FakeCollector(), + "https://example.com/wfs", + ) + + assert log["status"] == "200" + assert content is None + + def test_get_paged_wfs_runs_ogr2ogr_with_paging_config(tmp_path, mocker): output_path = tmp_path / "output.gpkg" captured = {}