diff --git a/.github/workflows/run_checks_build_and_test.yml b/.github/workflows/run_checks_build_and_test.yml index 05657c00..429c97cf 100644 --- a/.github/workflows/run_checks_build_and_test.yml +++ b/.github/workflows/run_checks_build_and_test.yml @@ -58,10 +58,20 @@ jobs: matrix: python-version: [ "3.14", + "3.13", + "3.12", + "3.15-dev", + "3.11", + "3.10", + "3.9", ] os: [ "ubuntu-24.04", ] + include: + - python-version: "3.14" + - os: "ubuntu-latest" + runs-on: ${{ matrix.os }} steps: - uses: actions/setup-python@v6 @@ -73,7 +83,6 @@ jobs: path: ./Pyshp - name: "Hypothesis tests" - if: uses: ./Pyshp/.github/actions/test with: extra_args: '-m hypothesis' diff --git a/README.md b/README.md index 8899510c..9ad10069 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ The Python Shapefile Library (PyShp) reads and writes ESRI Shapefiles in pure Py - **Author**: [Joel Lawhead](https://github.com/GeospatialPython) - **Maintainers**: [James Parrott](https://github.com/JamesParrott) & [Karim Bahgat](https://github.com/karimbahgat) -- **Version**: 3.1.4 -- **Date**: 29th June 2026 +- **Version**: 3.1.5 +- **Date**: 22nd July 2026 - **License**: [MIT](https://github.com/GeospatialPython/pyshp/blob/master/LICENSE.TXT) ## Contents @@ -93,6 +93,10 @@ part of your geospatial project. # Version Changes +## 3.1.5 +### Bug fix + - Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) + ## 3.1.4 ### Bug fix - Fix bug causing dates supplied as length 8 strings of digits to be encoded by the custom encoding, not ascii. diff --git a/changelog.txt b/changelog.txt index 3d46e338..3e0b2b32 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +VERSION 3.1.5 +2026-07-22 + * Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) + VERSION 3.1.4 2026-06-29 diff --git a/src/shapefile.py b/src/shapefile.py index 547a4c0a..718f2ef4 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -8,7 +8,7 @@ from __future__ import annotations -__version__ = "3.1.4" +__version__ = "3.1.5" import abc import array @@ -4431,7 +4431,9 @@ def _record(self, record: list[RecordValue]) -> None: if isinstance(value, list) and len(value) == 3: value = date(*value) if isinstance(value, date): - str_val = value.strftime("%Y%m%d") + # In Pythons using certain glibc versions. + # date.strftime does not preppend zeros. + str_val = value.strftime("%Y%m%d").zfill(8) # b"".join(ord(c).to_bytes() for c in s) elif value in MISSING: str_val = "0" * 8 # QGIS NULL for date type diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index 8b6381b8..6ebc72ed 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -2,6 +2,7 @@ import contextlib import datetime +import functools import io import itertools import os @@ -66,6 +67,7 @@ def ignore_warnings(category=None): ) + def coords_2D_list( min_size: int = 1, max_size: int | None = None, @@ -77,6 +79,7 @@ def coords_2D_list( ) + @pytest.mark.hypothesis @given(expected=point_2D, i=integers(min_value=1)) def test_Point_2D_roundtrips( @@ -563,20 +566,90 @@ def test_shx_reader_writer_roundtrip(codes_and_shapes)-> None: "utf-32-le", "cp1140", ] +POSSIBLY_NONINJECTIVE_CODECS = frozenset({ + 'big5', 'big5hkscs', 'cp932', 'cp949', 'cp950', + 'euc_jp', 'euc_jis_2004', 'euc_kr', 'gb2312', 'gbk', 'gb18030', + 'hz', 'iso2022_jp', 'iso2022_kr', 'shift_jis', 'shift_jis_2004' +}) + +@functools.cache +def _exclude_chars() -> dict[str,list[str]]: + + exclude_chars = {} #"iso2022" : ["\x1b"]} + """ + Not sure if bug in hypothesis generation, in core library, or just the way it is: + + Python taking a short cut with Control characters in ISO_2022 stateful codecs + Python 3.13.14 (main, Jul 18 2026, 17:02:37) [Clang 22.1.3 ] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> enc="iso2022_jp_1" + >>> s="\x1b" + >>> b=s.encode(enc) + >>> b + b'\x1b' + >>> b.decode(enc) + Traceback (most recent call last): + File "", line 1, in + b.decode(enc) + ~~~~~~~~^^^^^ + UnicodeDecodeError: 'iso2022_jp_1' codec can't decode byte 0x1b in position 0: incomplete multibyte sequence + decoding with 'iso2022_jp_1' codec failed + + Non injective encodings (dealt with below) + >>> enc="cp950" + >>> "•".encode(enc) + b'\xa1E' + >>> b=_ + >>> b.decode(enc) + '‧' + >>> s = _ + >>> s.encode(enc) + b'\xa1E' + + """ + -def _encodings() -> set[str]: from encodings.aliases import aliases - encs = set() for enc in aliases.values(): - if enc in encs: + if enc in exclude_chars: continue + # I'm not sure why any encoding would fail on an empty string, + # but I'd rather not have the tests get bogged by any that do exist. try: "".encode(enc) except (UnicodeEncodeError, LookupError): continue - encs.add(enc) - return encs -# assert _encodings() == {'utf_16_le', 'iso8859_7', 'cp437', 'iso2022_jp_3', 'shift_jis', 'cp775', 'cp1140', + + exclude_chars[enc] = [] + + if enc not in POSSIBLY_NONINJECTIVE_CODECS: + continue + + # find collisions of non-injective codecs + # Iterate over BMP + for code_point in range(0x10000): + char = chr(code_point) + try: + b = char.encode(enc) + except (UnicodeEncodeError, LookupError): + exclude_chars[enc].append(char) + continue + + decoded = None + try: + decoded = b.decode(enc) + except (UnicodeDecodeError, LookupError): + exclude_chars[enc].append(char) + continue + + if decoded != char: + exclude_chars[enc].append(char) + + + + + return exclude_chars +# assert set(_encodings()) == {'utf_16_le', 'iso8859_7', 'cp437', 'iso2022_jp_3', 'shift_jis', 'cp775', 'cp1140', # 'cp861', 'iso8859_11', 'iso8859_9', 'euc_jp', 'utf_16', 'cp950', 'mac_cyrillic', 'mac_turkish', 'iso2022_jp_1', 'iso8859_10', # 'iso2022_jp_2004', 'cp866', 'mac_greek', 'hz', 'cp1257', 'cp037', 'cp863', 'iso8859_4', 'utf_16_be', 'gb18030', 'cp1250', # 'cp850', 'iso8859_5', 'shift_jisx0213', 'iso8859_8', 'cp273', 'euc_jisx0213', 'cp932', 'cp862', 'tis_620', 'cp1125', 'koi8_r', @@ -586,25 +659,29 @@ def _encodings() -> set[str]: # 'iso2022_kr', 'cp1251', 'cp1255', 'mac_iceland', 'kz1048', 'iso8859_14', 'utf_32_be', 'ptcp154', 'iso8859_6', 'mac_roman', # 'utf_32', 'iso2022_jp_2', 'iso8859_16', 'mbcs', 'cp500', 'iso8859_2', 'cp949', 'cp852', 'utf_7', 'big5hkscs', 'johab'} -encodings = sampled_from(list(_encodings())) # if IN_CI else ENCODINGS) + +def strings_of_supported_code_points(encoding: str, min_size: int=1, max_size: int=10): + + return text( + alphabet=characters( + codec=encoding, + # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category + exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates + exclude_characters=_exclude_chars().get(encoding, []), + ), + min_size=min_size, + max_size=max_size, + ) + + +encodings = sampled_from(list(_exclude_chars())) # if IN_CI else ENCODINGS) @composite def _dbf_fields_strategy(draw, encoding: str) -> dict[str, str | int]: field_type, bounds_dict = draw(sampled_from(list(DBF_FIELD_TYPES.items()))) - name = draw( - text( - alphabet=characters( - codec=encoding, - # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category - exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates - # exclude_characters=[" "], - ), - min_size=1, - max_size=10, - ) - ) + name = draw(strings_of_supported_code_points(encoding)) max_length = bounds_dict.get("max_length", 254) min_length = bounds_dict.get("min_length", 1) @@ -688,14 +765,13 @@ def test_dbf_Field_roundtrips(encoding_and_dbf_field: dict) -> None: ascii_printable = string.ascii_letters + string.digits + string.punctuation + " " -def record_value_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): +def date_to_str(d: datetime.date) -> str: + return d.strftime("%Y%m%d").zfill(8) + +def record_value_strat_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): if field_type == "C": - return text( - alphabet=ascii_printable, - min_size=0, - max_size=size, - ) + return strings_of_supported_code_points(encoding, 0, size) if field_type in {"N", "F"}: int_digits = size if decimal == 0 else size - decimal - 1 @@ -715,7 +791,7 @@ def record_value_for_field(name: str, field_type: str, size: int, decimal: int, if field_type == "L": return sampled_from([True, False, None]) if field_type == "D": - return one_of(dates(), dates().map(lambda d: d.strftime("%Y%m%d"))) + return one_of(dates(), dates().map(date_to_str)) raise ValueError(f"Unsupported: {field_type=}") @@ -729,7 +805,7 @@ def _dbf_encoding_fields_and_record_strategy( fields = draw(lists(_dbf_fields_strategy(encoding), min_size=1, max_size=max_fields)) - record_strategy = tuples(*(record_value_for_field(encoding=encoding, **field) for field in fields)) + record_strategy = tuples(*(record_value_strat_for_field(encoding=encoding, **field) for field in fields)) return encoding, fields, record_strategy @@ -771,9 +847,9 @@ def _assert_reader_matches_expected_records(r, fields, written_records): decimal = field["decimal"] if field_type == "D": if isinstance(expected, datetime.date): - expected = expected.strftime("%Y%m%d") + expected = date_to_str(expected) if isinstance(actual, datetime.date): - actual = actual.strftime("%Y%m%d") + actual = date_to_str(actual) elif field_type in ("N", "F") and decimal >= 1: expected = float(format(expected, f".{decimal}f")) assert actual == expected, f"{actual=}, {expected=}, {field_type=}, {type(actual)=}, {type(expected)=}" diff --git a/uv.lock b/uv.lock index fa2ec976..4fc7f19d 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,8 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P1W" [options.exclude-newer-package] +hypothesis = false +sdna-plus = false pyshp-stubs = false [[package]] @@ -54,7 +56,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -78,9 +80,9 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "attrs", marker = "python_full_version < '3.10'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "sortedcontainers", marker = "python_full_version < '3.10'" }, + { name = "attrs" }, + { name = "exceptiongroup" }, + { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/20/8aa62b3e69fea68bb30d35d50be5395c98979013acd8152d64dc927e4cdb/hypothesis-6.141.1.tar.gz", hash = "sha256:8ef356e1e18fbeaa8015aab3c805303b7fe4b868e5b506e87ad83c0bf951f46f", size = 467389, upload-time = "2025-10-15T19:12:25.262Z" } wheels = [ @@ -95,8 +97,8 @@ resolution-markers = [ "python_full_version >= '3.10'", ] dependencies = [ - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "sortedcontainers", marker = "python_full_version >= '3.10'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/a3/10b04ba828ccde01b8a6806d8f18a16755825701fa7d1bcda294cccedeff/hypothesis-6.154.1.tar.gz", hash = "sha256:5956848819cada59aaecafd984d70e3577580e20f2fa6bfc202b0edb27dcecad", size = 476035, upload-time = "2026-05-28T07:27:14.599Z" } wheels = [ @@ -344,11 +346,11 @@ name = "pre-commit" version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cfgv", marker = "python_full_version >= '3.10'" }, - { name = "identify", marker = "python_full_version >= '3.10'" }, - { name = "nodeenv", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, - { name = "virtualenv", marker = "python_full_version >= '3.10'" }, + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8e/22/2de9408ac81acbb8a7d05d4cc064a152ccf33b3d480ebe0cd292153db239/pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9", size = 198525, upload-time = "2026-04-21T20:31:41.613Z" } wheels = [ @@ -434,13 +436,13 @@ resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } wheels = [ @@ -455,13 +457,13 @@ resolution-markers = [ "python_full_version >= '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ @@ -473,8 +475,8 @@ name = "python-discovery" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", marker = "python_full_version >= '3.10'" }, + { name = "filelock" }, + { name = "platformdirs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/60/e88788207d81e46362cfbef0d4aaf4c0f49efc3c12d4c3fa3f542c34ebec/python_discovery-1.3.1.tar.gz", hash = "sha256:62f6db28064c9613e7ca76cb3f00c38c839a07c31c00dfe7ed0986493d2150a6", size = 68011, upload-time = "2026-05-12T20:53:36.336Z" } wheels = [ @@ -656,11 +658,11 @@ name = "virtualenv" version = "21.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "distlib", marker = "python_full_version >= '3.10'" }, - { name = "filelock", marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", marker = "python_full_version >= '3.10'" }, - { name = "python-discovery", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/ba/1f6e8c957e4932be060dcdc482d339c12e0216351478add3645cdaa53c05/virtualenv-21.3.3.tar.gz", hash = "sha256:f5bda277e553b1c2b3c1a8debfc30496e1288cc93ce6b7b71b3280047e317328", size = 7613784, upload-time = "2026-05-13T18:01:30.19Z" } wheels = [