diff --git a/.github/workflows/run_checks_build_and_test.yml b/.github/workflows/run_checks_build_and_test.yml index 05657c00..429c97cf 100644 --- a/.github/workflows/run_checks_build_and_test.yml +++ b/.github/workflows/run_checks_build_and_test.yml @@ -58,10 +58,20 @@ jobs: matrix: python-version: [ "3.14", + "3.13", + "3.12", + "3.15-dev", + "3.11", + "3.10", + "3.9", ] os: [ "ubuntu-24.04", ] + include: + - python-version: "3.14" + - os: "ubuntu-latest" + runs-on: ${{ matrix.os }} steps: - uses: actions/setup-python@v6 @@ -73,7 +83,6 @@ jobs: path: ./Pyshp - name: "Hypothesis tests" - if: uses: ./Pyshp/.github/actions/test with: extra_args: '-m hypothesis' diff --git a/README.md b/README.md index 8899510c..9ad10069 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ The Python Shapefile Library (PyShp) reads and writes ESRI Shapefiles in pure Py - **Author**: [Joel Lawhead](https://github.com/GeospatialPython) - **Maintainers**: [James Parrott](https://github.com/JamesParrott) & [Karim Bahgat](https://github.com/karimbahgat) -- **Version**: 3.1.4 -- **Date**: 29th June 2026 +- **Version**: 3.1.5 +- **Date**: 22nd July 2026 - **License**: [MIT](https://github.com/GeospatialPython/pyshp/blob/master/LICENSE.TXT) ## Contents @@ -93,6 +93,10 @@ part of your geospatial project. # Version Changes +## 3.1.5 +### Bug fix + - Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) + ## 3.1.4 ### Bug fix - Fix bug causing dates supplied as length 8 strings of digits to be encoded by the custom encoding, not ascii. diff --git a/changelog.txt b/changelog.txt index 3d46e338..3e0b2b32 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +VERSION 3.1.5 +2026-07-22 + * Fixed another bug causing dates before the year 1000 to be encoded as less than 8 chars under "%Y%m%d (found by [Thomas Beierlein](https://github.com/GeospatialPython/pyshp/issues/435)) + VERSION 3.1.4 2026-06-29 diff --git a/src/shapefile.py b/src/shapefile.py index 547a4c0a..718f2ef4 100644 --- a/src/shapefile.py +++ b/src/shapefile.py @@ -8,7 +8,7 @@ from __future__ import annotations -__version__ = "3.1.4" +__version__ = "3.1.5" import abc import array @@ -4431,7 +4431,9 @@ def _record(self, record: list[RecordValue]) -> None: if isinstance(value, list) and len(value) == 3: value = date(*value) if isinstance(value, date): - str_val = value.strftime("%Y%m%d") + # In Pythons using certain glibc versions. + # date.strftime does not preppend zeros. + str_val = value.strftime("%Y%m%d").zfill(8) # b"".join(ord(c).to_bytes() for c in s) elif value in MISSING: str_val = "0" * 8 # QGIS NULL for date type diff --git a/tests/hypothesis_tests.py b/tests/hypothesis_tests.py index 8b6381b8..3b200952 100644 --- a/tests/hypothesis_tests.py +++ b/tests/hypothesis_tests.py @@ -2,6 +2,7 @@ import contextlib import datetime +import functools import io import itertools import os @@ -66,6 +67,7 @@ def ignore_warnings(category=None): ) + def coords_2D_list( min_size: int = 1, max_size: int | None = None, @@ -77,6 +79,7 @@ def coords_2D_list( ) + @pytest.mark.hypothesis @given(expected=point_2D, i=integers(min_value=1)) def test_Point_2D_roundtrips( @@ -563,20 +566,91 @@ def test_shx_reader_writer_roundtrip(codes_and_shapes)-> None: "utf-32-le", "cp1140", ] +POSSIBLY_NONINJECTIVE_CODECS = frozenset({ + 'big5', 'big5hkscs', 'cp932', 'cp936', 'cp949', 'cp950', + 'euc_jp', 'euc_jis_2004', 'euc_kr', 'gb2312', 'gbk', 'gb18030', + 'hz', 'iso2022_jp', 'iso2022_kr', 'shift_jis', 'shift_jis_2004' +}) + +@functools.cache +def _exclude_chars() -> dict[str,list[str]]: + + exclude_chars = {} + exclude_chars["iso2022"].append("\x1b") + """ + Not sure if bug in hypothesis generation, in core library, or just the way it is: + + Python taking a short cut with Control characters in ISO_2022 stateful codecs + Python 3.13.14 (main, Jul 18 2026, 17:02:37) [Clang 22.1.3 ] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> enc="iso2022_jp_1" + >>> s="\x1b" + >>> b=s.encode(enc) + >>> b + b'\x1b' + >>> b.decode(enc) + Traceback (most recent call last): + File "", line 1, in + b.decode(enc) + ~~~~~~~~^^^^^ + UnicodeDecodeError: 'iso2022_jp_1' codec can't decode byte 0x1b in position 0: incomplete multibyte sequence + decoding with 'iso2022_jp_1' codec failed + + Non injective encodings (dealt with below) + >>> enc="cp950" + >>> "•".encode(enc) + b'\xa1E' + >>> b=_ + >>> b.decode(enc) + '‧' + >>> s = _ + >>> s.encode(enc) + b'\xa1E' + + """ + -def _encodings() -> set[str]: from encodings.aliases import aliases - encs = set() for enc in aliases.values(): - if enc in encs: + if enc in exclude_chars: continue + # I'm not sure why any encoding would fail on an empty string, + # but I'd rather not have the tests get bogged by any that do exist. try: "".encode(enc) except (UnicodeEncodeError, LookupError): continue - encs.add(enc) - return encs -# assert _encodings() == {'utf_16_le', 'iso8859_7', 'cp437', 'iso2022_jp_3', 'shift_jis', 'cp775', 'cp1140', + + exclude_chars[enc] = [] + + if enc not in POSSIBLY_NONINJECTIVE_CODECS: + continue + + # find collisions of non-injective codecs + # Iterate over BMP + for code_point in range(0x10000): + char = chr(code_point) + try: + b = char.encode(enc) + except (UnicodeEncodeError, LookupError): + exclude_chars[enc].append(char) + continue + + decoded = None + try: + decoded = b.decode(enc) + except (UnicodeDecodeError, LookupError): + exclude_chars[enc].append(char) + continue + + if decoded != char: + exclude_chars[enc].append(char) + + + + + return exclude_chars +# assert set(_encodings()) == {'utf_16_le', 'iso8859_7', 'cp437', 'iso2022_jp_3', 'shift_jis', 'cp775', 'cp1140', # 'cp861', 'iso8859_11', 'iso8859_9', 'euc_jp', 'utf_16', 'cp950', 'mac_cyrillic', 'mac_turkish', 'iso2022_jp_1', 'iso8859_10', # 'iso2022_jp_2004', 'cp866', 'mac_greek', 'hz', 'cp1257', 'cp037', 'cp863', 'iso8859_4', 'utf_16_be', 'gb18030', 'cp1250', # 'cp850', 'iso8859_5', 'shift_jisx0213', 'iso8859_8', 'cp273', 'euc_jisx0213', 'cp932', 'cp862', 'tis_620', 'cp1125', 'koi8_r', @@ -586,25 +660,35 @@ def _encodings() -> set[str]: # 'iso2022_kr', 'cp1251', 'cp1255', 'mac_iceland', 'kz1048', 'iso8859_14', 'utf_32_be', 'ptcp154', 'iso8859_6', 'mac_roman', # 'utf_32', 'iso2022_jp_2', 'iso8859_16', 'mbcs', 'cp500', 'iso8859_2', 'cp949', 'cp852', 'utf_7', 'big5hkscs', 'johab'} -encodings = sampled_from(list(_encodings())) # if IN_CI else ENCODINGS) + +def strings_of_supported_code_points(encoding: str, min_size: int=1, max_size: int=10): + + for prefix, exclude_chars in _exclude_chars().items(): + if encoding.startswith(prefix): + break + else: + exclude_chars = [] + + return text( + alphabet=characters( + codec=encoding, + # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category + exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates + exclude_characters=exclude_chars, + ), + min_size=min_size, + max_size=max_size, + ) + + +encodings = sampled_from(list(_exclude_chars())) # if IN_CI else ENCODINGS) @composite def _dbf_fields_strategy(draw, encoding: str) -> dict[str, str | int]: field_type, bounds_dict = draw(sampled_from(list(DBF_FIELD_TYPES.items()))) - name = draw( - text( - alphabet=characters( - codec=encoding, - # https://en.wikipedia.org/wiki/Unicode_character_property#General_Category - exclude_categories=["Cs", "Co", "Cn"], # Cs - surrogates - # exclude_characters=[" "], - ), - min_size=1, - max_size=10, - ) - ) + name = draw(strings_of_supported_code_points(encoding)) max_length = bounds_dict.get("max_length", 254) min_length = bounds_dict.get("min_length", 1) @@ -688,14 +772,13 @@ def test_dbf_Field_roundtrips(encoding_and_dbf_field: dict) -> None: ascii_printable = string.ascii_letters + string.digits + string.punctuation + " " -def record_value_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): +def date_to_str(d: datetime.date) -> str: + return d.strftime("%Y%m%d").zfill(8) + +def record_value_strat_for_field(name: str, field_type: str, size: int, decimal: int, encoding: str): if field_type == "C": - return text( - alphabet=ascii_printable, - min_size=0, - max_size=size, - ) + return strings_of_supported_code_points(encoding, 0, size) if field_type in {"N", "F"}: int_digits = size if decimal == 0 else size - decimal - 1 @@ -715,7 +798,7 @@ def record_value_for_field(name: str, field_type: str, size: int, decimal: int, if field_type == "L": return sampled_from([True, False, None]) if field_type == "D": - return one_of(dates(), dates().map(lambda d: d.strftime("%Y%m%d"))) + return one_of(dates(), dates().map(date_to_str)) raise ValueError(f"Unsupported: {field_type=}") @@ -729,7 +812,7 @@ def _dbf_encoding_fields_and_record_strategy( fields = draw(lists(_dbf_fields_strategy(encoding), min_size=1, max_size=max_fields)) - record_strategy = tuples(*(record_value_for_field(encoding=encoding, **field) for field in fields)) + record_strategy = tuples(*(record_value_strat_for_field(encoding=encoding, **field) for field in fields)) return encoding, fields, record_strategy @@ -771,9 +854,9 @@ def _assert_reader_matches_expected_records(r, fields, written_records): decimal = field["decimal"] if field_type == "D": if isinstance(expected, datetime.date): - expected = expected.strftime("%Y%m%d") + expected = date_to_str(expected) if isinstance(actual, datetime.date): - actual = actual.strftime("%Y%m%d") + actual = date_to_str(actual) elif field_type in ("N", "F") and decimal >= 1: expected = float(format(expected, f".{decimal}f")) assert actual == expected, f"{actual=}, {expected=}, {field_type=}, {type(actual)=}, {type(expected)=}"