add additional python versions to repo #581
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,7 +3,7 @@ repos: | |
| rev: 24.8.0 | ||
| hooks: | ||
| - id: black | ||
| - repo: https://github.com/pycqa/flake8 | ||
| rev: 7.1.2 | ||
| hooks: | ||
| - id: flake8 | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| import csv | ||
| from charset_normalizer import from_bytes, from_path | ||
| import logging | ||
| import json_stream | ||
| import os | ||
|
|
@@ -21,20 +21,26 @@ class ConversionError(Exception): | |
| pass | ||
|
|
||
|
|
||
| def _best_encoding(best): | ||
| if not best: | ||
| return None | ||
| # charset-normalizer reports the base codec even when a BOM is present | ||
| # (best.bom=True); Python needs the "-sig" variant to strip it, or the | ||
| # BOM decodes as a leading U+FEFF character in the content. | ||
| return best.encoding + "-sig" if best.bom else best.encoding | ||
|
Comment on lines
+24
to
+30
There was a problem hiding this comment. 🎯 Functional Correctness | 🟠 Major | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -eu
python3 - <<'PY'
import codecs
from charset_normalizer import from_bytes
for source_encoding in ("utf-8", "utf-16", "utf-32"):
payload = "name\ncafé\n".encode(source_encoding)
match = from_bytes(payload, enable_fallback=False).best()
detected = match.encoding if match else None
has_bom = match.bom if match else False
candidate = f"{detected}-sig" if has_bom else detected
print(source_encoding, detected, has_bom, candidate)
if candidate:
codecs.lookup(candidate)
PYRepository: digital-land/digital-land-python Length of output: 301 🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- candidate files ---'
fd -t f '(^convert\.py$|requirements|pyproject|setup|poetry|Pipfile)' . | head -80
printf '%s\n' '--- convert.py symbols and relevant lines ---'
ast-grep outline digital_land/phase/convert.py --lang python 2>/dev/null || true
sed -n '1,90p' digital_land/phase/convert.py
printf '%s\n' '--- encoding uses ---'
rg -n -C 3 'encoding=|_best_encoding|from_path|from_fp|charset.?normalizer|bom' digital_land
printf '%s\n' '--- Python codec lookup ---'
python3 - <<'PY'
import codecs
for name in ("utf-8-sig", "utf_16-sig", "utf_32-sig", "utf-16", "utf-32"):
try:
info = codecs.lookup(name)
print(f"{name}: {info.name}")
except LookupError as exc:
print(f"{name}: LookupError: {exc}")
PYRepository: digital-land/digital-land-python Length of output: 10799 🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- declared charset-normalizer dependency ---'
rg -n -C 3 'charset.?normalizer' pyproject.toml setup.cfg
printf '%s\n' '--- repository tests for conversion and encoding ---'
rg -n -C 3 'detect_file_encoding|detect_encoding|_best_encoding|utf.?16|utf.?32|BOM|bom' tests digital_land 2>/dev/null | head -240 || true
printf '%s\n' '--- standard-library BOM and decoding behaviour ---'
python3 - <<'PY'
import codecs
text = "name\ncafé\n"
for source_encoding in ("utf-8", "utf-16", "utf-32"):
payload = text.encode(source_encoding)
print(source_encoding, payload[:4].hex())
for candidate in ("utf-8-sig", "utf-16", "utf-32"):
try:
decoded = payload.decode(candidate)
print(f" {candidate}: {decoded!r}")
except UnicodeDecodeError as exc:
print(f" {candidate}: UnicodeDecodeError")
PYRepository: digital-land/digital-land-python Length of output: 7694 🌐 Web query:
💡 Result: In the charset-normalizer library, Byte Order Mark (BOM) handling is integrated into the detection workflow, but it is treated differently for specific encodings [1][2]. 1. BOM Stripping Behavior: The library's core detection function, from_bytes, automatically strips the BOM from the payload for most encodings [1][2]. However, it explicitly does NOT strip the BOM for UTF-16 and UTF-32 [1][2][3]. 2. UTF-16/UTF-32 Handling: Because UTF-16 and UTF-32 require a BOM to determine endianness correctly, the library avoids stripping them automatically [4][5]. If an encoding such as utf_16 or utf_32 is being tested and no BOM/SIG is present, the library may skip testing that encoding as-is and instead attempt to check specific Little Endian (LE) or Big Endian (BE) sub-encoders [4][5]. 3. Identification Utilities: The library provides internal utilities for managing these signatures, including: - charset_normalizer.utils.identify_sig_or_bom(sequence): Used to identify and extract a SIG/BOM from a byte sequence [1][6][3]. - charset_normalizer.utils.should_strip_sig_or_bom(iana_encoding): A helper function that determines whether the BOM should be stripped based on the IANA encoding name [1][2][3]. For developers, the CharsetMatch object returned by the library includes a property has_sig_or_bom (also accessible via.bom or.byte_order_mark), which indicates whether a signature was detected in the original payload [7][8]. Citations:
Map only UTF-8 BOMs to a When 🤖 Prompt for AI Agents |
||
|
|
||
|
|
||
| def detect_file_encoding(path): | ||
| if not os.path.getsize(path): | ||
| return None | ||
| return _best_encoding(from_path(path).best()) | ||
|
|
||
|
|
||
| def detect_encoding(f): | ||
| data = f.read() | ||
| if not data: | ||
| return None | ||
| return _best_encoding(from_bytes(data).best()) | ||
|
|
||
|
|
||
| def load_csv(path, encoding="UTF-8", log=None): | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.