Skip to content

Commit db83ad0

Browse files
committed
Decode internal "char" type (OID 18) natively
Fixes #165. Any query touching PostgreSQL system catalogs (pg_type, pg_class, pg_attribute, pg_proc, ...) raised RustToPyValueMappingError because the internal "char" type — distinct from character(n)/BPCHAR — had no native decoder and fell through to other_postgres_bytes_to_py. Add an InternalChar(u8) wrapper next to the existing InternalUuid / InnerDecimal / InnerInterval helpers and wire it into postgres_bytes_to_py via two new match arms (Type::CHAR, Type::CHAR_ARRAY). The byte is read through tokio-postgres' i8 FromSql impl, cast back to u8, and mapped to a one-character Python str through char::from(u8) — i.e. Unicode code points 0..=255 (Latin-1 round-trip), matching psycopg2/psycopg3. The custom_decoders dispatch is intentionally unchanged: it stays keyed by column name per the existing documented contract. Tests: - python/tests/test_value_converter.py: * test_char_internal_type_pg_type_reproduction — exact snippet from #165 * test_char_internal_type_byte_spectrum — reachable ASCII bytes 0x20, 0x41, 0x61, 0x7E plus NULL (SQL chr() rejects NUL and re-encodes >=0x80 as multi-byte UTF-8) * test_char_internal_type_array — "char"[] decoded to list[str] - src/value_converter/models/internal_char.rs: * from_sql_round_trips_full_byte_range — full 0..=255 byte mapping the SQL test cannot reach * accepts_only_char_type — type guard rejects TEXT/VARCHAR/BPCHAR
1 parent 91d611a commit db83ad0

4 files changed

Lines changed: 173 additions & 2 deletions

File tree

python/tests/test_value_converter.py

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -625,6 +625,86 @@ class TestStrEnum(str, Enum):
625625
assert qs_result.result()[0]["test_mood2"] == TestStrEnum.OK
626626

627627

628+
async def test_char_internal_type_pg_type_reproduction(
629+
psql_pool: ConnectionPool,
630+
) -> None:
631+
"""Regression for issue #165.
632+
633+
The original repro queried system catalog columns of the internal
634+
``"char"`` type (OID 18). Prior to the fix this raised
635+
``RustToPyValueMappingError`` even when ``custom_decoders`` was supplied,
636+
because the type had no native decoder.
637+
"""
638+
pg_type_limit = 5
639+
async with psql_pool.acquire() as conn:
640+
result = await conn.execute(
641+
f"SELECT typname, typtype FROM pg_type LIMIT {pg_type_limit}",
642+
)
643+
rows = result.result()
644+
assert len(rows) == pg_type_limit
645+
for row in rows:
646+
assert isinstance(row["typname"], str)
647+
assert isinstance(row["typtype"], str)
648+
assert len(row["typtype"]) == 1
649+
650+
651+
async def test_char_internal_type_byte_spectrum(
652+
psql_pool: ConnectionPool,
653+
) -> None:
654+
"""Round-trip representative ASCII bytes through a ``"char"`` column.
655+
656+
The internal ``"char"`` type holds a single byte. SQL ``chr(N)`` rejects
657+
NUL (0x00) with "null character not permitted", and ``chr(N)`` for N >= 128
658+
produces multi-byte UTF-8 whose cast to ``"char"`` keeps only the first byte
659+
(e.g. chr(128)::"char" stores 0xC2, not 0x80). So this integration test
660+
covers the reachable ASCII slice. The full 0..=255 byte mapping is verified
661+
by the Rust unit test in models/internal_char.rs.
662+
"""
663+
bytes_under_test = [0x20, 0x41, 0x61, 0x7E]
664+
665+
async with psql_pool.acquire() as conn:
666+
await conn.execute("DROP TABLE IF EXISTS for_char_test")
667+
await conn.execute(
668+
'CREATE TABLE for_char_test (id INT, c "char")',
669+
)
670+
for i, b in enumerate(bytes_under_test):
671+
await conn.execute(
672+
'INSERT INTO for_char_test (id, c) VALUES ($1, chr($2)::"char")',
673+
[i, b],
674+
)
675+
await conn.execute(
676+
"INSERT INTO for_char_test (id, c) VALUES ($1, NULL)",
677+
[len(bytes_under_test)],
678+
)
679+
680+
result = await conn.execute(
681+
"SELECT id, c FROM for_char_test ORDER BY id",
682+
)
683+
rows = result.result()
684+
685+
decoded = {row["id"]: row["c"] for row in rows}
686+
for i, b in enumerate(bytes_under_test):
687+
value = decoded[i]
688+
assert isinstance(value, str)
689+
assert len(value) == 1
690+
assert (
691+
ord(value) == b
692+
), f"byte 0x{b:02x} round-tripped to ord(value)=0x{ord(value):02x}"
693+
assert decoded[len(bytes_under_test)] is None
694+
695+
696+
async def test_char_internal_type_array(
697+
psql_pool: ConnectionPool,
698+
) -> None:
699+
"""Decode an array of ``"char"`` (OID 1002) into a list of one-character strs."""
700+
async with psql_pool.acquire() as conn:
701+
result = await conn.execute(
702+
"SELECT ARRAY['a'::\"char\", 'b'::\"char\", 'c'::\"char\"] AS chars",
703+
)
704+
rows = result.result()
705+
assert rows[0]["chars"] == ["a", "b", "c"]
706+
707+
628708
async def test_custom_type_as_parameter(
629709
psql_pool: ConnectionPool,
630710
) -> None:
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
use postgres_types::FromSql;
2+
use pyo3::{types::PyString, Bound, IntoPyObject, Python};
3+
use tokio_postgres::types::Type;
4+
5+
use crate::exceptions::rust_errors::RustPSQLDriverError;
6+
7+
/// Wrapper around the single-byte payload of `PostgreSQL`'s internal `"char"`
8+
/// type (OID 18, distinct from `character(n)`/BPCHAR). Bytes 0..=255 map to
9+
/// Unicode code points 0..=255 (Latin-1 round-trip), matching psycopg2/psycopg3.
10+
#[derive(Clone, Copy)]
11+
pub struct InternalChar(u8);
12+
13+
impl<'py> IntoPyObject<'py> for InternalChar {
14+
type Target = PyString;
15+
type Output = Bound<'py, Self::Target>;
16+
type Error = RustPSQLDriverError;
17+
18+
fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
19+
let mut tmp = [0u8; 4];
20+
let s = char::from(self.0).encode_utf8(&mut tmp);
21+
Ok(PyString::new(py, s))
22+
}
23+
}
24+
25+
impl<'a> FromSql<'a> for InternalChar {
26+
fn from_sql(
27+
_ty: &Type,
28+
raw: &'a [u8],
29+
) -> Result<Self, Box<dyn std::error::Error + Sync + Send>> {
30+
// The `"char"` binary wire format is exactly one byte. Read it as `u8`
31+
// directly — the `i8`-then-cast route through tokio_postgres' `FromSql`
32+
// impl trips clippy::cast_sign_loss in pedantic mode for no gain.
33+
let [byte] = *raw else {
34+
return Err(format!("\"char\" expected 1 byte, got {}", raw.len()).into());
35+
};
36+
Ok(InternalChar(byte))
37+
}
38+
39+
fn accepts(ty: &Type) -> bool {
40+
*ty == Type::CHAR
41+
}
42+
}
43+
44+
#[cfg(test)]
45+
impl InternalChar {
46+
pub(crate) fn byte(self) -> u8 {
47+
self.0
48+
}
49+
}
50+
51+
#[cfg(test)]
52+
mod tests {
53+
use super::InternalChar;
54+
use postgres_types::{FromSql, Type};
55+
56+
#[test]
57+
fn from_sql_round_trips_full_byte_range() {
58+
// The signed-byte cast (i8 -> u8) inside from_sql must preserve every
59+
// raw byte. Cover all 256 values so a sign-extension or normalization
60+
// regression cannot slip through.
61+
for b in 0u16..=255 {
62+
let byte = b as u8;
63+
let buf = [byte];
64+
let decoded =
65+
<InternalChar as FromSql>::from_sql(&Type::CHAR, &buf).expect("char decode");
66+
assert_eq!(decoded.byte(), byte, "byte 0x{byte:02x} not preserved");
67+
}
68+
}
69+
70+
#[test]
71+
fn accepts_only_char_type() {
72+
assert!(<InternalChar as FromSql>::accepts(&Type::CHAR));
73+
assert!(!<InternalChar as FromSql>::accepts(&Type::TEXT));
74+
assert!(!<InternalChar as FromSql>::accepts(&Type::VARCHAR));
75+
assert!(!<InternalChar as FromSql>::accepts(&Type::BPCHAR));
76+
}
77+
}

src/value_converter/models/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pub mod decimal;
2+
pub mod internal_char;
23
pub mod interval;
34
pub mod serde_value;
45
pub mod uuid;

src/value_converter/to_python.rs

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ use crate::{
2121
RustRect,
2222
},
2323
models::{
24-
decimal::InnerDecimal, interval::InnerInterval, serde_value::InternalSerdeValue,
25-
uuid::InternalUuid,
24+
decimal::InnerDecimal, internal_char::InternalChar, interval::InnerInterval,
25+
serde_value::InternalSerdeValue, uuid::InternalUuid,
2626
},
2727
},
2828
};
@@ -191,6 +191,13 @@ fn postgres_bytes_to_py(
191191
composite_field_postgres_to_py::<Option<String>>(type_, buf, is_simple)?
192192
.into_py_any(py)?,
193193
),
194+
// Convert internal "char" (OID 18, single byte) into a one-character str.
195+
Type::CHAR => {
196+
match composite_field_postgres_to_py::<Option<InternalChar>>(type_, buf, is_simple)? {
197+
Some(ic) => Ok(ic.into_pyobject(py)?.unbind().into_any()),
198+
None => Ok(py.None()),
199+
}
200+
}
194201
// ---------- Boolean Types ----------
195202
// Convert BOOL type into bool
196203
Type::BOOL => Ok(
@@ -367,6 +374,12 @@ fn postgres_bytes_to_py(
367374
composite_field_postgres_to_py::<Option<Array<String>>>(type_, buf, is_simple)?,
368375
)
369376
.into_py_any(py)?),
377+
// Convert ARRAY of internal "char" into list[str] (each element is one byte).
378+
Type::CHAR_ARRAY => Ok(postgres_array_to_py(
379+
py,
380+
composite_field_postgres_to_py::<Option<Array<InternalChar>>>(type_, buf, is_simple)?,
381+
)
382+
.into_py_any(py)?),
370383
// ---------- Array Integer Types ----------
371384
// Convert ARRAY of SmallInt into Vec<i16>, then into list[int]
372385
Type::INT2_ARRAY => Ok(postgres_array_to_py(

0 commit comments

Comments
 (0)