From f2a1404928055755a32dff2796c95b44bc39de66 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 6 Jul 2026 20:41:10 -0600 Subject: [PATCH] Map bytea to a binary String instead of hex text A bytea value now reaches Ruby as a binary (ASCII-8BIT) String holding its exact bytes -- NUL-safe -- rather than as its \x... hex text output, and returning a String into a bytea takes the string's raw bytes verbatim (any encoding), with no hex/escape parsing. This matches PL/Python's bytea <-> bytes mapping. The conversion is intercepted at the datum level (before the type's text I/O) wherever a bytea crosses between SQL and Ruby: - scalar arguments and bytea[] arguments (plruby_func_build_args) - scalar and bytea[] return values (plruby_func_handler return path, which otherwise NUL-truncates via the text-input path) - composite fields, SPI result rows, and trigger $_TD (the shared plruby_hash_from_tuple) - fields, OUT params, and return_next rows via plruby_datum_from_value FROM-SQL uses PG_DETOAST_DATUM_PACKED + VARDATA_ANY/VARSIZE_ANY_EXHDR into an ASCII-8BIT string; TO-SQL builds the varlena directly from the string's raw bytes. bytea[] shares the array datum walk, generalized to convert elements binary-wise when no FromSQL transform applies. BREAKING: bodies that produced or consumed the hex text form must be updated (e.g. build bytes with [..].pack('C*') rather than a \x string). Rewrote sql/bytea for the new semantics (round-trips with NULs, high bytes, composite/SPI/array coverage). Full suite green on PG 12 and 18; jsonb_plruby and hstore_plruby suites still pass (shared array walk). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 12 ++++ doc/plruby.md | 13 ++++- expected/bytea.out | 136 +++++++++++++++++++++++++++++++++++---------- plruby.c | 44 +++++++++++++-- plruby_io.c | 65 ++++++++++++++++++++-- plruby_io.h | 12 +++- sql/bytea.sql | 73 +++++++++++++++++++----- 7 files changed, 298 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index deec06e..fb6e359 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,18 @@ and the project aims to follow [Semantic Versioning](https://semver.org/). independent and resets when the function is recompiled; an anonymous `DO` block gets a fresh, empty `$_SD` each run. +### Changed + +- **`bytea` maps to a binary `String`, not hex text (breaking).** A `bytea` + value now reaches Ruby as a binary (`ASCII-8BIT`) `String` of its raw bytes, + NUL-safe, instead of its `\x...` hex text; returning a `String` into a `bytea` + takes the string's raw bytes verbatim, with no hex/escape parsing. This + matches PL/Python's `bytea` <-> `bytes` mapping and applies wherever a `bytea` + crosses between SQL and Ruby (arguments, returns, `bytea[]` elements, + composite fields, and SPI result rows). Function bodies that previously + produced or consumed the hex text form must be updated (e.g. build bytes with + `[..].pack('C*')` rather than assembling a `\x` string). + ## [2.4.0] - 2026-07-06 Feature parity with the sibling PL/php 2.4: an `on_init` hook, the diff --git a/doc/plruby.md b/doc/plruby.md index 84e67c7..79487b1 100644 --- a/doc/plruby.md +++ b/doc/plruby.md @@ -98,6 +98,7 @@ function. | numeric | `String` (lossless) | `Numeric` or `String` | | boolean | `true` / `false` | `true` / `false` | | text / varchar / etc. | `String` | `String` | +| bytea | binary `String` (ASCII-8BIT) | `String` (raw bytes) | | arrays (e.g. `int[]`) | nested `Array` | nested `Array` | | composite / row / record | `Hash` (string keys) | `Hash` (or positional `Array`) | | NULL | `nil` | `nil` | @@ -115,8 +116,16 @@ Strings arrive tagged with the Ruby encoding that matches the database encoding (`UTF8` to `UTF-8`, `LATIN1` to `ISO-8859-1`, `EUC_JP` to `EUC-JP`, `WIN1251` to `Windows-1251`, and so on), so `length`, `reverse`, and regexps operate on characters, not bytes. Encodings Ruby does not recognize fall back to -`ASCII-8BIT` (binary), which is byte-preserving. `bytea` is passed as its -textual `\x...` hex representation (a `String`), not raw bytes. +`ASCII-8BIT` (binary), which is byte-preserving. + +`bytea` is the exception: it is a raw byte string, so it arrives as a binary +(`ASCII-8BIT`) `String` holding its exact bytes, including any NULs, rather than +as hex text. Returning a `String` into a `bytea` takes the string's raw bytes +verbatim (whatever its encoding), with no hex or escape parsing. This holds +wherever a `bytea` crosses between SQL and Ruby: plain arguments and returns, +`bytea[]` elements, fields of composite arguments and results, and SPI result +rows. To emit specific bytes, build the `String` directly, e.g. +`[0, 255, 16].pack('C*')`. ## Composite types and records diff --git a/expected/bytea.out b/expected/bytea.out index f869b93..8137538 100644 --- a/expected/bytea.out +++ b/expected/bytea.out @@ -1,54 +1,130 @@ -- --- bytea conversion. Like every type, a bytea reaches Ruby through its output --- function, so it arrives as its *textual* (hex) representation -- a String -- --- not as raw bytes. Returning a bytea feeds the String back through bytea's --- input function. +-- bytea conversion. A bytea is a raw byte string, so it reaches Ruby as a +-- binary (ASCII-8BIT) String holding its exact bytes -- NUL-safe -- rather than +-- as its hex text. Returning a Ruby String into a bytea takes the string's +-- raw bytes verbatim (any encoding), with no hex/escape parsing. -- SET bytea_output = 'hex'; --- A bytea argument is the hex text string (UTF-8 tagged), preserving NUL and --- high bytes in that representation. +-- A bytea argument is a binary String; its bytes match the input exactly. CREATE FUNCTION b_form(bytea) RETURNS text LANGUAGE plruby AS $$ - "#{args[0].class}:#{args[0]}" + "#{args[0].class}:#{args[0].encoding}:#{args[0].bytesize}" $$; SELECT b_form('\xdeadbeef'::bytea); - b_form -------------------- - String:\xdeadbeef + b_form +--------------------- + String:ASCII-8BIT:4 (1 row) -SELECT b_form('\x00ff01fe'::bytea); - b_form -------------------- - String:\x00ff01fe +SELECT b_form('\x00ff01fe'::bytea); -- NUL and high bytes preserved + b_form +--------------------- + String:ASCII-8BIT:4 (1 row) --- Round-trip: returning the hex string yields the identical bytea. +SELECT b_form('\x'::bytea); -- empty + b_form +--------------------- + String:ASCII-8BIT:0 +(1 row) + +-- The bytes are the raw content, so encode() of the echoed value round-trips. CREATE FUNCTION b_echo(bytea) RETURNS bytea LANGUAGE plruby AS $$ args[0] $$; -SELECT b_echo('\x00ff01fe'::bytea); - b_echo ------------- - \x00ff01fe +SELECT encode(b_echo('\x00ff01fe'::bytea), 'hex'); + encode +---------- + 00ff01fe +(1 row) + +-- The function can inspect and transform the raw bytes. +CREATE FUNCTION b_first_byte(bytea) RETURNS int LANGUAGE plruby AS $$ + args[0].bytes[0] +$$; +SELECT b_first_byte('\x41ff00'::bytea); -- 65 + b_first_byte +-------------- + 65 (1 row) --- Building a bytea from a Ruby-produced hex string. +-- Building a bytea from raw Ruby bytes. CREATE FUNCTION b_make() RETURNS bytea LANGUAGE plruby AS $$ - "\\x" + [0, 255, 16, 32].map { |n| "%02x" % n }.join + [0, 255, 16, 32].pack('C*') +$$; +SELECT encode(b_make(), 'hex'); + encode +---------- + 00ff1020 +(1 row) + +-- A Ruby String with embedded NULs becomes those exact bytes. +CREATE FUNCTION b_nul() RETURNS bytea LANGUAGE plruby AS $$ + "a\x00b".b $$; -SELECT b_make(); - b_make ------------- - \x00ff1020 +SELECT encode(b_nul(), 'hex'); + encode +-------- + 610062 (1 row) -- The empty bytea. CREATE FUNCTION b_empty() RETURNS bytea LANGUAGE plruby AS $$ - "\\x" + "".b +$$; +SELECT encode(b_empty(), 'hex'); + encode +-------- + +(1 row) + +-- bytea inside a composite: the field is a binary String too. +CREATE TYPE b_rec AS (tag text, blob bytea); +CREATE FUNCTION b_rec_in(b_rec) RETURNS int LANGUAGE plruby AS $$ + args[0]['blob'].bytesize +$$; +SELECT b_rec_in(ROW('x', '\x00ff01fe')::b_rec); -- 4 + b_rec_in +---------- + 4 +(1 row) + +CREATE FUNCTION b_rec_out() RETURNS b_rec LANGUAGE plruby AS $$ + {'tag' => 'y', 'blob' => [1, 2, 3].pack('C*')} +$$; +SELECT (b_rec_out()).tag, encode((b_rec_out()).blob, 'hex'); + tag | encode +-----+-------- + y | 010203 +(1 row) + +-- bytea read back through SPI is also a binary String. +CREATE FUNCTION b_via_spi() RETURNS int LANGUAGE plruby AS $$ + r = spi_exec("SELECT '\\x00ff01fe'::bytea AS b") + spi_fetch_row(r)['b'].bytesize +$$; +SELECT b_via_spi(); -- 4 + b_via_spi +----------- + 4 +(1 row) + +-- bytea[] arguments arrive as an Array of binary Strings, and a returned +-- Array of binary Strings becomes a bytea[]. +CREATE FUNCTION b_arr_in(bytea[]) RETURNS int LANGUAGE plruby AS $$ + args[0].map(&:bytesize).inject(0, :+) +$$; +SELECT b_arr_in(ARRAY['\x0000'::bytea, '\xffffff'::bytea]); -- 5 + b_arr_in +---------- + 5 +(1 row) + +CREATE FUNCTION b_arr_out() RETURNS bytea[] LANGUAGE plruby AS $$ + [[0, 255].pack('C*'), [16].pack('C*')] $$; -SELECT b_empty(); - b_empty ---------- - \x +SELECT encode((b_arr_out())[1], 'hex'), encode((b_arr_out())[2], 'hex'); + encode | encode +--------+-------- + 00ff | 10 (1 row) diff --git a/plruby.c b/plruby.c index 5074ae2..09c00ad 100644 --- a/plruby.c +++ b/plruby.c @@ -724,7 +724,26 @@ plruby_func_handler(FunctionCallInfo fcinfo, plruby_proc_desc *desc) case T_FIXNUM: case T_BIGNUM: case T_FLOAT: + retvalbuffer = plruby_value_to_cstring(result, false, false); + break; case T_STRING: + /* + * bytea takes the String's raw bytes (NUL-safe) via the datum + * path; every other scalar keeps the text-input path below. + */ + if (desc->ret_oid == BYTEAOID) + { + bool isnull; + Datum d = plruby_datum_from_value(result, desc->ret_oid, + (int32) -1, &isnull); + + if (isnull) + { + fcinfo->isnull = true; + return (Datum) 0; + } + return d; + } retvalbuffer = plruby_value_to_cstring(result, false, false); break; case T_ARRAY: @@ -733,14 +752,15 @@ plruby_func_handler(FunctionCallInfo fcinfo, plruby_proc_desc *desc) Oid elemtype = get_element_type(desc->ret_oid); /* - * Array of composite (or of a transformed type, e.g. - * jsonb[]): build the array datum directly so each element - * is assembled recursively/through its ToSQL function. - * Scalar arrays keep the (multidimensional-capable) text - * path below. + * Array of composite (or of a transformed type, e.g. jsonb[], + * or of bytea, whose elements are raw bytes): build the array + * datum directly so each element is assembled recursively / + * through its ToSQL function. Scalar arrays keep the + * (multidimensional-capable) text path below. */ if (OidIsValid(elemtype) && (type_is_rowtype(elemtype) || + elemtype == BYTEAOID || OidIsValid(plruby_transform_tosql(elemtype)))) { bool isnull; @@ -1202,6 +1222,20 @@ plruby_func_build_args(plruby_proc_desc *desc, FunctionCallInfo fcinfo) plruby_array_from_datum(PLRUBY_ARG_VALUE(fcinfo, j), elemtype, etrf)); } + else if (argtype == BYTEAOID) + { + /* bytea: raw binary String, not hex text */ + rb_ary_push(args, + plruby_binary_from_datum(PLRUBY_ARG_VALUE(fcinfo, j), + BYTEAOID)); + } + else if (elemtype == BYTEAOID) + { + /* bytea[]: elements as binary Strings */ + rb_ary_push(args, + plruby_array_from_datum(PLRUBY_ARG_VALUE(fcinfo, j), + BYTEAOID, InvalidOid)); + } else { char *tmp = OutputFunctionCall(&(desc->arg_out_func[i]), diff --git a/plruby_io.c b/plruby_io.c index 6958f07..8846fff 100644 --- a/plruby_io.c +++ b/plruby_io.c @@ -67,7 +67,7 @@ plruby_transform_fromsql(Oid typid) static VALUE plruby_array_from_datum_walk(Datum *elems, bool *nulls, int *idx, int ndims, const int *dims, int depth, - Oid fromsql_fn) + Oid elemtype, Oid fromsql_fn) { VALUE ary = rb_ary_new(); int i; @@ -78,20 +78,29 @@ plruby_array_from_datum_walk(Datum *elems, bool *nulls, int *idx, rb_ary_push(ary, plruby_array_from_datum_walk(elems, nulls, idx, ndims, dims, depth + 1, - fromsql_fn)); + elemtype, fromsql_fn)); else { if (nulls[*idx]) rb_ary_push(ary, Qnil); - else + else if (OidIsValid(fromsql_fn)) rb_ary_push(ary, (VALUE) OidFunctionCall1(fromsql_fn, elems[*idx])); + else + /* no transform: the only element-wise binary type is bytea */ + rb_ary_push(ary, plruby_binary_from_datum(elems[*idx], elemtype)); (*idx)++; } } return ary; } +/* + * An array datum -> nested Ruby Array. When fromsql_fn is valid, each element + * converts through that FromSQL transform; when it is InvalidOid, elements + * convert per plruby_binary_from_datum (used for bytea[], whose elements are + * raw byte strings). + */ VALUE plruby_array_from_datum(Datum d, Oid elemtype, Oid fromsql_fn) { @@ -112,7 +121,8 @@ plruby_array_from_datum(Datum d, Oid elemtype, Oid fromsql_fn) deconstruct_array(arr, elemtype, elmlen, elmbyval, elmalign, &elems, &nulls, &nelems); return plruby_array_from_datum_walk(elems, nulls, &idx, - ndims, ARR_DIMS(arr), 0, fromsql_fn); + ndims, ARR_DIMS(arr), 0, + elemtype, fromsql_fn); } /* --------------------------------------------------------------------- @@ -224,6 +234,29 @@ plruby_scalar_from_cstring(const char *str, Oid typeoid) } } +/* + * bytea -> Ruby. A bytea Datum is a raw varlena, so it maps to a binary + * (ASCII-8BIT) Ruby String holding its exact bytes -- NUL-safe -- rather than + * through bytea's hex text output. Returns Qundef when typeoid is not bytea, + * so callers can fall back to their normal text path. + */ +VALUE +plruby_binary_from_datum(Datum value, Oid typeoid) +{ + struct varlena *vl; + VALUE s; + + if (typeoid != BYTEAOID) + return Qundef; + + vl = PG_DETOAST_DATUM_PACKED(value); + s = rb_enc_str_new(VARDATA_ANY(vl), VARSIZE_ANY_EXHDR(vl), + rb_ascii8bit_encoding()); + if ((Pointer) vl != DatumGetPointer(value)) + pfree(vl); + return s; +} + /* --------------------------------------------------------------------- * PostgreSQL array text -> nested Ruby Array * ------------------------------------------------------------------- */ @@ -480,6 +513,20 @@ plruby_datum_from_value(VALUE val, Oid typeoid, int32 typmod, bool *isnull) (RB_TYPE_P(val, T_HASH) || RB_TYPE_P(val, T_ARRAY))) return plruby_composite_datum(val, typeoid, typmod); + /* + * bytea fed a Ruby String: take its raw bytes verbatim (any encoding), + * building the varlena directly rather than parsing hex/escape text. + */ + if (typeoid == BYTEAOID && RB_TYPE_P(val, T_STRING)) + { + long len = RSTRING_LEN(val); + bytea *result = (bytea *) palloc(len + VARHDRSZ); + + SET_VARSIZE(result, len + VARHDRSZ); + memcpy(VARDATA(result), RSTRING_PTR(val), len); + return PointerGetDatum(result); + } + /* Scalar leaf: convert via the target type's input function. */ { char *s = plruby_value_to_cstring(val, true, true); @@ -675,6 +722,16 @@ plruby_hash_from_tuple(HeapTuple tuple, TupleDesc tupdesc) continue; } + /* bytea (and bytea[]) map to raw binary Strings, not hex text. */ + v = plruby_binary_from_datum(attr, att->atttypid); + if (v == Qundef && get_element_type(att->atttypid) == BYTEAOID) + v = plruby_array_from_datum(attr, BYTEAOID, InvalidOid); + if (v != Qundef) + { + rb_hash_aset(h, rb_str_new_cstr(attname), v); + continue; + } + getTypeOutputInfo(att->atttypid, &typoutput, &typisvarlena); outputstr = OidOutputFunctionCall(typoutput, attr); diff --git a/plruby_io.h b/plruby_io.h index f3dd4bb..6460a44 100644 --- a/plruby_io.h +++ b/plruby_io.h @@ -36,9 +36,19 @@ extern int plruby_arg_ntransforms; extern Oid plruby_transform_tosql(Oid typid); extern Oid plruby_transform_fromsql(Oid typid); -/* An array datum -> nested Ruby Array, each element through fromsql_fn. */ +/* + * An array datum -> nested Ruby Array. A valid fromsql_fn converts each + * element through that FromSQL transform; InvalidOid means convert per + * plruby_binary_from_datum (bytea[] elements as binary Strings). + */ extern VALUE plruby_array_from_datum(Datum d, Oid elemtype, Oid fromsql_fn); +/* + * bytea Datum -> binary (ASCII-8BIT) Ruby String of its raw bytes, or Qundef + * if typeoid is not bytea (caller then uses the normal text path). + */ +extern VALUE plruby_binary_from_datum(Datum value, Oid typeoid); + /* Build a Ruby String from PG text tagged with the database encoding. */ extern VALUE plruby_str_from_pg(const char *str, long len); diff --git a/sql/bytea.sql b/sql/bytea.sql index f618d11..ed6d56e 100644 --- a/sql/bytea.sql +++ b/sql/bytea.sql @@ -1,33 +1,76 @@ -- --- bytea conversion. Like every type, a bytea reaches Ruby through its output --- function, so it arrives as its *textual* (hex) representation -- a String -- --- not as raw bytes. Returning a bytea feeds the String back through bytea's --- input function. +-- bytea conversion. A bytea is a raw byte string, so it reaches Ruby as a +-- binary (ASCII-8BIT) String holding its exact bytes -- NUL-safe -- rather than +-- as its hex text. Returning a Ruby String into a bytea takes the string's +-- raw bytes verbatim (any encoding), with no hex/escape parsing. -- SET bytea_output = 'hex'; --- A bytea argument is the hex text string (UTF-8 tagged), preserving NUL and --- high bytes in that representation. +-- A bytea argument is a binary String; its bytes match the input exactly. CREATE FUNCTION b_form(bytea) RETURNS text LANGUAGE plruby AS $$ - "#{args[0].class}:#{args[0]}" + "#{args[0].class}:#{args[0].encoding}:#{args[0].bytesize}" $$; SELECT b_form('\xdeadbeef'::bytea); -SELECT b_form('\x00ff01fe'::bytea); +SELECT b_form('\x00ff01fe'::bytea); -- NUL and high bytes preserved +SELECT b_form('\x'::bytea); -- empty --- Round-trip: returning the hex string yields the identical bytea. +-- The bytes are the raw content, so encode() of the echoed value round-trips. CREATE FUNCTION b_echo(bytea) RETURNS bytea LANGUAGE plruby AS $$ args[0] $$; -SELECT b_echo('\x00ff01fe'::bytea); +SELECT encode(b_echo('\x00ff01fe'::bytea), 'hex'); --- Building a bytea from a Ruby-produced hex string. +-- The function can inspect and transform the raw bytes. +CREATE FUNCTION b_first_byte(bytea) RETURNS int LANGUAGE plruby AS $$ + args[0].bytes[0] +$$; +SELECT b_first_byte('\x41ff00'::bytea); -- 65 + +-- Building a bytea from raw Ruby bytes. CREATE FUNCTION b_make() RETURNS bytea LANGUAGE plruby AS $$ - "\\x" + [0, 255, 16, 32].map { |n| "%02x" % n }.join + [0, 255, 16, 32].pack('C*') +$$; +SELECT encode(b_make(), 'hex'); + +-- A Ruby String with embedded NULs becomes those exact bytes. +CREATE FUNCTION b_nul() RETURNS bytea LANGUAGE plruby AS $$ + "a\x00b".b $$; -SELECT b_make(); +SELECT encode(b_nul(), 'hex'); -- The empty bytea. CREATE FUNCTION b_empty() RETURNS bytea LANGUAGE plruby AS $$ - "\\x" + "".b +$$; +SELECT encode(b_empty(), 'hex'); + +-- bytea inside a composite: the field is a binary String too. +CREATE TYPE b_rec AS (tag text, blob bytea); +CREATE FUNCTION b_rec_in(b_rec) RETURNS int LANGUAGE plruby AS $$ + args[0]['blob'].bytesize +$$; +SELECT b_rec_in(ROW('x', '\x00ff01fe')::b_rec); -- 4 + +CREATE FUNCTION b_rec_out() RETURNS b_rec LANGUAGE plruby AS $$ + {'tag' => 'y', 'blob' => [1, 2, 3].pack('C*')} +$$; +SELECT (b_rec_out()).tag, encode((b_rec_out()).blob, 'hex'); + +-- bytea read back through SPI is also a binary String. +CREATE FUNCTION b_via_spi() RETURNS int LANGUAGE plruby AS $$ + r = spi_exec("SELECT '\\x00ff01fe'::bytea AS b") + spi_fetch_row(r)['b'].bytesize +$$; +SELECT b_via_spi(); -- 4 + +-- bytea[] arguments arrive as an Array of binary Strings, and a returned +-- Array of binary Strings becomes a bytea[]. +CREATE FUNCTION b_arr_in(bytea[]) RETURNS int LANGUAGE plruby AS $$ + args[0].map(&:bytesize).inject(0, :+) +$$; +SELECT b_arr_in(ARRAY['\x0000'::bytea, '\xffffff'::bytea]); -- 5 + +CREATE FUNCTION b_arr_out() RETURNS bytea[] LANGUAGE plruby AS $$ + [[0, 255].pack('C*'), [16].pack('C*')] $$; -SELECT b_empty(); +SELECT encode((b_arr_out())[1], 'hex'), encode((b_arr_out())[2], 'hex');