Re: gs_group_1 crashing on 13beta2/s390x

Re: To PostgreSQL Hackers
> On the Debian s390x buildd, the 13beta2 build is crashing:
>
> 2020-07-15 01:19:59.149 UTC [859] LOG: server process (PID 1415) was terminated by signal 11: Segmentation fault
> 2020-07-15 01:19:59.149 UTC [859] DETAIL: Failed process was running: create table gs_group_1 as
> select g100, g10, sum(g::numeric), count(*), max(g::text)
> from gs_data_1 group by cube (g1000, g100,g10);

I wired gdb into the build process and got this backtrace:

2020-07-15 16:03:38.310 UTC [21073] LOG: server process (PID 21575) was terminated by signal 11: Segmentation fault
2020-07-15 16:03:38.310 UTC [21073] DETAIL: Failed process was running: create table gs_group_1 as
select g100, g10, sum(g::numeric), count(*), max(g::text)
from gs_data_1 group by cube (g1000, g100,g10);

******** build/src/bin/pg_upgrade/tmp_check/data.old/core ********
[New LWP 21575]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/s390x-linux-gnu/libthread_db.so.1".
Core was generated by `postgres: buildd regression [local] CREATE TABLE AS '.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 datumCopy (typByVal=false, typLen=-1, value=0) at ./build/../src/backend/utils/adt/datum.c:142
142 if (VARATT_IS_EXTERNAL_EXPANDED(vl))
#0 datumCopy (typByVal=false, typLen=-1, value=0) at ./build/../src/backend/utils/adt/datum.c:142
vl = 0x0
res = <optimized out>
res = <optimized out>
vl = <optimized out>
eoh = <optimized out>
resultsize = <optimized out>
resultptr = <optimized out>
realSize = <optimized out>
resultptr = <optimized out>
realSize = <optimized out>
resultptr = <optimized out>
#1 datumCopy (value=0, typByVal=false, typLen=-1) at ./build/../src/backend/utils/adt/datum.c:131
res = <optimized out>
vl = <optimized out>
eoh = <optimized out>
resultsize = <optimized out>
resultptr = <optimized out>
realSize = <optimized out>
resultptr = <optimized out>
#2 0x000002aa04423af8 in finalize_aggregate (aggstate=aggstate(at)entry=0x2aa05775920, peragg=peragg(at)entry=0x2aa056e02f0, resultVal=0x2aa056e0208, resultIsNull=0x2aa056e022a, pergroupstate=<optimized out>, pergroupstate=<optimized out>) at ./build/../src/backend/executor/nodeAgg.c:1128
fcinfodata = {fcinfo = {flinfo = 0x2aa056e0250, context = 0x2aa05775920, resultinfo = 0x0, fncollation = 0, isnull = false, nargs = 1, args = 0x3fff6a7b578}, fcinfo_data = "\000\000\002\252\005n\002P\000\000\002\252\005wY ", '\000' <repeats 13 times>, "\247\000\001\000\000\002\252\005t\265\250\000\000\003\377\211\341\207F\000\000\003\377\000\000\002\001\000\000\000\000\000\000\003\376\000\000\000\000\000\000\017\370\000\000\000\000\000\000\001\377\000\000\000\000\000\000\000\260\000\000\000k\000\000\000k\000\000\000\000\000\000 \000\000\000\003\377\213\016J \000\000\000p\000\000\000k\000\000\000\000\000\000\000\200\000\000\000\000\000\000\000\020", '\000' <repeats 11 times>, "w\000\000\000|\000\000\000\000\000\000\000\002\000\000\002\252\006&9\250\000\000\002\252\005wZh\000\000\002\252\005wZH\000\000\003\377\213\n_\210\000\000\000\000\000\000\000\000\000\000"...}
fcinfo = 0x3fff6a7b558
anynull = <optimized out>
oldContext = <optimized out>
i = <optimized out>
lc = <optimized out>
pertrans = <error reading variable pertrans (value has been optimized out)>
#3 0x000002aa04423ff4 in finalize_aggregates (aggstate=aggstate(at)entry=0x2aa05775920, peraggs=peraggs(at)entry=0x2aa056e0240, pergroup=0x2aa056c8ed8) at ./build/../src/backend/executor/nodeAgg.c:1345
peragg = 0x2aa056e02f0
transno = <optimized out>
pergroupstate = 0x2aa056c8ef8
econtext = <optimized out>
aggvalues = 0x2aa056e01f8
aggnulls = 0x2aa056e0228
aggno = 2
transno = <optimized out>
#4 0x000002aa04424f5c in agg_retrieve_direct (aggstate=0x2aa05775920) at ./build/../src/backend/executor/nodeAgg.c:2480
econtext = 0x2aa05776080
firstSlot = 0x2aa062639a8
numGroupingSets = <optimized out>
node = <optimized out>
tmpcontext = 0x2aa05775d60
peragg = 0x2aa056e0240
outerslot = <optimized out>
nextSetSize = <optimized out>
pergroups = 0x2aa056c8ea8
result = <optimized out>
hasGroupingSets = <optimized out>
currentSet = <optimized out>
numReset = <optimized out>
i = <optimized out>
node = <optimized out>
econtext = <optimized out>
tmpcontext = <optimized out>
peragg = <optimized out>
pergroups = <optimized out>
outerslot = <optimized out>
firstSlot = <optimized out>
result = <optimized out>
hasGroupingSets = <optimized out>
numGroupingSets = <optimized out>
currentSet = <optimized out>
nextSetSize = <optimized out>
numReset = <optimized out>
i = <optimized out>
#5 ExecAgg (pstate=0x2aa05775920) at ./build/../src/backend/executor/nodeAgg.c:2140
node = 0x2aa05775920
result = 0x0
#6 0x000002aa0441001a in ExecProcNode (node=0x2aa05775920) at ./build/../src/include/executor/executor.h:245
No locals.
#7 ExecutePlan (execute_once=<optimized out>, dest=0x2aa0565fa58, direction=<optimized out>, numberTuples=0, sendTuples=<optimized out>, operation=CMD_SELECT, use_parallel_mode=<optimized out>, planstate=0x2aa05775920, estate=0x2aa057756f8) at ./build/../src/backend/executor/execMain.c:1646
slot = <optimized out>
current_tuple_count = 0
slot = <optimized out>
current_tuple_count = <optimized out>
#8 standard_ExecutorRun (queryDesc=0x2aa062df508, direction=<optimized out>, count=0, execute_once=<optimized out>) at ./build/../src/backend/executor/execMain.c:364
estate = 0x2aa057756f8
operation = CMD_SELECT
dest = 0x2aa0565fa58
sendTuples = <optimized out>
oldcontext = 0x2aa0565f830
__func__ = "standard_ExecutorRun"
#9 0x000002aa043933fa in ExecCreateTableAs (pstate=pstate(at)entry=0x2aa0565f948, stmt=stmt(at)entry=0x2aa055bea28, params=params(at)entry=0x0, queryEnv=queryEnv(at)entry=0x0, qc=0x3fff6a7ca30) at ./build/../src/backend/commands/createas.c:354
query = <optimized out>
into = <optimized out>
is_matview = <optimized out>
dest = 0x2aa0565fa58
save_userid = 0
save_sec_context = 0
save_nestlevel = 0
address = {classId = 70885274, objectId = 1023, objectSubId = -156778696}
rewritten = <optimized out>
plan = 0x2aa062df3f8
queryDesc = 0x2aa062df508
__func__ = "ExecCreateTableAs"
#10 0x000002aa0459f378 in ProcessUtilitySlow (pstate=pstate(at)entry=0x2aa0565f948, pstmt=pstmt(at)entry=0x2aa055bead8, queryString=queryString(at)entry=0x2aa055bcfb8 "create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*), max(g::text)\nfrom gs_data_1 group by cube (g1000, g100,g10);", context=context(at)entry=PROCESS_UTILITY_TOPLEVEL, params=params(at)entry=0x0, queryEnv=0x0, qc=0x3fff6a7ca30, dest=<error reading variable: value has been optimized out>) at ./build/../src/backend/tcop/utility.c:1600
_save_exception_stack = 0x3fff6a7c820
_save_context_stack = 0x0
_local_sigjmp_buf = {{__jmpbuf = {{__gregs = {0, 2929258264904, 2929257605848, 0, 2929257605672, 2929257598904, 4396084256648, 1, 18739560736704083, 18738779339296963}, __fpregs = {2929259797352, 2929259797352, 2929257598904, 2929258351376, 0, 4397889735684, 4397889735216, 0}}}, __mask_was_saved = 0, __saved_mask = {__val = {4397889733480, 10, 2929257605400, 0, 0, 2929257606120, 4096, 1, 0, 4396084256648, 2929242951088, 2929239505962, 4397889733120, 18736975094525723, 2929257598904, 18446744069489843728}}}}
_do_rethrow = false
parsetree = 0x2aa055bea28
isTopLevel = true
isCompleteQuery = true
needCleanup = false
commandCollected = false
address = {classId = 1023, objectId = 4138189632, objectSubId = 682}
secondaryObject = {classId = 0, objectId = 0, objectSubId = 0}
__func__ = "ProcessUtilitySlow"
#11 0x000002aa0459dd36 in standard_ProcessUtility (pstmt=0x2aa055bead8, queryString=0x2aa055bcfb8 "create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*), max(g::text)\nfrom gs_data_1 group by cube (g1000, g100,g10);", context=<optimized out>, params=0x0, queryEnv=<optimized out>, dest=0x2aa057d5b68, qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/utility.c:1069
parsetree = 0x2aa055bea28
isTopLevel = <optimized out>
isAtomicContext = <optimized out>
pstate = 0x2aa0565f948
readonly_flags = <optimized out>
__func__ = "standard_ProcessUtility"
#12 0x000002aa0459e874 in ProcessUtility (pstmt=pstmt(at)entry=0x2aa055bead8, queryString=<optimized out>, context=context(at)entry=PROCESS_UTILITY_TOPLEVEL, params=<optimized out>, queryEnv=queryEnv(at)entry=0x0, dest=0x2aa057d5b68, qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/utility.c:524
No locals.
#13 0x000002aa0459b210 in PortalRunUtility (portal=0x2aa05620008, pstmt=0x2aa055bead8, isTopLevel=<optimized out>, setHoldSnapshot=<optimized out>, dest=<optimized out>, qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/pquery.c:1157
utilityStmt = <optimized out>
snapshot = 0x2aa05674c58
#14 0x000002aa0459bca0 in PortalRunMulti (portal=portal(at)entry=0x2aa05620008, isTopLevel=isTopLevel(at)entry=true, setHoldSnapshot=setHoldSnapshot(at)entry=false, dest=<optimized out>, dest(at)entry=0x2aa057d5b68, altdest=<optimized out>, altdest(at)entry=0x2aa057d5b68, qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/pquery.c:1303
pstmt = <optimized out>
stmtlist_item__state = {l = 0x2aa057d5b18, i = 0}
active_snapshot_set = false
stmtlist_item = 0x2aa057d5b30
#15 0x000002aa0459c9f4 in PortalRun (portal=portal(at)entry=0x2aa05620008, count=count(at)entry=9223372036854775807, isTopLevel=isTopLevel(at)entry=true, run_once=run_once(at)entry=true, dest=dest(at)entry=0x2aa057d5b68, altdest=0x2aa057d5b68, qc=0x3fff6a7ca30) at ./build/../src/backend/tcop/pquery.c:779
_save_exception_stack = 0x3fff6a7ccc0
_save_context_stack = 0x0
_local_sigjmp_buf = {{__jmpbuf = {{__gregs = {2929258004488, 2, 2929258004488, 2929167695872, 2929257605768, 1, 4396084256648, 4397889736544, 18739560736712267, 18738779339319315}, __fpregs = {89, 0, 2929257598904, 2929258351376, 4397889735216, 4397889735684, 4397889735214, 0}}}, __mask_was_saved = 0, __saved_mask = {__val = {0, 8192, 2, 1, 1, 2929243650206, 2929258004488, 2929257605768, 2, 4396084256648, 4397889736544, 2929237235946, 4397889734816, 4397889736544, 2929239411110, 4397889734656}}}}
_do_rethrow = <optimized out>
result = <optimized out>
nprocessed = <optimized out>
saveTopTransactionResourceOwner = 0x2aa055e9bf0
saveTopTransactionContext = 0x2aa05674b10
saveActivePortal = 0x0
saveResourceOwner = 0x2aa055e9bf0
savePortalContext = 0x0
saveMemoryContext = 0x2aa05674b10
__func__ = "PortalRun"
#16 0x000002aa0459830a in exec_simple_query (query_string=<optimized out>) at ./build/../src/backend/tcop/postgres.c:1239
snapshot_set = <optimized out>
per_parsetree_context = <optimized out>
plantree_list = <optimized out>
parsetree = 0x2aa055bea58
commandTag = <optimized out>
qc = {commandTag = CMDTAG_UNKNOWN, nprocessed = 0}
querytree_list = <optimized out>
portal = 0x2aa05620008
receiver = 0x2aa057d5b68
format = 0
parsetree_item__state = {l = 0x2aa055bea88, i = 0}
dest = DestRemote
oldcontext = <optimized out>
parsetree_list = 0x2aa055bea88
parsetree_item = 0x2aa055beaa0
save_log_statement_stats = false
was_logged = false
use_implicit_block = false
msec_str = "\000\000\002\252\000\000\000Q\000\000\002\252\005[ϸ\000\000\002\252\000\000\000\000\000\000\003\377\213\n_\210"
__func__ = "exec_simple_query"
#17 0x000002aa0459a05e in PostgresMain (argc=<optimized out>, argv=argv(at)entry=0x2aa055e8190, dbname=<optimized out>, username=<optimized out>) at ./build/../src/backend/tcop/postgres.c:4315
query_string = 0x2aa055bcfb8 "create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*), max(g::text)\nfrom gs_data_1 group by cube (g1000, g100,g10);"
firstchar = 81
input_message = {data = 0x2aa055bcfb8 "create table gs_group_1 as\nselect g100, g10, sum(g::numeric), count(*), max(g::text)\nfrom gs_data_1 group by cube (g1000, g100,g10);", len = 133, maxlen = 1024, cursor = 133}
local_sigjmp_buf = {{__jmpbuf = {{__gregs = {8388608, 64, 2929244860962, 2929257583240, 2929244863032, 2929244863024, 4396084256648, 4397889736544, 18739560736695925, 18738779339318659}, __fpregs = {4397889736544, 4397889736532, 6, 4397419976768, 2929690573168, 2930182562592, 4397889736696, 0}}}, __mask_was_saved = 1, __saved_mask = {__val = {0, 2929239505782, 2929257748144, 4397889736288, 4396084256648, 4397889736544, 2929242274994, 4397889735960, 1024, 4396084256648, 4397889736544, 2929242135450, 4, 0, 4397889736320, 4397889736160}}}}
send_ready_for_query = false
disable_idle_in_transaction_timeout = false
__func__ = "PostgresMain"
#18 0x000002aa04512066 in BackendRun (port=0x2aa055e16b0, port=0x2aa055e16b0) at ./build/../src/backend/postmaster/postmaster.c:4523
av = 0x2aa055e8190
maxac = <optimized out>
ac = 1
i = 1
av = <optimized out>
maxac = <optimized out>
ac = <optimized out>
i = <optimized out>
__func__ = "BackendRun"
__errno_location = <optimized out>
__errno_location = <optimized out>
__errno_location = <optimized out>
#19 BackendStartup (port=0x2aa055e16b0) at ./build/../src/backend/postmaster/postmaster.c:4215
bn = <optimized out>
pid = <optimized out>
bn = <optimized out>
pid = <optimized out>
__func__ = "BackendStartup"
__errno_location = <optimized out>
__errno_location = <optimized out>
save_errno = <optimized out>
__errno_location = <optimized out>
__errno_location = <optimized out>
#20 ServerLoop () at ./build/../src/backend/postmaster/postmaster.c:1727
port = 0x2aa055e16b0
i = <optimized out>
rmask = {fds_bits = {32, 0 <repeats 15 times>}}
selres = <optimized out>
now = <optimized out>
readmask = {fds_bits = {32, 0 <repeats 15 times>}}
nSockets = 0
last_lockfile_recheck_time = 1594829011
last_touch_time = 1594829011
__func__ = "ServerLoop"
#21 0x000002aa04513128 in PostmasterMain (argc=<optimized out>, argv=<optimized out>) at ./build/../src/backend/postmaster/postmaster.c:1400
opt = <optimized out>
status = <optimized out>
userDoption = <optimized out>
listen_addr_saved = false
i = <optimized out>
output_config_variable = <optimized out>
__func__ = "PostmasterMain"
#22 0x000002aa04243fb4 in main (argc=<optimized out>, argv=0x2aa055b71b0) at ./build/../src/backend/main/main.c:210
do_check_root = <optimized out>

Christoph