Re: BUG #14917: process hang on create index

I don't know how to create a small, reproducible example, customer
personal data is contained. Sorry for the long listing.
We have about 12k tables in this DB. Problem has only been observed with
one user's table.

After a fresh DB start and CREATE INDEX idx_table123_fast ON table123 USING
btree (k, n), strace looks normal to a point:

[...snip...]
read(41, "\20\317\0\0\270Y\372v\0\0\0\0\330\0X\1\0 \4
\0\0\0\0h\237(dot)\1\310\236(at)\1"..., 8192) = 8192
read(41, "\20\317\0\0`g\372v\0\0\0\0x\0\330\20\0 \4
\0\0\0\0`\237:\1\300\236>\1"..., 8192) = 8192

(stops for a minute)

--- SIGALRM {si_signo=SIGALRM, si_code=SI_KERNEL} ---
rt_sigreturn() = 2

(after a few minutes, I press CTRL+C in pgsql shell to stop the operation)

--- SIGINT {si_signo=SIGINT, si_code=SI_USER, si_pid=4264, si_uid=106} ---
rt_sigreturn() = 8
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
write(2, "2017-11-18 08:27:59 UTC [4239-1]"..., 221) = 221
sendto(11, "E\0\0\0aSERROR\0C57014\0Mcanceling st"..., 98, 0, NULL, 0) = 98
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
munmap(0x7f138b926000, 8392704) = 0
munmap(0x7f138c127000, 8392704) = 0
...

Now I rerun the CREATE INDEX... and all I get now is:

) = 1 ([{fd=11, revents=POLLIN}])
recvfrom(11, "Q\0\0\0ICREATE INDEX idx_space02448"..., 8192, 0, NULL, NULL)
= 74
gettimeofday({1510993750, 627472}, NULL) = 0
gettimeofday({1510993750, 627541}, NULL) = 0
open("base/82144111/82144135", O_RDONLY) = -1 ENOENT (No such file or
directory)
stat("base/82144111", {st_mode=S_IFDIR|0700, st_size=267, ...}) = 0
open("base/82144111/82144135", O_RDWR|O_CREAT|O_EXCL, 0600) = 32
close(32) = 0
open("base/82144111/82144135", O_RDWR) = 32
lseek(32, 0, SEEK_END) = 0
lseek(41, 0, SEEK_END) = 734953472
brk(0x7f13d5b92000) = 0x7f13d5b92000
brk(0x7f13d5bd2000) = 0x7f13d5bd2000
brk(0x7f13d5c6a000) = 0x7f13d5c6a000
brk(0x7f13d5c9a000) = 0x7f13d5c9a000
brk(0x7f13d5cfa000) = 0x7f13d5cfa000
brk(0x7f13d5dca000) = 0x7f13d5dca000
brk(0x7f13d5e8a000) = 0x7f13d5e8a000
brk(0x7f13d608a000) = 0x7f13d608a000
brk(0x7f13d620a000) = 0x7f13d620a000
brk(0x7f13d660a000) = 0x7f13d660a000
brk(0x7f13d6e0a000) = 0x7f13d6e0a000
brk(0x7f13d710a000) = 0x7f13d710a000
brk(0x7f13d790a000) = 0x7f13d790a000
brk(0x7f13d7f0a000) = 0x7f13d7f0a000
brk(0x7f13d870a000) = 0x7f13d870a000
brk(0x7f13d8f0a000) = 0x7f13d8f0a000
brk(0x7f13d970a000) = 0x7f13d970a000
mmap(NULL, 12587008, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f13cc2c2000
brk(0x7f13d9f0a000) = 0x7f13d9f0a000
brk(0x7f13da70a000) = 0x7f13da70a000
brk(0x7f13daf0a000) = 0x7f13daf0a000
brk(0x7f13db70a000) = 0x7f13db70a000
brk(0x7f13dbf0a000) = 0x7f13dbf0a000
mremap(0x7f13cc2c2000, 12587008, 25169920, MREMAP_MAYMOVE) = 0x7f13caac1000
brk(0x7f13dc70a000) = 0x7f13dc70a000
brk(0x7f13dcf0a000) = 0x7f13dcf0a000
brk(0x7f13dd70a000) = 0x7f13dd70a000
brk(0x7f13ddf0a000) = 0x7f13ddf0a000
brk(0x7f13de70a000) = 0x7f13de70a000
brk(0x7f13def0a000) = 0x7f13def0a000
brk(0x7f13df70a000) = 0x7f13df70a000
brk(0x7f13dff0a000) = 0x7f13dff0a000
brk(0x7f13e070a000) = 0x7f13e070a000
brk(0x7f13e0f0a000) = 0x7f13e0f0a000
brk(0x7f13e170a000) = 0x7f13e170a000
mremap(0x7f13caac1000, 25169920, 50335744, MREMAP_MAYMOVE) = 0x7f13c7ac0000
brk(0x7f13e1f0a000) = 0x7f13e1f0a000
brk(0x7f13e270a000) = 0x7f13e270a000
brk(0x7f13e2f0a000) = 0x7f13e2f0a000
brk(0x7f13e370a000) = 0x7f13e370a000
brk(0x7f13e3f0a000) = 0x7f13e3f0a000
brk(0x7f13e470a000) = 0x7f13e470a000
brk(0x7f13e4f0a000) = 0x7f13e4f0a000
brk(0x7f13e570a000) = 0x7f13e570a000
brk(0x7f13e5f0a000) = 0x7f13e5f0a000
brk(0x7f13e670a000) = 0x7f13e670a000
brk(0x7f13e6f0a000) = 0x7f13e6f0a000
brk(0x7f13e770a000) = 0x7f13e770a000
brk(0x7f13e7f0a000) = 0x7f13e7f0a000
brk(0x7f13e870a000) = 0x7f13e870a000
brk(0x7f13e8f0a000) = 0x7f13e8f0a000
brk(0x7f13e970a000) = 0x7f13e970a000
brk(0x7f13e9f0a000) = 0x7f13e9f0a000
brk(0x7f13ea70a000) = 0x7f13ea70a000
brk(0x7f13eaf0a000) = 0x7f13eaf0a000
brk(0x7f13eb70a000) = 0x7f13eb70a000
brk(0x7f13ebf0a000) = 0x7f13ebf0a000
mremap(0x7f13c7ac0000, 50335744, 100667392, MREMAP_MAYMOVE) =
0x7f13c1abf000
brk(0x7f13ec70a000) = 0x7f13ec70a000
brk(0x7f13ecf0a000) = 0x7f13ecf0a000
brk(0x7f13ed70a000) = 0x7f13ed70a000
brk(0x7f13edf0a000) = 0x7f13edf0a000
brk(0x7f13ee70a000) = 0x7f13ee70a000
brk(0x7f13eef0a000) = 0x7f13eef0a000
brk(0x7f13ef70a000) = 0x7f13ef70a000
brk(0x7f13eff0a000) = 0x7f13eff0a000
brk(0x7f13f070a000) = 0x7f13f070a000
brk(0x7f13f0f0a000) = 0x7f13f0f0a000
brk(0x7f13f170a000) = 0x7f13f170a000
brk(0x7f13f1f0a000) = 0x7f13f1f0a000
brk(0x7f13f270a000) = 0x7f13f270a000
brk(0x7f13f2f0a000) = 0x7f13f2f0a000
brk(0x7f13f370a000) = 0x7f13f370a000
brk(0x7f13f3f0a000) = 0x7f13f3f0a000
brk(0x7f13f470a000) = 0x7f13f470a000
brk(0x7f13f4f0a000) = 0x7f13f4f0a000
brk(0x7f13f570a000) = 0x7f13f570a000
brk(0x7f13f5f0a000) = 0x7f13f5f0a000
brk(0x7f13f670a000) = 0x7f13f670a000
brk(0x7f13f6f0a000) = 0x7f13f6f0a000
brk(0x7f13f770a000) = 0x7f13f770a000
brk(0x7f13f7f0a000) = 0x7f13f7f0a000
brk(0x7f13f870a000) = 0x7f13f870a000
brk(0x7f13f8f0a000) = 0x7f13f8f0a000
brk(0x7f13f970a000) = 0x7f13f970a000
brk(0x7f13f9f0a000) = 0x7f13f9f0a000
brk(0x7f13fa70a000) = 0x7f13fa70a000
brk(0x7f13faf0a000) = 0x7f13faf0a000
brk(0x7f13fb70a000) = 0x7f13fb70a000
brk(0x7f13fbf0a000) = 0x7f13fbf0a000
brk(0x7f13fc70a000) = 0x7f13fc70a000
brk(0x7f13fcf0a000) = 0x7f13fcf0a000
brk(0x7f13fd70a000) = 0x7f13fd70a000
brk(0x7f13fdf0a000) = 0x7f13fdf0a000
brk(0x7f13fe70a000) = 0x7f13fe70a000
brk(0x7f13fef0a000) = 0x7f13fef0a000
brk(0x7f13ff70a000) = 0x7f13ff70a000
brk(0x7f13fff0a000) = 0x7f13fff0a000
brk(0x7f140070a000) = 0x7f140070a000
brk(0x7f1400f0a000) = 0x7f1400f0a000
mremap(0x7f13c1abf000, 100667392, 201330688, MREMAP_MAYMOVE) =
0x7f13b5abe000
brk(0x7f140170a000) = 0x7f140170a000
brk(0x7f1401f0a000) = 0x7f1401f0a000
brk(0x7f140270a000) = 0x7f140270a000
brk(0x7f1402f0a000) = 0x7f1402f0a000
brk(0x7f140370a000) = 0x7f140370a000
brk(0x7f1403f0a000) = 0x7f1403f0a000
brk(0x7f140470a000) = 0x7f140470a000
brk(0x7f1404f0a000) = 0x7f1404f0a000
brk(0x7f140570a000) = 0x7f140570a000
brk(0x7f1405f0a000) = 0x7f1405f0a000
brk(0x7f140670a000) = 0x7f140670a000
brk(0x7f1406f0a000) = 0x7f1406f0a000
brk(0x7f140770a000) = 0x7f140770a000
brk(0x7f1407f0a000) = 0x7f1407f0a000
brk(0x7f140870a000) = 0x7f140870a000
brk(0x7f1408f0a000) = 0x7f1408f0a000
brk(0x7f140970a000) = 0x7f140970a000

(hangs until CTRL+C)

If I restart the DB, everything can be repeated.

I tried exporting the table and importing into a new database, no change.
Tested with Ubuntu 14.04 and 16.04, also CentOS 7.4, 3.x and 4.4.x kernels.
AWS i3.4xlarge instance, 128GB RAM, 16 cores.

Boris

On Fri, Nov 17, 2017 at 2:58 PM, Michael Paquier <michael(dot)paquier(at)gmail(dot)com>
wrote:

> On Fri, Nov 17, 2017 at 8:26 PM, <sagadin(at)gmail(dot)com> wrote:
> > This happens just on one customer table, UTF-8 is used. Can recreate and
> > provide more details.
>
> Self-contained examples aimed at reproducing a problem or a set of
> problems are always welcome if you think that there is a bug of some
> kind. Without more details, there is not much that can be done anyway.
> --
> Michael
>