segmentation fault with AMBER topology (cvs code 03.03.2010)

From: Cojocaru,Vlad (vlad.cojocaru_at_h-its.org)
Date: Thu Mar 04 2010 - 05:28:38 CST

Dear all,

As people reported already, the newest CVSS codes of NAMD 2.7b2 fail to start simulations with AMBER topologies. I did a number of tests to see if I can get rid of the problem, but so far no success.
A segmentation error occurs and this appears to be independent of the MPI version (openmpi or mvapich2), tcl version (8.3, 8.4 or 8.5), or charmm version (6.1.2 or 6.1.3).
In all these tests I used the intel compiler 11.0.083 .
Below you have the complete error message (from strace) . Maybe it can help to track the problem.

Does anybody have new info about these segmentation faults ? Are the developers able to reproduce this ?

Thanks
Vlad

read(12, "PARM file in AMBER 7 format\n"..., 4096) = 28
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "PARM file in AMBER 7 format\n"..., 28) = 28
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Warning: Encounter 10-12 H-bond t"..., 4096) = 69
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Warning: Encounter 10-12 H-bond t"..., 69) = 69
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Info: SUMMARY OF PARAMETERS:\nInfo"..., 4096) = 146
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Info: SUMMARY OF PARAMETERS:\nInfo"..., 146) = 146
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Info: TIME FOR READING PDB FILE: "..., 4096) = 49
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Info: TIME FOR READING PDB FILE: "..., 49) = 49
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Info: Reading from binary file 1r"..., 4096) = 59
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Info: Reading from binary file 1r"..., 59) = 59
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Info: ***************************"..., 4096) = 570
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Info: ***************************"..., 570) = 570
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Info: \nInfo: Entering startup at "..., 4096) = 72
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Info: \nInfo: Entering startup at "..., 72) = 72
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(12, "Info: Startup phase 0 took 0.0003"..., 4096) = 70
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(1, "Info: Startup phase 0 took 0.0003"..., 70) = 70
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=16, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([{fd=16, revents=POLLIN}])
read(16, "[node-08-13:23147] *** Process re"..., 4096) = 207
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=12, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(2, "[node-08-13:23147] *** Process re"..., 207[node-08-13:23147] *** Process received signal ***
[node-08-13:23147] Signal: Segmentation fault (11)
[node-08-13:23147] Signal code: Address not mapped (1)
[node-08-13:23147] Failing at address: 0xffffffff
) = 207
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=12, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 1 ([...])
read(16, "[node-08-13:23147] [ 0] /lib/libp"..., 4096) = 3399
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=12, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 38, 1000) = 1 ([...])
write(2, "[node-08-13:23147] [ 0] /lib/libp"..., 3399[node-08-13:23147] [ 0] /lib/libpthread.so.0 [0x2ae85a272a80]
[node-08-13:23147] [ 1] /lib/libc.so.6(memcpy+0xa0) [0x2ae85a4fcf50]
[node-08-13:23147] [ 2] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_intel_fast_memcpy.A+0x6) [0xb2e1c6]
[node-08-13:23147] [ 3] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN8MOStream3PutEPcm+0x65) [0x8c7cd5]
[node-08-13:23147] [ 4] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN10Parameters15send_ParametersEP8MOStream+0xb32) [0x9113c2]
[node-08-13:23147] [ 5] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN4Node7startupEv+0xe65) [0x902885]
[node-08-13:23147] [ 6] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN12CkIndex_Node18_call_startup_voidEPvP4Node+0x12) [0x901a12]
[node-08-13:23147] [ 7] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(CkDeliverMessageFree+0x25) [0xa10c15]
[node-08-13:23147] [ 8] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_Z15_processHandlerPvP11CkCoreState+0x520) [0xa10190]
[node-08-13:23147] [ 9] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(CmiHandleMessage+0x2a) [0xa90c0a]
[node-08-13:23147] [10] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(CsdScheduleForever+0x54) [0xa90b74]
[node-08-13:23147] [11] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(CsdScheduler+0x1c) [0xa90b1c]
node-08-13:23147] [12] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN7BackEnd7suspendEv+0xb) [0x4d5f3b]
[node-08-13:23147] [13] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN9ScriptTcl7Tcl_runEPvP10Tcl_InterpiPPc+0x42c) [0x972b3c]
[node-08-13:23147] [14] /sw/mcm/app/vlad/tcl/2010-02-23/x86_64_infinib/8.5.8/lib/libtcl8.5.so(TclInvokeStringCommand+0x78) [0x2ae8589c1488]
[node-08-13:23147] [15] /sw/mcm/app/vlad/tcl/2010-02-23/x86_64_infinib/8.5.8/lib/libtcl8.5.so [0x2ae8589c26db]
[node-08-13:23147] [16] /sw/mcm/app/vlad/tcl/2010-02-23/x86_64_infinib/8.5.8/lib/libtcl8.5.so [0x2ae8589c32ad]
[node-08-13:23147] [17] /sw/mcm/app/vlad/tcl/2010-02-23/x86_64_infinib/8.5.8/lib/libtcl8.5.so(Tcl_EvalEx+0x16) [0x2ae8589c3826]
[node-08-13:23147] [18] /sw/mcm/app/vlad/tcl/2010-02-23/x86_64_infinib/8.5.8/lib/libtcl8.5.so(Tcl_FSEvalFileEx+0x1d1) [0x2ae858a30841]
[node-08-13:23147] [19] /sw/mcm/app/vlad/tcl/2010-02-23/x86_64_infinib/8.5.8/lib/libtcl8.5.so(Tcl_EvalFile+0x2f) [0x2ae858a309cf]
[node-08-13:23147] [20] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZN9ScriptTcl3runEPc+0x13) [0x973753]
[node-08-13:23147] [21] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(main+0x26a) [0x4cfe4a]
[node-08-13:23147] [22] /lib/libc.so.6(__libc_start_main+0xe6) [0x2ae85a49e1a6]
[node-08-13:23147] [23] /sw/mcm/app/vlad/namd/2010-03-03/namd2_openmpi-1.4.1_tcl8.5/Linux-x86_64-intel-openmpi/namd2(_ZNSt8ios_base4InitD1Ev+0x41) [0x4cae09]
[node-08-13:23147] *** End of error message ***
) = 3399
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=12, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 37, 1000) = 2 ([{fd=12, revents=POLLHUP}, ...])
read(12, 0x7fff13aa7084, 4096) = -1 EIO (Input/output error)
close(12) = 0
read(16, ""..., 4096) = 0
close(16) = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=41, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=0, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 35, 1000) = 1 ([{fd=0,revents=POLLIN}])
readv(0, [{"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 36}], 1) = 0
close(0) = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=41, events=POLLIN}, {fd=18, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=39, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 34, 1000) = 1 ([{fd=18, revents=POLLHUP}])
--- SIGCHLD (Child exited) @ 0 (0) ---
rt_sigaction(SIGCHLD, {0x2b21974a7780, [CHLD], SA_RESTORER|SA_RESTART, 0x2b2198432f60}, {0x2b21974a7780, [CHLD], SA_RESTORER|SA_RESTART, 0x2b2198432f60}, 8) = 0
sendto(3, "a"..., 1, 0, NULL, 0) = 1
rt_sigreturn(0x3) = 1
wait4(-1, [{WIFSIGNALED(s) && WTERMSIG(s) == SIGSEGV && WCOREDUMP(s)}], WNOHANG, NULL) = 23147
wait4(-1, 0x7fff13aa80a0, WNOHANG, NULL) = 0
read(18, ""..., 4096) = 0
close(18) = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=41, events=POLLIN}, {fd=36, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=39, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 33, 0) = 1 ([{fd=4, revents=POLLIN}])
recvfrom(4, "a"..., 100, 0, NULL, NULL) = 1
stat("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/0", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
open("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 0
fstat(0, {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
getdents(0, /* 2 entries */, 4096) = 48
getdents(0, /* 0 entries */, 4096) = 0
close(0) = 0
open("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 0
fstat(0, {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
getdents(0, /* 2 entries */, 4096) = 48
getdents(0, /* 0 entries */, 4096) = 0
close(0) = 0
rmdir("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/0") = 0
stat("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1", {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
open("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 0
fstat(0, {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
getdents(0, /* 11 entries */, 4096) = 320
unlink("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/shared_mem_btl_module.node-08-13")= 0
unlink("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/shared_mem_pool.node-08-13") = 0
getdents(0, /* 0 entries */, 4096) = 0
close(0) = 0
open("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 0
fstat(0, {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
getdents(0, /* 9 entries */, 4096) = 216
close(0) = 0
stat("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0", {st_mode=S_IFDIR|0700, st_size=4096, ...})= 0
open("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC)= 0
fstat(0, {st_mode=S_IFDIR|0700, st_size=4096, ...}) = 0
getdents(0, /* 3 entries */, 4096) = 80
close(0) = 0
open("/tmp/14542.1.cluster8.q/openmpi-sessions-cojocavd_at_node-08-13_0/21272/1/0", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=41, events=POLLIN}, {fd=36, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=39, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 33, 0) = 0 (Timeout)
write(9, "\1\0\0\0"..., 4) = 4
close(9) = 0
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6, events=POLLIN}, {fd=8, events=POLLIN}, {fd=10, events=POLLIN}, {fd=42, events=POLLIN}, {fd=41, events=POLLIN}, {fd=36, events=POLLIN}, {fd=13, events=POLLIN}, {fd=20, events=POLLIN}, {fd=22, events=POLLIN}, {fd=14, events=POLLIN}, {fd=23, events=POLLIN}, {fd=25, events=POLLIN}, {fd=17, events=POLLIN}, {fd=26, events=POLLIN}, {fd=28, events=POLLIN}, {fd=19, events=POLLIN}, {fd=29, events=POLLIN}, {fd=31, events=POLLIN}, {fd=21, events=POLLIN}, {fd=32, events=POLLIN}, {fd=34, events=POLLIN}, {fd=24, events=POLLIN}, {fd=35, events=POLLIN}, {fd=37, events=POLLIN}, {fd=27, events=POLLIN}, {fd=38, events=POLLIN}, {fd=40, events=POLLIN}, {fd=39, events=POLLIN}, {fd=15, events=POLLIN}, {fd=30, events=POLLIN}, ...], 33, 0) = 1 ([{fd=8, revents=POLLIN|POLLHUP}])
futex(0x2b21987503a0, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/sw/mcm/app/vlad/mpi/2010-03-03/x86_64_infinib/openmpi/1.4.1/share/openmpi/help-orterun.txt", O_RDONLY) = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff13aa7b60) = -1 ENOTTY (Inappropriate ioctl for device)
fstat(0, {st_mode=S_IFREG|0644, st_size=13590, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b219d207000
read(0, "# -*- text -*-\n#\n# Copyright (c) "..., 8192) = 8192
close(0) = 0
munmap(0x2b219d207000, 8192) = 0
write(2, "---------------------------------"..., 261--------------------------------------------------------------------------
mpirun noticed that process rank 0 with PID 23147 on node node-08-13 exited on signal 11 (Segmentation fault).
--------------------------------------------------------------------------

This archive was generated by hypermail 2.1.6 : Wed Feb 29 2012 - 15:53:51 CST