Developers of Sympa

[Thomas Berry <address@concealed>]

This morning, task_manager.pl started dying for unknown reasons.

Any ideas?

Thomas


Included below is some data pulled from sympa.log and strace:


Here's an excerpt of sympa.log up to the point of failure (log level 3):

Jan 15 08:45:01 list task_manager[11601]: mail::sending() mail::sending()
Jan 15 08:45:01 list task_manager[11601]: mail::smtpto() 
mail::smtpto(address@concealed, 
address@concealed,  )
Jan 15 08:45:01 list task_manager[11601]: mail::smtpto() Open = 1
Jan 15 08:45:01 list task_manager[11601]: List::send_notify_to_owner() 
List::send_notify_to_owner(jpl.org.343, automatic_bounce_management)
Jan 15 08:45:01 list task_manager[11601]: List::get_owners_email() 
List::get_owners_email(jpl.org.343)
Jan 15 08:45:01 list task_manager[11601]: List::get_owners() 
List::get_owners(jpl.org.343)
Jan 15 08:45:01 list task_manager[11601]: List::get_first_admin_user() 
List::get_first_admin_user(jpl.org.343,owner,domain,0,0)
Jan 15 08:45:01 list task_manager[11601]: Lock::new() 
Lock::new(/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user,) 

Jan 15 08:45:01 list task_manager[11601]: Lock::lock() 
Lock::lock(/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user.lock,read) 

Jan 15 08:45:01 list task_manager[11601]: Lock::_lock_file() 
Lock::_lock_file(/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user.lock,read,20) 

Jan 15 08:45:01 list task_manager[11601]: Lock::_lock_file() Got lock 
for read on 
/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user.lock 

Jan 15 08:45:01 list task_manager[11601]: List::get_first_admin_user() 
SQL: SELECT user_admin AS email, comment_admin AS gecos, reception_admin 
AS reception, UNIX_TIMESTAMP(date_admin) AS date, 
UNIX_TIMESTAMP(update_admin) AS update_date, info_admin AS info, 
profile_admin AS profile, subscribed_admin AS subscribed, included_admin 
AS included, include_sources_admin AS id, REVERSE(SUBSTRING(user_admin 
FROM position('@' IN user_admin) FOR 50)) AS dom FROM admin_table WHERE 
(list_admin = 'jpl.org.343' AND robot_admin = 'list.jpl.nasa.gov' AND 
role_admin = 'owner' ) ORDER BY dom
Jan 15 08:45:01 list task_manager[11601]: List::get_next_admin_user() 
List::get_next_admin_user
Jan 15 08:45:01 list task_manager[11601]: Lock::new() 
Lock::new(/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user,) 

Jan 15 08:45:01 list task_manager[11601]: Lock::unlock() 
Lock::unlock(/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user.lock) 

Jan 15 08:45:01 list task_manager[11601]: Lock::_unlock_file() 
Lock::_unlock_file(/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user.lock) 

Jan 15 08:45:01 list task_manager[11601]: Lock::_unlock_file() Release 
lock on 
/usr/local/jpl/msg/list/sympa/expl/list.jpl.nasa.gov/jpl.org.343/include_admin_user.lock 




Here's the last bit from an strace of task_manager.pl.


open("/usr/local/jpl/msg/list/lib/perl5/5.8.7/i386-linux/CORE/libresolv.so.2", 
O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib/libresolv.so.2", O_RDONLY)   = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360S\332"..., 
512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=79488, ...}) = 0
old_mmap(0xda3000, 75944, PROT_READ|PROT_EXEC, 
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xda3000
old_mmap(0xdb2000, 8192, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0xdb2000
old_mmap(0xdb4000, 6312, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xdb4000
close(3)                                = 0
munmap(0xb7d3f000, 76705)               = 0
open("/etc/nsswitch.conf", O_RDONLY)    = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=1623, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 
0) = 0xb7d51000
read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1623
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0xb7d51000, 4096)                = 0
open("/usr/local/jpl/msg/list/lib/perl5/5.8.7/i386-linux/CORE/libnss_files.so.2", 
O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=76705, ...}) = 0
old_mmap(NULL, 76705, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7d3f000
close(3)                                = 0
open("/lib/libnss_files.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\33\0"..., 
512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=45889, ...}) = 0
old_mmap(NULL, 41608, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 
0) = 0x2fe000
old_mmap(0x307000, 8192, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x307000
close(3)                                = 0
munmap(0xb7d3f000, 76705)               = 0
open("/etc/services", O_RDONLY)         = 3
fcntl64(3, F_GETFD)                     = 0
fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=20373, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 
0) = 0xb7d51000
read(3, "# /etc/services:\n# $Id: services"..., 4096) = 4096
read(3, "123/tcp\nntp\t\t123/udp\t\t\t\t# Networ"..., 4096) = 4096
read(3, "\t\t873/tcp\t\t\t\t# rsync\nrsync\t\t873/"..., 4096) = 4096
read(3, "pserver\t2401/tcp\t\t\t# CVS client/"..., 4096) = 4096
close(3)                                = 0
munmap(0xb7d51000, 4096)                = 0
rt_sigaction(SIGPIPE, {SIG_IGN}, {SIG_DFL}, 8) = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 3
fcntl64(3, F_SETFL, O_RDONLY)           = 0
fcntl64(3, F_GETFL)                     = 0x2 (flags O_RDWR)
gettimeofday({1200415162, 175004}, NULL) = 0
open("/etc/resolv.conf", O_RDONLY)      = 4
fstat64(4, {st_mode=S_IFREG|0644, st_size=69, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 
0) = 0xb7d51000
read(4, "search jpl.nasa.gov\nnameserver 1"..., 4096) = 69
read(4, "", 4096)                       = 0
close(4)                                = 0
munmap(0xb7d51000, 4096)                = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 4
fcntl64(4, F_GETFL)                     = 0x2 (flags O_RDWR)
fcntl64(4, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
connect(4, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 0
poll([{fd=4, events=POLLOUT|POLLERR|POLLHUP, revents=POLLOUT}], 1, 5000) = 1
send(4, "\2\0\0\0\r\0\0\0\6\0\0\0hosts\0\0\0", 20, MSG_NOSIGNAL) = 20
poll([{fd=4, events=POLLIN|POLLERR|POLLHUP, revents=POLLIN}], 1, 5000) = 1
recvmsg(4, {msg_name(0)=NULL, msg_iov(1)=[{"hosts\0", 6}], 
msg_controllen=16, {cmsg_len=16, cmsg_level=SOL_SOCKET, 
cmsg_type=SCM_RIGHTS, {5}}, msg_flags=0}, 0) = 6
fstat64(5, {st_mode=S_IFREG|0600, st_size=217016, ...}) = 0
pread64(5, "\1\0\0\0h\0\0\0b\351\4\0\1\0\0\0\240$\224F\0\0\0\0\323"..., 
104, 0) = 104
mmap2(NULL, 217016, PROT_READ, MAP_SHARED, 5, 0) = 0xb7d1d000
close(5)                                = 0
close(4)                                = 0
connect(3, {sa_family=AF_INET, sin_port=htons(4192), 
sin_addr=inet_addr("137.78.160.28")}, 16) = 0
setsockopt(3, SOL_IP, IP_TOS, [8], 4)   = 0
setsockopt(3, SOL_TCP, TCP_NODELAY, [1], 4) = 0
setsockopt(3, SOL_SOCKET, SO_KEEPALIVE, [1], 4) = 0
read(3, "<\0\0\0", 4)                   = 4
read(3, "\n5.0.7-beta-log\0\10V\1\0I@Cn\"\\*Z\0,\242\10"..., 60) = 60
stat64("/usr/share/mysql/charsets/Index.xml", {st_mode=S_IFREG|0644, 
st_size=17846, ...}) = 0
open("/usr/share/mysql/charsets/Index.xml", O_RDONLY|O_LARGEFILE) = 4
read(4, "<?xml version=\'1.0\' encoding=\"ut"..., 17846) = 17846
close(4)                                = 0
write(3, "K\0\0\1\217\242\0\0\0\0\0@\10\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 
79) = 79
read(3, "\7\0\0\2", 4)                  = 4
read(3, "\0\0\0\2\0\0\0", 7)            = 7
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
read(3, 0x9399fe0, 8192)                = -1 EAGAIN (Resource 
temporarily unavailable)
fcntl64(3, F_SETFL, O_RDWR)             = 0
write(3, "\21\0\0\0\3set autocommit=1", 21) = 21
read(3, "\7\0\0\1", 4)                  = 4
read(3, "\0\0\0\2\0\0\0", 7)            = 7
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
read(3, 0x9399fe0, 8192)                = -1 EAGAIN (Resource 
temporarily unavailable)
fcntl64(3, F_SETFL, O_RDWR)             = 0
write(3, "\21\0\0\0\3SET NAMES \'utf8\'", 21) = 21
read(3, "\7\0\0\1", 4)                  = 4
read(3, "\0\0\0\2\0\0\0", 7)            = 7
stat64("/usr/local/jpl/msg/list/sympa/etc/data_structure.version", 
{st_mode=S_IFREG|0640, st_size=137, ...}) = 0
open("/usr/local/jpl/msg/list/sympa/etc/data_structure.version", 
O_RDONLY|O_LARGEFILE) = 4
ioctl(4, SNDCTL_TMR_TIMEBASE or TCGETS, 0xbfee92e8) = -1 ENOTTY 
(Inappropriate ioctl for device)
_llseek(4, 0, [0], SEEK_CUR)            = 0
fstat64(4, {st_mode=S_IFREG|0640, st_size=137, ...}) = 0
fcntl64(4, F_SETFD, FD_CLOEXEC)         = 0
read(4, "# This file is automatically cre"..., 4096) = 137
close(4)                                = 0
stat64("/usr/sbin/sendmail", {st_mode=S_IFREG|S_ISGID|0755, 
st_size=733912, ...}) = 0
stat64("/usr/local/jpl/msg/list/ssl/bin/openssl", {st_mode=S_IFREG|0755, 
st_size=1336756, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool", {st_mode=S_IFDIR|0770, 
st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/msg", {st_mode=S_IFDIR|0770, 
st_size=65536, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/automatic", 
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/digest", 
{st_mode=S_IFDIR|0770, st_size=12288, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/moderation", 
{st_mode=S_IFDIR|0770, st_size=40960, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/topic", 
{st_mode=S_IFDIR|0750, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/auth", 
{st_mode=S_IFDIR|0770, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/outgoing", 
{st_mode=S_IFDIR|0770, st_size=20480, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/bounce", 
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/subscribe", 
{st_mode=S_IFDIR|0750, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/task", 
{st_mode=S_IFDIR|0770, st_size=65536, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/distribute", 
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/tmp", {st_mode=S_IFDIR|0770, 
st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/msg/bad", 
{st_mode=S_IFDIR|0750, st_size=57344, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/distribute/bad", 
{st_mode=S_IFDIR|0750, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/spool/automatic/bad", 
{st_mode=S_IFDIR|0750, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/ssl/certs/ca-bundle.crt", 
{st_mode=S_IFREG|0644, st_size=249373, ...}) = 0
stat64("/usr/local/jpl/msg/list/ssl/certs/ca-bundle.crt", 
{st_mode=S_IFREG|0644, st_size=249373, ...}) = 0
stat64("/usr/local/jpl/msg/list/ssl/certs", {st_mode=S_IFDIR|0755, 
st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/ssl/certs", {st_mode=S_IFDIR|0755, 
st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content", 
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/index.html", 
{st_mode=S_IFREG|0640, st_size=0, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css", 
{st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/style.css", 
{st_mode=S_IFREG|0644, st_size=19781, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/bin/etc/web_tt2/css.tt2", 
{st_mode=S_IFREG|0444, st_size=21408, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/style.css", 
{st_mode=S_IFREG|0644, st_size=19781, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/print.css", 
{st_mode=S_IFREG|0644, st_size=1, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/bin/etc/web_tt2/css.tt2", 
{st_mode=S_IFREG|0444, st_size=21408, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/print.css", 
{st_mode=S_IFREG|0644, st_size=1, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/fullPage.css", 
{st_mode=S_IFREG|0644, st_size=1, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/bin/etc/web_tt2/css.tt2", 
{st_mode=S_IFREG|0444, st_size=21408, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/fullPage.css", 
{st_mode=S_IFREG|0644, st_size=1, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/print-preview.css", 
{st_mode=S_IFREG|0644, st_size=1, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/bin/etc/web_tt2/css.tt2", 
{st_mode=S_IFREG|0444, st_size=21408, ...}) = 0
stat64("/usr/local/jpl/msg/list/sympa/static_content/css/print-preview.css", 
{st_mode=S_IFREG|0644, st_size=1, ...}) = 0
open("/dev/null", O_WRONLY|O_APPEND|O_CREAT|O_LARGEFILE, 0666) = 4
_llseek(4, 0, [0], SEEK_END)            = 0
ioctl(4, SNDCTL_TMR_TIMEBASE or TCGETS, 0xbfee92e8) = -1 ENOTTY 
(Inappropriate ioctl for device)
_llseek(4, 0, [0], SEEK_CUR)            = 0
fstat64(4, {st_mode=S_IFCHR|0666, st_rdev=makedev(1, 3), ...}) = 0
dup2(4, 2)                              = 2
close(4)                                = 0
fcntl64(2, F_SETFD, 0)                  = 0
open("/dev/null", O_WRONLY|O_APPEND|O_CREAT|O_LARGEFILE, 0666) = 4
_llseek(4, 0, [0], SEEK_END)            = 0
ioctl(4, SNDCTL_TMR_TIMEBASE or TCGETS, 0xbfee92e8) = -1 ENOTTY 
(Inappropriate ioctl for device)
_llseek(4, 0, [0], SEEK_CUR)            = 0
fstat64(4, {st_mode=S_IFCHR|0666, st_rdev=makedev(1, 3), ...}) = 0
dup2(4, 1)                              = 1
close(4)                                = 0
fcntl64(1, F_SETFD, 0)                  = 0
open("/dev/tty", O_RDONLY|O_LARGEFILE)  = 4
ioctl(4, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo 
...}) = 0
_llseek(4, 0, 0xbfee9320, SEEK_CUR)     = -1 ESPIPE (Illegal seek)
fstat64(4, {st_mode=S_IFCHR|0666, st_rdev=makedev(5, 0), ...}) = 0
fcntl64(4, F_SETFD, FD_CLOEXEC)         = 0
ioctl(4, 0x20007471, 0)                 = -1 EINVAL (Invalid argument)
close(4)                                = 0
setpgid(0, 0)                           = 0
clone(child_stack=0, 
flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, 
child_tidptr=0xb7f52708) = 8761
write(1, "Starting task_manager daemon, pi"..., 40) = 40
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
read(3, 0x9399fe0, 8192)                = -1 EAGAIN (Resource 
temporarily unavailable)
fcntl64(3, F_SETFL, O_RDWR)             = 0
write(3, "\1\0\0\0\1", 5)               = 5
shutdown(3, 2 /* send and receive */)   = 0
close(3)                                = 0
stat64("/usr/local/jpl/msg/list/lib/perl5/site_perl/5.8.7/i386-linux/auto/DBI/DESTROY.al", 
0x256fa0) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/sympa/bin/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/5.8.7/i386-linux/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/5.8.7/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/site_perl/5.8.7/i386-linux/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/site_perl/5.8.7/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/site_perl/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/vendor_perl/5.8.7/i386-linux/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/vendor_perl/5.8.7/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/vendor_perl/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("/usr/local/jpl/msg/list/lib/perl5/5.8.0/auto/DBI/DESTROY.al", 
O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such file or directory)
open("./auto/DBI/DESTROY.al", O_RDONLY|O_LARGEFILE) = -1 ENOENT (No such 
file or directory)
exit_group(0)                           = ?
Process 8760 detached