Empty System Logs in Linux – How to Fix Empty /var/log/messages and /var/log/secure

journaldlinuxlog-filesredhatrsyslog

I found that rsyslog stopped writing on logs (messages; secure; cron;etc)

System information:
NAME="Red Hat Enterprise Linux Server"
VERSION="7.4 (Maipo)"
ID="rhel"
ID_LIKE="fedora"
VARIANT="Server"
VARIANT_ID="server"
VERSION_ID="7.4"

already restarted rsyslog with no results. Other logs processed by rsyslogd are being writen without issues.

Also restarted systemd-journald.

rsyslog.conf (Omitting commented lines):

$ModLoad imuxsock
$ModLoad imjournal
$WorkDirectory /var/lib/rsyslog
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$IncludeConfig /etc/rsyslog.d/*.conf
$OmitLocalLogging on
$IMJournalStateFile imjournal.state
*.info;mail.none;authpriv.none;cron.none                /var/log/messages
authpriv.*                                              /var/log/secure
mail.*                                                  -/var/log/maillog
cron.*                                                  /var/log/cron
*.emerg                                                 :omusrmsg:*
uucp,news.crit                                          /var/log/spooler
local7.*                                                /var/log/boot.log
local3.*                                                /var/log/sshd-second.log

Journald.conf (Omitting commented lines):

[Journal]

Already deleted /run/log/journal/* and restarted journald
Already deleted imjournal.state and restarted rsyslog

output of the command rsyslogd -N 1:

rsyslogd: version 8.24.0, config validation run (level 1), master config /etc/rsyslog.conf

rsyslogd: End of config validation run. Bye.

output of the command rsyslogd -N 6:

rsyslogd: version 8.24.0, config validation run (level 6), master config /etc/rsyslog.conf

rsyslogd: End of config validation run. Bye.

Already tried all the solutions described in https://unix.stackexchange.com/questions/124942/rsyslog-not-logging with no results

Output of lsof -p 

COMMAND   PID USER   FD      TYPE             DEVICE SIZE/OFF     NODE NAME
rsyslogd 5820 root  cwd       DIR              202,2      236       64 /
rsyslogd 5820 root  rtd       DIR              202,2      236       64 /
rsyslogd 5820 root  txt       REG              202,2   663872   552421 /usr/sbin/rsyslogd
rsyslogd 5820 root  mem       REG               0,18  8388608    27215 /run/log/journal/ca23e130dda846d5b2a30e4ab9461e43/system.journal
rsyslogd 5820 root  mem       REG              202,2    29352 13067688 /usr/lib64/rsyslog/imudp.so
rsyslogd 5820 root  mem       REG              202,2    68192    34595 /usr/lib64/libbz2.so.1.0.6
rsyslogd 5820 root  mem       REG              202,2    99944    34673 /usr/lib64/libelf-0.168.so
rsyslogd 5820 root  mem       REG              202,2   402384    34477 /usr/lib64/libpcre.so.1.2.0
rsyslogd 5820 root  mem       REG              202,2    19888    34705 /usr/lib64/libattr.so.1.1.0
rsyslogd 5820 root  mem       REG              202,2   297328    45575 /usr/lib64/libdw-0.168.so
rsyslogd 5820 root  mem       REG              202,2   111080  1274500 /usr/lib64/libresolv-2.17.so
rsyslogd 5820 root  mem       REG              202,2    19384    34688 /usr/lib64/libgpg-error.so.0.10.0
rsyslogd 5820 root  mem       REG              202,2   535064    34703 /usr/lib64/libgcrypt.so.11.8.2
rsyslogd 5820 root  mem       REG              202,2   157400    34499 /usr/lib64/liblzma.so.5.2.2
rsyslogd 5820 root  mem       REG              202,2   155752    34476 /usr/lib64/libselinux.so.1
rsyslogd 5820 root  mem       REG              202,2  1139680    34473 /usr/lib64/libm-2.17.so
rsyslogd 5820 root  mem       REG              202,2    20032    34709 /usr/lib64/libcap.so.2.22
rsyslogd 5820 root  mem       REG              202,2    24928 13067682 /usr/lib64/rsyslog/imjournal.so
rsyslogd 5820 root  mem       REG              202,2    38032 13067689 /usr/lib64/rsyslog/imuxsock.so
rsyslogd 5820 root  mem       REG              202,2    24416 13067690 /usr/lib64/rsyslog/lmnet.so
rsyslogd 5820 root  mem       REG              202,2  2127336    23375 /usr/lib64/libc-2.17.so
rsyslogd 5820 root  mem       REG              202,2    88720  1233870 /usr/lib64/libgcc_s-4.8.5-20150702.so.1
rsyslogd 5820 root  mem       REG              202,2    20040    35554 /usr/lib64/libuuid.so.1.3.0
rsyslogd 5820 root  mem       REG              202,2    40824   374355 /usr/lib64/libfastjson.so.4.0.0
rsyslogd 5820 root  mem       REG              202,2    15424   392270 /usr/lib64/libestr.so.0.0.0
rsyslogd 5820 root  mem       REG              202,2    44448    23398 /usr/lib64/librt-2.17.so
rsyslogd 5820 root  mem       REG              202,2    19776    34471 /usr/lib64/libdl-2.17.so
rsyslogd 5820 root  mem       REG              202,2   144792  1274481 /usr/lib64/libpthread-2.17.so
rsyslogd 5820 root  mem       REG              202,2    90632    34489 /usr/lib64/libz.so.1.2.7
rsyslogd 5820 root  mem       REG              202,2   164112    23368 /usr/lib64/ld-2.17.so
rsyslogd 5820 root  mem       REG              202,2   162560     3600 /usr/lib64/libsystemd.so.0.6.0
rsyslogd 5820 root    0r      CHR                1,3      0t0     1041 /dev/null
rsyslogd 5820 root    1w      CHR                1,3      0t0     1041 /dev/null
rsyslogd 5820 root    2w      CHR                1,3      0t0     1041 /dev/null
rsyslogd 5820 root    3u     IPv4              28378      0t0      UDP *:syslog
rsyslogd 5820 root    4u     IPv6              28379      0t0      UDP *:syslog
rsyslogd 5820 root    5r  a_inode                0,9        0     5987 inotify
rsyslogd 5820 root    6u     unix 0xffff8800da61a400      0t0    28380 socket
rsyslogd 5820 root    7r      REG               0,18  8388608    27215 /run/log/journal/ca23e130dda846d5b2a30e4ab9461e43/system.journal
rsyslogd 5820 root    8u  a_inode                0,9        0     5987 [eventpoll]
rsyslogd 5820 root    9w      REG              202,2   193240     8457 /var/log/haproxy.log

Does anyone have any clues?

Best Answer

well... almost 2 years had gone by and finally found the solution for the issue. If anyone has the same issue please try the steps below.

syslogd version rsyslogd 8.24.0-38.el7

The issue was related to the imjournal module. I've remove all the entries of the rsyslog.conf related to imjournal and switched the OmitLocalLogging to off

After that I've restarted the rsyslog service using systemctl restart rsyslog and the log entries started to be populated to the log file that I've configured in rsyslog.conf.

The rsyslog.conf file now looks like this:

$ModLoad imuxsock
#$ModLoad imjournal
$ModLoad immark 
$WorkDirectory /var/lib/rsyslog
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
$IncludeConfig /etc/rsyslog.d/*.conf
$OmitLocalLogging off
#$IMJournalStateFile imjournal.state
*.info;mail.none;authpriv.none;cron.none                /var/log/messages
authpriv.*                                              /var/log/secure
mail.*                                                  -/var/log/maillog
cron.*                                                  /var/log/cron
*.emerg                                                 :omusrmsg:*
uucp,news.crit                                          /var/log/spooler
local7.*                                                /var/log/boot.log
local3.*                                                /var/log/sshd-second.log
Related Topic