I have a light memory leak apparently due to semaphores which are not released by Apache (configured as a reverse proxy). When the platform gets more loaded with additional traffic this leak gets more important over time.
Memory can be released when we issue the command below as well as restarting Apache with a hard restart. Graceful restart doesn’t release memory.
/usr/bin/ipcrm sem $(/usr/bin/ipcs -s | grep www-data | awk '{print$2}')
If we don’t force the release for the semaphores we get into an issue similar to the one described in the post below.
What are semaphores and how are they caused?
Apache then refuses to restart and we get no error message in the logs.
We already made several tests with no success in fixing the problem. We changed the sysctl.conf file with the kernel.sem values
Current values are :
$ cat /proc/sys/kernel/sem
500 64000 64 256
I deactivated the modules below one by one (and changed the Apache configuration files accordingly):
LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so
LoadModule geoip_module /usr/lib/apache2/modules/mod_geoip.so
LoadModule pagespeed_module /usr/lib/apache2/modules/mod_pagespeed_ap24.so
LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so
LoadModule proxy_protocol_module /usr/lib/apache2/modules/mod_proxy_protocol.so
Then I tried to move from mpm_event to mpm_worker, without much success as well.
The current versions of Apache and the modules are:
$ apache2 -V
Server version: Apache/2.4.7 (Ubuntu)
Server built: Mar 10 2015 13:05:59
Server's Module Magic Number: 20120211:27
Server loaded: APR 1.5.1-dev, APR-UTIL 1.5.3
Compiled using: APR 1.5.1-dev, APR-UTIL 1.5.3
Architecture: 64-bit
Server MPM: worker
threaded: yes (fixed thread count)
forked: yes (variable process count)
Server compiled with:
-D APR_HAS_SENDFILE
-D APR_HAS_MMAP
-D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
-D APR_USE_SYSVSEM_SERIALIZE
-D APR_USE_PTHREAD_SERIALIZE
-D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-D APR_HAS_OTHER_CHILD
-D AP_HAVE_RELIABLE_PIPED_LOGS
-D DYNAMIC_MODULE_LIMIT=256
-D HTTPD_ROOT="/etc/apache2"
-D SUEXEC_BIN="/usr/lib/apache2/suexec"
-D DEFAULT_PIDLOG="/var/run/apache2.pid"
-D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
-D DEFAULT_ERRORLOG="logs/error_log"
-D AP_TYPES_CONFIG_FILE="mime.types"
-D SERVER_CONFIG_FILE="apache2.conf"
Ubuntu packages:
apache2 : 2.4.7-1ubuntu4.4
apache2-bin : 2.4.7-1ubuntu4.4
apache2-data : 2.4.7-1ubuntu4.4
apache2-dev : 2.4.7-1ubuntu4.4
libapache2-mod-evasive: 1.10.1-2
libapache2-mod-security2: 2.7.7-2
libapache2-modsecurity: 2.7.7-2
libgeoip-dev : 1.6.0-1
Other compiled packages
geoip-api-mod_geoip2 : 1.2.9
mod_pagespeed : latest stable version
proxy-protocol (https://github.com/roadrunner2/mod-proxy-protocol): latest version
Here is my apache2.conf
# GENERIC DECLARATIONS
User www-data
Group www-data
Listen 80
ServerAdmin admin@xxx
ServerName server:80
ServerRoot "/etc/apache2"
# MODULES LOADING
# Generic modulesE
#LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
LoadModule mpm_worker_module /usr/lib/apache2/modules/mod_mpm_worker.so
LoadModule ratelimit_module /usr/lib/apache2/modules/mod_ratelimit.so
LoadModule reqtimeout_module /usr/lib/apache2/modules/mod_reqtimeout.so
LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so
LoadModule setenvif_module /usr/lib/apache2/modules/mod_setenvif.so
LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
LoadModule proxy_connect_module /usr/lib/apache2/modules/mod_proxy_connect.so
LoadModule proxy_http_module /usr/lib/apache2/modules/mod_proxy_http.so
LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so
LoadModule status_module /usr/lib/apache2/modules/mod_status.so
LoadModule autoindex_module /usr/lib/apache2/modules/mod_autoindex.so
LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
LoadModule authz_host_module /usr/lib/apache2/modules/mod_authz_host.so
LoadModule unique_id_module /usr/lib/apache2/modules/mod_unique_id.so
LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so
LoadModule substitute_module /usr/lib/apache2/modules/mod_substitute.so
LoadModule filter_module /usr/lib/apache2/modules/mod_filter.so
LoadModule deflate_module /usr/lib/apache2/modules/mod_deflate.so
# Specific modules
LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so
LoadModule geoip_module /usr/lib/apache2/modules/mod_geoip.so
LoadModule pagespeed_module /usr/lib/apache2/modules/mod_pagespeed_ap24.so
#LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so
LoadModule proxy_protocol_module /usr/lib/apache2/modules/mod_proxy_protocol.so
#### PROXY SETTINGS ####
# LOG FORMATS
LogFormat "%t \"%r\" %>s %I %O [%{GEOIP_COUNTRY_CODE}e] %{UNIQUE_ID}e" o
CustomLog "/var/log/access_log" o
# TIMEOUT SETTINGS
Timeout 60
ProxyTimeout 60
# ERROR LOGS
ErrorLog "/var/log/error_log"
LogLevel error
#### WEB PAGES CONFIGURATION ####
DocumentRoot "/var/www"
<Directory "/var/www">
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
DirectoryIndex index.html
<Files ".ht*">
Require all denied
</Files>
TypesConfig /etc/apache2/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
#### MODULES CONFIGURATION ####
# Server-pool management (MPM specific)
Include /etc/apache2/httpd-mpm.conf
# Secure (SSL/TLS) connections
Include /etc/apache2/httpd-ssl.conf
#### GEOIP CONF ####
GeoIPEnable On
GeoIPScanProxyHeaders On
GeoIPDBFile /usr/share/GeoIP/GeoIP.dat IndexCache
#### MODSECURITY CONF ####
SecPcreMatchLimit 15000
SecPcreMatchLimitRecursion 15000
SecTmpDir /tmp/
SecDataDir /tmp/
#### MODPAGESPEED CONF ####
ModPagespeed on
ModPagespeedInheritVHostConfig on
AddOutputFilterByType MOD_PAGESPEED_OUTPUT_FILTER text/html
AddOutputFilterByType MOD_PAGESPEED_OUTPUT_FILTER text/html
ModPagespeedXHeaderValue "Powered"
ModPagespeedRewriteLevel OptimizeForBandwidth
ModPagespeedFileCachePath "/var/cache/mod_pagespeed/"
ModPagespeedFileCacheInodeLimit 500000
ModPagespeedStatisticsLogging off
ModPagespeedMessageBufferSize 100000
Include /etc/apache2/pagespeed_libraries.conf
#### DDOS PROTECTION ####
#DOSHashTableSize 3097
#DOSPageCount 2
#DOSPageInterval 1
#DOSSiteCount 150
#DOSSiteInterval 1
#DOSBlockingPeriod 10
#DOSLogDir "/var/lock/mod_evasive"
#### VHOSTS #####
Include /etc/apache2/sites/*.conf
httpd-mpm.conf :
PidFile "/var/log/apache2/httpd.pid"
<IfModule mod_mpm_event.c>
StartServers 3
MinSpareThreads 75
MaxSpareThreads 250
ThreadsPerChild 25
MaxRequestWorkers 400
MaxConnectionsPerChild 10000
</IfModule>
<IfModule mod_mpm_worker.c>
# Default values
ServerLimit 16
MaxMemFree 2048
StartServers 2
MinSpareThreads 75
MaxSpareThreads 250
# Custom values
MaxConnectionsPerChild 10000
ThreadsPerChild 1000
MaxRequestWorkers 16000
</IfModule>
Vhosts configuration example :
<VirtualHost *:80>
#### SSL FALLBACK ####
ServerName server
ProxyProtocol on
RewriteEngine On
RewriteRule ^(.*)$ https://%{HTTP_HOST}$1 [R=301,L]
######################
</VirtualHost>
<VirtualHost *:443>
#### GENERIC CONFIGURATION ####
ServerName server
ProxyProtocol on
ProxyPreserveHost On
RewriteEngine On
RewriteRule "^/errors/(.*)" "https://server/v1/errors/$1" [R]
ProxyPass / https://server/ keepalive=On
ProxyPassReverse / https://server/
###############################
#### COMPRESSION ####
ModPagespeedFetchHttps enable
DeflateFilterNote Input instream
DeflateFilterNote Output outstream
DeflateFilterNote Ratio ratio
#####################
#### LOG FORMATS ####
LogFormat "[3] [%{Host}i] %t \"%r\" %>s [response_time %D] [request_size %I] [response_size_before_compression %O] [response_size_after_compression %{outstream}n] [country \"%{GEOIP_COUNTRY_CODE}e\"] [uuid \"%{uuid}C\"] [user-agent \"%{User-agent}i\"] [unique_id \"%{UNIQUE_ID}e\"]" O
LogFormat "[3] [%{Host}i] %t [pid %{pid}P:tid %{tid}P] [tag \"GEOBLOCKING\"] [tag \"%{GEOIP_COUNTRY_CODE}e\"] [client %a] [uri \"%U\"] [unique_id \"%{UNIQUE_ID}e\"]" GeoIPblocked
ErrorLog "/var/log/error_log"
CustomLog "/var/log/access_log" O
##########################
#### SSL CONFIGURATION ####
SSLEngine on
SSLCertificateFile "/etc/apache2/sites/3/3.crt"
SSLCertificateKeyFile "/etc/apache2/sites/3/3.key"
###########################
#### DDOS PROTECTION ####
#DOSHashTableSize 3097
#DOSPageCount 10
#DOSSiteCount 150
#DOSPageInterval 1
#DOSSiteInterval 1
#DOSBlockingPeriod 10
#############################
#### MODSECURITY ####
SecRuleEngine on
SecDefaultAction "setenv:unique_id=%{UNIQUE_ID},phase:2,redirect:/errors/block?unique_id=%{unique_id}&site_id=3"
Include /etc/apache2/modsecurity.conf
Include /etc/apache2/sites/3/3-exceptions.conf
Include /etc/apache2/sites/3/3-cms-exceptions.conf
Include /etc/apache2/sites/3/3-vp.conf
##########################
#### SCRIPT INJECTION ####
SetInputFilter DEFLATE
AddOutputFilterByType SUBSTITUTE text/html
Substitute "s\</body>\<script src=\"script_source"></script><script type=\"text/javascript\"> 1kB script </script></body>\in"
#########################################
Modsecurity.conf
SecRequestBodyAccess On
SecRule REQUEST_HEADERS:Content-Type "text/xml" "id:'200000',phase:1,nolog,allow"
SecRule REQUEST_HEADERS:Content-Type "application/json" "id:'200001',phase:1,nolog,allow"
SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 13107200
SecRequestBodyInMemoryLimit 13107200
SecRequestBodyLimitAction Reject
SecRule REQBODY_ERROR "!@eq 0" \
"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"
SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
"id:'200003',phase:2,t:none,log,deny,status:400, \
msg:'Multipart request body failed strict validation: \
PE %{REQBODY_PROCESSOR_ERROR}, \
BQ %{MULTIPART_BOUNDARY_QUOTED}, \
BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
DB %{MULTIPART_DATA_BEFORE}, \
DA %{MULTIPART_DATA_AFTER}, \
HF %{MULTIPART_HEADER_FOLDING}, \
LF %{MULTIPART_LF_LINE}, \
SM %{MULTIPART_MISSING_SEMICOLON}, \
IQ %{MULTIPART_INVALID_QUOTING}, \
IP %{MULTIPART_INVALID_PART}, \
IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"
SecRule TX:/^MSC_/ "!@streq 0" \
"id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged: %{MATCHED_VAR_NAME}'"
SecResponseBodyAccess Off
SecArgumentSeparator &
SecCookieFormat 0
SecUnicodeMapFile unicode.mapping 20127
Include /etc/apache2/rules/*.conf
Apache rules are the standard crs rules with about 50 exceptions set via the SecRuleRemoveById directive.
Best Answer
We use Apache a lot in a reverse proxy configuration and used to see memory leaks which were noticeable as the server only had 512MB RAM.
I don't know if we had a semaphore-related leak or something else, but we cured it by reducing the MaxConnectionsPerChild directive to one quarter of the default i.e. 2500. At our traffic levels this meant child processes were recycled about twice an hour, which cured the problem. There was no noticeable performance impact.