Update: Problem is reported on version 1.9.1 and is not present in up-to-date versions of Docker.
I have an docker app running in elasticbeanstalk that hangs after about 10-11 days of uptime.
The app runs a cron job every minute. There is no long-running application process (of mine).
Once it has hung, it no longer responds to HTTP requests.
The following docker
commands hang: top
, exec
, restart
docker logs
responds but shows no error.
I do not know how to debug further from this point. Any ideas?
docker inspect
shows:
{
"Id": "a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444",
"Created": "2016-06-16T22:29:57.448398251Z",
"Path": "/usr/local/bin/init.sh",
"Args": [],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 4406,
"ExitCode": 0,
"Error": "",
"StartedAt": "2016-06-16T22:29:58.459393626Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "6aa1c5468fbe4102e7428d4325f666868498b9880c9dc3f14c30c2a6804a25a5",
"ResolvConfPath": "/var/lib/docker/containers/a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444/hostname",
"HostsPath": "/var/lib/docker/containers/a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444/hosts",
"LogPath": "/var/lib/docker/containers/a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444/a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444-json.log",
"Name": "/cranky_hugle",
"RestartCount": 0,
"Driver": "devicemapper",
"ExecDriver": "native-0.2",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": [
"8de1c20fc2c90e7a41c709e226ecd3401db18c6e0044527b4cc205d15167ac6d"
],
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LxcConf": [],
"Memory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"KernelMemory": 0,
"CpuShares": 0,
"CpuPeriod": 0,
"CpusetCpus": "",
"CpusetMems": "",
"CpuQuota": 0,
"BlkioWeight": 0,
"OomKillDisable": false,
"MemorySwappiness": -1,
"Privileged": false,
"PortBindings": {},
"Links": null,
"PublishAllPorts": false,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"VolumesFrom": null,
"Devices": [],
"NetworkMode": "default",
"IpcMode": "",
"PidMode": "",
"UTSMode": "",
"CapAdd": null,
"CapDrop": null,
"GroupAdd": null,
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"SecurityOpt": null,
"ReadonlyRootfs": false,
"Ulimits": null,
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"CgroupParent": "",
"ConsoleSize": [
0,
0
],
"VolumeDriver": ""
},
"GraphDriver": {
"Name": "devicemapper",
"Data": {
"DeviceId": "46",
"DeviceName": "docker-202:1-263679-a9e7ed019ce543073c16507036d1a5579817a0e0abf79e6f79949bcc96dca444",
"DeviceSize": "107374182400"
}
},
"Mounts": [],
"Config": {
"Hostname": "a9e7ed019ce5",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"443/tcp": {},
"80/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"ENV=prd",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"APACHE_RUN_USER=www-data",
"APACHE_RUN_GROUP=www-data",
"APACHE_LOG_DIR=/var/log/apache2",
"GIT_REV=76bf2a3207482a0dfc06551374056480946825ff"
],
"Cmd": [
"/usr/local/bin/init.sh"
],
"Image": "6aa1c5468fbe",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"OnBuild": null,
"Labels": {},
"StopSignal": "SIGTERM"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "*********",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"443/tcp": null,
"80/tcp": null
},
"SandboxKey": "/var/run/docker/netns/*********",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "*********",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "*********",
"Networks": {
"bridge": {
"EndpointID": "*********",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "*********"
}
}
}
}
]
Best Answer
The version of Docker you're running is chock full of bugs that can cause problems managing containers. We were whacking containers left and right until Docker 1.11 came out, the new architecture of which makes is much more stable and reliable in this sense. I strongly recommend upgrading.