Solved my problem as such.
Create two separate notification scripts in Icinga in commands.cfg
define command {
command_name Custom-Host-Notify
command_line /etc/icinga/Custom-Host-Notify "$HOSTNAME$" "$HOSTSTATE$" "$SHORTDATETIME$" "$HOSTOUTPUT$" "$HOSTADDRESS$" "$NOTIFICATIONTYPE$" "$HOSTDISPLAYNAME" "$NOTIFICATIONAUTHOR$" "$NOTIFICATIONCOMMENT$" "$HOSTNOTESURL$" "$CONTACTPAGER$" "$CONTACTEMAIL$"
register 1
}
define command {
command_name Custom-Service-Notify
command_line /etc/icinga/Custom-Service-Notify "$HOSTNAME$" "$HOSTALIAS$" "$SERVICEDESC$" "$SERVICESTATE$" "$SHORTDATETIME$" "$SERVICEOUTPUT$" "$HOSTADDRESS$" "$NOTIFICATIONTYPE$" "$SERVICEDISPLAYNAME$" "$NOTIFICATIONAUTHOR$" "$NOTIFICATIONNAME$" "$SERVICENOTESURL$" "$CONTACTPAGER$" "$CONTACTEMAIL$"
register 1
}
Define contacts.cfg as such
define contact {
contact_name John Doe
alias Senior Systems Engineer
contactgroups SystemAdministrators
host_notifications_enabled 1
service_notifications_enabled 1
host_notification_period 24x7
service_notification_period 24x7
host_notification_options d,u,r,f,s
service_notification_options w,u,c,r,f,s
host_notification_commands Custom-Host-Notify
service_notification_commands Custom-Service-Notify
email john@doe.com
##This is how we send SMS to verizon phones using email.
pager 5555555555@vtext.com
register 1
}
In icinga.cfg, enable this -
enable_environment_macros=1
In /etc/icinga/, create the two scripts that are being called -
custom-host-notify
#!/bin/bash
##Icinga passes variables in this format - Custom-Host-Notify "$HOSTNAME$" "$HOSTSTATE$" "$SHORTDATETIME$" "$HOSTOUTPUT$" "$HOSTADDRESS$" "$NOTIFICATIONTYPE$" "$HOSTDISPLAYNAME" "$NOTIFICATIONAUTHOR$" "$NOTIFICATIONCOMMENT$" $HOSTNOTESURL$" "$CONTACTPAGER$" "$CONTACTEMAIL$"
##The quotes are essential since some of the variables have spaces and we need to capture the variable as a single variable. Bash sees space separated words as separate variables.
#Define variables
HOSTNAME="$1"
HOSTSTATE="$2"
SHORTDATETIME="$3"
HOSTOUTPUT="$4"
HOSTADDRESS="$5"
NOTIFICATIONTYPE="$6"
HOSTDISPLAYNAME="$7"
NOTIFICATIONAUTHORS="$8"
NOTIFICATIONCOMMENT="$9"
HOSTNOTESURL="${10}"
CONTACTPAGER="${11}"
CONTACTEMAIL="${12}"
ENV=`echo $1 | awk '{print substr($0,8,1)}'`
##Evaluate if the environment is Production.Please note, your logic may differ here. This is based on our naming convention.
if [ "$ENV" == "p" ];then
##If Production, then send SMS
/usr/bin/printf "%b" "$NOTIFICATIONTYPE - $HOSTNAME is $HOSTSTATE $SHORTDATETIME - Info:$HOSTOUTPUT ($HOSTADDRESS$)" | /bin/mail $CONTACTPAGER$
else
/usr/bin/printf "%b" "$SHORTDATETIME - Info:\n\n$HOSTOUTPUT\n\n$NOTIFICATIONTYPE - Host $HOSTNAME is $HOSTSTATE\n\n(Address: $HOSTADDRESS Displayname: $HOSTDISPLAYNAME)\n\nNotes: $NOTIFICATIONAUTHOR $NOTIFICATIONCOMMENT\n$HOSTNOTESURL\n\nUrl:\nhttps://icinga-tech.apartmentsext.com/icinga/cgi-bin/extinfo.cgi?type=1&host=$HOSTNAME\n\n" | /bin/mail -s "$NOTIFICATIONTYPE - $HOSTNAME is $HOSTSTATE" $CONTACTEMAIL
fi
custom-service-notify
#!bin/bash
##Icinga passes variables in this format - Custom-Service-Notify "$HOSTNAME$" "$HOSTALIAS$" "$SERVICEDESC$" "$SERVICESTATE$" "$SHORTDATETIME$" "$SERVICEOUTPUT$" "$HOSTADDRESS$" "$NOTIFICATIONTYPE$" "$SERVICEDISPLAYNAME" "$NOTIFICATIONAUTHOR$" "$NOTIFICATIONNAME$" $SERVICENOTESURL$" "$CONTACTPAGER$" "$CONTACTEMAIL$"
##The quotes are essential since some of the variables have spaces and we need to capture the variable as a single variable. Bash sees space separated words as separate variables.
#Define variables
HOSTNAME="$1"
HOSTALIAS="$2"
SERVICEDESC="$3"
SERVICESTATE="$4"
SHORTDATETIME="$5"
SERVICEOUTPUT="$6"
HOSTADDRESS="$7"
NOTIFICATIONTYPE="$8"
SERVICEDISPLAYNAME="$9"
NOTIFICATIONAUTHORS="${10}"
NOTIFICATIONNAME="${11}"
SERVICENOTESURL="${12}"
CONTACTPAGER="${13}"
CONTACTEMAIL="${14}"
ENV=`echo $1 | awk '{print substr($0,8,1)}'`
##Evaluate if the environment is Production.Please note, your logic may differ here. This is based on our naming convention.
if [ "$ENV" == "p" ];then
##If Production, then send SMS
echo "$NOTIFICATIONTYPE - $HOSTALIAS/$SERVICEDESC is $SERVICESTATE on $HOSTADDRESS at $SHORTDATETIME ($SERVICEOUTPUT)" | /bin/mail $CONTACTPAGER
else
##If not production, send email
/usr/bin/printf "%b" "$SHORTDATETIME - Info:\n\n$SERVICEOUTPUT\n\n$NOTIFICATIONTYPE - Service $SERVICEDESC on Host $HOSTALIAS is $SERVICESTATE\n\n(Address: $HOSTADDRESS Displayname: $SERVICEDISPLAYNAME)\n\nNotes: $NOTIFICATIONAUTHOR $NOTIFICATIONCOMMENT\n$SERVICENOTESURL\n\nUrl:\nhttps://icinga-tech.apartmentsext.com/icinga/cgi-bin/status.cgi?host=$HOSTNAME\n\n" | /bin/mail -s "$NOTIFICATIONTYPE - $HOSTALIAS/$SERVICEDESC is $SERVICESTATE" $CONTACTEMAIL
fi
You can always customize the messages and variables to your liking.
Restart Icinga and you are good to go.
Edit: I decided to forgo implied inheritance and just set the notifications in the service-templates. You can set the notifications in the services itself.
Best Answer
This behavior has been fixed in 2.3.10, so multiple host dependencies can exist. (https://dev.icinga.org/issues/10058)
There is a whole chapter in the docs: http://docs.icinga.org/icinga2/latest/doc/module/icinga2/chapter/monitoring-basics#dependencies
What you basically do two dependencies: