FreeRadius + Active Directory + Google Authenticator

I have FreeRadius 3.0.13 installed on CentOS 7.3 which also has SSSD 1.14.0 which is being used to communicate with our Windows 2012 Domain controller. We are able to authenticate using AD via radius.
We also have google authenticator installed on this Radius server.
We are able to connect to our openvpn server and authentication using AD and Google is good, have no issues here.

I am however having issues trying to only allow users in a certain AD group to authenicate.
I'm using the 'pam_sss' module to do the authentication against AD.

This is my 'pam.d/radius' config

#%PAM-1.0
auth       requisite    pam_google_authenticator.so forward_pass
auth       required     pam_sss.so use_first_pass
account    required     pam_nologin.so
account    include      password-auth
session    include      password-auth

Here is my 'raddb/users' config

    DEFAULT Auth-Type := PAM
    #DEFAULT Group == "remoteaccess", Auth-Type := Reject
    #        Reply-Message = "You are a member of the Correct remoteaccess Group"

    DEFAULT Framed-Protocol == PPP
            Framed-Protocol = PPP,
            Framed-Compression = Van-Jacobson-TCP-IP
    DEFAULT Hint == "CSLIP"
            Framed-Protocol = SLIP,
            Framed-Compression = Van-Jacobson-TCP-IP

And this is my 'raddb/default' config

server default {
listen {
        type = auth
        ipaddr = *
        port = 0
        limit {
              max_connections = 16
              lifetime = 0
              idle_timeout = 300
        }
}
listen {
        ipaddr = *
        port = 0
        type = acct

        limit {
                idle_timeout = 300
        }
}
listen {
        type = auth
        ipv6addr = ::   # any.  ::1 == localhost
        port = 0
        limit {
              max_connections = 16
              lifetime = 0
              idle_timeout = 300
        }
}
listen {
        ipv6addr = ::
        port = 0
        type = acct
        limit {
        }
}
authorize {
        filter_username
        preprocess
        chap
        mschap
        digest
        suffix
        eap {
                ok = return
        }
        files
        -sql
        -ldap
        expiration
        logintime
        pap
}
authenticate {
        Auth-Type PAP {
                pap
        }
        Auth-Type CHAP {
                chap
        }
        Auth-Type MS-CHAP {
                mschap
        }
        mschap
        digest
        pam
        eap
}
preacct {
        preprocess
        acct_unique
        suffix
        files
}
accounting {
        detail
        unix
        -sql
        exec
        attr_filter.accounting_response
}
session {
}
post-auth {
        update {
                &reply: += &session-state:
        }
        -sql
        exec
        remove_reply_message_if_eap
        Post-Auth-Type REJECT {
                -sql
                attr_filter.access_reject
                eap
                remove_reply_message_if_eap
        }
        Post-Auth-Type Challenge {
        }

}
pre-proxy {
}
post-proxy {
        eap
}
}

And here is the raddb/radius config

prefix = /usr
exec_prefix = /usr
sysconfdir = /etc
localstatedir = /var
sbindir = /usr/sbin
logdir = ${localstatedir}/log/radius
raddbdir = ${sysconfdir}/raddb
radacctdir = ${logdir}/radacct
name = radiusd
confdir = ${raddbdir}
modconfdir = ${confdir}/mods-config
certdir = ${confdir}/certs
cadir   = ${confdir}/certs
run_dir = ${localstatedir}/run/${name}
db_dir = ${localstatedir}/lib/radiusd
debug_level = 9
libdir = /usr/lib64/freeradius
pidfile = ${run_dir}/${name}.pid
correct_escapes = true
max_request_time = 30
cleanup_delay = 5
max_requests = 16384
hostname_lookups = no
log {
        destination = files
        colourise = yes
        file = ${logdir}/radius.log
        syslog_facility = daemon
        stripped_names = yes
        auth = yes
        auth_badpass = yes
        auth_goodpass = yes
        msg_denied = "You are already logged in - access denied"
}
checkrad = ${sbindir}/checkrad
security {
        user = root
        group = root
        allow_core_dumps = no
        max_attributes = 200
        reject_delay = 1
        status_server = yes
}
proxy_requests  = yes
$INCLUDE proxy.conf
$INCLUDE clients.conf
thread pool {
        start_servers = 5
        max_servers = 32
        min_spare_servers = 3
        max_spare_servers = 10
        max_requests_per_server = 0
        auto_limit_acct = no
}
modules {
        $INCLUDE mods-enabled/
}

instantiate {
}

policy {
        $INCLUDE policy.d/
}
$INCLUDE sites-enabled/

Anyone got any ideas on how to check for AD group and reject depending on what group the user is or is not in?