diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh index 231d0ecd5..1e0d68137 100755 --- a/data/Dockerfiles/watchdog/watchdog.sh +++ b/data/Dockerfiles/watchdog/watchdog.sh @@ -19,9 +19,11 @@ fi if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then SMTP_VERBOSE="--verbose" + CURL_VERBOSE="--verbose" set -xv else SMTP_VERBOSE="" + CURL_VERBOSE="" exec 2>/dev/null fi @@ -97,7 +99,9 @@ log_msg() { echo $(date) $(printf '%s\n' "${1}") } -function mail_error() { +function notify_error() { + # Check if one of the notification options is enabled + [[ -z ${WATCHDOG_NOTIFY_EMAIL} ]] && [[ -z ${WATCHDOG_NOTIFY_WEBHOOK} ]] && return 0 THROTTLE= [[ -z ${1} ]] && return 1 # If exists, body will be the content of "/tmp/${1}", even if ${2} is set @@ -122,37 +126,57 @@ function mail_error() { else SUBJECT="${WATCHDOG_SUBJECT}: ${1}" fi - IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}" - for rcpt in "${MAIL_RCPTS[@]}"; do - RCPT_DOMAIN= - RCPT_MX= - RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'}) - CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx) - if [[ -z ${CHECK_FOR_VALID_MX} ]]; then - log_msg "Cannot determine MX for ${rcpt}, skipping email notification..." + + # Send mail notification if enabled + if [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]]; then + IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}" + for rcpt in "${MAIL_RCPTS[@]}"; do + RCPT_DOMAIN= + RCPT_MX= + RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'}) + CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx) + if [[ -z ${CHECK_FOR_VALID_MX} ]]; then + log_msg "Cannot determine MX for ${rcpt}, skipping email notification..." + return 1 + fi + [ -f "/tmp/${1}" ] && BODY="/tmp/${1}" + timeout 10s ./smtp-cli --missing-modules-ok \ + "${SMTP_VERBOSE}" \ + --charset=UTF-8 \ + --subject="${SUBJECT}" \ + --body-plain="${BODY}" \ + --add-header="X-Priority: 1" \ + --to=${rcpt} \ + --from="watchdog@${MAILCOW_HOSTNAME}" \ + --hello-host=${MAILCOW_HOSTNAME} \ + --ipv4 + if [[ $? -eq 1 ]]; then # exit code 1 is fine + log_msg "Sent notification email to ${rcpt}" + else + if [[ "${SMTP_VERBOSE}" == "" ]]; then + log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf." + else + log_msg "Error while sending notification email to ${rcpt}." + fi + fi + done + fi + + # Send webhook notification if enabled + if [[ ! -z ${WATCHDOG_NOTIFY_WEBHOOK} ]]; then + if [[ -z ${WATCHDOG_NOTIFY_WEBHOOK_BODY} ]]; then + log_msg "No webhook body set, skipping webhook notification..." return 1 fi - [ -f "/tmp/${1}" ] && BODY="/tmp/${1}" - timeout 10s ./smtp-cli --missing-modules-ok \ - "${SMTP_VERBOSE}" \ - --charset=UTF-8 \ - --subject="${SUBJECT}" \ - --body-plain="${BODY}" \ - --add-header="X-Priority: 1" \ - --to=${rcpt} \ - --from="watchdog@${MAILCOW_HOSTNAME}" \ - --hello-host=${MAILCOW_HOSTNAME} \ - --ipv4 - if [[ $? -eq 1 ]]; then # exit code 1 is fine - log_msg "Sent notification email to ${rcpt}" - else - if [[ "${SMTP_VERBOSE}" == "" ]]; then - log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf." - else - log_msg "Error while sending notification email to ${rcpt}." - fi - fi - done + + # Replace subject and body placeholders + WEBHOOK_BODY=$(echo ${WATCHDOG_NOTIFY_WEBHOOK_BODY} | sed "s|\$SUBJECT\|\${SUBJECT}|$SUBJECT|g" | sed "s|\$BODY\|\${BODY}|$BODY|") + + # POST to webhook + curl -X POST -H "Content-Type: application/json" ${CURL_VERBOSE} -d "${WEBHOOK_BODY}" ${WATCHDOG_NOTIFY_WEBHOOK} + + log_msg "Sent notification using webhook" + fi } get_container_ip() { @@ -197,7 +221,7 @@ get_container_ip() { # One-time check if grep -qi "$(echo ${IPV6_NETWORK} | cut -d: -f1-3)" <<< "$(ip a s)"; then if [[ -z "$(get_ipv6)" ]]; then - mail_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection." + notify_error "ipv6-config" "enable_ipv6 is true in docker-compose.yml, but an IPv6 link could not be established. Please verify your IPv6 connection." fi fi @@ -746,9 +770,7 @@ olefy_checks() { } # Notify about start -if [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]]; then - mail_error "watchdog-mailcow" "Watchdog started monitoring mailcow." -fi +notify_error "watchdog-mailcow" "Watchdog started monitoring mailcow." # Create watchdog agents @@ -1029,33 +1051,33 @@ while true; do fi if [[ ${com_pipe_answer} == "ratelimit" ]]; then log_msg "At least one ratelimit was applied" - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" + notify_error "${com_pipe_answer}" elif [[ ${com_pipe_answer} == "mail_queue_status" ]]; then log_msg "Mail queue status is critical" - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" + notify_error "${com_pipe_answer}" elif [[ ${com_pipe_answer} == "external_checks" ]]; then log_msg "Your mailcow is an open relay!" # Define $2 to override message text, else print service was restarted at ... - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please stop mailcow now and check your network configuration!" + notify_error "${com_pipe_answer}" "Please stop mailcow now and check your network configuration!" elif [[ ${com_pipe_answer} == "mysql_repl_checks" ]]; then log_msg "MySQL replication is not working properly" # Define $2 to override message text, else print service was restarted at ... # Once mail per 10 minutes - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check the SQL replication status" 600 + notify_error "${com_pipe_answer}" "Please check the SQL replication status" 600 elif [[ ${com_pipe_answer} == "dovecot_repl_checks" ]]; then log_msg "Dovecot replication is not working properly" # Define $2 to override message text, else print service was restarted at ... # Once mail per 10 minutes - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check the Dovecot replicator status" 600 + notify_error "${com_pipe_answer}" "Please check the Dovecot replicator status" 600 elif [[ ${com_pipe_answer} == "certcheck" ]]; then log_msg "Certificates are about to expire" # Define $2 to override message text, else print service was restarted at ... # Only mail once a day - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please renew your certificate" 86400 + notify_error "${com_pipe_answer}" "Please renew your certificate" 86400 elif [[ ${com_pipe_answer} == "acme-mailcow" ]]; then log_msg "acme-mailcow did not complete successfully" # Define $2 to override message text, else print service was restarted at ... - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" "Please check acme-mailcow for further information." + notify_error "${com_pipe_answer}" "Please check acme-mailcow for further information." elif [[ ${com_pipe_answer} == "fail2ban" ]]; then F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null)) if [[ ! -z "${F2B_RES}" ]]; then @@ -1065,7 +1087,7 @@ while true; do log_msg "Banned ${host}" rm /tmp/fail2ban 2> /dev/null timeout 2s whois "${host}" > /tmp/fail2ban - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && [[ ${WATCHDOG_NOTIFY_BAN} =~ ^([yY][eE][sS]|[yY])+$ ]] && mail_error "${com_pipe_answer}" "IP ban: ${host}" + [[ ${WATCHDOG_NOTIFY_BAN} =~ ^([yY][eE][sS]|[yY])+$ ]] && notify_error "${com_pipe_answer}" "IP ban: ${host}" done fi elif [[ ${com_pipe_answer} =~ .+-mailcow ]]; then @@ -1085,7 +1107,7 @@ while true; do else log_msg "Sending restart command to ${CONTAINER_ID}..." curl --silent --insecure -XPOST https://dockerapi/containers/${CONTAINER_ID}/restart - [[ ! -z ${WATCHDOG_NOTIFY_EMAIL} ]] && mail_error "${com_pipe_answer}" + notify_error "${com_pipe_answer}" log_msg "Wait for restarted container to settle and continue watching..." sleep 35 fi @@ -1095,3 +1117,4 @@ while true; do kill -USR1 ${BACKGROUND_TASKS[*]} fi done + diff --git a/docker-compose.yml b/docker-compose.yml index 730e5fdf8..05fc1f2d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -457,7 +457,7 @@ services: - /lib/modules:/lib/modules:ro watchdog-mailcow: - image: mailcow/watchdog:1.98 + image: mailcow/watchdog:1.99 dns: - ${IPV4_NETWORK:-172.22.1}.254 tmpfs: @@ -487,6 +487,8 @@ services: - WATCHDOG_NOTIFY_EMAIL=${WATCHDOG_NOTIFY_EMAIL:-} - WATCHDOG_NOTIFY_BAN=${WATCHDOG_NOTIFY_BAN:-y} - WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT} + - WATCHDOG_NOTIFY_WEBHOOK=${WATCHDOG_NOTIFY_WEBHOOK} + - WATCHDOG_NOTIFY_WEBHOOK_BODY=${WATCHDOG_NOTIFY_WEBHOOK_BODY} - WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n} - WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n} - WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n} diff --git a/generate_config.sh b/generate_config.sh index 10c54c93a..72cbf0833 100755 --- a/generate_config.sh +++ b/generate_config.sh @@ -398,6 +398,13 @@ USE_WATCHDOG=y #WATCHDOG_NOTIFY_EMAIL=a@example.com,b@example.com,c@example.com #WATCHDOG_NOTIFY_EMAIL= +# Send notifications to a webhook URL that receives a POST request with the content type "application/json". +# You can use this to send notifications to services like Discord, Slack and others. +#WATCHDOG_NOTIFY_WEBHOOK=https://discord.com/api/webhooks/XXXXXXXXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# JSON body included in the webhook POST request. Needs to be in single quotes. +# Following variables are available: SUBJECT, BODY +#WATCHDOG_NOTIFY_WEBHOOK_BODY='{"username": "mailcow Watchdog", "content": "**${SUBJECT}**\n${BODY}"}' + # Notify about banned IP (includes whois lookup) WATCHDOG_NOTIFY_BAN=n diff --git a/update.sh b/update.sh index f636b21ec..cb018b4fd 100755 --- a/update.sh +++ b/update.sh @@ -441,6 +441,8 @@ CONFIG_ARRAY=( "SKIP_SOGO" "USE_WATCHDOG" "WATCHDOG_NOTIFY_EMAIL" + "WATCHDOG_NOTIFY_WEBHOOK" + "WATCHDOG_NOTIFY_WEBHOOK_BODY" "WATCHDOG_NOTIFY_BAN" "WATCHDOG_EXTERNAL_CHECKS" "WATCHDOG_SUBJECT" @@ -623,6 +625,20 @@ for option in ${CONFIG_ARRAY[@]}; do echo "#MAILDIR_SUB=Maildir" >> mailcow.conf echo "MAILDIR_SUB=" >> mailcow.conf fi + elif [[ ${option} == "WATCHDOG_NOTIFY_WEBHOOK" ]]; then + if ! grep -q ${option} mailcow.conf; then + echo "Adding new option \"${option}\" to mailcow.conf" + echo '# Send notifications to a webhook URL that receives a POST request with the content type "application/json".' >> mailcow.conf + echo '# You can use this to send notifications to services like Discord, Slack and others.' >> mailcow.conf + echo '#WATCHDOG_NOTIFY_WEBHOOK=https://discord.com/api/webhooks/XXXXXXXXXXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' >> mailcow.conf + fi + elif [[ ${option} == "WATCHDOG_NOTIFY_WEBHOOK_BODY" ]]; then + if ! grep -q ${option} mailcow.conf; then + echo "Adding new option \"${option}\" to mailcow.conf" + echo '# JSON body included in the webhook POST request. Needs to be in single quotes.' >> mailcow.conf + echo '# Following variables are available: SUBJECT, BODY' >> mailcow.conf + echo '#WATCHDOG_NOTIFY_WEBHOOK_BODY=\'{"username": "mailcow Watchdog", "content": "**${SUBJECT}**\n${BODY}"}\'' >> mailcow.conf + fi elif [[ ${option} == "WATCHDOG_NOTIFY_BAN" ]]; then if ! grep -q ${option} mailcow.conf; then echo "Adding new option \"${option}\" to mailcow.conf"