iReminder feature - reed-alert - Lightweight agentless alerting system for server Err bitreich.org 70 hgit clone git://bitreich.org/reed-alert/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/reed-alert/ URL:git://bitreich.org/reed-alert/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/reed-alert/ bitreich.org 70 1Log /scm/reed-alert/log.gph bitreich.org 70 1Files /scm/reed-alert/files.gph bitreich.org 70 1Refs /scm/reed-alert/refs.gph bitreich.org 70 1Tags /scm/reed-alert/tag bitreich.org 70 1README /scm/reed-alert/file/README.gph bitreich.org 70 1LICENSE /scm/reed-alert/file/LICENSE.gph bitreich.org 70 i--- Err bitreich.org 70 1commit bd556831403537df5f315d15421ec9a7bfe5a102 /scm/reed-alert/commit/bd556831403537df5f315d15421ec9a7bfe5a102.gph bitreich.org 70 1parent a98fb0c772193a5b2bda84c49dbf51d140c6b89e /scm/reed-alert/commit/a98fb0c772193a5b2bda84c49dbf51d140c6b89e.gph bitreich.org 70 hAuthor: Solene Rapenne URL:mailto:solene@perso.pw bitreich.org 70 iDate: Tue, 15 Jan 2019 17:21:18 +0100 Err bitreich.org 70 i Err bitreich.org 70 iReminder feature Err bitreich.org 70 i Err bitreich.org 70 iDiffstat: Err bitreich.org 70 i M README | 10 +++++++++- Err bitreich.org 70 i M functions.lisp | 21 +++++++++++++++++---- Err bitreich.org 70 i Err bitreich.org 70 i2 files changed, 26 insertions(+), 5 deletions(-) Err bitreich.org 70 i--- Err bitreich.org 70 1diff --git a/README b/README /scm/reed-alert/file/README.gph bitreich.org 70 i@@ -95,7 +95,8 @@ The Notification System Err bitreich.org 70 i Err bitreich.org 70 i When a check return a failure, a previously defined notifier will be Err bitreich.org 70 i called. This will be triggered only after reed-alert find **3** Err bitreich.org 70 i-failures (not more or less) in a row for this check, this is a default Err bitreich.org 70 i+failures (not more or less, but this can be changed globally by Err bitreich.org 70 i+modifying *tries* variable) in a row for this check, this is a default Err bitreich.org 70 i value that can be changed per probe with the :try parameter as Err bitreich.org 70 i explained later in this document. This is to prevent reed-alert to Err bitreich.org 70 i spam notifications for a long time (number of failures very high, like Err bitreich.org 70 i@@ -108,6 +109,13 @@ reed-alert will use the notifier system when it reach its try number Err bitreich.org 70 i and when the problem is fixed, so you know when it begins and when it Err bitreich.org 70 i ends. Err bitreich.org 70 i Err bitreich.org 70 i+It is possible to be reminded about a failure every n tries by setting Err bitreich.org 70 i+the keyword :reminder and using a number. This is useful if you want Err bitreich.org 70 i+to be reminded from time to time if a problem is not fixed, using some Err bitreich.org 70 i+alerts like mails can be easily overlooked or lost in a huge mail Err bitreich.org 70 i+amount. The :reminder is a setting per check. For a global reminder Err bitreich.org 70 i+setting, one can set *reminder* variable. Err bitreich.org 70 i+ Err bitreich.org 70 i reed-alert keep tracks of the count of failures with one file per Err bitreich.org 70 i probe failing in the "states" folder. To ensure unique filenames, the Err bitreich.org 70 i following format is used (+ means it's concatenated) : Err bitreich.org 70 1diff --git a/functions.lisp b/functions.lisp /scm/reed-alert/file/functions.lisp.gph bitreich.org 70 i@@ -3,6 +3,7 @@ Err bitreich.org 70 i (require 'asdf)) Err bitreich.org 70 i Err bitreich.org 70 i (defparameter *tries* 3) Err bitreich.org 70 i+(defparameter *reminder* 0) Err bitreich.org 70 i (defparameter *alerts* '()) Err bitreich.org 70 i (defparameter *states-dir* "~/.reed-alert/states/") Err bitreich.org 70 i (ensure-directories-exist *states-dir*) Err bitreich.org 70 i@@ -69,7 +70,10 @@ Err bitreich.org 70 i (defun trigger-alert(level function params result state) Err bitreich.org 70 i (let* ((notifier-command (assoc level *alerts*)) Err bitreich.org 70 i (command-string (cadr notifier-command))) Err bitreich.org 70 i- (setf command-string (replace-all command-string "%state%" (if (eql 'error state) "Start" "End"))) Err bitreich.org 70 i+ (setf command-string (replace-all command-string "%state%" (cond Err bitreich.org 70 i+ ((eql state 'START) "Begin") Err bitreich.org 70 i+ ((eql state 'REMINDER) "Reminder") Err bitreich.org 70 i+ (t "End")))) Err bitreich.org 70 i (setf command-string (replace-all command-string "%result%" (format nil "~a" result))) Err bitreich.org 70 i (setf command-string (replace-all command-string "%hostname%" (machine-instance))) Err bitreich.org 70 i (setf command-string (replace-all command-string "%os%" (software-type))) Err bitreich.org 70 i@@ -84,7 +88,7 @@ Err bitreich.org 70 i (get-decoded-time) Err bitreich.org 70 i (format nil "~a/~a/~a ~a:~a:~a" year month day hour minute second)))) Err bitreich.org 70 i command-string)) Err bitreich.org 70 i- Err bitreich.org 70 i+ Err bitreich.org 70 i (defmacro stop-if-error(&body body) Err bitreich.org 70 i `(progn Err bitreich.org 70 i (and ,@body))) Err bitreich.org 70 i@@ -129,12 +133,21 @@ Err bitreich.org 70 i t) Err bitreich.org 70 i Err bitreich.org 70 i ;; failure handling Err bitreich.org 70 i- (let ((trigger-now? (= (+ 1 tries) (getf params :try *tries*)))) ; we add +1 because it's failing right now Err bitreich.org 70 i+ (let ((trigger-now? (or Err bitreich.org 70 i+ ;; we add +1 to tries because it's failing right now Err bitreich.org 70 i+ (and (= (+ 1 tries) (getf params :try *tries*)) Err bitreich.org 70 i+ 'START) ;; it starts failing Err bitreich.org 70 i+ Err bitreich.org 70 i+ ;; if reminder is set and a valid value (> 0) Err bitreich.org 70 i+ (when (< 0 (getf params :reminder *reminder*)) Err bitreich.org 70 i+ (and (= 0 (mod (+ 1 tries) (getf params :reminder *reminder*))) Err bitreich.org 70 i+ 'REMINDER))))) ;; do we need to remind it's failing? Err bitreich.org 70 i+ Err bitreich.org 70 i (format t " => ~aerror (~a failure(s) before)~a~a~%" *red* tries *white* (if trigger-now? " NOTIFIED" "")) Err bitreich.org 70 i Err bitreich.org 70 i ;; more error than limit, send alert once Err bitreich.org 70 i (when trigger-now? Err bitreich.org 70 i- (uiop:run-program (trigger-alert level fonction params (cadr result) 'error) :output t)) Err bitreich.org 70 i+ (uiop:run-program (trigger-alert level fonction params (cadr result) trigger-now?) :output t)) Err bitreich.org 70 i ;; increment the number of tries by 1 Err bitreich.org 70 i (with-open-file (stream-out filepath :direction :output Err bitreich.org 70 i :if-exists :supersede) Err bitreich.org 70 .