From 4db00b00ed85e927276ccd67cbc6fb1964e0be65 Mon Sep 17 00:00:00 2001 From: Gibheer Date: Sun, 18 Nov 2018 21:38:04 +0100 Subject: moncheck: fix error reporting of daedlines The deadlines were wrong all over the place, so this commit finally fixes that. --- cmd/moncheck/main.go | 97 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 18 deletions(-) (limited to 'cmd') diff --git a/cmd/moncheck/main.go b/cmd/moncheck/main.go index dba3099..2e549b7 100644 --- a/cmd/moncheck/main.go +++ b/cmd/moncheck/main.go @@ -4,11 +4,15 @@ import ( "bytes" "context" "database/sql" + "database/sql/driver" "encoding/json" "flag" + "fmt" "io/ioutil" "log" "os/exec" + "strconv" + "strings" "sync" "syscall" "time" @@ -25,6 +29,8 @@ type ( DB string `json:"db"` Wait string `json:"wait"` } + + States []int ) func main() { @@ -73,7 +79,7 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) { var ( id int64 cmdLine []string - states []int64 + states States notify bool ) found := false @@ -83,7 +89,7 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) { tx.Rollback() break } - if err := rows.Scan(&id, pq.Array(&cmdLine), pq.Array(&states), ¬ify); err != nil { + if err := rows.Scan(&id, pq.Array(&cmdLine), &states, ¬ify); err != nil { log.Printf("could not scan values: %s", err) tx.Rollback() break @@ -95,46 +101,44 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) { tx.Rollback() continue } - statePos := 5 - if len(states) < 6 { - statePos = len(states) - } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) cmd := exec.CommandContext(ctx, cmdLine[0], cmdLine[1:]...) output := bytes.NewBuffer([]byte{}) cmd.Stdout = output cmd.Stderr = output - if err != nil && err == context.Canceled { + err = cmd.Run() + if err != nil && ctx.Err() == context.DeadlineExceeded { log.Printf("[%d] check took too long: %s", id, err) + cancel() // TODO which state to choose? // TODO add notification handler // TODO all this casting should be done better - states = append([]int64{1}, states[:statePos]...) + states.Add(99) + output.Write([]byte(ctx.Err().Error())) } else if err != nil { cancel() - exitErr, ok := err.(*exec.ExitError) + status, ok := cmd.ProcessState.Sys().(syscall.WaitStatus) if !ok { log.Printf("[%d]error running check: %s", id, err) - states = append([]int64{1}, states[:statePos]...) + states.Add(1) } else { - status, ok := exitErr.Sys().(syscall.WaitStatus) - if !ok { - } else { - states = append([]int64{int64(status.ExitStatus())}, states[:statePos]...) - } + log.Printf("%s", cmd.ProcessState.String()) + states.Add(status.ExitStatus()) } } else { cancel() - states = append([]int64{0}, states[:statePos]...) + states.Add(0) } + msg := output.String() - if _, err := tx.Exec("update active_checks set next_time = now() + intval, states = $2 where check_id = $1", id, pq.Array(&states)); err != nil { + if _, err := tx.Exec("update active_checks set next_time = now() + intval, states = $2 where check_id = $1", id, &states); err != nil { log.Printf("[%d] could not update row '%d': %s", thread, id, err) tx.Rollback() continue } + log.Printf("meh %s", output.String()) if notify { - if _, err := tx.Exec("insert into notifications(check_id, states, output) values ($1, $2, $3);", &id, pq.Array(&states), output.Bytes()); err != nil { + if _, err := tx.Exec("insert into notifications(check_id, states, output) values ($1, $2, $3);", &id, &states, &msg); err != nil { log.Printf("[%d] could not create notification for '%d': %s", thread, id, err) tx.Rollback() continue @@ -143,3 +147,60 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) { tx.Commit() } } + +func (s *States) Value() (driver.Value, error) { + last := len(*s) + if last == 0 { + return "{}", nil + } + result := strings.Builder{} + _, err := result.WriteString("{") + if err != nil { + return "", fmt.Errorf("could not write to buffer: %s", err) + } + for i, state := range *s { + if _, err := fmt.Fprintf(&result, "%d", state); err != nil { + return "", fmt.Errorf("could not write to buffer: %s", err) + } + if i < last-1 { + if _, err := result.WriteString(","); err != nil { + return "", fmt.Errorf("could not write to buffer: %s", err) + } + } + } + if _, err := result.WriteString("}"); err != nil { + return "", fmt.Errorf("could not write to buffer: %s", err) + } + return result.String(), nil +} + +func (s *States) Scan(src interface{}) error { + switch src := src.(type) { + case []byte: + tmp := bytes.Trim(src, "{}") + states := bytes.Split(tmp, []byte(",")) + result := make([]int, len(states)) + for i, state := range states { + var err error + result[i], err = strconv.Atoi(string(state)) + if err != nil { + return fmt.Errorf("could not parse element %s: %s", state, err) + } + } + *s = result + return nil + default: + return fmt.Errorf("could not convert %T to states", src) + } +} + +// Append prepends the new state before all others. +func (s *States) Add(state int) { + vals := *s + statePos := 5 + if len(vals) < 6 { + statePos = len(vals) + } + *s = append([]int{state}, vals[:statePos]...) + return +} -- cgit v1.2.3-70-g09d2