moncheck: fix error reporting of daedlines
The deadlines were wrong all over the place, so this commit finally fixes that.
This commit is contained in:
parent
2a4332951a
commit
4db00b00ed
@ -4,11 +4,15 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
@ -25,6 +29,8 @@ type (
|
||||
DB string `json:"db"`
|
||||
Wait string `json:"wait"`
|
||||
}
|
||||
|
||||
States []int
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -73,7 +79,7 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) {
|
||||
var (
|
||||
id int64
|
||||
cmdLine []string
|
||||
states []int64
|
||||
states States
|
||||
notify bool
|
||||
)
|
||||
found := false
|
||||
@ -83,7 +89,7 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) {
|
||||
tx.Rollback()
|
||||
break
|
||||
}
|
||||
if err := rows.Scan(&id, pq.Array(&cmdLine), pq.Array(&states), ¬ify); err != nil {
|
||||
if err := rows.Scan(&id, pq.Array(&cmdLine), &states, ¬ify); err != nil {
|
||||
log.Printf("could not scan values: %s", err)
|
||||
tx.Rollback()
|
||||
break
|
||||
@ -95,46 +101,44 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) {
|
||||
tx.Rollback()
|
||||
continue
|
||||
}
|
||||
statePos := 5
|
||||
if len(states) < 6 {
|
||||
statePos = len(states)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
cmd := exec.CommandContext(ctx, cmdLine[0], cmdLine[1:]...)
|
||||
output := bytes.NewBuffer([]byte{})
|
||||
cmd.Stdout = output
|
||||
cmd.Stderr = output
|
||||
if err != nil && err == context.Canceled {
|
||||
err = cmd.Run()
|
||||
if err != nil && ctx.Err() == context.DeadlineExceeded {
|
||||
log.Printf("[%d] check took too long: %s", id, err)
|
||||
cancel()
|
||||
// TODO which state to choose?
|
||||
// TODO add notification handler
|
||||
// TODO all this casting should be done better
|
||||
states = append([]int64{1}, states[:statePos]...)
|
||||
states.Add(99)
|
||||
output.Write([]byte(ctx.Err().Error()))
|
||||
} else if err != nil {
|
||||
cancel()
|
||||
exitErr, ok := err.(*exec.ExitError)
|
||||
status, ok := cmd.ProcessState.Sys().(syscall.WaitStatus)
|
||||
if !ok {
|
||||
log.Printf("[%d]error running check: %s", id, err)
|
||||
states = append([]int64{1}, states[:statePos]...)
|
||||
states.Add(1)
|
||||
} else {
|
||||
status, ok := exitErr.Sys().(syscall.WaitStatus)
|
||||
if !ok {
|
||||
} else {
|
||||
states = append([]int64{int64(status.ExitStatus())}, states[:statePos]...)
|
||||
}
|
||||
log.Printf("%s", cmd.ProcessState.String())
|
||||
states.Add(status.ExitStatus())
|
||||
}
|
||||
} else {
|
||||
cancel()
|
||||
states = append([]int64{0}, states[:statePos]...)
|
||||
states.Add(0)
|
||||
}
|
||||
msg := output.String()
|
||||
|
||||
if _, err := tx.Exec("update active_checks set next_time = now() + intval, states = $2 where check_id = $1", id, pq.Array(&states)); err != nil {
|
||||
if _, err := tx.Exec("update active_checks set next_time = now() + intval, states = $2 where check_id = $1", id, &states); err != nil {
|
||||
log.Printf("[%d] could not update row '%d': %s", thread, id, err)
|
||||
tx.Rollback()
|
||||
continue
|
||||
}
|
||||
log.Printf("meh %s", output.String())
|
||||
if notify {
|
||||
if _, err := tx.Exec("insert into notifications(check_id, states, output) values ($1, $2, $3);", &id, pq.Array(&states), output.Bytes()); err != nil {
|
||||
if _, err := tx.Exec("insert into notifications(check_id, states, output) values ($1, $2, $3);", &id, &states, &msg); err != nil {
|
||||
log.Printf("[%d] could not create notification for '%d': %s", thread, id, err)
|
||||
tx.Rollback()
|
||||
continue
|
||||
@ -143,3 +147,60 @@ func check(thread int, db *sql.DB, waitDuration time.Duration) {
|
||||
tx.Commit()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *States) Value() (driver.Value, error) {
|
||||
last := len(*s)
|
||||
if last == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
result := strings.Builder{}
|
||||
_, err := result.WriteString("{")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("could not write to buffer: %s", err)
|
||||
}
|
||||
for i, state := range *s {
|
||||
if _, err := fmt.Fprintf(&result, "%d", state); err != nil {
|
||||
return "", fmt.Errorf("could not write to buffer: %s", err)
|
||||
}
|
||||
if i < last-1 {
|
||||
if _, err := result.WriteString(","); err != nil {
|
||||
return "", fmt.Errorf("could not write to buffer: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := result.WriteString("}"); err != nil {
|
||||
return "", fmt.Errorf("could not write to buffer: %s", err)
|
||||
}
|
||||
return result.String(), nil
|
||||
}
|
||||
|
||||
func (s *States) Scan(src interface{}) error {
|
||||
switch src := src.(type) {
|
||||
case []byte:
|
||||
tmp := bytes.Trim(src, "{}")
|
||||
states := bytes.Split(tmp, []byte(","))
|
||||
result := make([]int, len(states))
|
||||
for i, state := range states {
|
||||
var err error
|
||||
result[i], err = strconv.Atoi(string(state))
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse element %s: %s", state, err)
|
||||
}
|
||||
}
|
||||
*s = result
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("could not convert %T to states", src)
|
||||
}
|
||||
}
|
||||
|
||||
// Append prepends the new state before all others.
|
||||
func (s *States) Add(state int) {
|
||||
vals := *s
|
||||
statePos := 5
|
||||
if len(vals) < 6 {
|
||||
statePos = len(vals)
|
||||
}
|
||||
*s = append([]int{state}, vals[:statePos]...)
|
||||
return
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user