add container functions

This is the first major container draft and implements most of the
functions to create containers, remove them, show and set attributes on
them.

It also implements the special case in dim, that a list of all
containers and their free space is returned.
It currently does not implement a partial output when a subnet or
layer3domain is returned.
First I want to know how well it actually scales and works as this is a
major pain point in the python implementation.

At the moment the output also has the problem that it can grow quite
large in memory as the tree is built in the middleware. A better way
would be to build the json directly in the database so it can be
returned directly. We will have to see when this becomes a major issue.

container.go

@@ -0,0 +1,263 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"fmt"
+
+	"dim/query"
+	"dim/types"
+	"github.com/lib/pq"
+)
+
+// ContainerCreate will create a new container.
+func containerCreate(c *Context, req Request, res *Response) error {
+	subnet := types.Subnet{}
+	options := struct {
+		Attributes       types.FieldMap `json:"attributes"`
+		DisallowChildren bool           `json:"disallow_children"`
+		Layer3Domain     string         `json:"layer3domain"`
+		AllowOverlap     bool           `json:"allow_overlap"`
+	}{
+		Attributes: types.FieldMap{},
+	}
+	if err := req.ParseAtLeast(1, &subnet, &options); err != nil {
+		res.AddMessage(LevelError, "could not parse options: %s", err)
+		return nil
+	}
+	if options.Layer3Domain == "" {
+		res.AddMessage(LevelError, "layer3domain name is empty")
+		return nil
+	}
+	attrs, err := options.Attributes.MarshalJSON()
+	if err != nil {
+		res.AddMessage(LevelError, "could not encode attributes to json: %s", err)
+		return nil
+	}
+
+	l3id := 0
+	err = c.tx.QueryRow(`select id from layer3domains where name = $1`, options.Layer3Domain).Scan(&l3id)
+	if err != nil {
+		res.AddMessage(LevelError, "could not resolve layer3domain")
+		return fmt.Errorf("could not resolve layer3domain '%s': %#v", options.Layer3Domain, err)
+	}
+
+	_, err = c.tx.Exec(`insert into containers(layer3domain_id, subnet, created_by, modified_by, attributes)
+		values ($1, $2, $3, $3, $4)`, l3id, subnet.String(), c.username, attrs)
+	if err != nil {
+		res.AddMessage(LevelError, "could not create new container")
+		return fmt.Errorf("could not create new container '%s': %#v", subnet.String(), err)
+	}
+	res.AddMessage(LevelInfo, "created container '%s'", subnet.String())
+	return nil
+}
+
+// ContainerDelete deletes a container.
+func containerDelete(c *Context, req Request, res *Response) error {
+	subnet := types.Subnet{}
+	options := struct {
+		Layer3Domain string `json:"layer3domain"`
+		Recursive    bool   `json:"recursive"`
+	}{}
+	if err := req.ParseAtLeast(1, &subnet, &options); err != nil {
+		res.AddMessage(LevelError, "could not parse options: %s", err)
+		return nil
+	}
+
+	l3id := 0
+	err := c.tx.QueryRow(`select id from layer3domains where name = $1 for update`, options.Layer3Domain).
+		Scan(&l3id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			res.AddMessage(LevelError, "layer3domain '%s' does not exist", options.Layer3Domain)
+			return nil
+		}
+		res.AddMessage(LevelError, "could not fetch layer3domain '%s'", options.Layer3Domain)
+		return fmt.Errorf("could not fetch layer3domain '%s': %#v", options.Layer3Domain, err)
+	}
+	// TODO implement Recursive (delete all containers and IPs in this subnet)
+	_, err = c.tx.Exec(`delete from containers where subnet = $1 and layer3domain_id = $2`, subnet.String(), l3id)
+	if err != nil {
+		res.AddMessage(LevelError, "could not delete container '%s' in layer3domain '%s'", subnet.String(), options.Layer3Domain)
+		return fmt.Errorf("could not delete subnets for container '%s': %#v", subnet.String(), err)
+	}
+	res.AddMessage(LevelInfo, "deleted container '%s' in layer3domain '%s'", subnet.String(), options.Layer3Domain)
+	return nil
+}
+
+// ContainerList lists all containers with the requested fields.
+func containerList(c *Context, req Request, res *Response) error {
+	options := struct {
+		Attributes   types.FieldList `json:"attributes"`
+		Layer3Domain string          `json:"layer3domain"`
+		IPBlock      types.Subnet    `json:"ipblock"`
+		IPVersion    types.IPVersion `json:"version"`
+		Depth        int             `json:"depth"`
+		Status       string          `json:"status"` // TODO not supported yet
+	}{
+		Attributes: types.NewFieldList("subnet", "", "subnets"),
+	}
+	if err := req.ParseAtLeast(0, &options); err != nil {
+		res.AddMessage(LevelError, "could not parse options")
+		return nil
+	}
+
+	rawQuery := `select cfl.layer3domain_id, cfl.subnet, cfl.parents, c.attributes, cfl.state
+		from containers_free_list cfl
+		left join containers c on cfl.layer3domain_id = c.layer3domain_id and cfl.subnet = c.subnet`
+	args := []interface{}{}
+	if options.Layer3Domain != "" {
+		l3id := 0
+		err := c.tx.QueryRow(`select id from layer3domains where name = $1`, options.Layer3Domain).Scan(&l3id)
+		if err != nil {
+			res.AddMessage(LevelError, "could not resolve layer3domain")
+			return fmt.Errorf("could not resolve layer3domain '%s': %#v", options.Layer3Domain, err)
+		}
+		args = append(args, l3id)
+		rawQuery += " where layer3domain_id = $1"
+	}
+
+	rows, err := c.tx.Query(rawQuery, args...)
+	if err != nil {
+		res.AddMessage(LevelError, "could not fetch containers")
+		return fmt.Errorf("could not fetch container tree: %#v", err)
+	}
+	defer rows.Close()
+	type (
+		Container struct {
+			Status     string               `json:"status"`
+			Attributes json.RawMessage      `json:"attributes"`
+			Containers map[string]Container `json:"containers"`
+		}
+	)
+	result := map[int]Container{}
+	for rows.Next() {
+		cont := Container{Containers: map[string]Container{}}
+		parents := &[]string{}
+		l3id := 0
+		subnet := ""
+		attr := sql.NullString{}
+		if err := rows.Scan(&l3id, &subnet, pq.Array(parents), &attr, &cont.Status); err != nil {
+			res.AddMessage(LevelError, "could not scan containers")
+			return fmt.Errorf("could not scan containers: %#v", err)
+		}
+		if _, found := result[l3id]; !found {
+			result[l3id] = Container{Status: "layer3domain", Containers: map[string]Container{}}
+		}
+		current := result[l3id]
+		for _, parent := range *parents {
+			if _, found := current.Containers[parent]; !found {
+				current.Containers[parent] = Container{Status: "container", Containers: map[string]Container{}}
+			}
+			current = current.Containers[parent]
+		}
+		if attr.Valid {
+			cont.Attributes = json.RawMessage(attr.String)
+		}
+		current.Containers[subnet] = cont
+	}
+	res.Result = result
+	return nil
+}
+
+func containerGetAttr(c *Context, req Request, res *Response) error {
+	subnet := types.Subnet{}
+	options := struct {
+		Layer3Domain string
+	}{}
+	if err := req.ParseAtLeast(2, &subnet, &options); err != nil {
+		res.AddMessage(LevelError, "could not parse options: %s", err)
+		return nil
+	}
+
+	result := json.RawMessage{}
+	selClause := query.FieldsToJSON("c", map[string]string{
+		"subnet":       "c.subnet",
+		"modified_by":  "c.modified_by",
+		"modified_at":  "c.modified_at",
+		"created_by":   "c.created_by",
+		"created_at":   "c.created_at",
+		"layer3domain": "l.name",
+	})
+	queryStr := fmt.Sprintf(`select %s from containers c join layer3domains l
+		on c.layer3domain_id = l.id
+		where c.subnet = $1 and l.name = $2`, selClause)
+	err := c.tx.QueryRow(queryStr, subnet, options.Layer3Domain).Scan(&result)
+	if err != nil {
+		res.AddMessage(LevelError, "could not return result")
+		return fmt.Errorf("could not get container '%s': %s - query: %s", subnet, err, queryStr)
+	}
+	res.Result = result
+	return nil
+}
+
+func containerSetAttr(c *Context, req Request, res *Response) error {
+	subnet := types.Subnet{}
+	attrs := types.FieldMap{}
+	options := struct {
+		Layer3Domain string
+	}{}
+	if err := req.ParseAtLeast(3, &subnet, &attrs, &options); err != nil {
+		res.AddMessage(LevelError, "could not parse options: %s", err)
+		return nil
+	}
+	if attrs.Size() == 0 {
+		res.AddMessage(LevelError, "no key/value pairs provided to update")
+		return nil
+	}
+	if options.Layer3Domain == "" {
+		res.AddMessage(LevelError, "layer3domain is required")
+		return nil
+	}
+	if attrs.Contains("layer3domain") {
+		res.AddMessage(LevelError, "can't change the layer3domain of a subnet")
+		return nil
+	}
+
+	// TODO this is ugly. Can we have better API somehow?
+	fieldMap := map[string]string{
+		"layer3domain_id": "",
+		"subnet":          "",
+		"modified_by":     "",
+		"modified_at":     "",
+		"created_by":      "",
+		"created_at":      "",
+	}
+	if attrs.Contains("subnets") {
+		res.AddMessage(LevelError, "can not set subnets as attributes")
+		return nil
+	}
+
+	l3name := options.Layer3Domain
+	l3id := 0
+	err := c.tx.QueryRow(`select id from layer3domains where name = $1`, l3name).Scan(&l3id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			res.AddMessage(LevelError, "layer3domain '%s' does not exist", l3name)
+			return nil
+		}
+		res.AddMessage(LevelError, "could not get layer3domain")
+		return fmt.Errorf("could not fetch layer3domain id for name '%s': %#v", l3name, err)
+	}
+
+	setClause, args, err := query.FieldMapToUpdate(attrs, fieldMap)
+	if err != nil {
+		res.AddMessage(LevelError, "could not encode requested attributes: %s", err)
+		return nil
+	}
+	queryStr := fmt.Sprintf("update containers p set %s where subnet = $%d::cidr and layer3domain_id = $%d returning subnet", setClause, len(args)+1, len(args)+2)
+	args = append(args, subnet) // don't forget to add the where clause parameter
+	args = append(args, l3id)   // don't forget to add the layer3domain to the where clause
+	c.Debugf(LevelInfo, "query: %s - args: %#v", queryStr, args)
+	changed := ""
+	if err := c.tx.QueryRow(queryStr, args...).Scan(&changed); err != nil {
+		if err == sql.ErrNoRows {
+			res.AddMessage(LevelError, "subnet '%s' in layer3domain '%s' does not exist", subnet.String(), l3name)
+			return nil
+		}
+		res.AddMessage(LevelError, "could not set attributes")
+		c.Logf(LevelError, "could not set attributes on subnet '%s': %s - query: `%s` - args: `%#v`", subnet, err, queryStr, args)
+		return nil
+	}
+	return nil
+}

main.go

@@ -23,6 +23,7 @@ type (
 			Type       string `toml:"type"`
 			Connection string `toml:"conn"`
 		} `toml:"db"`
+		Debug bool `toml:"debug_mode"`
 	}
 )
 
@@ -59,7 +60,7 @@ func main() {
 		return
 	}
 
-	s, err := NewServer(db)
+	s, err := NewServer(db, cfg.Debug)
 	if err != nil {
 		log.Fatalf("could not create server instance: %s", err)
 		return

query/query.go

@@ -80,6 +80,9 @@ func FieldMapToUpdate(fm types.FieldMap, nameMap map[string]string) (string, []i
 	i := 0
 	for key, val := range fm.Fields() {
 		if name, found := nameMap[key]; found {
+			if name == "" {
+				continue
+			}
 			i++
 			setClause = append(setClause, fmt.Sprintf("%s = $%d", name, i))
 			if val == "" {

schema/01_initial.sql

@@ -1,7 +1,7 @@
 create table if not exists layer3domains(
   id serial not null primary key,
   name varchar(128) not null unique,
-  attributes jsonb not null default '{}',
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -12,7 +12,7 @@ create table if not exists pools(
   id serial not null,
   layer3domain_id integer not null references layer3domains(id),
   name varchar(128) unique,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -24,7 +24,7 @@ create table containers(
 	layer3domain_id integer not null references layer3domains(id),
  	subnet cidr not null,
  	pool_id integer,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -50,7 +50,7 @@ create table if not exists ips(
   layer3domain_id integer not null,
   version smallint not null,
   address inet not null,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -61,7 +61,7 @@ create table if not exists ips(
 create table if not exists zones(
   id serial not null primary key,
   name varchar not null unique,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -80,7 +80,7 @@ create table if not exists zoneviews(
   retry integer not null default 900,
   expire integer not null default 604800,
   minimum bigint not null default 86400,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -94,7 +94,7 @@ create table if not exists records(
   type varchar(11) not null,
   ttl integer,
   value text not null,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -105,7 +105,7 @@ create table if not exists records(
 create table if not exists outputgroups(
   id serial not null primary key,
   name varchar(128) not null unique,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),
@@ -124,7 +124,7 @@ create table if not exists outputs(
   plugin varchar(20) not null,
   db_uri varchar(250) not null,
   status varchar(250) not null,
-  attributes jsonb not null default '{}'::jsonb,
+  attributes jsonb default '{}'::jsonb constraint attributes_not_null check(attributes is not null and attributes != 'null'::jsonb),
   created_at timestamptz not null default now(),
   created_by varchar(128) not null,
   modified_at timestamptz not null default now(),

server.go

@@ -21,6 +21,7 @@ type (
 	Server struct {
 		db     *sql.DB
 		routes map[string]Handler
+		debug  bool
 	}
 
 	// Handler is a function receiving a Context to process a request.
@@ -31,9 +32,10 @@ type (
 	// It contains a prepared transaction for usage and important details like
 	// the user account.
 	Context struct {
-		id  string
+		id    string
-		req *http.Request
+		req   *http.Request
-		w   http.ResponseWriter
+		w     http.ResponseWriter
+		debug bool // print debug output to the console
 
 		username string
 		tx       *sql.Tx
@@ -59,13 +61,14 @@ type (
 )
 
 // NewServer creates a new server handler.
-func NewServer(db *sql.DB) (*Server, error) {
+func NewServer(db *sql.DB, debug bool) (*Server, error) {
 	if db == nil {
 		return nil, fmt.Errorf("database connection is not set")
 	}
 	return &Server{
 		db:     db,
 		routes: map[string]Handler{},
+		debug:  debug,
 	}, nil
 }
 
@@ -96,6 +99,7 @@ func (s *Server) Handle(w http.ResponseWriter, r *http.Request) {
 		id:       id,
 		req:      r,
 		w:        w,
+		debug:    s.debug,
 		username: "unknown",
 	}
 
@@ -172,6 +176,13 @@ func (c *Context) Logf(level, msg string, args ...interface{}) {
 	log.Printf("%s - %s - %s", c.id, level, fmt.Sprintf(msg, args...))
 }
 
+// Debugf logs output only when the server is set into debug mode.
+func (c *Context) Debugf(level, msg string, args ...interface{}) {
+	if c.debug {
+		log.Printf("%s - %s - %s", c.id, level, fmt.Sprintf(msg, args...))
+	}
+}
+
 // Generate a useable request ID, so that it can be found in the logs.
 func newIdent() (string, error) {
 	b := make([]byte, 16)

types/ip.go

@@ -58,7 +58,7 @@ func (i IP) Is6() bool {
 //
 // This function is needed so that a subnet can be inserted into
 // the database without much casting.
-func (s *Subnet) Value() (driver.Value, error) {
+func (s Subnet) Value() (driver.Value, error) {
 	return s.String(), nil
 }