Improve and expose config format handling

config/persistence.go

@@ -2,8 +2,8 @@ package config
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
+	"path"
 	"strings"
 
 	"github.com/safing/portbase/log"
@@ -72,70 +72,124 @@ func JSONToMap(jsonData []byte) (map[string]interface{}, error) {
 		return nil, err
 	}
 
-	flatten(loaded, loaded, "")
+	return Flatten(loaded), nil
-	return loaded, nil
 }
 
-func flatten(rootMap, subMap map[string]interface{}, subKey string) {
+// Flatten returns a flattened copy of the given hierarchical config.
+func Flatten(config map[string]interface{}) (flattenedConfig map[string]interface{}) {
+	flattenedConfig = make(map[string]interface{})
+	flattenMap(flattenedConfig, config, "")
+	return flattenedConfig
+}
+
+func flattenMap(rootMap, subMap map[string]interface{}, subKey string) {
 	for key, entry := range subMap {
 
 		// get next level key
-		subbedKey := key
+		subbedKey := path.Join(subKey, key)
-		if subKey != "" {
-			subbedKey = fmt.Sprintf("%s/%s", subKey, key)
-		}
 
 		// check for next subMap
 		nextSub, ok := entry.(map[string]interface{})
 		if ok {
-			flatten(rootMap, nextSub, subbedKey)
+			flattenMap(rootMap, nextSub, subbedKey)
-			delete(rootMap, key)
+		} else {
-		} else if subKey != "" {
 			// only set if not on root level
 			rootMap[subbedKey] = entry
 		}
 	}
 }
 
-// MapToJSON expands a flattened map and returns it as json. The map is altered in the process.
+// MapToJSON expands a flattened map and returns it as json.
-func MapToJSON(values map[string]interface{}) ([]byte, error) {
+func MapToJSON(config map[string]interface{}) ([]byte, error) {
-	expand(values)
+	return json.MarshalIndent(Expand(config), "", "  ")
-	return json.MarshalIndent(values, "", "  ")
 }
 
-// expand expands a flattened map.
+// Expand returns a hierarchical copy of the given flattened config.
-func expand(mapData map[string]interface{}) {
+func Expand(flattenedConfig map[string]interface{}) (config map[string]interface{}) {
-	var newMaps []map[string]interface{}
+	config = make(map[string]interface{})
-	for key, entry := range mapData {
+	for key, entry := range flattenedConfig {
-		if strings.Contains(key, "/") {
+		PutValueIntoHierarchicalConfig(config, key, entry)
-			parts := strings.SplitN(key, "/", 2)
+	}
-			if len(parts) == 2 {
+	return config
+}
 
-				// get subMap
+// PutValueIntoHierarchicalConfig injects a configuration entry into an hierarchical config map. Conflicting entries will be replaced.
-				var subMap map[string]interface{}
+func PutValueIntoHierarchicalConfig(config map[string]interface{}, key string, value interface{}) {
-				v, ok := mapData[parts[0]]
+	parts := strings.Split(key, "/")
+
+	// create/check maps for all parts except the last one
+	subMap := config
+	for i := 0; i < len(parts)-1; i++ {
+		var nextSubMap map[string]interface{}
+		// get value
+		value, ok := subMap[parts[i]]
+		if !ok {
+			// create new map and assign it
+			nextSubMap = make(map[string]interface{})
+			subMap[parts[i]] = nextSubMap
+		} else {
+			nextSubMap, ok = value.(map[string]interface{})
+			if !ok {
+				// create new map and assign it
+				nextSubMap = make(map[string]interface{})
+				subMap[parts[i]] = nextSubMap
+			}
+		}
+
+		// assign for next parts loop
+		subMap = nextSubMap
+	}
+
+	// assign value to last submap
+	subMap[parts[len(parts)-1]] = value
+}
+
+// CleanFlattenedConfig removes all inexistent configuration options from the given flattened config map.
+func CleanFlattenedConfig(flattenedConfig map[string]interface{}) {
+	optionsLock.RLock()
+	defer optionsLock.RUnlock()
+
+	for key := range flattenedConfig {
+		_, ok := options[key]
+		if !ok {
+			delete(flattenedConfig, key)
+		}
+	}
+}
+
+// CleanHierarchicalConfig removes all inexistent configuration options from the given hierarchical config map.
+func CleanHierarchicalConfig(config map[string]interface{}) {
+	optionsLock.RLock()
+	defer optionsLock.RUnlock()
+
+	cleanSubMap(config, "")
+}
+
+func cleanSubMap(subMap map[string]interface{}, subKey string) (empty bool) {
+	var foundValid int
+	for key, value := range subMap {
+		value, ok := value.(map[string]interface{})
+		if ok {
+			// we found another section
+			isEmpty := cleanSubMap(value, path.Join(subKey, key))
+			if isEmpty {
+				delete(subMap, key)
+			} else {
+				foundValid++
+			}
+		} else {
+			// we found an option value
+			if strings.Contains(key, "/") {
+				delete(subMap, key)
+			} else {
+				_, ok := options[path.Join(subKey, key)]
 				if ok {
-					subMap, ok = v.(map[string]interface{})
+					foundValid++
-					if !ok {
-						subMap = make(map[string]interface{})
-						newMaps = append(newMaps, subMap)
-						mapData[parts[0]] = subMap
-					}
 				} else {
-					subMap = make(map[string]interface{})
+					delete(subMap, key)
-					newMaps = append(newMaps, subMap)
-					mapData[parts[0]] = subMap
 				}
-
-				// set entry
-				subMap[parts[1]] = entry
-				// delete entry from
-				delete(mapData, key)
-
 			}
 		}
 	}
-	for _, entry := range newMaps {
+	return foundValid == 0
-		expand(entry)
-	}
 }

config/persistence_test.go

@@ -2,12 +2,12 @@ package config
 
 import (
 	"bytes"
+	"encoding/json"
 	"testing"
 )
 
-func TestJSONMapConversion(t *testing.T) {
+var (
-
+	jsonData = `{
-	jsonData := `{
   "a": "b",
   "c": {
     "d": "e",
@@ -22,9 +22,9 @@ func TestJSONMapConversion(t *testing.T) {
   },
   "p": "q"
 }`
-	jsonBytes := []byte(jsonData)
+	jsonBytes = []byte(jsonData)
 
-	mapData := map[string]interface{}{
+	mapData = map[string]interface{}{
 		"a":       "b",
 		"p":       "q",
 		"c/d":     "e",
@@ -33,32 +33,62 @@ func TestJSONMapConversion(t *testing.T) {
 		"c/h/k":   "l",
 		"c/h/m/n": "o",
 	}
+)
 
-	m, err := JSONToMap(jsonBytes)
+func TestJSONMapConversion(t *testing.T) {
-	if err != nil {
-		t.Fatal(err)
-	}
 
+	// convert to json
 	j, err := MapToJSON(mapData)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	// check if to json matches
 	if !bytes.Equal(jsonBytes, j) {
 		t.Errorf("json does not match, got %s", j)
 	}
 
+	// convert to map
+	m, err := JSONToMap(jsonBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// and back
 	j2, err := MapToJSON(m)
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	// check if double convert matches
 	if !bytes.Equal(jsonBytes, j2) {
 		t.Errorf("json does not match, got %s", j)
 	}
-
+}
-	// fails for some reason
+
-	// if !reflect.DeepEqual(mapData, m) {
+func TestConfigCleaning(t *testing.T) {
-	// 	t.Errorf("maps do not match, got %s", m)
+	// load
-	// }
+	configFlat, err := JSONToMap(jsonBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// clean everything
+	CleanFlattenedConfig(configFlat)
+	if len(configFlat) != 0 {
+		t.Errorf("should be empty: %+v", configFlat)
+	}
+
+	// load manuall for hierarchical config
+	configHier := make(map[string]interface{})
+	err = json.Unmarshal(jsonBytes, &configHier)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// clean everything
+	CleanHierarchicalConfig(configHier)
+	if len(configHier) != 0 {
+		t.Errorf("should be empty: %+v", configHier)
+	}
 }

config/perspective.go

@@ -19,7 +19,7 @@ type perspectiveOption struct {
 // NewPerspective parses the given config and returns it as a new perspective.
 func NewPerspective(config map[string]interface{}) (*Perspective, error) {
 	// flatten config structure
-	flatten(config, config, "")
+	config = Flatten(config)
 
 	perspective := &Perspective{
 		config: make(map[string]*perspectiveOption),