From 09e4c68f7b5eadcd7c2678937454cf675d59f2fe Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 28 Apr 2020 09:47:02 +0200 Subject: [PATCH] Improve and expose config format handling --- config/persistence.go | 140 +++++++++++++++++++++++++------------ config/persistence_test.go | 58 +++++++++++---- config/perspective.go | 2 +- 3 files changed, 142 insertions(+), 58 deletions(-) diff --git a/config/persistence.go b/config/persistence.go index 4ffbde3..75a6a47 100644 --- a/config/persistence.go +++ b/config/persistence.go @@ -2,8 +2,8 @@ package config import ( "encoding/json" - "fmt" "io/ioutil" + "path" "strings" "github.com/safing/portbase/log" @@ -72,70 +72,124 @@ func JSONToMap(jsonData []byte) (map[string]interface{}, error) { return nil, err } - flatten(loaded, loaded, "") - return loaded, nil + return Flatten(loaded), nil } -func flatten(rootMap, subMap map[string]interface{}, subKey string) { +// Flatten returns a flattened copy of the given hierarchical config. +func Flatten(config map[string]interface{}) (flattenedConfig map[string]interface{}) { + flattenedConfig = make(map[string]interface{}) + flattenMap(flattenedConfig, config, "") + return flattenedConfig +} + +func flattenMap(rootMap, subMap map[string]interface{}, subKey string) { for key, entry := range subMap { // get next level key - subbedKey := key - if subKey != "" { - subbedKey = fmt.Sprintf("%s/%s", subKey, key) - } + subbedKey := path.Join(subKey, key) // check for next subMap nextSub, ok := entry.(map[string]interface{}) if ok { - flatten(rootMap, nextSub, subbedKey) - delete(rootMap, key) - } else if subKey != "" { + flattenMap(rootMap, nextSub, subbedKey) + } else { // only set if not on root level rootMap[subbedKey] = entry } } } -// MapToJSON expands a flattened map and returns it as json. The map is altered in the process. -func MapToJSON(values map[string]interface{}) ([]byte, error) { - expand(values) - return json.MarshalIndent(values, "", " ") +// MapToJSON expands a flattened map and returns it as json. +func MapToJSON(config map[string]interface{}) ([]byte, error) { + return json.MarshalIndent(Expand(config), "", " ") } -// expand expands a flattened map. -func expand(mapData map[string]interface{}) { - var newMaps []map[string]interface{} - for key, entry := range mapData { - if strings.Contains(key, "/") { - parts := strings.SplitN(key, "/", 2) - if len(parts) == 2 { +// Expand returns a hierarchical copy of the given flattened config. +func Expand(flattenedConfig map[string]interface{}) (config map[string]interface{}) { + config = make(map[string]interface{}) + for key, entry := range flattenedConfig { + PutValueIntoHierarchicalConfig(config, key, entry) + } + return config +} - // get subMap - var subMap map[string]interface{} - v, ok := mapData[parts[0]] +// PutValueIntoHierarchicalConfig injects a configuration entry into an hierarchical config map. Conflicting entries will be replaced. +func PutValueIntoHierarchicalConfig(config map[string]interface{}, key string, value interface{}) { + parts := strings.Split(key, "/") + + // create/check maps for all parts except the last one + subMap := config + for i := 0; i < len(parts)-1; i++ { + var nextSubMap map[string]interface{} + // get value + value, ok := subMap[parts[i]] + if !ok { + // create new map and assign it + nextSubMap = make(map[string]interface{}) + subMap[parts[i]] = nextSubMap + } else { + nextSubMap, ok = value.(map[string]interface{}) + if !ok { + // create new map and assign it + nextSubMap = make(map[string]interface{}) + subMap[parts[i]] = nextSubMap + } + } + + // assign for next parts loop + subMap = nextSubMap + } + + // assign value to last submap + subMap[parts[len(parts)-1]] = value +} + +// CleanFlattenedConfig removes all inexistent configuration options from the given flattened config map. +func CleanFlattenedConfig(flattenedConfig map[string]interface{}) { + optionsLock.RLock() + defer optionsLock.RUnlock() + + for key := range flattenedConfig { + _, ok := options[key] + if !ok { + delete(flattenedConfig, key) + } + } +} + +// CleanHierarchicalConfig removes all inexistent configuration options from the given hierarchical config map. +func CleanHierarchicalConfig(config map[string]interface{}) { + optionsLock.RLock() + defer optionsLock.RUnlock() + + cleanSubMap(config, "") +} + +func cleanSubMap(subMap map[string]interface{}, subKey string) (empty bool) { + var foundValid int + for key, value := range subMap { + value, ok := value.(map[string]interface{}) + if ok { + // we found another section + isEmpty := cleanSubMap(value, path.Join(subKey, key)) + if isEmpty { + delete(subMap, key) + } else { + foundValid++ + } + } else { + // we found an option value + if strings.Contains(key, "/") { + delete(subMap, key) + } else { + _, ok := options[path.Join(subKey, key)] if ok { - subMap, ok = v.(map[string]interface{}) - if !ok { - subMap = make(map[string]interface{}) - newMaps = append(newMaps, subMap) - mapData[parts[0]] = subMap - } + foundValid++ } else { - subMap = make(map[string]interface{}) - newMaps = append(newMaps, subMap) - mapData[parts[0]] = subMap + delete(subMap, key) } - - // set entry - subMap[parts[1]] = entry - // delete entry from - delete(mapData, key) - } } } - for _, entry := range newMaps { - expand(entry) - } + return foundValid == 0 } diff --git a/config/persistence_test.go b/config/persistence_test.go index ad4f008..70584c5 100644 --- a/config/persistence_test.go +++ b/config/persistence_test.go @@ -2,12 +2,12 @@ package config import ( "bytes" + "encoding/json" "testing" ) -func TestJSONMapConversion(t *testing.T) { - - jsonData := `{ +var ( + jsonData = `{ "a": "b", "c": { "d": "e", @@ -22,9 +22,9 @@ func TestJSONMapConversion(t *testing.T) { }, "p": "q" }` - jsonBytes := []byte(jsonData) + jsonBytes = []byte(jsonData) - mapData := map[string]interface{}{ + mapData = map[string]interface{}{ "a": "b", "p": "q", "c/d": "e", @@ -33,32 +33,62 @@ func TestJSONMapConversion(t *testing.T) { "c/h/k": "l", "c/h/m/n": "o", } +) - m, err := JSONToMap(jsonBytes) - if err != nil { - t.Fatal(err) - } +func TestJSONMapConversion(t *testing.T) { + // convert to json j, err := MapToJSON(mapData) if err != nil { t.Fatal(err) } + // check if to json matches if !bytes.Equal(jsonBytes, j) { t.Errorf("json does not match, got %s", j) } + // convert to map + m, err := JSONToMap(jsonBytes) + if err != nil { + t.Fatal(err) + } + + // and back j2, err := MapToJSON(m) if err != nil { t.Fatal(err) } + // check if double convert matches if !bytes.Equal(jsonBytes, j2) { t.Errorf("json does not match, got %s", j) } - - // fails for some reason - // if !reflect.DeepEqual(mapData, m) { - // t.Errorf("maps do not match, got %s", m) - // } +} + +func TestConfigCleaning(t *testing.T) { + // load + configFlat, err := JSONToMap(jsonBytes) + if err != nil { + t.Fatal(err) + } + + // clean everything + CleanFlattenedConfig(configFlat) + if len(configFlat) != 0 { + t.Errorf("should be empty: %+v", configFlat) + } + + // load manuall for hierarchical config + configHier := make(map[string]interface{}) + err = json.Unmarshal(jsonBytes, &configHier) + if err != nil { + t.Fatal(err) + } + + // clean everything + CleanHierarchicalConfig(configHier) + if len(configHier) != 0 { + t.Errorf("should be empty: %+v", configHier) + } } diff --git a/config/perspective.go b/config/perspective.go index e863449..76b826d 100644 --- a/config/perspective.go +++ b/config/perspective.go @@ -19,7 +19,7 @@ type perspectiveOption struct { // NewPerspective parses the given config and returns it as a new perspective. func NewPerspective(config map[string]interface{}) (*Perspective, error) { // flatten config structure - flatten(config, config, "") + config = Flatten(config) perspective := &Perspective{ config: make(map[string]*perspectiveOption),