From 91cf9b805a8f680e7772f2af301c6c849f962da2 Mon Sep 17 00:00:00 2001 From: Emmanuel Date: Thu, 21 Jan 2021 03:07:21 +0100 Subject: [PATCH] Avancement parser --- internal/ansible/play.go | 19 ++++++--- internal/ansible/playbook.go | 69 +++++++++++++++++-------------- internal/ansible/playbook_test.go | 65 +++++++++++++++++++++++++---- internal/ansible/role.go | 46 +++++++++++++-------- internal/ansible/task.go | 8 +--- internal/ansible/task_test.go | 27 ------------ internal/utils/converter.go | 3 ++ internal/utils/set.go | 4 +- main.go | 38 ++++++++--------- 9 files changed, 163 insertions(+), 116 deletions(-) diff --git a/internal/ansible/play.go b/internal/ansible/play.go index bf86b83..c3dd436 100644 --- a/internal/ansible/play.go +++ b/internal/ansible/play.go @@ -1,15 +1,24 @@ package ansible import ( + "fmt" "github.com/ca-gip/dploy/internal/utils" ) type Play struct { - Hosts string `json:"hosts" yaml:"hosts"` - Roles []Role `yaml:"roles,omitempty"` - RawTags interface{} `json:"tags,inline" yaml:"tags,inline"` + Hosts string `yaml:"hosts"` + Roles []Role `yaml:"roles"` + Tags utils.Set `yaml:"tags"` } -func (play *Play) Tags() []string { - return utils.InferSlice(play.RawTags) +func (play *Play) AllTags() (tags *utils.Set) { + tags = utils.NewSet() + for _, role := range play.Roles { + tags = tags.Concat(role.AllTags().List()) + fmt.Println("role loop tags list is: ", tags.List()) + + } + tags.Concat(play.Tags.List()) + fmt.Println("play tags list is: ", tags.List()) + return } diff --git a/internal/ansible/playbook.go b/internal/ansible/playbook.go index 434fb4c..668c74a 100644 --- a/internal/ansible/playbook.go +++ b/internal/ansible/playbook.go @@ -10,41 +10,52 @@ import ( "strings" ) -type Tasks struct { - Tags interface{} `yaml:"tags,omitempty"` -} - type Playbook struct { - AbsolutePath string - RootPath *string + absolutePath string + rootPath *string Plays []Play - AllTags utils.Set +} + +const decoderTagName = "tags" + +func (playbook *Playbook) AllTags() (tags *utils.Set) { + tags = utils.NewSet() + for _, play := range playbook.Plays { + tags.Concat(play.AllTags().List()) + } + return } func (playbook *Playbook) RelativePath() string { - return strings.TrimPrefix(playbook.AbsolutePath, *playbook.RootPath+"/") + return strings.TrimPrefix(playbook.absolutePath, *playbook.rootPath+"/") } -func ReadFromFile(osPathname string) (plays []Play) { +func ReadFromFile(osPathname string) (playbook Playbook) { // Try to check playbook content binData, err := ioutil.ReadFile(osPathname) + + // IMPORTANT: Yaml and Json parser need a root element, + // They can't read a raw list. + content := fmt.Sprintf("plays:\n%s", string(binData)) + if err != nil { - klog.Error("Cannot read playbook", osPathname, ". Error: ", err.Error()) + fmt.Println("Cannot read playbook", osPathname, ". Error: ", err.Error()) return } - err = yaml.Unmarshal([]byte(binData), plays) + err = yaml.Unmarshal([]byte(content), &playbook) if err != nil { - klog.Error("Skip", osPathname, " not an inventory ") + fmt.Println("Skip", osPathname, " not a playbook ", err.Error()) return } - if plays == nil || len(plays) == 0 { - klog.Info("No play found inside the playbook: ", osPathname) + if len(playbook.Plays) == 0 { + fmt.Println("No play found inside the playbook: ", osPathname) return } - if (plays[0]).Hosts == utils.EmptyString { - klog.V(8).Info("No play found inside the playbook: ", osPathname) + if playbook.Plays[0].Hosts == utils.EmptyString { + fmt.Println("No play found inside the playbook: ", osPathname) return } + return } @@ -58,6 +69,7 @@ func readPlaybook(rootPath string) (result []*Playbook, err error) { return } + fmt.Println("reading playbook") // Merge Play, Role and Task Tags for a playbook allTags := utils.NewSet() @@ -72,28 +84,25 @@ func readPlaybook(rootPath string) (result []*Playbook, err error) { } // Try to check playbook content - plays := ReadFromFile(osPathname) + playbook := ReadFromFile(osPathname) // Browse Role Tags - for _, play := range plays { + for _, play := range playbook.Plays { - allTags.Concat(play.Tags()) - fmt.Println("Play tags are: ", play.Tags()) + allTags.Concat(play.AllTags().List()) + fmt.Println("Play tags are: ", play.Tags) for _, role := range play.Roles { - role.ReadRole(rootPath) - log.Info(" Role info", role.Tags()) - allTags.Concat(role.Tags()) + role.ReadRoleTasks(rootPath) + fmt.Println(" Role info", role.AllTags()) + allTags.Concat(role.AllTags().List()) } } - playbook := Playbook{ - RootPath: &rootPath, - AbsolutePath: osPathname, - Plays: plays, - AllTags: *allTags, - } + playbook.absolutePath = osPathname + playbook.rootPath = &rootPath + result = append(result, &playbook) - log.Debug("Available tags are :", playbook.AllTags) + fmt.Println("Available tags are :", playbook.AllTags()) return nil }, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { diff --git a/internal/ansible/playbook_test.go b/internal/ansible/playbook_test.go index 4f416ec..5e9b5b1 100644 --- a/internal/ansible/playbook_test.go +++ b/internal/ansible/playbook_test.go @@ -1,13 +1,14 @@ package ansible import ( - "github.com/ghodss/yaml" + "github.com/ca-gip/dploy/internal/utils" "github.com/go-test/deep" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" "testing" ) -const validPlaybook = ` +const validPlaybook1 = ` - hosts: aws-node gather_facts: no roles: @@ -17,24 +18,70 @@ const validPlaybook = ` tags: always,alwaystest ` -func TestReadFromFile(t *testing.T) { +var expectedValidPlaybook1 = Play{ + Hosts: "aws-node", + Roles: []Role{ + { + Name: "add-aws-facts", + Tags: *utils.NewSetFromSlice("add-aws-facts"), + }, + }, + Tags: *utils.NewSetFromSlice("always", "alwaystest"), +} + +func Test(t *testing.T) { - t.Run("with a valid play and tags", func(t *testing.T) { - binData := []byte(validPlaybook) + t.Run("with a valid play data", func(t *testing.T) { + binData := []byte(validPlaybook1) var plays []Play err := yaml.Unmarshal([]byte(binData), &plays) assert.Nil(t, err) assert.NotNil(t, plays) assert.NotEmpty(t, plays) - deep.Equal(plays[0], Play{ + //deep.CompareUnexportedFields = false + if diff := deep.Equal(plays[0], expectedValidPlaybook1); diff != nil { + t.Error(diff) + } + assert.Equal(t, plays[0], expectedValidPlaybook1) + + // Not so deep ? + if diff := deep.Equal(plays[0].Roles[0].Tags.List(), expectedValidPlaybook1.Roles[0].Tags.List()); diff != nil { + t.Error(diff) + } + + if diff := deep.Equal(plays[0].Roles[0].Tags.List(), expectedValidPlaybook1.Roles[0].Tags.List()); diff != nil { + t.Error(diff) + } + }) + + t.Run("with two different play should be different deep.Equals", func(t *testing.T) { + //deep.CompareUnexportedFields = false + var left = Play{ Hosts: "aws-node", Roles: []Role{ - {Name: "add-aws-facts"}, + { + Name: "add-aws-facts", + Tags: *utils.NewSetFromSlice("left"), + }, }, - RawTags: []string{"add-aws-facts"}, - }) + Tags: *utils.NewSetFromSlice("left", "Left"), + } + var right = Play{ + Hosts: "aws-node", + Roles: []Role{ + { + Name: "add-aws-facts", + Tags: *utils.NewSetFromSlice("left"), + }, + }, + Tags: *utils.NewSetFromSlice("right", "Right"), + } + // Not so deep ? + if diff := deep.Equal(left, right); len(diff) != 0 { + t.Error(diff) + } }) } diff --git a/internal/ansible/role.go b/internal/ansible/role.go index 4f13649..0636df8 100644 --- a/internal/ansible/role.go +++ b/internal/ansible/role.go @@ -1,66 +1,76 @@ package ansible import ( + "fmt" "github.com/ca-gip/dploy/internal/utils" - "github.com/ghodss/yaml" "github.com/karrick/godirwalk" + "gopkg.in/yaml.v2" "io/ioutil" - "k8s.io/klog/v2" "path/filepath" "strings" ) type Role struct { AbsolutePath string - Name string `json:"name" yaml:"role,flow"` - rawTags interface{} `json:"tags" yaml:"tags,flow"` - Tasks []Tasks + Name string `yaml:"role"` + Tasks []Task `yaml:"tasks"` + Tags utils.Set `yaml:"tags"` } -func (role *Role) Tags() []string { - return utils.InferSlice(role.rawTags) +func (role *Role) AllTags() (tags *utils.Set) { + tags = utils.NewSet() + for _, task := range role.Tasks { + tags.Concat(task.Tags.List()) + } + fmt.Println("tags:::", tags.List()) + tags.Concat(role.Tags.List()) + return } // Gather inventory files from a Parent directory // Using a recursive scan. All non inventory files are ignored ( not .ini file ) // All sub parent directory added like label in the inventory -func (role *Role) ReadRole(rootPath string, pathTags ...string) (err error) { +func (role *Role) ReadRoleTasks(rootPath string, pathTags ...string) (err error) { absRoot, err := filepath.Abs(rootPath + "/roles/" + role.Name) + fmt.Println("reading ", role.Name, "at: ", absRoot) if err != nil { - klog.Error("The role ", role.Name, "can't be read. Error:", err.Error()) + fmt.Println("The role ", role.Name, "can't be read. Error:", err.Error()) return } + fmt.Println(role.Name) err = godirwalk.Walk(absRoot, &godirwalk.Options{ Callback: func(osPathname string, de *godirwalk.Dirent) error { - tags := utils.NewSet() - if !strings.Contains(filepath.Base(osPathname), ".yml") { return nil } binData, err := ioutil.ReadFile(osPathname) if err != nil { - klog.Error("Cannot read file: ", osPathname, ". Error:", err.Error()) + fmt.Println("Cannot read file: ", osPathname, ". Error:", err.Error()) } var tasks []Task err = yaml.Unmarshal([]byte(binData), &tasks) + + if err != nil { + fmt.Println("error readin role", osPathname, "err:", err.Error()) + } else { + fmt.Println("task is", tasks) + } + for _, task := range tasks { - tags.Concat(task.Tags()) + role.Tasks = append(role.Tasks, task) } - klog.Info("tags in role tags:", role.rawTags) + fmt.Println(osPathname, "tags in role tags:", role.AllTags()) - tasks = append(tasks, Task{rawTags: tags.List()}) - if len(tags.List()) > 0 { - klog.Info("Task tags:", tags.List()) - } return nil }, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { + fmt.Println(err.Error()) return godirwalk.SkipNode }, Unsorted: true, diff --git a/internal/ansible/task.go b/internal/ansible/task.go index 19ef543..86a2b78 100644 --- a/internal/ansible/task.go +++ b/internal/ansible/task.go @@ -3,10 +3,6 @@ package ansible import "github.com/ca-gip/dploy/internal/utils" type Task struct { - Role string `json:"role"` - rawTags interface{} `json:"tags,omitempty" yaml:"tags,omitempty"` -} - -func (task *Task) Tags() []string { - return utils.InferSlice(task.rawTags) + Name string `yaml:"name,omitempty"` + Tags utils.Set } diff --git a/internal/ansible/task_test.go b/internal/ansible/task_test.go index 0532ec7..5ba79e8 100644 --- a/internal/ansible/task_test.go +++ b/internal/ansible/task_test.go @@ -20,15 +20,6 @@ const validTaskWithoutName = ` tags: tasktag1 ` -const validMultiTask = ` -- template: src="source" dest="destination" owner=root - tags: tasktag1 - -- name: task2 - template: src="source" dest="destination" owner=root - tags: tasktag2 -` - var expectedValidTask1 = Task{ Name: "Task1", Tags: *utils.NewSetFromSlice("tasktag1"), @@ -109,22 +100,4 @@ func TestTask(t *testing.T) { } }) - t.Run("with multitask should have tags", func(t *testing.T) { - binData := []byte(validMultiTask) - var tasks []Task - err := yaml.Unmarshal([]byte(binData), &tasks) - - assert.Nil(t, err) - assert.NotNil(t, tasks) - assert.NotEmpty(t, tasks) - assert.Len(t, tasks, 2) - - expected := *utils.NewSetFromSlice("tasktag1", "tasktag2") - actual := utils.NewSet().Concat(tasks[0].Tags.List()).Concat(tasks[1].Tags.List()) - - if diff := deep.Equal(expected.List(), actual.List()); len(diff) != 0 { - t.Error(diff) - } - }) - } diff --git a/internal/utils/converter.go b/internal/utils/converter.go index 4f984bf..41cb34c 100644 --- a/internal/utils/converter.go +++ b/internal/utils/converter.go @@ -22,6 +22,9 @@ func InferSlice(input interface{}) (slice []string) { return case reflect.String: slice = strings.Split(value.String(), ",") + fmt.Println("String find type for", value) + fmt.Println("String find type for", slice) + return slice default: fmt.Println("cannot find type for", value) } diff --git a/internal/utils/set.go b/internal/utils/set.go index 56d06e3..5e59f42 100644 --- a/internal/utils/set.go +++ b/internal/utils/set.go @@ -66,7 +66,7 @@ func (s *Set) UnmarshalYAML(unmarshal func(i interface{}) error) (err error) { for _, v := range tmpSlice { s.m[v] = emptyType{} // add 1 to k as it is starting at base 0 } - fmt.Println("slice string read", s) + fmt.Println("slicedd", s) return nil } else if err = unmarshal(&tmpString); err == nil { strSplits := strings.Split(tmpString, ",") @@ -75,7 +75,7 @@ func (s *Set) UnmarshalYAML(unmarshal func(i interface{}) error) (err error) { for _, v := range strSplits { s.m[v] = emptyType{} // add 1 to k as it is starting at base 0 } - fmt.Println("string", s) + fmt.Println("slice1", s) return nil } return err diff --git a/main.go b/main.go index be526e7..4638625 100644 --- a/main.go +++ b/main.go @@ -9,25 +9,25 @@ import ( func main() { - //home, _ := os.UserHomeDir() - //path := fmt.Sprintf("%s/%s", home, "Projects/ansible-kube") - //k8s := services.LoadFromPath(path) - //fmt.Println("Filtering ", len(k8s.Inventories), "/", len(k8s.Inventories)) - // - //fmt.Println("Playbooks") - // - //tpl := services.AnsibleCommandTpl{ - // Inventory: k8s.Inventories, - // Playbook: k8s.Playbooks[1], - // Tags: []string{"tag1", "tag2"}, - // Limit: []string{"limit1,limit2"}, - // SkipTags: []string{"testt"}, - // Check: true, - // Diff: true, - // VaultPasswordFile: "/path/to/passwordfile", - // AskVaultPass: false, - //} - //tpl.GenerateCmd() + home, _ := os.UserHomeDir() + path := fmt.Sprintf("%s/%s", home, "Projects/ansible-mock") + k8s := ansible.LoadFromPath(path) + fmt.Println("Filtering ", len(k8s.Inventories), "/", len(k8s.Inventories)) + + fmt.Println("Playbooks") + + tpl := ansible.AnsibleCommandTpl{ + Inventory: k8s.Inventories, + Playbook: k8s.Playbooks[0], + Tags: []string{"tag1", "tag2"}, + Limit: []string{"limit1,limit2"}, + SkipTags: []string{"testt"}, + Check: true, + Diff: true, + VaultPasswordFile: "/path/to/passwordfile", + AskVaultPass: false, + } + tpl.GenerateCmd() cmd.Execute() }