diff --git a/internal/ansible/play.go b/internal/ansible/play.go index bf86b83..c3dd436 100644 --- a/internal/ansible/play.go +++ b/internal/ansible/play.go @@ -1,15 +1,24 @@ package ansible import ( + "fmt" "github.com/ca-gip/dploy/internal/utils" ) type Play struct { - Hosts string `json:"hosts" yaml:"hosts"` - Roles []Role `yaml:"roles,omitempty"` - RawTags interface{} `json:"tags,inline" yaml:"tags,inline"` + Hosts string `yaml:"hosts"` + Roles []Role `yaml:"roles"` + Tags utils.Set `yaml:"tags"` } -func (play *Play) Tags() []string { - return utils.InferSlice(play.RawTags) +func (play *Play) AllTags() (tags *utils.Set) { + tags = utils.NewSet() + for _, role := range play.Roles { + tags = tags.Concat(role.AllTags().List()) + fmt.Println("role loop tags list is: ", tags.List()) + + } + tags.Concat(play.Tags.List()) + fmt.Println("play tags list is: ", tags.List()) + return } diff --git a/internal/ansible/playbook.go b/internal/ansible/playbook.go index d504bfc..668c74a 100644 --- a/internal/ansible/playbook.go +++ b/internal/ansible/playbook.go @@ -3,49 +3,59 @@ package ansible import ( "fmt" "github.com/ca-gip/dploy/internal/utils" - "github.com/ghodss/yaml" "github.com/karrick/godirwalk" + "gopkg.in/yaml.v2" "io/ioutil" - "k8s.io/klog/v2" "path/filepath" "strings" ) -type Tasks struct { - Tags interface{} `yaml:"tags,omitempty"` -} - type Playbook struct { - AbsolutePath string - RootPath *string + absolutePath string + rootPath *string Plays []Play - AllTags utils.Set +} + +const decoderTagName = "tags" + +func (playbook *Playbook) AllTags() (tags *utils.Set) { + tags = utils.NewSet() + for _, play := range playbook.Plays { + tags.Concat(play.AllTags().List()) + } + return } func (playbook *Playbook) RelativePath() string { - return strings.TrimPrefix(playbook.AbsolutePath, *playbook.RootPath+"/") + return strings.TrimPrefix(playbook.absolutePath, *playbook.rootPath+"/") } -func ReadFromFile(osPathname string) (plays []Play) { +func ReadFromFile(osPathname string) (playbook Playbook) { // Try to check playbook content binData, err := ioutil.ReadFile(osPathname) + + // IMPORTANT: Yaml and Json parser need a root element, + // They can't read a raw list. + content := fmt.Sprintf("plays:\n%s", string(binData)) + if err != nil { - klog.Error("Cannot read playbook", osPathname, ". Error: ", err.Error()) + fmt.Println("Cannot read playbook", osPathname, ". Error: ", err.Error()) return } - err = yaml.Unmarshal([]byte(binData), plays) + err = yaml.Unmarshal([]byte(content), &playbook) if err != nil { - klog.Error("Skip", osPathname, " not an inventory ") + fmt.Println("Skip", osPathname, " not a playbook ", err.Error()) return } - if plays == nil || len(plays) == 0 { - klog.Info("No play found inside the playbook: ", osPathname) + if len(playbook.Plays) == 0 { + fmt.Println("No play found inside the playbook: ", osPathname) return } - if (plays[0]).Hosts == utils.EmptyString { - klog.V(8).Info("No play found inside the playbook: ", osPathname) + if playbook.Plays[0].Hosts == utils.EmptyString { + fmt.Println("No play found inside the playbook: ", osPathname) return } + return } @@ -59,6 +69,7 @@ func readPlaybook(rootPath string) (result []*Playbook, err error) { return } + fmt.Println("reading playbook") // Merge Play, Role and Task Tags for a playbook allTags := utils.NewSet() @@ -73,28 +84,25 @@ func readPlaybook(rootPath string) (result []*Playbook, err error) { } // Try to check playbook content - plays := ReadFromFile(osPathname) + playbook := ReadFromFile(osPathname) // Browse Role Tags - for _, play := range plays { + for _, play := range playbook.Plays { - allTags.Concat(play.Tags()) - fmt.Println("Play tags are: ", play.Tags()) + allTags.Concat(play.AllTags().List()) + fmt.Println("Play tags are: ", play.Tags) for _, role := range play.Roles { - role.ReadRole(rootPath) - klog.Info(" Role info", role.Tags()) - allTags.Concat(role.Tags()) + role.ReadRoleTasks(rootPath) + fmt.Println(" Role info", role.AllTags()) + allTags.Concat(role.AllTags().List()) } } - playbook := Playbook{ - RootPath: &rootPath, - AbsolutePath: osPathname, - Plays: plays, - AllTags: *allTags, - } + playbook.absolutePath = osPathname + playbook.rootPath = &rootPath + result = append(result, &playbook) - klog.V(8).Info("Available tags are :", playbook.AllTags) + fmt.Println("Available tags are :", playbook.AllTags()) return nil }, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { diff --git a/internal/ansible/playbook_test.go b/internal/ansible/playbook_test.go index 4f416ec..5e9b5b1 100644 --- a/internal/ansible/playbook_test.go +++ b/internal/ansible/playbook_test.go @@ -1,13 +1,14 @@ package ansible import ( - "github.com/ghodss/yaml" + "github.com/ca-gip/dploy/internal/utils" "github.com/go-test/deep" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" "testing" ) -const validPlaybook = ` +const validPlaybook1 = ` - hosts: aws-node gather_facts: no roles: @@ -17,24 +18,70 @@ const validPlaybook = ` tags: always,alwaystest ` -func TestReadFromFile(t *testing.T) { +var expectedValidPlaybook1 = Play{ + Hosts: "aws-node", + Roles: []Role{ + { + Name: "add-aws-facts", + Tags: *utils.NewSetFromSlice("add-aws-facts"), + }, + }, + Tags: *utils.NewSetFromSlice("always", "alwaystest"), +} + +func Test(t *testing.T) { - t.Run("with a valid play and tags", func(t *testing.T) { - binData := []byte(validPlaybook) + t.Run("with a valid play data", func(t *testing.T) { + binData := []byte(validPlaybook1) var plays []Play err := yaml.Unmarshal([]byte(binData), &plays) assert.Nil(t, err) assert.NotNil(t, plays) assert.NotEmpty(t, plays) - deep.Equal(plays[0], Play{ + //deep.CompareUnexportedFields = false + if diff := deep.Equal(plays[0], expectedValidPlaybook1); diff != nil { + t.Error(diff) + } + assert.Equal(t, plays[0], expectedValidPlaybook1) + + // Not so deep ? + if diff := deep.Equal(plays[0].Roles[0].Tags.List(), expectedValidPlaybook1.Roles[0].Tags.List()); diff != nil { + t.Error(diff) + } + + if diff := deep.Equal(plays[0].Roles[0].Tags.List(), expectedValidPlaybook1.Roles[0].Tags.List()); diff != nil { + t.Error(diff) + } + }) + + t.Run("with two different play should be different deep.Equals", func(t *testing.T) { + //deep.CompareUnexportedFields = false + var left = Play{ Hosts: "aws-node", Roles: []Role{ - {Name: "add-aws-facts"}, + { + Name: "add-aws-facts", + Tags: *utils.NewSetFromSlice("left"), + }, }, - RawTags: []string{"add-aws-facts"}, - }) + Tags: *utils.NewSetFromSlice("left", "Left"), + } + var right = Play{ + Hosts: "aws-node", + Roles: []Role{ + { + Name: "add-aws-facts", + Tags: *utils.NewSetFromSlice("left"), + }, + }, + Tags: *utils.NewSetFromSlice("right", "Right"), + } + // Not so deep ? + if diff := deep.Equal(left, right); len(diff) != 0 { + t.Error(diff) + } }) } diff --git a/internal/ansible/role.go b/internal/ansible/role.go index 4f13649..0636df8 100644 --- a/internal/ansible/role.go +++ b/internal/ansible/role.go @@ -1,66 +1,76 @@ package ansible import ( + "fmt" "github.com/ca-gip/dploy/internal/utils" - "github.com/ghodss/yaml" "github.com/karrick/godirwalk" + "gopkg.in/yaml.v2" "io/ioutil" - "k8s.io/klog/v2" "path/filepath" "strings" ) type Role struct { AbsolutePath string - Name string `json:"name" yaml:"role,flow"` - rawTags interface{} `json:"tags" yaml:"tags,flow"` - Tasks []Tasks + Name string `yaml:"role"` + Tasks []Task `yaml:"tasks"` + Tags utils.Set `yaml:"tags"` } -func (role *Role) Tags() []string { - return utils.InferSlice(role.rawTags) +func (role *Role) AllTags() (tags *utils.Set) { + tags = utils.NewSet() + for _, task := range role.Tasks { + tags.Concat(task.Tags.List()) + } + fmt.Println("tags:::", tags.List()) + tags.Concat(role.Tags.List()) + return } // Gather inventory files from a Parent directory // Using a recursive scan. All non inventory files are ignored ( not .ini file ) // All sub parent directory added like label in the inventory -func (role *Role) ReadRole(rootPath string, pathTags ...string) (err error) { +func (role *Role) ReadRoleTasks(rootPath string, pathTags ...string) (err error) { absRoot, err := filepath.Abs(rootPath + "/roles/" + role.Name) + fmt.Println("reading ", role.Name, "at: ", absRoot) if err != nil { - klog.Error("The role ", role.Name, "can't be read. Error:", err.Error()) + fmt.Println("The role ", role.Name, "can't be read. Error:", err.Error()) return } + fmt.Println(role.Name) err = godirwalk.Walk(absRoot, &godirwalk.Options{ Callback: func(osPathname string, de *godirwalk.Dirent) error { - tags := utils.NewSet() - if !strings.Contains(filepath.Base(osPathname), ".yml") { return nil } binData, err := ioutil.ReadFile(osPathname) if err != nil { - klog.Error("Cannot read file: ", osPathname, ". Error:", err.Error()) + fmt.Println("Cannot read file: ", osPathname, ". Error:", err.Error()) } var tasks []Task err = yaml.Unmarshal([]byte(binData), &tasks) + + if err != nil { + fmt.Println("error readin role", osPathname, "err:", err.Error()) + } else { + fmt.Println("task is", tasks) + } + for _, task := range tasks { - tags.Concat(task.Tags()) + role.Tasks = append(role.Tasks, task) } - klog.Info("tags in role tags:", role.rawTags) + fmt.Println(osPathname, "tags in role tags:", role.AllTags()) - tasks = append(tasks, Task{rawTags: tags.List()}) - if len(tags.List()) > 0 { - klog.Info("Task tags:", tags.List()) - } return nil }, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { + fmt.Println(err.Error()) return godirwalk.SkipNode }, Unsorted: true, diff --git a/internal/ansible/task.go b/internal/ansible/task.go index 19ef543..86a2b78 100644 --- a/internal/ansible/task.go +++ b/internal/ansible/task.go @@ -3,10 +3,6 @@ package ansible import "github.com/ca-gip/dploy/internal/utils" type Task struct { - Role string `json:"role"` - rawTags interface{} `json:"tags,omitempty" yaml:"tags,omitempty"` -} - -func (task *Task) Tags() []string { - return utils.InferSlice(task.rawTags) + Name string `yaml:"name,omitempty"` + Tags utils.Set } diff --git a/internal/ansible/task_test.go b/internal/ansible/task_test.go new file mode 100644 index 0000000..5ba79e8 --- /dev/null +++ b/internal/ansible/task_test.go @@ -0,0 +1,103 @@ +package ansible + +import ( + "fmt" + "github.com/ca-gip/dploy/internal/utils" + "github.com/go-test/deep" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + "testing" +) + +const validTask1 = ` +- name: Task1 + template: src="source" dest="destination" owner=root + tags: tasktag1 +` + +const validTaskWithoutName = ` +- template: src="source" dest="destination" owner=root + tags: tasktag1 +` + +var expectedValidTask1 = Task{ + Name: "Task1", + Tags: *utils.NewSetFromSlice("tasktag1"), +} + +func TestTask(t *testing.T) { + + t.Run("with a valid task data should be equals", func(t *testing.T) { + binData := []byte(validTask1) + var task []Task + err := yaml.Unmarshal([]byte(binData), &task) + assert.Nil(t, err) + assert.NotNil(t, task) + assert.NotEmpty(t, task) + + //deep.CompareUnexportedFields = false + if diff := deep.Equal(task[0], expectedValidTask1); diff != nil { + t.Error(diff) + } + assert.Equal(t, task[0], expectedValidTask1) + + // Not so deep ? + if diff := deep.Equal(task[0].Tags, expectedValidTask1.Tags); diff != nil { + t.Error(diff) + } + + }) + + t.Run("with two different task tags should fail", func(t *testing.T) { + left := Task{ + Name: "Task1", + Tags: *utils.NewSetFromSlice("tasktag1"), + } + + right := Task{ + Name: "Task1", + Tags: *utils.NewSetFromSlice("tasktag2"), + } + + if diff := deep.Equal(left, right); len(diff) != 0 { + t.Error(diff) + } + }) + + t.Run("with two different task name should fail", func(t *testing.T) { + left := Task{ + Name: "Task1", + Tags: *utils.NewSetFromSlice("tasktag1"), + } + + right := Task{ + Name: "Task2", + Tags: *utils.NewSetFromSlice("tasktag1"), + } + + if diff := deep.Equal(left, right); len(diff) == 0 { + t.Error(diff) + } + }) + + t.Run("without name should have tags", func(t *testing.T) { + binData := []byte(validTaskWithoutName) + var task []Task + err := yaml.Unmarshal([]byte(binData), &task) + + assert.Nil(t, err) + assert.NotNil(t, task) + assert.NotEmpty(t, task) + + expected := Task{ + Name: "", + Tags: *utils.NewSetFromSlice("tasktag1"), + } + + fmt.Println("task read", task) + if diff := deep.Equal(expected, task[0]); len(diff) != 0 { + t.Error(diff) + } + }) + +} diff --git a/internal/utils/converter.go b/internal/utils/converter.go index 4f984bf..41cb34c 100644 --- a/internal/utils/converter.go +++ b/internal/utils/converter.go @@ -22,6 +22,9 @@ func InferSlice(input interface{}) (slice []string) { return case reflect.String: slice = strings.Split(value.String(), ",") + fmt.Println("String find type for", value) + fmt.Println("String find type for", slice) + return slice default: fmt.Println("cannot find type for", value) } diff --git a/internal/utils/set.go b/internal/utils/set.go index 88dfe94..5e59f42 100644 --- a/internal/utils/set.go +++ b/internal/utils/set.go @@ -1,6 +1,13 @@ package utils -var exists = struct{}{} +import ( + "fmt" + "strings" +) + +type emptyType = struct{} + +var empty = emptyType{} type Set struct { m map[string]struct{} @@ -12,8 +19,14 @@ func NewSet() *Set { return s } +func NewSetFromSlice(elements ...string) *Set { + s := &Set{} + s.m = make(map[string]struct{}) + return s.Concat(elements) +} + func (s *Set) Add(value string) *Set { - s.m[value] = exists + s.m[value] = empty return s } @@ -40,3 +53,30 @@ func (s *Set) Contains(value string) bool { _, c := s.m[value] return c } + +// Read Struct Field to return the associated Tag name +// omitempty can be specified +func (s *Set) UnmarshalYAML(unmarshal func(i interface{}) error) (err error) { + var tmpSlice []string + var tmpString string + + if err = unmarshal(&tmpSlice); err == nil { + s.m = make(map[string]emptyType, len(tmpSlice)) + + for _, v := range tmpSlice { + s.m[v] = emptyType{} // add 1 to k as it is starting at base 0 + } + fmt.Println("slicedd", s) + return nil + } else if err = unmarshal(&tmpString); err == nil { + strSplits := strings.Split(tmpString, ",") + s.m = make(map[string]emptyType, len(tmpSlice)) + + for _, v := range strSplits { + s.m[v] = emptyType{} // add 1 to k as it is starting at base 0 + } + fmt.Println("slice1", s) + return nil + } + return err +} diff --git a/main.go b/main.go index feda97e..4638625 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,7 @@ import ( func main() { home, _ := os.UserHomeDir() - path := fmt.Sprintf("%s/%s", home, "Projects/ansible-kube") + path := fmt.Sprintf("%s/%s", home, "Projects/ansible-mock") k8s := ansible.LoadFromPath(path) fmt.Println("Filtering ", len(k8s.Inventories), "/", len(k8s.Inventories))