Rename lists and fix .resource_types purge (#196)

* rename whitelist to allowlist and blacklist to skiplist

* rename blacklist to skiplist

* rename whitelist to allowlist and blacklist to skiplist

* rename whitelist to allowlist and blacklist to skiplist

* rename whitelist to allowlist and blacklist to skiplist

* rename whitelist to allowlist and blacklist to skiplist

* rename whitelist to allowlist

* fix unintended .resource_types purge, because it was checked in the wrong section

* add local.yaml for testing

* add detectGitRemoteUrlChange (#197)

* add detectGitRemoteUrlChange, fixes git remote url changes for control repo #185

* comment debug print format

* only purge control repo and fix typo

* only print this in debug

* satisfy GitHub Actions test

.gitignore

@@ -14,3 +14,4 @@ external_modules/
 .scannerwork/
 tests/github-test-private/
 coverage.txt
+local.yaml

README.md

@@ -439,14 +439,14 @@ sources:
     prefix: true
 ```
 
-Starting with [v.0.7.1](https://github.com/xorpaul/g10k/releases/tag/v0.7.1) g10k supports `purge_blacklist` feature to remove unnecessary files from the sync / Puppetservers.
+Starting with [v.0.7.1](https://github.com/xorpaul/g10k/releases/tag/v0.7.1) g10k supports `purge_skiplist` feature to remove unnecessary files from the sync / Puppetservers.
 
 Example:
 
 ```
 ---
 deploy:
-  purge_blacklist: [ 'spec', 'readmes', 'examples', '*.markdown', '*.md', 'junit', 'docs' ]
+  purge_skiplist: [ 'spec', 'readmes', 'examples', '*.markdown', '*.md', 'junit', 'docs' ]
 
 sources:
   example:

config.go

@@ -104,12 +104,12 @@ func readConfigfile(configFile string) ConfigSettings {
 	if !reflect.DeepEqual(config.Deploy, emptyDeploy) {
 		Debugf("detected deploy configuration hash, which takes precedence over the non-deploy scoped settings")
 		config.PurgeLevels = config.Deploy.PurgeLevels
-		config.PurgeWhitelist = config.Deploy.PurgeWhitelist
-		config.DeploymentPurgeWhitelist = config.Deploy.DeploymentPurgeWhitelist
+		config.PurgeAllowList = config.Deploy.PurgeAllowList
+		config.DeploymentPurgeAllowList = config.Deploy.DeploymentPurgeAllowList
 		config.WriteLock = config.Deploy.WriteLock
 		config.GenerateTypes = config.Deploy.GenerateTypes
 		config.PuppetPath = config.Deploy.PuppetPath
-		config.PurgeBlacklist = config.Deploy.PurgeBlacklist
+		config.PurgeSkiplist = config.Deploy.PurgeSkiplist
 		config.Deploy = emptyDeploy
 	}
 

g10k.go

@@ -88,25 +88,25 @@ type ConfigSettings struct {
 	PostRunCommand              []string       `yaml:"postrun"`
 	Deploy                      DeploySettings `yaml:"deploy"`
 	PurgeLevels                 []string       `yaml:"purge_levels"`
-	PurgeWhitelist              []string       `yaml:"purge_whitelist"`
-	DeploymentPurgeWhitelist    []string       `yaml:"deployment_purge_whitelist"`
+	PurgeAllowList              []string       `yaml:"purge_allowlist"`
+	DeploymentPurgeAllowList    []string       `yaml:"deployment_purge_allowlist"`
 	WriteLock                   string         `yaml:"write_lock"`
 	GenerateTypes               bool           `yaml:"generate_types"`
 	PuppetPath                  string         `yaml:"puppet_path"`
-	PurgeBlacklist              []string       `yaml:"purge_blacklist"`
+	PurgeSkiplist              []string       `yaml:"purge_skiplist"`
 	CloneGitModules             bool           `yaml:"clone_git_modules"`
 }
 
 // DeploySettings is a struct for settings for controlling how g10k deploys behave.
 // Trying to emulate r10k https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy
 type DeploySettings struct {
 	PurgeLevels              []string `yaml:"purge_levels"`
-	PurgeWhitelist           []string `yaml:"purge_whitelist"`
-	DeploymentPurgeWhitelist []string `yaml:"deployment_purge_whitelist"`
+	PurgeAllowList           []string `yaml:"purge_allowlist"`
+	DeploymentPurgeAllowList []string `yaml:"deployment_purge_allowlist"`
 	WriteLock                string   `yaml:"write_lock"`
 	GenerateTypes            bool     `yaml:"generate_types"`
 	PuppetPath               string   `yaml:"puppet_path"`
-	PurgeBlacklist           []string `yaml:"purge_blacklist"`
+	PurgeSkiplist           []string `yaml:"purge_skiplist"`
 }
 
 // Forge is a simple struct that contains the base URL of

g10k_test.go

@@ -197,8 +197,8 @@ func TestConfigDeploy(t *testing.T) {
 		Forge:   Forge{Baseurl: "https://forgeapi.puppet.com"},
 		Sources: s, Timeout: 5, Maxworker: 50, MaxExtractworker: 20,
 		PurgeLevels:              []string{"deployment"},
-		PurgeWhitelist:           []string{"custom.json", "**/*.xpp"},
-		DeploymentPurgeWhitelist: []string{"full_hiera_*"}}
+		PurgeAllowList:           []string{"custom.json", "**/*.xpp"},
+		DeploymentPurgeAllowList: []string{"full_hiera_*"}}
 
 	if !reflect.DeepEqual(got, expected) {
 		fmt.Println("### Expected:")
@@ -316,25 +316,25 @@ func TestResolveStatic(t *testing.T) {
 
 }
 
-func TestResolveStaticBlacklist(t *testing.T) {
+func TestResolveStaticSkiplist(t *testing.T) {
 	path, err := exec.LookPath("hashdeep")
 	if err != nil {
 		t.Skip("Skipping full Puppet environment resolve test, because package hashdeep is missing")
 	}
 
 	quiet = true
-	purgeDir("./cache", "TestResolvStaticBlacklist()")
-	purgeDir("./example", "TestResolvStaticBlacklist()")
-	config = readConfigfile("tests/TestConfigStaticBlacklist.yaml")
+	purgeDir("./cache", "TestResolvStaticSkiplist()")
+	purgeDir("./example", "TestResolvStaticSkiplist()")
+	config = readConfigfile("tests/TestConfigStaticSkiplist.yaml")
 	// increase maxworker to finish the test quicker
 	config.Maxworker = 500
-	branchParam = "blacklist"
+	branchParam = "skiplist"
 	resolvePuppetEnvironment(false, "")
 
 	// remove timestamps from .g10k-deploy.json otherwise hash sum would always differ
-	removeTimestampsFromDeployfile("example/example_blacklist/.g10k-deploy.json")
+	removeTimestampsFromDeployfile("example/example_skiplist/.g10k-deploy.json")
 
-	cmd := exec.Command(path, "-vv", "-l", "-r", "-a", "-k", "tests/hashdeep_example_static_blacklist.hashdeep", "./example")
+	cmd := exec.Command(path, "-vv", "-l", "-r", "-a", "-k", "tests/hashdeep_example_static_skiplist.hashdeep", "./example")
 	out, err := cmd.CombinedOutput()
 	exitCode := 0
 	if msg, ok := err.(*exec.ExitError); ok { // there is error code
@@ -349,19 +349,19 @@ func TestResolveStaticBlacklist(t *testing.T) {
 	Debugf("hashdeep output:" + string(out))
 
 	expectedMissingFiles := []string{
-		"example/example_blacklist/external_modules/stdlib/spec",
-		"example/example_blacklist/external_modules/stdlib/readmes",
-		"example/example_blacklist/external_modules/stdlib/examples",
+		"example/example_skiplist/external_modules/stdlib/spec",
+		"example/example_skiplist/external_modules/stdlib/readmes",
+		"example/example_skiplist/external_modules/stdlib/examples",
 	}
 	for _, expectedMissingFile := range expectedMissingFiles {
 		if fileExists(expectedMissingFile) {
-			t.Errorf("blacklisted directory still exists that should have been purged! " + expectedMissingFile)
+			t.Errorf("skiplisted directory still exists that should have been purged! " + expectedMissingFile)
 		}
 	}
 
-	purgeDir("example/example_blacklist/Puppetfile", "TestResolveStaticBlacklist()")
+	purgeDir("example/example_skiplist/Puppetfile", "TestResolveStaticSkiplist()")
 
-	cmd = exec.Command(path, "-l", "-r", "-a", "-k", "tests/hashdeep_example_static_blacklist.hashdeep", "./example")
+	cmd = exec.Command(path, "-l", "-r", "-a", "-k", "tests/hashdeep_example_static_skiplist.hashdeep", "./example")
 	out, err = cmd.CombinedOutput()
 	exitCode = 0
 	if msg, ok := err.(*exec.ExitError); ok { // there is error code
@@ -1274,7 +1274,7 @@ func TestConfigRetryGitCommands(t *testing.T) {
 	if exitCode != 0 {
 		t.Errorf("terminated with %v, but we expected exit status %v", exitCode, 0)
 	}
-	//fmt.Println(string(out))
+	// fmt.Println(string(out))
 	if !strings.Contains(string(out), "WARN: git command failed: git --git-dir /tmp/g10k/modules/https-__github.com_puppetlabs_puppetlabs-firewall.git remote update --prune deleting local cached repository and retrying...") {
 		t.Errorf("terminated with the correct exit code, but the expected output was missing. out: %s", string(out))
 	}
@@ -2027,7 +2027,7 @@ func TestCheckDirPermissions(t *testing.T) {
 	purgeDir("/tmp/example", funcName)
 }
 
-func TestPurgeWhitelist(t *testing.T) {
+func TestPurgeAllowList(t *testing.T) {
 	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
 	cacheDir := "/tmp/g10k"
 	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
@@ -2094,7 +2094,7 @@ func TestPurgeWhitelist(t *testing.T) {
 
 	for _, expectedFile := range expectedFiles {
 		if !fileExists(expectedFile) {
-			t.Errorf("purge_whitelist item was purged: " + expectedFile)
+			t.Errorf("purge_allowlist item was purged: " + expectedFile)
 		}
 	}
 
@@ -2116,7 +2116,7 @@ func TestPurgeWhitelist(t *testing.T) {
 	purgeDir("/tmp/example", funcName)
 }
 
-func TestPurgeWhitelistRecursive(t *testing.T) {
+func TestPurgeAllowListRecursive(t *testing.T) {
 	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
 	cacheDir := "/tmp/g10k"
 	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
@@ -2180,7 +2180,7 @@ func TestPurgeWhitelistRecursive(t *testing.T) {
 
 	for _, expectedFile := range expectedFiles {
 		if !fileExists(expectedFile) {
-			t.Errorf("purge_whitelist item was purged: " + expectedFile)
+			t.Errorf("purge_allowlist item was purged: " + expectedFile)
 		}
 	}
 
@@ -2544,12 +2544,12 @@ func TestPurgeStaleDeploymentOnly(t *testing.T) {
 	purgeDir("/tmp/full", funcName)
 }
 
-func TestPurgeStaleDeploymentOnlyWithWhitelist(t *testing.T) {
+func TestPurgeStaleDeploymentOnlyWithAllowList(t *testing.T) {
 	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
 	cacheDir := "/tmp/g10k"
 	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
 		debug = true
-		config = readConfigfile("tests/TestConfigFullworkingPurgeDeploymentWithWhitelist.yaml")
+		config = readConfigfile("tests/TestConfigFullworkingPurgeDeploymentWithAllowList.yaml")
 		branchParam = ""
 		resolvePuppetEnvironment(false, "")
 		return

git.go

@@ -115,7 +115,11 @@ func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool
 		gitCmd = "git clone --single-branch --branch " + gitModule.tree + " " + gitModule.git + " " + workDir
 	}
 	if isDir(workDir) {
-		gitCmd = "git --git-dir " + workDir + " remote update --prune"
+		if detectGitRemoteURLChange(workDir, gitModule.git) && isControlRepo {
+			purgeDir(workDir, "git remote url changed")
+		} else {
+			gitCmd = "git --git-dir " + workDir + " remote update --prune"
+		}
 	}
 
 	if explicitlyLoadSSHKey {
@@ -317,3 +321,21 @@ func detectDefaultBranch(gitDir string) string {
 	//fmt.Println(defaultBranch)
 	return defaultBranch
 }
+
+func detectGitRemoteURLChange(d string, url string) bool {
+	gitRemoteCmd := "git --git-dir " + d + " remote -v"
+
+	er := executeCommand(gitRemoteCmd, config.Timeout, false)
+	if er.returnCode != 0 {
+		Warnf("WARN: Could not detect remote URL for git repository " + d + " trying to purge it and mirror it again")
+		return true
+	}
+
+	f := strings.Fields(er.output)
+	if len(f) < 3 {
+		Warnf("WARN: Could not detect remote URL for git repository " + d + " trying to purge it and mirror it again")
+		return true
+	}
+	configuredRemote := f[1]
+	return configuredRemote != url
+}

modules.go

@@ -24,17 +24,17 @@ func unTar(r io.Reader, targetBaseDir string) {
 		// get the individual filename and extract to the current directory
 		filename := header.Name
 		// check if currently extracting archive is a forge or a git module
-		// we need to remove the module name from the filename otherwise the blacklist pattern would not match
+		// we need to remove the module name from the filename otherwise the skiplist pattern would not match
 		// e.g puppetlabs-stdlib-6.0.0/MAINTAINERS.md for a forge module
 		// and MAINTAINERS.md for a git module
-		blacklistFilename := filename
+		skiplistFilename := filename
 		if targetBaseDir == config.ForgeCacheDir {
-			blacklistFilenameComponents := strings.SplitAfterN(filename, "/", 2)
-			if len(blacklistFilenameComponents) > 1 {
-				blacklistFilename = blacklistFilenameComponents[1]
+			skiplistFilenameComponents := strings.SplitAfterN(filename, "/", 2)
+			if len(skiplistFilenameComponents) > 1 {
+				skiplistFilename = skiplistFilenameComponents[1]
 			}
 		}
-		if matchBlacklistContent(blacklistFilename) {
+		if matchSkiplistContent(skiplistFilename) {
 			continue
 		}
 		targetFilename := filepath.Join(targetBaseDir, filename)
@@ -122,14 +122,14 @@ func unTar(r io.Reader, targetBaseDir string) {
 	}
 }
 
-func matchBlacklistContent(filePath string) bool {
-	for _, blPattern := range config.PurgeBlacklist {
+func matchSkiplistContent(filePath string) bool {
+	for _, blPattern := range config.PurgeSkiplist {
 		filepathResult, _ := filepath.Match(blPattern, filePath)
 		if strings.HasPrefix(filePath, blPattern) || filepathResult {
-			Debugf("skipping file " + filePath + " because purge_blacklist pattern '" + blPattern + "' matches")
+			Debugf("skipping file " + filePath + " because purge_skiplist pattern '" + blPattern + "' matches")
 			return true
 		}
 	}
-	//Debugf("not skipping file " + filePath + " because no purge_blacklist pattern matches")
+	//Debugf("not skipping file " + filePath + " because no purge_skiplist pattern matches")
 	return false
 }

puppetfile.go

@@ -428,9 +428,6 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
 	if stringSliceContains(config.PurgeLevels, "puppetfile") {
 		if len(exisitingModuleDirs) > 0 && len(moduleParam) == 0 {
 			for d := range exisitingModuleDirs {
-				if strings.HasSuffix(d, ".resource_types") && isDir(d) {
-					continue
-				}
 				Infof("Removing unmanaged path " + d)
 				if !dryRun {
 					purgeDir(d, "purge_level puppetfile")

stale.go

@@ -33,13 +33,13 @@ func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[stri
 				Debugf("Glob'ing with path " + globPath)
 				environments, _ := filepath.Glob(globPath)
 
-				whitelistEnvironments := []string{}
-				if len(config.DeploymentPurgeWhitelist) > 0 {
-					for _, wlpattern := range config.DeploymentPurgeWhitelist {
-						whitelistGlobPath := filepath.Join(basedir, wlpattern)
-						Debugf("deployment_purge_whitelist Glob'ing with path " + whitelistGlobPath)
-						we, _ := filepath.Glob(whitelistGlobPath)
-						whitelistEnvironments = append(whitelistEnvironments, we...)
+				allowlistEnvironments := []string{}
+				if len(config.DeploymentPurgeAllowList) > 0 {
+					for _, wlpattern := range config.DeploymentPurgeAllowList {
+						allowlistGlobPath := filepath.Join(basedir, wlpattern)
+						Debugf("deployment_purge_allowlist Glob'ing with path " + allowlistGlobPath)
+						we, _ := filepath.Glob(allowlistGlobPath)
+						allowlistEnvironments = append(allowlistEnvironments, we...)
 					}
 				}
 
@@ -61,8 +61,8 @@ func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[stri
 						Debugf("Checking if environment should exist: " + envName)
 						if allEnvironments[envName] {
 							Debugf("Not purging environment " + envName)
-						} else if stringSliceContains(whitelistEnvironments, filepath.Join(basedir, envName)) {
-							Debugf("Not purging environment " + envName + " due to deployment_purge_whitelist match")
+						} else if stringSliceContains(allowlistEnvironments, filepath.Join(basedir, envName)) {
+							Debugf("Not purging environment " + envName + " due to deployment_purge_allowlist match")
 						} else {
 							Infof("Removing unmanaged environment " + envName)
 							if !dryRun {
@@ -82,36 +82,40 @@ func purgeUnmanagedContent(allBasedirs map[string]bool, allEnvironments map[stri
 }
 
 func checkForStaleContent(workDir string) {
-	// add purge whitelist
-	if len(config.PurgeWhitelist) > 0 {
-		Debugf("interpreting purge whitelist globs: " + strings.Join(config.PurgeWhitelist, " "))
-		for _, wlItem := range config.PurgeWhitelist {
+	// add purge allowlist
+	if len(config.PurgeAllowList) > 0 {
+		for _, wlItem := range config.PurgeAllowList {
+			Debugf("interpreting purge allowlist globs: " + strings.Join(config.PurgeAllowList, " "))
 			globPath := filepath.Join(workDir, wlItem)
-			Debugf("Glob'ing with purge whitelist glob " + globPath)
+			Debugf("Glob'ing with purge allowlist glob " + globPath)
 			wlPaths, _ := filepathx.Glob(globPath)
-			Debugf("additional purge whitelist items: " + strings.Join(wlPaths, " "))
-				desiredContent = append(desiredContent, wlPaths...)
+			Debugf("additional purge allowlist items: " + strings.Join(wlPaths, " "))
+			desiredContent = append(desiredContent, wlPaths...)
 		}
 	}
 
 	checkForStaleContent := func(path string, info os.FileInfo, err error) error {
 		//Debugf("filepath.Walk'ing found path: " + path)
 		stale := true
-		for _, desiredFile := range desiredContent {
-			for _, unchangedModuleDir := range unchangedModuleDirs {
-				if strings.HasPrefix(path, unchangedModuleDir) {
+		if strings.HasSuffix(path, ".resource_types") && isDir(path) {
+			stale = false
+		} else {
+			for _, desiredFile := range desiredContent {
+				for _, unchangedModuleDir := range unchangedModuleDirs {
+					if strings.HasPrefix(path, unchangedModuleDir) {
+						stale = false
+						break
+					}
+				}
+				if path == desiredFile || path == workDir {
 					stale = false
 					break
 				}
 			}
-			if path == desiredFile || path == workDir {
-				stale = false
-				break
-			}
 		}
 
 		if stale {
-			Infof("Removing unmanaged path " + path)
+			Debugf("Removing unmanaged path " + path)
 			purgeDir(path, "checkForStaleContent()")
 		}
 		return nil

tests/TestConfigDeploy.yaml

@@ -3,8 +3,8 @@
 
 deploy:
   purge_levels: ['deployment']
-  purge_whitelist: [ 'custom.json', '**/*.xpp' ]
-  deployment_purge_whitelist: ['full_hiera_*']
+  purge_allowlist: [ 'custom.json', '**/*.xpp' ]
+  deployment_purge_allowlist: ['full_hiera_*']
 
 sources:
   full:

tests/TestConfigExamplePurgeAllLevels.yaml

@@ -2,7 +2,7 @@
 :cachedir: '/tmp/g10k'
 deploy:
   purge_levels: ['deployment', 'puppetfile', 'environment']
-  purge_whitelist: [ '.latest_revision', '.resource_types' ]
+  purge_allowlist: [ '.latest_revision', '.resource_types' ]
 
 sources:
   example: