From ecbcd6db79e83824a1a535b6ca9e4db8122cd246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fede=20Fern=C3=A1ndez?= Date: Tue, 18 Apr 2017 17:42:17 +0200 Subject: [PATCH] Update docs and commit files in release process (#174) * Integrates the update docs task in the after ci process * Removes the commit files task and improve the update files task * Fixes scripted tests --- .../sbtorgpolicies/github/GitHubOps.scala | 9 ++- .../sbtorgpolicies/io/ReplaceTextEngine.scala | 3 +- .../sbtorgpolicies/github/GitHubOpsTest.scala | 8 +- .../sbtorgpolicies/OrgPoliciesKeys.scala | 15 ++-- .../settings/DefaultSettings.scala | 2 +- .../scala/sbtorgpolicies/settings/bash.scala | 19 ----- .../scala/sbtorgpolicies/settings/files.scala | 75 ++++++++++++------- .../sbtorgpolicies/settings/release.scala | 6 +- .../sbtorgpolicies/update-doc-files/build.sbt | 18 +++-- .../update-doc-files/docs/README.md | 4 +- .../docs/src/main/resources/index.md | 2 +- .../update-doc-files/non-docs/README.md | 4 +- .../update-doc-files/other-docs/README.md | 4 +- .../other-docs/src/main/resources/index.md | 2 +- 14 files changed, 95 insertions(+), 76 deletions(-) diff --git a/core/src/main/scala/sbtorgpolicies/github/GitHubOps.scala b/core/src/main/scala/sbtorgpolicies/github/GitHubOps.scala index 2f968d1..7b3fabb 100644 --- a/core/src/main/scala/sbtorgpolicies/github/GitHubOps.scala +++ b/core/src/main/scala/sbtorgpolicies/github/GitHubOps.scala @@ -26,6 +26,7 @@ import com.github.marklister.base64.Base64.Encoder import github4s.Github import github4s.GithubResponses._ import github4s.free.domain._ +import sbt.IO import sbtorgpolicies.exceptions.{GitHubException, OrgPolicyException} import sbtorgpolicies.github.instances._ import sbtorgpolicies.github.syntax._ @@ -58,12 +59,16 @@ class GitHubOps(owner: String, repo: String, accessToken: Option[String]) { baseDir: File, branch: String, message: String, - files: List[String]): Either[OrgPolicyException, Option[Ref]] = { + files: List[File]): Either[OrgPolicyException, Option[Ref]] = { + + def relativePath(file: File): String = IO.relativize(baseDir, file).getOrElse(file.getName) def readFileContents: IOResult[List[(String, String)]] = { files.foldLeft[IOResult[List[(String, String)]]](Right(Nil)) { case (Right(partialResult), file) => - fileReader.getFileContent(baseDir.getAbsolutePath.ensureFinalSlash + file).map((file, _) :: partialResult) + fileReader.getFileContent(file.getAbsolutePath).map { content => + (relativePath(file), content) :: partialResult + } case (Left(e), _) => Left(e) } } diff --git a/core/src/main/scala/sbtorgpolicies/io/ReplaceTextEngine.scala b/core/src/main/scala/sbtorgpolicies/io/ReplaceTextEngine.scala index 34e0abf..01dead5 100644 --- a/core/src/main/scala/sbtorgpolicies/io/ReplaceTextEngine.scala +++ b/core/src/main/scala/sbtorgpolicies/io/ReplaceTextEngine.scala @@ -94,8 +94,7 @@ class ReplaceTextEngine { case (Some(startMatch), Some(endMatch)) if startMatch.end < endMatch.start => val textToBeReplaced = unprocessed.subStr(startMatch.end, endMatch.start) val replaced = replacements.foldLeft(textToBeReplaced) { - case (text, (target, replacement)) => - text.replaceAllLiterally(target, replacement) + case (text, (target, replacement)) => text.replaceAll(target, replacement) } val newContent = unprocessed.subStr(0, startMatch.end) + replaced + unprocessed.subStr( endMatch.start, diff --git a/core/src/test/scala/sbtorgpolicies/github/GitHubOpsTest.scala b/core/src/test/scala/sbtorgpolicies/github/GitHubOpsTest.scala index 953ccde..adbd5bb 100644 --- a/core/src/test/scala/sbtorgpolicies/github/GitHubOpsTest.scala +++ b/core/src/test/scala/sbtorgpolicies/github/GitHubOpsTest.scala @@ -16,6 +16,8 @@ package sbtorgpolicies.github +import java.io.File + import cats.data.NonEmptyList import cats.free.Free import cats.syntax.either._ @@ -130,7 +132,7 @@ class GitHubOpsTest extends TestOps { } val result: Either[OrgPolicyException, Option[Ref]] = - githubOps.commitFiles(baseDir, branch, sampleMessage, filesAndContents.map(_._1)) + githubOps.commitFiles(baseDir, branch, sampleMessage, filesAndContents.map(t => new File(baseDir, t._1))) (nelRefResponse, refCommitResponse) match { case (Left(e), _) => @@ -191,7 +193,7 @@ class GitHubOpsTest extends TestOps { .thenReturn(Free.pure[GitHub4s, GHResponse[Ref]](updateReferenceR)) val result: Either[OrgPolicyException, Option[Ref]] = - githubOps.commitFiles(baseDir, branch, sampleMessage, filesAndContents.map(_._1)) + githubOps.commitFiles(baseDir, branch, sampleMessage, filesAndContents.map(t => new File(baseDir, t._1))) (nelRefR, refCommitR, treeResultR, createCommitR, updateReferenceR) match { case (Left(e), _, _, _, _) => @@ -224,7 +226,7 @@ class GitHubOpsTest extends TestOps { when(fileReaderMock.getFileContent(any[String])).thenReturn(ioException.asLeft) val result: Either[OrgPolicyException, Option[Ref]] = - githubOps.commitFiles(baseDir, branch, sampleMessage, filesAndContents.map(_._1)) + githubOps.commitFiles(baseDir, branch, sampleMessage, filesAndContents.map(t => new File(baseDir, t._1))) result shouldBe ioException.asLeft diff --git a/src/main/scala/sbtorgpolicies/OrgPoliciesKeys.scala b/src/main/scala/sbtorgpolicies/OrgPoliciesKeys.scala index 2ed9461..468b61e 100644 --- a/src/main/scala/sbtorgpolicies/OrgPoliciesKeys.scala +++ b/src/main/scala/sbtorgpolicies/OrgPoliciesKeys.scala @@ -89,10 +89,15 @@ sealed trait OrgPoliciesSettingsKeys { settingKey[List[Validation]]("Validation list the plugin must check") val orgUpdateDocFilesSetting: SettingKey[List[File]] = - settingKey[List[File]]("List of files and directories whose replace blocks will be replaced with the new values") + settingKey[List[File]]("List of files and directories whose replace blocks will be replaced with the new values.") + + val orgUpdateDocFilesCommitSetting: SettingKey[Boolean] = + settingKey[Boolean]("Determines if the files should be committed after the update. 'true' by default.") val orgUpdateDocFilesReplacementsSetting: SettingKey[Map[String, String]] = - settingKey[Map[String, String]]("Replacements for the replace blocks") + settingKey[Map[String, String]]( + "Replacements for the replace blocks. " + + "By default, the regular expression \"\\\\d+.\\\\d+.\\\\d+\" will be replaced by the project version.") } sealed trait OrgPoliciesTaskKeys { @@ -101,8 +106,6 @@ sealed trait OrgPoliciesTaskKeys { val orgCheckSettings: TaskKey[Unit] = taskKey[Unit]("Task to check the project settings.") - val orgCommitPolicyFiles: TaskKey[Unit] = taskKey[Unit]("Commits the policy files into the specified branch.") - val orgCompile: TaskKey[Unit] = taskKey[Unit]("Just a (compile in Compile) but ignoring the result (Analysis type) and returning Unit.") @@ -122,7 +125,9 @@ sealed trait OrgPoliciesTaskKeys { val orgValidateFiles: TaskKey[Unit] = taskKey[Unit]("Validates all files according to a set of policy rules.") val orgUpdateDocFiles: TaskKey[Unit] = - taskKey[Unit]("Updates all replace blocks in the defined files and directories") + taskKey[Unit]( + "Updates all replace blocks in the defined files and directories and commits both, " + + "the modified files and the policy files, only if `orgUpdateDocFilesCommitSetting` is `true`") } diff --git a/src/main/scala/sbtorgpolicies/settings/DefaultSettings.scala b/src/main/scala/sbtorgpolicies/settings/DefaultSettings.scala index 7f774c7..698b026 100644 --- a/src/main/scala/sbtorgpolicies/settings/DefaultSettings.scala +++ b/src/main/scala/sbtorgpolicies/settings/DefaultSettings.scala @@ -114,7 +114,7 @@ trait DefaultSettings extends AllSettings { }, orgAfterCISuccessTaskListSetting := List( orgCreateFiles.toOrgTask, - orgCommitPolicyFiles.toOrgTask, + orgUpdateDocFiles.toOrgTask, depUpdateDependencyIssues.toOrgTask, orgPublishReleaseTask.toOrgTask(allModulesScope = true, crossScalaVersionsScope = true) ), diff --git a/src/main/scala/sbtorgpolicies/settings/bash.scala b/src/main/scala/sbtorgpolicies/settings/bash.scala index cb2894a..ed79f3b 100644 --- a/src/main/scala/sbtorgpolicies/settings/bash.scala +++ b/src/main/scala/sbtorgpolicies/settings/bash.scala @@ -31,25 +31,6 @@ trait bash { val orgBashTasks = Seq( - orgCommitPolicyFiles := Def.task { - onlyRootUnitTask(baseDirectory.value, (baseDirectory in LocalRootProject).value, streams.value.log) { - val ghOps: GitHubOps = orgGithubOpsSetting.value - ghOps.commitFiles( - baseDir = (baseDirectory in LocalRootProject).value, - branch = orgCommitBranchSetting.value, - message = s"${orgCommitMessageSetting.value} [ci skip]", - files = orgEnforcedFilesSetting.value.map(_.outputPath) - ) match { - case Right(Some(_)) => - streams.value.log.info("Policy files committed successfully") - case Right(None) => - streams.value.log.info("No changes detected in policy files. Skipping commit") - case Left(e) => - streams.value.log.error(s"Error committing files") - e.printStackTrace() - } - } - }.value, orgPublishReleaseTask := Def.task { val scalaV = scalaVersion.value s"sbt ++$scalaV $orgPublishReleaseCommandKey".! diff --git a/src/main/scala/sbtorgpolicies/settings/files.scala b/src/main/scala/sbtorgpolicies/settings/files.scala index cf7c56d..2c9c5ce 100644 --- a/src/main/scala/sbtorgpolicies/settings/files.scala +++ b/src/main/scala/sbtorgpolicies/settings/files.scala @@ -20,6 +20,7 @@ import cats.syntax.either._ import sbt.Keys._ import sbt._ import sbtorgpolicies.OrgPoliciesKeys._ +import sbtorgpolicies.github.GitHubOps import sbtorgpolicies.io._ import sbtorgpolicies.templates.utils._ @@ -28,8 +29,9 @@ import scala.util.matching.Regex trait files { val orgFilesSettings = Seq( - orgUpdateDocFilesSetting := List(baseDirectory.value / "docs"), - orgUpdateDocFilesReplacementsSetting := Map.empty + orgUpdateDocFilesSetting := List(baseDirectory.value / "docs", baseDirectory.value / "README.md"), + orgUpdateDocFilesCommitSetting := true, + orgUpdateDocFilesReplacementsSetting := Map("\"\\d+\\.\\d+\\.\\d+\"(-SNAPSHOT)?" -> ("\"" + version.value + "\"")) ) val orgFilesTasks = @@ -58,34 +60,57 @@ trait files { }.value, orgUpdateDocFiles := Def.task { onlyRootUnitTask(baseDirectory.value, (baseDirectory in LocalRootProject).value, streams.value.log) { - val replaceTextEngine = new ReplaceTextEngine - val blockTitle: String = "Replace" - val startBlockRegex: Regex = markdownComment(blockTitle, scape = true).r - val endBlockRegex: Regex = markdownComment(blockTitle, start = false, scape = true).r - val isFileSupported: (File) => Boolean = file => { - file.getName.indexOf('.') < 0 || file.getName.endsWith(".md") - } + val modifiedDocFiles: List[File] = if (!version.value.endsWith("-SNAPSHOT")) { + val replaceTextEngine = new ReplaceTextEngine + val blockTitle: String = "Replace" + val startBlockRegex: Regex = markdownComment(blockTitle, scape = true).r + val endBlockRegex: Regex = markdownComment(blockTitle, start = false, scape = true).r - val replaced = replaceTextEngine.replaceBlocks( - startBlockRegex, - endBlockRegex, - orgUpdateDocFilesReplacementsSetting.value, - orgUpdateDocFilesSetting.value, - isFileSupported) + val isFileSupported: (File) => Boolean = file => { + file.getName.indexOf('.') < 0 || file.getName.endsWith(".md") + } - val errorFiles = replaced.filter(_.status.failure).map(_.file.getAbsolutePath) - if (errorFiles.nonEmpty) { - streams.value.log.warn(printList("The following files where processed with errors:", errorFiles)) - } + val replaced = replaceTextEngine.replaceBlocks( + startBlockRegex, + endBlockRegex, + orgUpdateDocFilesReplacementsSetting.value, + orgUpdateDocFilesSetting.value, + isFileSupported) - val modified = replaced.filter(f => f.status.success && f.status.modified).map(_.file.getAbsolutePath) - if (modified.nonEmpty) { - streams.value.log.info(printList("The following files where modified:", modified)) - } + val errorFiles = replaced.filter(_.status.failure).map(_.file.getAbsolutePath) + if (errorFiles.nonEmpty) { + streams.value.log.warn(printList("The following files where processed with errors:", errorFiles)) + } + + replaced.filter(f => f.status.success && f.status.modified).map(_.file) + } else Nil + + val baseDir: File = (baseDirectory in LocalRootProject).value + val policyFiles: List[File] = orgEnforcedFilesSetting.value.map(f => baseDir / f.outputPath) + val allFiles: List[File] = policyFiles ++ modifiedDocFiles - if (errorFiles.isEmpty && modified.isEmpty) { - streams.value.log.info("No files updated") + if (allFiles.nonEmpty) { + if (orgUpdateDocFilesCommitSetting.value) { + streams.value.log.info(printList("Committing files", allFiles.map(_.getAbsolutePath))) + val ghOps: GitHubOps = orgGithubOpsSetting.value + ghOps.commitFiles( + baseDir = baseDir, + branch = orgCommitBranchSetting.value, + message = s"${orgCommitMessageSetting.value} [ci skip]", + files = allFiles + ) match { + case Right(Some(_)) => + streams.value.log.info("Files committed successfully") + case Right(None) => + streams.value.log.info("No changes detected in docs and policy files. Skipping commit") + case Left(e) => + streams.value.log.error(s"Error committing files") + e.printStackTrace() + } + } else streams.value.log.info("orgUpdateDocFilesCommitSetting set to `false`. Skipping commit") + } else { + streams.value.log.info("No files to be committed") } } }.value diff --git a/src/main/scala/sbtorgpolicies/settings/release.scala b/src/main/scala/sbtorgpolicies/settings/release.scala index a1281a8..c28f2b8 100644 --- a/src/main/scala/sbtorgpolicies/settings/release.scala +++ b/src/main/scala/sbtorgpolicies/settings/release.scala @@ -20,7 +20,7 @@ import cats.syntax.either._ import org.joda.time.{DateTime, DateTimeZone} import sbt.Keys.{baseDirectory, packageOptions, version} import sbt.Package.ManifestAttributes -import sbt.{LocalRootProject, Project, Setting, State} +import sbt.{File, LocalRootProject, Project, Setting, State} import sbtorgpolicies.github.GitHubOps import sbtorgpolicies.io.FileHelper import sbtorgpolicies.OrgPoliciesKeys._ @@ -124,7 +124,7 @@ trait release { baseDir = baseDir, branch = branch, message = s"$commitMessage [ci skip]", - files = List(fileType.outputPath)) + files = List(new File(baseDir, fileType.outputPath))) } yield maybeRef) match { case Right(Some(_)) => st.log.info("Update Change Log was finished successfully") @@ -150,7 +150,7 @@ trait release { val commitMessage = s"$orgVersionCommitMessage to ${vs._2}" - ghOps.commitFiles(baseDir, branch, commitMessage, List(file.getName)) match { + ghOps.commitFiles(baseDir, branch, commitMessage, List(file)) match { case Right(Some(_)) => st.log.info("Next version was committed successfully") case Right(None) => diff --git a/src/sbt-test/sbtorgpolicies/update-doc-files/build.sbt b/src/sbt-test/sbtorgpolicies/update-doc-files/build.sbt index 9348d72..a17cd21 100644 --- a/src/sbt-test/sbtorgpolicies/update-doc-files/build.sbt +++ b/src/sbt-test/sbtorgpolicies/update-doc-files/build.sbt @@ -1,18 +1,22 @@ scalaVersion := "2.12.1" +version := "2.0.0" + +orgUpdateDocFilesCommitSetting := false + val docsFiles = List(new File("docs/README.md"), new File("docs/src/main/resources/index.md")) val otherDocsFiles = List(new File("other-docs/README.md"), new File("other-docs/src/main/resources/index.md")) val nonDocsFiles = List(new File("non-docs/README.md")) -def checkFiles(list: List[File]): Boolean = { +def checkFiles(version: String, list: List[File]): Boolean = { list.foldLeft(true) { case (b, f) => val content = IO.readLines(f).mkString("\n") - b && content.contains("2.0") && !content.contains("1.2.1") + b && content.contains(version) && !content.contains("1.2.1") } && nonDocsFiles.foldLeft(true) { case (b, f) => val content = IO.readLines(f).mkString("\n") - b && !content.contains("2.0") && content.contains("1.2.1") + b && !content.contains(version) && content.contains("1.2.1") } } @@ -20,7 +24,7 @@ lazy val testCheckSettings = TaskKey[Unit]("testCheckSettings") lazy val testCheckSettings2 = TaskKey[Unit]("testCheckSettings2") testCheckSettings := Def.task { - if (checkFiles(docsFiles)) { + if (checkFiles(version.value, docsFiles)) { streams.value.log.info("Test succeeded.") } else { sys.error("Error validating docs files") @@ -28,11 +32,9 @@ testCheckSettings := Def.task { }.value testCheckSettings2 := Def.task { - if (checkFiles(otherDocsFiles)) { + if (checkFiles(version.value, otherDocsFiles)) { streams.value.log.info("Test succeeded.") } else { sys.error("Error validating other docs files") } -}.value - -orgUpdateDocFilesReplacementsSetting := Map("1.2.1" -> "2.0") \ No newline at end of file +}.value \ No newline at end of file diff --git a/src/sbt-test/sbtorgpolicies/update-doc-files/docs/README.md b/src/sbt-test/sbtorgpolicies/update-doc-files/docs/README.md index 13a3543..6a12422 100644 --- a/src/sbt-test/sbtorgpolicies/update-doc-files/docs/README.md +++ b/src/sbt-test/sbtorgpolicies/update-doc-files/docs/README.md @@ -1,11 +1,11 @@ [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" [comment]: # (End Replace) [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" [comment]: # (End Replace) diff --git a/src/sbt-test/sbtorgpolicies/update-doc-files/docs/src/main/resources/index.md b/src/sbt-test/sbtorgpolicies/update-doc-files/docs/src/main/resources/index.md index c0c4b03..1363162 100644 --- a/src/sbt-test/sbtorgpolicies/update-doc-files/docs/src/main/resources/index.md +++ b/src/sbt-test/sbtorgpolicies/update-doc-files/docs/src/main/resources/index.md @@ -1,7 +1,7 @@ Hello World! [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" Another stuff [comment]: # (End Replace) diff --git a/src/sbt-test/sbtorgpolicies/update-doc-files/non-docs/README.md b/src/sbt-test/sbtorgpolicies/update-doc-files/non-docs/README.md index 13a3543..6a12422 100644 --- a/src/sbt-test/sbtorgpolicies/update-doc-files/non-docs/README.md +++ b/src/sbt-test/sbtorgpolicies/update-doc-files/non-docs/README.md @@ -1,11 +1,11 @@ [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" [comment]: # (End Replace) [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" [comment]: # (End Replace) diff --git a/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/README.md b/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/README.md index 13a3543..6a12422 100644 --- a/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/README.md +++ b/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/README.md @@ -1,11 +1,11 @@ [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" [comment]: # (End Replace) [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" [comment]: # (End Replace) diff --git a/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/src/main/resources/index.md b/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/src/main/resources/index.md index c0c4b03..1363162 100644 --- a/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/src/main/resources/index.md +++ b/src/sbt-test/sbtorgpolicies/update-doc-files/other-docs/src/main/resources/index.md @@ -1,7 +1,7 @@ Hello World! [comment]: # (Start Replace) -Version 1.2.1 +Version "1.2.1" Another stuff [comment]: # (End Replace)