From a72e88ced9f09e0c8897e5707598252f2d54184c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 20 Jun 2022 17:46:47 +0200 Subject: [PATCH 1/5] refactor(settings): simplify SystemConfig.getVersion #7000 Instead of trying to read a built time file from Maven, use MicroProfile Config to retrieve the version and build number. The version is by default set via microprofile-config.properties (or overridden by an env var in a container). The build number is still read from either BuildNumber.properties or, if not present, from MicroProfile Config, defaulting to empty. This also avoids copying extra files into containers to retrieve the version string. --- .../iq/dataverse/util/SystemConfig.java | 175 ++++-------------- .../iq/dataverse/util/SystemConfigTest.java | 36 ++++ 2 files changed, 77 insertions(+), 134 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index bd27405fae5..25dd3dd6138 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -1,18 +1,28 @@ package edu.harvard.iq.dataverse.util; import com.ocpsoft.pretty.PrettyContext; - import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DvObjectContainer; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinAuthenticationProvider; import edu.harvard.iq.dataverse.authorization.providers.oauth2.AbstractOAuth2AuthenticationProvider; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.validation.PasswordValidatorUtil; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; +import org.passay.CharacterRule; + +import javax.ejb.EJB; +import javax.ejb.Stateless; +import javax.inject.Named; +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.json.JsonString; +import javax.json.JsonValue; import java.io.StringReader; import java.net.InetAddress; import java.net.UnknownHostException; @@ -23,25 +33,11 @@ import java.util.List; import java.util.Map; import java.util.MissingResourceException; -import java.util.Properties; import java.util.ResourceBundle; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.ejb.EJB; -import javax.ejb.Stateless; -import javax.inject.Named; -import javax.json.Json; -import javax.json.JsonArray; -import javax.json.JsonObject; -import javax.json.JsonReader; -import javax.json.JsonString; -import javax.json.JsonValue; - -import org.passay.CharacterRule; -import org.apache.commons.io.IOUtils; - /** * System-wide configuration */ @@ -50,6 +46,7 @@ public class SystemConfig { private static final Logger logger = Logger.getLogger(SystemConfig.class.getCanonicalName()); + private static final Config config = ConfigProvider.getConfig(); @EJB SettingsServiceBean settingsService; @@ -109,9 +106,8 @@ public class SystemConfig { public static final long defaultZipDownloadLimit = 104857600L; // 100MB private static final int defaultMultipleUploadFilesLimit = 1000; private static final int defaultLoginSessionTimeout = 480; // = 8 hours - - private static String appVersionString = null; - private static String buildNumberString = null; + + private String buildNumber = null; private static final String JVM_TIMER_SERVER_OPTION = "dataverse.timerServer"; @@ -132,127 +128,38 @@ public String getVersion() { // candidate for being moved into some kind of an application-scoped caching // service... some CachingService @Singleton - ? (L.A. 5.8) public String getVersion(boolean withBuildNumber) { - - if (appVersionString == null) { - - // The Version Number is no longer supplied in a .properties file - so - // we can't just do - // return BundleUtil.getStringFromBundle("version.number", null, ResourceBundle.getBundle("VersionNumber", Locale.US)); - // - // Instead, we'll rely on Maven placing the version number into the - // Manifest, and getting it from there: - // (this is considered a better practice, and will also allow us - // to maintain this number in only one place - the pom.xml file) - // -- L.A. 4.0.2 - - // One would assume, that once the version is in the MANIFEST.MF, - // as Implementation-Version:, it would be possible to obtain - // said version simply as - // appVersionString = getClass().getPackage().getImplementationVersion(); - // alas - that's not working, for whatever reason. (perhaps that's - // only how it works with jar-ed packages; not with .war files). - // People on the interwebs suggest that one should instead - // open the Manifest as a resource, then extract its attributes. - // There were some complications with that too. Plus, relying solely - // on the MANIFEST.MF would NOT work for those of the developers who - // are using "in place deployment" (i.e., where - // Netbeans runs their builds directly from the local target - // directory, bypassing the war file deployment; and the Manifest - // is only available in the .war file). For that reason, I am - // going to rely on the pom.properties file, and use java.util.Properties - // to read it. We have to look for this file in 2 different places - // depending on whether this is a .war file deployment, or a - // developers build. (the app-level META-INF is only populated when - // a .war file is built; the "maven-archiver" directory, on the other - // hand, is only available when it's a local build deployment). - // So, long story short, I'm resorting to the convoluted steps below. - // It may look hacky, but it should actually be pretty solid and - // reliable. - - - // First, find the absolute path url of the application persistence file - // always supplied with the Dataverse app: - java.net.URL fileUrl = Thread.currentThread().getContextClassLoader().getResource("META-INF/persistence.xml"); - String filePath = null; - - - if (fileUrl != null) { - filePath = fileUrl.getFile(); - if (filePath != null) { - InputStream mavenPropertiesInputStream = null; - String mavenPropertiesFilePath; - Properties mavenProperties = new Properties(); - - - filePath = filePath.replaceFirst("/[^/]*$", "/"); - // Using a relative path, find the location of the maven pom.properties file. - // First, try to look for it in the app-level META-INF. This will only be - // available if it's a war file deployment: - mavenPropertiesFilePath = filePath.concat("../../../META-INF/maven/edu.harvard.iq/dataverse/pom.properties"); - - try { - mavenPropertiesInputStream = new FileInputStream(mavenPropertiesFilePath); - } catch (IOException ioex) { - // OK, let's hope this is a local dev. build. - // In that case the properties file should be available in - // the maven-archiver directory: - - mavenPropertiesFilePath = filePath.concat("../../../../maven-archiver/pom.properties"); - - // try again: - - try { - mavenPropertiesInputStream = new FileInputStream(mavenPropertiesFilePath); - } catch (IOException ioex2) { - logger.warning("Failed to find and/or open for reading the pom.properties file."); - mavenPropertiesInputStream = null; - } - } - - if (mavenPropertiesInputStream != null) { - try { - mavenProperties.load(mavenPropertiesInputStream); - appVersionString = mavenProperties.getProperty("version"); - } catch (IOException ioex) { - logger.warning("caught IOException trying to read and parse the pom properties file."); - } finally { - IOUtils.closeQuietly(mavenPropertiesInputStream); - } - } - - } else { - logger.warning("Null file path representation of the location of persistence.xml in the webapp root directory!"); - } - } else { - logger.warning("Could not find the location of persistence.xml in the webapp root directory!"); - } - - - if (appVersionString == null) { - // still null? - defaulting to 4.0: - appVersionString = "4.0"; - } - } + // Retrieve the version via MPCONFIG + // NOTE: You may override the version via all methods of MPCONFIG. + // It will default to read from microprofile-config.properties source, + // which contains in the source a Maven property reference to ${project.version}. + // When packaging the app to deploy it, Maven will replace this, rendering it a static entry. + // NOTE: MicroProfile Config will cache the entry for us in internal maps. + String appVersion = JvmSettings.VERSION.lookup(); if (withBuildNumber) { - if (buildNumberString == null) { - // (build number is still in a .properties file in the source tree; it only - // contains a real build number if this war file was built by - // Jenkins) - + if (buildNumber == null) { + // (build number is still in a .properties file in the source tree; it only + // contains a real build number if this war file was built by Jenkins) + // TODO: might be replaced with same trick as for version via Maven property w/ empty default try { - buildNumberString = ResourceBundle.getBundle("BuildNumber").getString("build.number"); + buildNumber = ResourceBundle.getBundle("BuildNumber").getString("build.number"); } catch (MissingResourceException ex) { - buildNumberString = null; + buildNumber = null; + } + + // Also try to read the build number via MicroProfile Config if not already present from the + // properties file (so can be overridden by env var or other source) + if (buildNumber == null || buildNumber.isEmpty()) { + buildNumber = JvmSettings.BUILD.lookupOptional().orElse(""); } } - if (buildNumberString != null && !buildNumberString.equals("")) { - return appVersionString + " build " + buildNumberString; - } - } + if (!buildNumber.equals("")) { + return appVersion + " build " + buildNumber; + } + } - return appVersionString; + return appVersion; } public String getSolrHostColonPort() { diff --git a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java index 891b029f521..b8ad0a57748 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java @@ -1,13 +1,49 @@ package edu.harvard.iq.dataverse.util; +import edu.harvard.iq.dataverse.settings.JvmSettings; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; class SystemConfigTest { + SystemConfig systemConfig = new SystemConfig(); + + @Test + void testGetVersion() { + // given + String version = "100.100"; + System.setProperty(JvmSettings.VERSION.getScopedKey(), version); + + // when + String result = systemConfig.getVersion(false); + + // then + assertEquals(version, result); + } + + @Test + void testGetVersionWithBuild() { + // given + String version = "100.100"; + String build = "FOOBAR"; + System.setProperty(JvmSettings.VERSION.getScopedKey(), version); + System.setProperty(JvmSettings.BUILD.getScopedKey(), build); + + // when + String result = systemConfig.getVersion(true); + + // then + assertTrue(result.startsWith(version), "'" + result + "' not starting with " + version); + assertTrue(result.contains("build")); + + // Cannot test this here - there might be the bundle file present which is not under test control + //assertTrue(result.endsWith(build), "'" + result + "' not ending with " + build); + } + @Test void testGetLongLimitFromStringOrDefault_withNullInput() { long defaultValue = 5L; From 5f925edf6668893c96df5117157086ef641a5b44 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 30 Jun 2022 21:57:49 +0200 Subject: [PATCH 2/5] docs(dev): add some tips about new options dataverse.build and dataverse.version #7000 --- doc/sphinx-guides/source/developers/tips.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 3fff3e76ea8..2b15948bd34 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -173,6 +173,13 @@ commit id in your test deployment webpages on the bottom right corner next to th When you prefer manual updates, there is another script, see above: :ref:`custom_build_num_script`. +An alternative to that is using *MicroProfile Config* and set the option ``dataverse.build`` via a system property, +environment variable (``DATAVERSE_BUILD``) or `one of the other config sources +`__. + +You could even override the version itself with the option ``dataverse.version`` in the same way, which is usually +picked up from a build time source. + Sample Data ----------- From a7fe29c8e2e088fff71a13327e28c7cbb9595c15 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 16 Sep 2022 10:32:10 +0200 Subject: [PATCH 3/5] test(settings): make SystemConfigTest version testing use JvmSetting extension --- .../harvard/iq/dataverse/util/SystemConfigTest.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java index b8ad0a57748..3bbe331a361 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/SystemConfigTest.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.testing.JvmSetting; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -26,22 +27,18 @@ void testGetVersion() { } @Test + @JvmSetting(key = JvmSettings.VERSION, value = "100.100") + @JvmSetting(key = JvmSettings.BUILD, value = "FOOBAR") void testGetVersionWithBuild() { - // given - String version = "100.100"; - String build = "FOOBAR"; - System.setProperty(JvmSettings.VERSION.getScopedKey(), version); - System.setProperty(JvmSettings.BUILD.getScopedKey(), build); - // when String result = systemConfig.getVersion(true); // then - assertTrue(result.startsWith(version), "'" + result + "' not starting with " + version); + assertTrue(result.startsWith("100.100"), "'" + result + "' not starting with 100.100"); assertTrue(result.contains("build")); // Cannot test this here - there might be the bundle file present which is not under test control - //assertTrue(result.endsWith(build), "'" + result + "' not ending with " + build); + //assertTrue(result.endsWith("FOOBAR"), "'" + result + "' not ending with FOOBAR"); } @Test From 4b60983e360b3ee4b5a50535b769852fc9ea67ef Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 6 Dec 2022 09:25:42 +0100 Subject: [PATCH 4/5] refactor(settings): remove unused Config var in SystemConfig #7000 --- src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index fe95f53d293..fc7fd7beb06 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -10,8 +10,6 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.validation.PasswordValidatorUtil; -import org.eclipse.microprofile.config.Config; -import org.eclipse.microprofile.config.ConfigProvider; import org.passay.CharacterRule; import javax.ejb.EJB; @@ -46,7 +44,6 @@ public class SystemConfig { private static final Logger logger = Logger.getLogger(SystemConfig.class.getCanonicalName()); - private static final Config config = ConfigProvider.getConfig(); @EJB SettingsServiceBean settingsService; @@ -133,7 +130,6 @@ public String getVersion(boolean withBuildNumber) { // It will default to read from microprofile-config.properties source, // which contains in the source a Maven property reference to ${project.version}. // When packaging the app to deploy it, Maven will replace this, rendering it a static entry. - // NOTE: MicroProfile Config will cache the entry for us in internal maps. String appVersion = JvmSettings.VERSION.lookup(); if (withBuildNumber) { From 4ed101364bf3762320e85030d8b6e030127db6b5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 16 Dec 2022 15:22:02 -0500 Subject: [PATCH 5/5] cross link some docs #7000 --- .../source/developers/making-releases.rst | 11 +++++++++++ doc/sphinx-guides/source/developers/tips.rst | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 55f5f550dd9..a2575bb5f50 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -95,6 +95,8 @@ At this point you can send around the draft release for any final feedback. Link Make corrections to the draft, if necessary. It will be out of sync with the .md file, but that's ok (`#7988 `_ is tracking this). +.. _run-build-create-war: + Run a Build to Create the War File ---------------------------------- @@ -110,6 +112,15 @@ Click "Save" then "Build Now". The build number will appear in ``/api/info/version`` (along with the commit mentioned above) from a running installation (e.g. ``{"version":"5.10.1","build":"907-b844672``). +Note that the build number comes from script in an early build step... + +.. code-block:: bash + + COMMIT_SHA1=`echo $GIT_COMMIT | cut -c-7` + echo "build.number=${BUILD_NUMBER}-${COMMIT_SHA1}" > $WORKSPACE/src/main/java/BuildNumber.properties + +... but we can explore alternative methods of specifying the build number, as described in :ref:`auto-custom-build-number`. + Build Installer (dvinstall.zip) ------------------------------- diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst index 2b15948bd34..bf75a05f84e 100755 --- a/doc/sphinx-guides/source/developers/tips.rst +++ b/doc/sphinx-guides/source/developers/tips.rst @@ -58,6 +58,8 @@ From the root of the git repo, run the following command to set the build number This should update or place a file at ``src/main/java/BuildNumber.properties``. +(See also :ref:`auto-custom-build-number` for other ways of changing the build number.) + Then, from Netbeans, click "Run" and then "Clean and Build Project (dataverse)". After this completes successfully, click "Run" and then "Run Project (dataverse)" Confirm the Change Was Deployed @@ -164,6 +166,8 @@ Git on Mac On a Mac, you won't have git installed unless you have "Command Line Developer Tools" installed but running ``git clone`` for the first time will prompt you to install them. +.. _auto-custom-build-number: + Automation of Custom Build Number on Webpage ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -180,6 +184,8 @@ environment variable (``DATAVERSE_BUILD``) or `one of the other config sources You could even override the version itself with the option ``dataverse.version`` in the same way, which is usually picked up from a build time source. +See also discussion of version numbers in :ref:`run-build-create-war`. + Sample Data -----------