diff --git a/.travis.yml b/.travis.yml index dfe28fb..f7b69a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,32 @@ -language: java -sudo: false -jdk: openjdk8 +jobs: + include: + - os: linux + jdk: openjdk8 + language: java + - os: windows + language: shell + +env: + - MAVEN_VERSION=3.6.3 JAVA_VERSION=8.242.8.1 before_install: - - "echo $JAVA_OPTS" + - if [ "$TRAVIS_OS_NAME" = "windows" ]; then + echo "Installing java ${JAVA_VERSION}"; + choco install openjdk8 --version ${JAVA_VERSION}; + echo "Installing maven ${MAVEN_VERSION}"; + choco install maven --version ${MAVEN_VERSION}; + export JAVA_HOME="/c/Program Files/OpenJDK/openjdk-8u242-b08"; + export MAVEN_HOME="/c/ProgramData/chocolatey/lib/maven/apache-maven-${MAVEN_VERSION}/bin"; + export PATH="${PATH}:${JAVA_HOME}/bin:${MAVEN_HOME}"; + fi + - "echo ${PATH}" + - "mvn -N io.takari:maven:0.7.7:wrapper -Dmaven=${MAVEN_VERSION}" + script: - - mvn install -B -V - - mvn javadoc:jar - - mvn javadoc:test-aggregate + - ./mvnw install -B -V + - ./mvnw javadoc:jar + - ./mvnw javadoc:test-aggregate after_success: - - bash <(curl -s https://codecov.io/bash) + - if [ "$TRAVIS_OS_NAME" = "linux" ]; then bash <(curl -s https://codecov.io/bash); fi diff --git a/src/main/java/org/duraspace/bagit/BagWriter.java b/src/main/java/org/duraspace/bagit/BagWriter.java index 83d1a41..c2dcbe4 100644 --- a/src/main/java/org/duraspace/bagit/BagWriter.java +++ b/src/main/java/org/duraspace/bagit/BagWriter.java @@ -4,19 +4,12 @@ */ package org.duraspace.bagit; -import static org.duraspace.bagit.BagItDigest.MD5; -import static org.duraspace.bagit.BagItDigest.SHA1; -import static org.duraspace.bagit.BagItDigest.SHA256; -import static org.duraspace.bagit.BagItDigest.SHA512; - -import java.io.BufferedWriter; import java.io.File; -import java.io.FileOutputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.OutputStream; -import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; +import java.security.DigestOutputStream; import java.security.MessageDigest; import java.util.HashMap; import java.util.Map; @@ -38,6 +31,12 @@ public class BagWriter { private Map> tagFileRegistry; private Map> tagRegistry; + /** + * This map provides a way to retrieve all ongoing MessageDigests so that multiple checksums + * can easily be run and retrieved + */ + private Map activeStreams; + /** * Version of the BagIt specification implemented */ @@ -64,6 +63,8 @@ public BagWriter(final File bagDir, final Set algorithms) { bagitValues.put("BagIt-Version", BAGIT_VERSION); bagitValues.put("Tag-File-Character-Encoding", "UTF-8"); tagRegistry.put("bagit.txt", bagitValues); + + activeStreams = new HashMap<>(); } /** @@ -109,15 +110,15 @@ public Map getTags(final String key) { * @throws IOException when an I/O error occurs */ public void write() throws IOException { - writeManifests("manifest", payloadRegistry); + writeManifests("manifest", payloadRegistry, true); for (String tagFile : tagRegistry.keySet()) { writeTagFile(tagFile); } - writeManifests("tagmanifest", tagFileRegistry); + writeManifests("tagmanifest", tagFileRegistry, false); } - private void writeManifests(final String prefix, final Map> registry) - throws IOException { + private void writeManifests(final String prefix, final Map> registry, + final boolean registerToTags) throws IOException { final String delimiter = " "; final char backslash = '\\'; final char bagitSeparator = '/'; @@ -127,12 +128,21 @@ private void writeManifests(final String prefix, final Map filemap = registry.get(algorithm); if (filemap != null) { final File f = new File(bagDir, prefix + "-" + algorithm.bagitName() + ".txt"); - try (PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(f)))) { + try (OutputStream out = streamFor(f.toPath())) { for (final File payload : filemap.keySet()) { // replace all occurrences of backslashes, which are not allowed per the bagit spec final String relative = bag.relativize(payload.toPath()).toString() .replace(backslash, bagitSeparator); - out.println(filemap.get(payload) + delimiter + relative); + final String line = filemap.get(payload) + delimiter + relative; + out.write(line.getBytes()); + out.write("\n".getBytes()); + + if (registerToTags) { + for (Map.Entry entry : activeStreams.entrySet()) { + addTagChecksum(entry.getKey(), f, entry.getValue().getMessageDigest()); + } + } + activeStreams.clear(); } } } @@ -144,48 +154,41 @@ private void writeTagFile(final String key) throws IOException { if (values != null) { final File f = new File(bagDir, key); - MessageDigest md5 = null; - MessageDigest sha1 = null; - MessageDigest sha256 = null; - MessageDigest sha512 = null; - if (algorithms.contains(MD5)) { - md5 = MD5.messageDigest(); - } - if (algorithms.contains(SHA1)) { - sha1 = SHA1.messageDigest(); - } - if (algorithms.contains(SHA256)) { - sha256 = SHA256.messageDigest(); - } - if (algorithms.contains(SHA512)) { - sha512 = SHA512.messageDigest(); - } - - try (OutputStream out = new FileOutputStream(f)) { + try (OutputStream out = streamFor(f.toPath())) { for (final String field : values.keySet()) { final byte[] bytes = (field + ": " + values.get(field) + "\n").getBytes(); out.write(bytes); - - if (md5 != null) { - md5.update(bytes); - } - if (sha1 != null) { - sha1.update(bytes); - } - if (sha256 != null) { - sha256.update(bytes); - } - if (sha512 != null) { - sha512.update(bytes); - } } } - addTagChecksum(MD5, f, md5); - addTagChecksum(SHA1, f, sha1); - addTagChecksum(SHA256, f, sha256); - addTagChecksum(SHA512, f, sha512); + for (Map.Entry entry : activeStreams.entrySet()) { + addTagChecksum(entry.getKey(), f, entry.getValue().getMessageDigest()); + } } + + activeStreams.clear(); + } + + /** + * Create an {@link OutputStream} for a given {@link Path} which can be used to write data to the file. + * This wraps the returned {@link OutputStream} with {@link DigestOutputStream}s in order to create a checksum + * for the file as it is being written. There is one {@link DigestOutputStream} per {@link BagItDigest} in this + * classes registered {@code algorithms}. Each {@link DigestOutputStream} is stored in the {@code activeStreams} so + * that it can be retrieved later on. + * + * @param file the {@link Path} to create an {@link OutputStream} for + * @return the {@link OutputStream} + * @throws IOException if there is an error creating the {@link OutputStream} + */ + private OutputStream streamFor(final Path file) throws IOException { + OutputStream lastStream = Files.newOutputStream(file); + for (BagItDigest algorithm : algorithms) { + final DigestOutputStream dos = new DigestOutputStream(lastStream, algorithm.messageDigest()); + activeStreams.put(algorithm, dos); + lastStream = dos; + } + + return lastStream; } private void addTagChecksum(final BagItDigest algorithm, final File f, final MessageDigest digest) { diff --git a/src/test/java/org/duraspace/bagit/BagWriterTest.java b/src/test/java/org/duraspace/bagit/BagWriterTest.java index f1c8dd2..2252ca0 100644 --- a/src/test/java/org/duraspace/bagit/BagWriterTest.java +++ b/src/test/java/org/duraspace/bagit/BagWriterTest.java @@ -18,10 +18,12 @@ import java.nio.file.Paths; import java.security.MessageDigest; import java.time.LocalDate; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Stream; import gov.loc.repository.bagit.domain.Bag; import gov.loc.repository.bagit.exceptions.CorruptChecksumException; @@ -142,6 +144,17 @@ public void write() throws IOException { final List extraLines = Files.readAllLines(extra); assertThat(extraLines).contains("test-key: test-value"); + // Assert that tagmanifest-{sha1,sha256,sha512}.txt contain the manifest checksums + final String manifestRegex = sha1.bagitName() + "|" + sha256.bagitName() + "|" + sha512.bagitName(); + for (Path tagmanifest : Sets.newHashSet(sha1Tagmanifest, sha256Tagmanifest, sha512Tagmanifest)) { + try (Stream lines = Files.lines(tagmanifest)) { + assertThat(lines) + .filteredOn(line -> line.contains("manifest")) + .hasSize(3) + .allSatisfy(entry -> assertThat(entry).containsPattern(manifestRegex)); + } + } + // Finally, pass BagProfile validation and BagIt validation final BagReader reader = new BagReader(); final BagVerifier verifier = new BagVerifier(); @@ -158,4 +171,19 @@ public void write() throws IOException { fail("Unable to verify bag:\n" + e.getMessage()); } } + + @Test(expected = RuntimeException.class) + public void testAddInvalidAlgorithm() throws IOException { + // The message digests to use + final BagItDigest sha1 = BagItDigest.SHA1; + final BagItDigest sha256 = BagItDigest.SHA256; + + // Create a writer with 3 manifest algorithms + Files.createDirectories(bag); + final BagWriter writer = new BagWriter(bag.toFile(), Sets.newHashSet(sha1)); + + // we don't need to pass any files, just the errant BagItDigest + writer.registerChecksums(sha256, Collections.emptyMap()); + } + } \ No newline at end of file