diff --git a/.github/workflows/build-2.x.yml b/.github/workflows/build-2.x.yml new file mode 100644 index 00000000..bc7e50b8 --- /dev/null +++ b/.github/workflows/build-2.x.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + braches: [2.x] + pull_request: + branches: [2.x] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + path: build_dir + - uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: '11' + check-latest: true + - name: Gradle Build + run: | + cd $GITHUB_WORKSPACE/build_dir + ./gradlew --stacktrace build + - name: Gradle Docs + run: | + cd $GITHUB_WORKSPACE/build_dir + ./gradlew --stacktrace docs + - name: Codecov report + run: | + cd $GITHUB_WORKSPACE/build_dir + ./gradlew codeCoverageReport + - name: Codecov + uses: codecov/codecov-action@v1 + - name: Upload Archives + run: | + cd $GITHUB_WORKSPACE/build_dir + ./gradlew uploadArchives -PossrhUsername='${{ secrets.SONATYPE_USERNAME }}' -PossrhPassword='${{ secrets.SONATYPE_PASSWORD}}' + if: steps.extract_branch.outputs.branch == 'main' diff --git a/.gitignore b/.gitignore index 168c02e0..2f466fc1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ build bin/ .bloop/ .metals/ +islandora-alpaca-app/src/main/resources/alpaca.properties diff --git a/README.md b/README.md index 4876220e..d1e2e1cc 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,267 @@ Event-driven middleware based on [Apache Camel](http://camel.apache.org/) that s ## Requirements -This project requires Java 8 and can be built with [Gradle](https://gradle.org). To build and test locally, use `./gradlew build`. +This project requires Java 11 and can be built with [Gradle](https://gradle.org). + +To build and test locally, clone this repository and then change into the Alpaca directory. +Next run `./gradlew clean build shadowJar`. + +The main executable jar is available in the `islandora-alpaca-app/build/libs` directory, with the classifier `-all`. + +ie. +> islandora-alpaca-app/build/libs/islandora-alpaca-app-2.0.0-all.jar + +## Configuration + +Alpaca is made up of several services, each of these can be enabled or disabled individually. + +Alpaca takes an external file to configure its behaviour. + +Look at the [`example.properties`](example.properties) file to see some example settings. + +The properties are: + +``` +# Common options +error.maxRedeliveries=4 +``` +This defines how many times to retry a message before failing completely. + +There are also common ActiveMQ properties to setup the connection. + +``` +# ActiveMQ options +jms.brokerUrl=tcp://localhost:61616 +``` + +This defines the url to the ActiveMQ broker. + +``` +jms.username= +jms.password= +``` +This defines the login credentials (if required) + +``` +jms.connections=10 +``` +This defines the pool of connections to the ActiveMQ instance. + +``` +jms.concurrent-consumers=1 +``` +This defines how many messages to process simultaneously. + +### islandora-indexing-fcrepo + +This service manages a Drupal node into a corresponding Fedora resource. + +It's properties are: + +``` +# Fcrepo indexer options +fcrepo.indexer.enabled=true +``` + +This defines whether the Fedora indexer is enabled or not. + +``` +fcrepo.indexer.node=queue:islandora-indexing-fcrepo-content +fcrepo.indexer.delete=queue:islandora-indexing-fcrepo-delete +fcrepo.indexer.media=queue:islandora-indexing-fcrepo-media +fcrepo.indexer.external=queue:islandora-indexing-fcrepo-file-external +``` + +These define the various queues to listen on for the indexing/deletion +messages. The part after `queue:` should match your Islandora instance "Actions". + +``` +fcrepo.indexer.milliner.baseUrl=http://localhost:8000/milliner +``` +This defines the location of your Milliner microservice. + +``` +fcrepo.indexer.concurrent-consumers=1 +fcrepo.indexer.max-concurrent-consumers=1 +``` +These define the default number of concurrent consumers and maximum number of concurrent +consumers working off your ActiveMQ instance. +A value of `-1` means no setting is applied. + +``` +fcrepo.indexer.async-consumer=true +``` + +This property allows the concurrent consumers to process concurrently; otherwise, the consumers will wait to the previous message has been processed before executing. + +### islandora-indexing-triplestore + +This service indexes the Drupal node into the configured triplestore + +It's properties are: + +``` +# Triplestore indexer options +triplestore.indexer.enabled=false +``` + +This defines whether the Triplestore indexer is enabled or not. + +``` +triplestore.index.stream=queue:islandora-indexing-triplestore-index +triplestore.delete.stream=queue:islandora-indexing-triplestore-delete +``` + +These define the various queues to listen on for the indexing/deletion +messages. The part after `queue:` should match your Islandora instance "Actions". + +``` +triplestore.baseUrl=http://localhost:8080/bigdata/namespace/kb/sparql +``` + +This defines the location of your triplestore's SPARQL update endpoint. + +``` +triplestore.indexer.concurrent-consumers=1 +triplestore.indexer.max-concurrent-consumers=1 +``` + +These define the default number of concurrent consumers and maximum number of concurrent +consumers working off your ActiveMQ instance. +A value of `-1` means no setting is applied. + + +``` +triplestore.indexer.async-consumer=true +``` + +This property allows the concurrent consumers to process concurrently; otherwise, the consumers will wait to the previous message has been processed before executing. + +### islandora-connector-derivative + +This service is used to configure an external microservice. This service will deploy multiple copies of its routes +with different configured inputs and outputs based on properties. + +The routes to be configured are defined with the property `derivative.systems.installed` which expects +a comma separated list. Each item in the list defines a new route and must also define 3 additional properties. + +``` +derivative..enabled=true +``` + +This defines if the `item` service is enabled. + +``` +derivative..in.stream=queue:islandora-item-connector.index +``` + +This is the input queue for the derivative microservice. +The part after `queue:` should match your Islandora instance "Actions". + +``` +derivative..service.url=http://example.org/derivative/convert +``` + +This is the microservice URL to process the request. + +``` +derivative..concurrent-consumers=1 +derivative..max-concurrent-consumers=1 +``` + +These define the default number of concurrent consumers and maximum number of concurrent +consumers working off your ActiveMQ instance. +A value of `-1` means no setting is applied. + + +``` +derivative..async-consumer=true +``` + +This property allows the concurrent consumers to process concurrently; otherwise, the consumers will wait to the previous message has been processed before executing. + +For example, with two services defined (houdini and crayfits) my configuration would have + +``` +derivative.systems.installed=houdini,fits + +derivative.houdini.enabled=true +derivative.houdini.in.stream=queue:islandora-connector-houdini +derivative.houdini.service.url=http://127.0.0.1:8000/houdini/convert +derivative.houdini.concurrent-consumers=1 +derivative.houdini.max-concurrent-consumers=4 +derivative.houdini.async-consumer=true + +derivative.fits.enabled=true +derivative.fits.in.stream=queue:islandora-connector-fits +derivative.fits.service.url=http://127.0.0.1:8000/crayfits +derivative.fits.concurrent-consumers=2 +derivative.fits.max-concurrent-consumers=2 +derivative.fits.async-consumer=false +``` + +### Customizing HTTP client timeouts + +You can alter the HTTP client from the defaults for its request, connection and socket timeouts. +To do this you want to enable the request configurer. + +```shell +request.configurer.enabled=true +``` + +Then set the next 3 timeouts (measured in milliseconds) to the desired timeout. + +```shell +request.timeout=-1 +connection.timeout=-1 +socket.timeout=-1 +``` + +The default for all three is `-1` which indicates no timeout. + +## Deploying/Running + +You can see the options by passing the `-h|--help` flag + +```shell +> java -jar islandora-alpaca-app/build/libs/islandora-alpaca-app-2.0.0-all.jar -h +Usage: alpaca [-hV] [-c=] + -h, --help Show this help message and exit. + -V, --version Print version information and exit. + -c, --config= + The path to the configuration file +``` + +Using the `-V|--version` flag will just return the current version of the application. + +```shell +> java -jar islandora-alpaca-app/build/libs/islandora-alpaca-app-2.0.0-all.jar -v +2.0.0 +``` + +To start Alpaca you would pass the external property file with the `-c|--config` flag. + +For example if you are using an external properties file located at `/opt/my.properties`, +you would run: + +```shell +java -jar islandora-alpaca-app-2.0.0-all.jar -c /opt/my.properties +``` + +## Debugging/Troubleshooting + +Logging is done to the console, and defaults to the INFO level. To get more verbose logging you +can use the Java property `islandora.alpaca.log` + +i.e. + +```shell +java -Dislandora.alpaca.log=DEBUG -jar islandora-alpaca-app-2.0.0-all.jar -c /opt/my.properties +``` ## Documentation -Further documentation for this module is available on the [Islandora 8 documentation site](https://islandora.github.io/documentation/). +Further documentation for this module is available on the [Islandora documentation site](https://islandora.github.io/documentation/). ## Troubleshooting/Issues diff --git a/build.gradle b/build.gradle index 509390af..cf789706 100644 --- a/build.gradle +++ b/build.gradle @@ -1,11 +1,17 @@ plugins { id 'com.github.hierynomus.license' version '0.15.0' id 'net.researchgate.release' version '2.8.1' + id 'java-library' + id 'com.github.johnrengelman.shadow' version '6.1.0' apply(false) } +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent + allprojects { - apply plugin: 'maven' + apply plugin: 'maven-publish' apply plugin: 'jacoco' + apply plugin: 'java' group = 'ca.islandora.alpaca' @@ -28,15 +34,16 @@ allprojects { } subprojects { - apply plugin: 'java' apply plugin: 'maven-publish' apply plugin: 'signing' apply plugin: 'checkstyle' apply plugin: 'com.github.hierynomus.license' apply plugin: 'pmd' - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + java { + sourceCompatibility(JavaVersion.VERSION_11) + targetCompatibility(JavaVersion.VERSION_11) + } ext { vendor = 'Islandora Foundation' @@ -46,24 +53,26 @@ subprojects { license = 'MIT' versions = [ - activemq: '5.15.0', + activemq: '5.16.0', asmCommons: '5.0.3', - camel: '2.20.4', - commonsIo: '2.4', + camel: '3.7.6', + commonsIo: '2.8.0', fcrepoCamel: '5.0.0', fcrepoCamelToolbox: '5.0.0', - httpClientOsgi: '4.5.3', - httpCoreOsgi: '4.4.6', - jackson: '2.8.0', + httpClient: '4.5.13', + httpCore: '4.4.14', + jackson: '2.13.0', + javaxApi: '1.3.2', javaxInject: '1', - junit: '4.12', - junitToolbox: '2.3', - karaf: '4.0.6', - osgiCore: '6.0.0', - osgiFramework: '1.9.0', - paxExam: '4.13.2', - slf4j: '1.7.28', - xercesServiceMix: '2.11.0_1' + javaxJms: '2.0.1', + jena: '3.17.0', + jsonSmart: '2.4.7', + junit4: '4.13.2', + logback: '1.2.6', + picocli: '4.6.1', + slf4j: '1.7.32', + spring: '5.3.0', + xerces : '2.4.0' ] /* OSGi */ @@ -71,13 +80,6 @@ subprojects { projectOsgiVersion = project.version.replaceAll("-SNAPSHOT", ".SNAPSHOT") } - task processConfig(type: Copy) { - from('src/main/cfg') { - include '**/*.cfg' - } - into 'build/cfg/main' - } - task sourceJar(type: Jar) { classifier 'sources' from sourceSets.main.allSource @@ -93,18 +95,17 @@ subprojects { archives sourceJar } - classes { - classes.dependsOn processConfig - } - checkstyle { - configFile = rootProject.file('gradle/checkstyle/checkstyle.xml') - configProperties.checkstyleConfigDir = rootProject.file('gradle/checkstyle') + config = resources.text.fromFile("${rootProject.projectDir}/gradle/checkstyle/checkstyle.xml") + configProperties("checkStyleDir": "${rootProject.projectDir}/gradle/checkstyle") + toolVersion "7.6.1" ignoreFailures false } license { include "**/*.java" + // Ignore copied Sparql processors and ConditionOnProperty classes from fcrepo-camel and fcrepo-camel-toolbox + excludes(["**/indexing/triplestore/processors/*.java", "**/support/config/ConditionOnProperty*.java"]) header rootProject.file('gradle/license/header.txt') mapping { java = 'SLASHSTAR_STYLE' @@ -112,99 +113,133 @@ subprojects { } pmd { - toolVersion = '6.19.0' + toolVersion = '6.21.0' // Only run on src/main sourceSets = [sourceSets.main] ignoreFailures = true + ruleSets = [] ruleSetFiles = rootProject.files('gradle/pmd/ruleset.xml') } publishing { - publications { - maven(MavenPublication) { - from components.java - } - } - } - - signing { - required { !version.endsWith("SNAPSHOT") && gradle.taskGraph.hasTask("uploadArchives") } - sign configurations.archives - } - - uploadArchives { - repositories.mavenDeployer { + repositories { def sonatypeUsername = project.hasProperty('ossrhUsername') ? ossrhUsername : "" def sonatypePassword = project.hasProperty('ossrhPassword') ? ossrhPassword : "" - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: sonatypeUsername, password: sonatypePassword) - } - - snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { - authentication(userName: sonatypeUsername, password: sonatypePassword) - } - - pom.project { - packaging 'jar' - url project.homepage - inceptionYear project.inceptionYear - name 'Islandora :: Alpaca' - description 'Event driven middleware based on Apache Camel that synchronizes a Fedora 4 with Drupal.' - - organization { - name project.vendor - url project.homepage + def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/" + def snapshotRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/" + maven { + url version.endsWith("SNAPSHOT") ? snapshotRepoUrl : releasesRepoUrl + credentials(PasswordCredentials) { + username = sonatypeUsername + password = sonatypePassword } + } + } + publications { + maven(MavenPublication) { + from components.java + pom { + setPackaging('jar') + inceptionYear = project.inceptionYear + name = 'Islandora :: Alpaca' + description = 'Event driven middleware based on Apache Camel that synchronizes a Fedora with Drupal.' + + organization { + name = project.vendor + url = project.homepage + } - scm { - connection 'scm:git:git@github.com:Islandora/Alpaca.git' - developerConnection 'scm:git:git@github.com:Islandora/Alpaca.git' - url 'https://github.com/islandora/Alpaca' - tag 'HEAD' - } + scm { + connection = 'scm:git:git@github.com:Islandora/Alpaca.git' + developerConnection = 'scm:git:git@github.com:Islandora/Alpaca.git' + url = 'https://github.com/islandora/Alpaca' + tag = 'HEAD' + } - mailingLists { - mailingList { - name 'islandora-dev' - subscribe 'islandora-dev+subscribe@googlegroups.com' - unsubscribe 'islandora-dev+unsubscribe@googlegroups.com' - post 'islandora-dev@googlegroups.com' - archive 'https://groups.google.com/d/forum/islandora-dev' + mailingLists { + mailingList { + name = 'islandora-dev' + subscribe = 'islandora-dev+subscribe@googlegroups.com' + unsubscribe = 'islandora-dev+unsubscribe@googlegroups.com' + post = 'islandora-dev@googlegroups.com' + archive = 'https://groups.google.com/d/forum/islandora-dev' + } } - } - issueManagement { - system 'GitHub' - url 'https://github.com/Islandora/documentation/issues' - } + issueManagement { + system = 'GitHub' + url = 'https://github.com/Islandora/documentation/issues' + } - developers { - developer { - id 'dannylamb' - name 'Daniel Lamb' - email 'dlamb @ (domain of organization url)' - organization 'Islandora Foundation' - organizationUrl 'http://islandora.ca' - roles { - role 'developer' + developers { + developer { + id = 'dannylamb' + name = 'Daniel Lamb' + email = 'dlamb @ (domain of organization url)' + organization = 'Born Digital' + organizationUrl ='http://born-digital.com' + timezone = '-4' + } + developer { + id = 'whikloj' + name = 'Jared Whiklo' + email = 'jared.whiklo @ (domain of organization url)' + organization = ' University of Manitoba' + organizationUrl = 'https://umanitoba.ca' + timezone = '-5' } - timezone '-4' } - } - licenses { - license { - name 'MIT' - url 'https://opensource.org/licenses/MIT' - comments 'Copyright (c) 2015 Islandora Foundation' + licenses { + license { + name = 'MIT' + url = 'https://opensource.org/licenses/MIT' + distribution = 'https://opensource.org/licenses/MIT' + comments = 'Copyright (c) 2015 Islandora Foundation' + } } } } } } + + signing { + required { !version.endsWith("SNAPSHOT") && gradle.taskGraph.hasTask("uploadArchives") } + sign configurations.archives + } + + tasks.named('test') { + useJUnit() + } + + tasks.withType(Test) { + testLogging { + // set options for log level LIFECYCLE + events TestLogEvent.FAILED, + TestLogEvent.PASSED, + TestLogEvent.SKIPPED + //TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + showExceptions true + showCauses true + showStackTraces true + + // set options for log level DEBUG and INFO + debug { + events TestLogEvent.STARTED, + TestLogEvent.FAILED, + TestLogEvent.PASSED, + TestLogEvent.SKIPPED, + TestLogEvent.STANDARD_ERROR, + TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + } + info.events = debug.events + info.exceptionFormat = debug.exceptionFormat + } + } + } task docs(type: Javadoc) { diff --git a/example.properties b/example.properties new file mode 100644 index 00000000..5d5ad8c3 --- /dev/null +++ b/example.properties @@ -0,0 +1,64 @@ +# Common options +error.maxRedeliveries=5 +jms.brokerUrl=tcp://localhost:61616 +jms.username= +jms.password= +jms.connections=10 + +# Custom Http client options +# All timeouts in milliseconds +request.configurer.enabled=false +request.timeout=-1 +connection.timeout=-1 +socket.timeout=-1 + +# Fedora indexer options +fcrepo.indexer.enabled=true +fcrepo.indexer.node=queue:islandora-indexing-fcrepo-content +fcrepo.indexer.delete=queue:islandora-indexing-fcrepo-delete +fcrepo.indexer.media=queue:islandora-indexing-fcrepo-media +fcrepo.indexer.external=queue:islandora-indexing-fcrepo-file-external +fcrepo.indexer.milliner.baseUrl=http://127.0.0.1:8000/milliner/ +fcrepo.indexer.concurrent-consumers=-1 +fcrepo.indexer.max-concurrent-consumers=-1 +fcrepo.indexer.async-consumer=false + +# Triplestore indexer options +triplestore.indexer.enabled=true +triplestore.baseUrl=http://127.0.0.1:8080/bigdata/namespace/kb/sparql +triplestore.index.stream=queue:islandora-indexing-triplestore-index +triplestore.delete.stream=queue:islandora-indexing-triplestore-delete +triplestore.indexer.concurrent-consumers=-1 +triplestore.indexer.max-concurrent-consumers=-1 +triplestore.indexer.async-consumer=false + +# Derivative services +derivative.systems.installed=fits,homarus,houdini,ocr + +derivative.fits.enabled=true +derivative.fits.in.stream=queue:islandora-connector-fits +derivative.fits.service.url=http://localhost:8000/crayfits +derivative.fits.concurrent-consumers=-1 +derivative.fits.max-concurrent-consumers=-1 +derivative.fits.async-consumer=false + +derivative.homarus.enabled=true +derivative.homarus.in.stream=queue:islandora-connector-homarus +derivative.homarus.service.url=http://127.0.0.1:8000/homarus/convert +derivative.homarus.concurrent-consumers=-1 +derivative.homarus.max-concurrent-consumers=-1 +derivative.homarus.async-consumer=false + +derivative.houdini.enabled=true +derivative.houdini.in.stream=queue:islandora-connector-houdini +derivative.houdini.service.url=http://127.0.0.1:8000/houdini/convert +derivative.houdini.concurrent-consumers=-1 +derivative.houdini.max-concurrent-consumers=-1 +derivative.houdini.async-consumer=false + +derivative.ocr.enabled=true +derivative.ocr.in.stream=queue:islandora-connector-ocr +derivative.ocr.service.url=http://localhost:8000/hypercube +derivative.ocr.concurrent-consumers=-1 +derivative.ocr.max-concurrent-consumers=-1 +derivative.ocr.async-consumer=false diff --git a/gradle.properties b/gradle.properties index 7adefe8a..f1859671 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1 @@ -version = 1.0.2 -org.gradle.parallel +version = 2.0.0 diff --git a/gradle/checkstyle/checkstyle.xml b/gradle/checkstyle/checkstyle.xml index a08a41da..5cbd924e 100644 --- a/gradle/checkstyle/checkstyle.xml +++ b/gradle/checkstyle/checkstyle.xml @@ -5,7 +5,7 @@ - + diff --git a/gradle/pmd/ruleset.xml b/gradle/pmd/ruleset.xml index 53d8a9f4..76998b30 100644 --- a/gradle/pmd/ruleset.xml +++ b/gradle/pmd/ruleset.xml @@ -5,7 +5,7 @@ xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 http://pmd.sourceforge.net/ruleset_2_0_0.xsd"> - Islandora 8 Alpaca PMD ruleset + Islandora Alpaca PMD ruleset @@ -26,7 +26,12 @@ - + + + + + @@ -61,7 +66,11 @@ - + + + + + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 647564f6..7244eee2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ #Fri Nov 01 09:16:17 CDT 2019 -distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStorePath=wrapper/dists diff --git a/islandora-alpaca-app/build.gradle b/islandora-alpaca-app/build.gradle new file mode 100644 index 00000000..e349ce3d --- /dev/null +++ b/islandora-alpaca-app/build.gradle @@ -0,0 +1,65 @@ +plugins { + id 'com.github.johnrengelman.shadow' +} + +import com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer + +description = 'Islandora Alpaca application' + +dependencies { + implementation "info.picocli:picocli:${versions.picocli}" + implementation "org.apache.camel:camel-spring-javaconfig:${versions.camel}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "org.springframework:spring-context:${versions.spring}" + implementation project(':islandora-support') + implementation project(':islandora-connector-derivative') + implementation project(':islandora-indexing-fcrepo') + implementation project(':islandora-indexing-triplestore') + + runtimeOnly "ch.qos.logback:logback-classic:${versions.logback}" + +} + +sourceSets { + main { + resources { + srcDirs = ['src/main/resources'] + } + } +} + +shadowJar { + mergeServiceFiles() + transform(AppendingTransformer) { + resource = "META-INF/services/org/apache/camel/TypeConverterLoader" + } +} + +jar { + manifest { + attributes( + "Main-Class": "ca.islandora.alpaca.driver.AlpacaDriver" + ) + } + + File outputFileDir = project.file( getProjectDir().toString() + '/src/main/resources' ) + File outputFile = new File( outputFileDir, 'alpaca.properties' ) + + doFirst { + outputFileDir.mkdirs() + + Properties properties = new Properties(); + + // Add properties, the file is needed by VersionProvider to determine the current version of Alpaca. + properties.setProperty("groupId", "${project.group}"); + properties.setProperty("artifactId", "${project.name}"); + properties.setProperty("version", "${project.version}"); + FileOutputStream outputStream = new FileOutputStream(outputFile); + try { + properties.store(outputStream, "Generated from Gradle") + } + finally { + outputStream.close() + } + } +} diff --git a/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/AlpacaConfig.java b/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/AlpacaConfig.java new file mode 100644 index 00000000..a3ebadfe --- /dev/null +++ b/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/AlpacaConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.driver; + +import org.apache.camel.spring.javaconfig.CamelConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +/** + * A configuration class for the application, scans for other configuration classes. + * + * @author dbernstein + * @author whikloj + */ +@Configuration +@ComponentScan(basePackages = {"ca.islandora.alpaca"}) +public class AlpacaConfig extends CamelConfiguration { + +} diff --git a/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/AlpacaDriver.java b/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/AlpacaDriver.java new file mode 100644 index 00000000..caa193d0 --- /dev/null +++ b/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/AlpacaDriver.java @@ -0,0 +1,83 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.driver; + +import static ca.islandora.alpaca.support.config.PropertyConfig.ALPACA_CONFIG_PROPERTY; +import static org.slf4j.LoggerFactory.getLogger; + +import java.nio.file.Path; +import java.util.concurrent.Callable; + +import org.slf4j.Logger; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +import picocli.CommandLine; + +/** + * Command line application class + * @author whikloj + */ +@CommandLine.Command(name = "alpaca", mixinStandardHelpOptions = true, sortOptions = false, + versionProvider = VersionProvider.class) +public class AlpacaDriver implements Callable { + + /** + * Logger instance. + */ + private static final Logger LOGGER = getLogger(AlpacaDriver.class); + + /** + * Configuration file. + */ + @CommandLine.Option(names = {"--config", "-c"}, required = false, order = 1, + description = "The path to the configuration file") + private Path configurationFilePath; + + @Override + public Integer call() throws Exception { + if (configurationFilePath != null) { + System.setProperty(ALPACA_CONFIG_PROPERTY, configurationFilePath.toFile().getAbsolutePath()); + } + final var appContext = new AnnotationConfigApplicationContext("ca.islandora.alpaca"); + try { + appContext.start(); + LOGGER.info("Alpaca started."); + + while (appContext.isRunning()) { + try { + Thread.sleep(1000); + } catch (final InterruptedException e) { + throw new RuntimeException("This should never happen"); + } + } + return 0; + } finally { + appContext.close(); + } + } + + /** + * @param args Command line arguments + */ + public static void main(final String[] args) { + final AlpacaDriver driver = new AlpacaDriver(); + final CommandLine cmd = new CommandLine(driver); + cmd.execute(args); + } + +} diff --git a/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/VersionProvider.java b/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/VersionProvider.java new file mode 100644 index 00000000..7e37e8e8 --- /dev/null +++ b/islandora-alpaca-app/src/main/java/ca/islandora/alpaca/driver/VersionProvider.java @@ -0,0 +1,47 @@ +package ca.islandora.alpaca.driver;/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import picocli.CommandLine; + +/** + * Provides the current version of Alpaca to picocli + * @author whikloj + */ +public class VersionProvider implements CommandLine.IVersionProvider { + + /** + * Name of the file to locate the version in. + */ + private static final String VERSION_FILENAME = "alpaca.properties"; + + @Override + public String[] getVersion() throws Exception { + final var filestream = getClass().getClassLoader().getResourceAsStream(VERSION_FILENAME); + final String version = new BufferedReader( + new InputStreamReader(filestream, StandardCharsets.UTF_8)) + .lines() + .filter(a -> a.startsWith("version")) + .map(a -> a.split("=")[1]) + .map(String::trim) + .findAny().orElse("0"); + return new String[] {version}; + } +} diff --git a/islandora-alpaca-app/src/main/resources/logback.xml b/islandora-alpaca-app/src/main/resources/logback.xml new file mode 100644 index 00000000..cc047963 --- /dev/null +++ b/islandora-alpaca-app/src/main/resources/logback.xml @@ -0,0 +1,23 @@ + + + + + + %p %d{HH:mm:ss.SSS} [%thread] \(%c{0}\) %m%n + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/islandora-connector-derivative/build.gradle b/islandora-connector-derivative/build.gradle index 5c1c59bc..cbbd2f18 100644 --- a/islandora-connector-derivative/build.gradle +++ b/islandora-connector-derivative/build.gradle @@ -1,41 +1,24 @@ -apply plugin: 'osgi' -description = 'Islandora 8 Derivative Connector' +description = 'Islandora Derivative Connector' dependencies { - implementation group: 'commons-io', name: 'commons-io', version: versions.commonsIo - implementation group: 'org.apache.camel', name: 'camel-blueprint', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-core', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-jackson', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-http4', version: versions.camel - implementation group: 'org.slf4j', name: 'slf4j-api', version: versions.slf4j - implementation project(':islandora-event-support') - testImplementation group: 'org.apache.camel', name: 'camel-test-blueprint', version: versions.camel - testImplementation group: 'org.apache.servicemix.bundles', name: 'org.apache.servicemix.bundles.xerces', version: versions.xercesServiceMix - testImplementation group: 'org.ow2.asm', name: 'asm-commons', version: versions.asmCommons - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: versions.slf4j -} + implementation "ch.qos.logback:logback-core:${versions.logback}" + implementation "commons-io:commons-io:${versions.commonsIo}" + implementation "javax.annotation:javax.annotation-api:${versions.javaxApi}" + implementation "javax.inject:javax.inject:${versions.javaxInject}" + implementation "org.apache.camel:camel-activemq:${versions.camel}" + implementation "org.apache.camel:camel-core:${versions.camel}" + implementation "org.apache.camel:camel-http:${versions.camel}" + implementation "org.apache.camel:camel-jackson:${versions.camel}" + implementation "org.apache.camel:camel-jsonpath:${versions.camel}" + implementation "org.apache.camel:camel-spring-javaconfig:${versions.camel}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation project(':islandora-support') -jar { - manifest { - description project.description - docURL project.docURL - vendor project.vendor - license project.license + runtimeOnly "ch.qos.logback:logback-classic:${versions.logback}" - instruction 'Import-Package', 'org.apache.camel.component.http4,' + - "org.apache.camel;version=\"${versions.camel}\"," + - "ca.islandora.alpaca.support.event;version=\"${project.version}\"," + - defaultOsgiImports - instruction 'Export-Package', 'ca.islandora.alpaca.connector.derivative' - } + testImplementation "org.apache.camel:camel-test-spring:${versions.camel}" + testImplementation "org.ow2.asm:asm-commons:${versions.asmCommons}" + testImplementation "junit:junit:${versions.junit4}" } -test { - testLogging { - // Uncomment the below line while debugging. - // events 'standard_out', 'standard_error' - exceptionFormat = 'full' - displayGranularity = 0 - } -} diff --git a/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeConnector.java b/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeConnector.java index 556bd0a4..4d2f9560 100644 --- a/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeConnector.java +++ b/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeConnector.java @@ -18,15 +18,17 @@ package ca.islandora.alpaca.connector.derivative; +import static org.apache.camel.LoggingLevel.DEBUG; import static org.apache.camel.LoggingLevel.ERROR; import static org.slf4j.LoggerFactory.getLogger; -import ca.islandora.alpaca.support.event.AS2Event; -import org.apache.camel.builder.RouteBuilder; import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; import org.apache.camel.model.dataformat.JsonLibrary; import org.slf4j.Logger; +import ca.islandora.alpaca.support.event.AS2Event; + /** * @author dhlamb */ @@ -37,21 +39,66 @@ public class DerivativeConnector extends RouteBuilder { */ private static final Logger LOGGER = getLogger(DerivativeConnector.class); + /** + * Input source. + */ + private final String inputStream; + + /** + * Output target. + */ + private final String outputStream; + + /** + * The name of this connector instance + */ + private final String connectorName; + + /** + * The common derivative configuration. + */ + private DerivativeOptions config; + + /** + * Basic constructor + * + * @param name + * The derivative connector name. + * @param inputSource + * The input stream name. + * @param outputSource + * The output target name. + * @param configuration + * The common configuration options. + */ + public DerivativeConnector(final String name, final String inputSource, final String outputSource, + final DerivativeOptions configuration) { + super(); + connectorName = name; + inputStream = inputSource; + outputStream = outputSource; + config = configuration; + } + @Override public void configure() { + LOGGER.info("DerivativeConnector (" + connectorName + ") routes starting"); + // Global exception handler for the indexer. // Just logs after retrying X number of times. onException(Exception.class) - .maximumRedeliveries("{{error.maxRedeliveries}}") + .maximumRedeliveries(config.getMaxRedeliveries()) .log( ERROR, LOGGER, - "Error connecting generating derivative with {{derivative.service.url}}: " + + "(" + connectorName + ") Error connecting generating derivative with " + outputStream + ": " + "${exception.message}\n\n${exception.stacktrace}" ); - from("{{in.stream}}") - .routeId("IslandoraConnectorDerivative") + from(inputStream) + .routeId("IslandoraConnectorDerivative-" + connectorName) + + .log(DEBUG, LOGGER, "Received message on IslandoraConnectorDerivative-" + connectorName) // Parse the event into a POJO. .unmarshal().json(JsonLibrary.Jackson, AS2Event.class) @@ -66,13 +113,13 @@ public void configure() { .setHeader("X-Islandora-Args", simple("${exchangeProperty.event.attachment.content.args}")) .setHeader("Apix-Ldp-Resource", simple("${exchangeProperty.event.attachment.content.sourceUri}")) .setBody(simple("${null}")) - .to("{{derivative.service.url}}?connectionClose=true") + .to(outputStream) // PUT the media. .removeHeaders("*", "Authorization", "Content-Type") .setHeader("Content-Location", simple("${exchangeProperty.event.attachment.content.fileUploadUri}")) .setHeader(Exchange.HTTP_METHOD, constant("PUT")) - .toD("${exchangeProperty.event.attachment.content.destinationUri}?connectionClose=true"); + .toD(config.addHttpOptions("${exchangeProperty.event.attachment.content.destinationUri}")); } } diff --git a/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeOptions.java b/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeOptions.java new file mode 100644 index 00000000..2802ce4a --- /dev/null +++ b/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/DerivativeOptions.java @@ -0,0 +1,196 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.connector.derivative; + +import static org.slf4j.LoggerFactory.getLogger; + +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.annotation.PostConstruct; + +import org.apache.camel.CamelContext; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +import ca.islandora.alpaca.support.config.PropertyConfig; + +/** + * Base derivative configuration class. + * @author whikloj + */ +@Configuration +public class DerivativeOptions extends PropertyConfig { + + private static final Logger LOGGER = getLogger(DerivativeOptions.class); + + private static final String DERIVATIVE_LIST_PROPERTY = "derivative.systems.installed"; + private static final String DERIVATIVE_PREFIX = "derivative"; + private static final String DERIVATIVE_ENABLED_PROPERTY = "enabled"; + private static final String DERIVATIVE_INPUT_PROPERTY = "in.stream"; + private static final String DERIVATIVE_OUTPUT_PROPERTY = "service.url"; + private static final String DERIVATIVE_CONCURRENT_PROPERTY = "concurrent-consumers"; + private static final String DERIVATIVE_MAX_CONCURRENT_PROPERTY = "max-concurrent-consumers"; + private static final String DERIVATIVE_ASYNC_CONSUMER = "async-consumer"; + + @Autowired + private Environment environment; + + @Autowired + private CamelContext camelContext; + + @Value("${" + DERIVATIVE_LIST_PROPERTY + ":#{null}}") + private String derivativeSystems; + + /** + * Register additional beans for derivative routes. + * + * @param camelContext + * The camel context to add derivative service routes to. + * @throws Exception + * When unable to add routes to the camel context. + */ + @PostConstruct + private void processAllServices() throws Exception { + if (derivativeSystems != null && !derivativeSystems.isBlank()) { + final var systemNames = derivativeSystems.contains(",") ? + Arrays.stream(derivativeSystems.split(",")).filter(o -> !o.isBlank()).map(String::trim) + .collect(Collectors.toSet()) : Set.of(derivativeSystems); + for (final var system : systemNames) { + final var enabled = + environment.getProperty(enabledProperty(system), Boolean.class, false); + if (enabled) { + startDerivativeService(system); + } else { + LOGGER.debug("Derivative connector (" + system + ") is disabled, skipping"); + } + } + } + } + + /** + * Attempt to start the routes and add them to the camel context. + * + * @param camelContext + * The current camel context. + * @param serviceName + * The derivative service name. + * @throws Exception + * When unable to add routes to the camel context. + */ + private void startDerivativeService(final String serviceName) throws Exception { + final var input = environment.getProperty(inputProperty(serviceName), ""); + final var output = environment.getProperty(outputProperty(serviceName), ""); + if (!input.isBlank() && !output.isBlank()) { + final int concurrentConsumers = environment.getProperty(concurrentConsumerProperty(serviceName), + Integer.class, -1); + final int maxConcurrentConsumers = environment.getProperty(maxConcurrentConsumerProperty(serviceName), + Integer.class, -1); + final boolean asyncConsumer = environment.getProperty(asyncConsumerProperty(serviceName), + Boolean.class, false); + // Add concurrent/max-concurrent + final String finalInput = addJmsOptions(addBrokerName(input), concurrentConsumers, maxConcurrentConsumers, + asyncConsumer); + // Add connectionClose and other http options. + final String finalOutput = addHttpOptions(output); + camelContext.addRoutes(new DerivativeConnector(serviceName, finalInput, finalOutput, this)); + } else { + final StringBuilder message = new StringBuilder(); + if (input.isBlank()) { + message.append(inputProperty(serviceName)).append(" is blank"); + } + if (output.isBlank()) { + if (message.length() > 0) { + message.append(" and "); + } + message.append(outputProperty(serviceName)).append(" is blank"); + } + message.append(", skipping"); + LOGGER.debug(message.toString()); + } + + } + + /** + * Just adds the JMS broker name to the provided queue/topic. + * @param queueName the provided queue/topic. + * @return the full endpoint including the broker name. + */ + private String addBrokerName(final String queueName) { + return JMS_ENDPOINT_NAME + ":" + queueName; + } + + /** + * Return the expected enabled property + * @param systemName the derivative system name + * @return the property + */ + private String enabledProperty(final String systemName) { + return DERIVATIVE_PREFIX + "." + systemName + "." + DERIVATIVE_ENABLED_PROPERTY; + } + + /** + * Return the expected input topic/queue property + * @param systemName the derivative system name + * @return the property + */ + private String inputProperty(final String systemName) { + return DERIVATIVE_PREFIX + "." + systemName + "." + DERIVATIVE_INPUT_PROPERTY; + } + + /** + * Return the expected output service url property + * @param systemName the derivative system name + * @return the property + */ + private String outputProperty(final String systemName) { + return DERIVATIVE_PREFIX + "." + systemName + "." + DERIVATIVE_OUTPUT_PROPERTY; + } + + /** + * Return the expected concurrent consumers property. + * @param systemName the derivative system name + * @return the property + */ + private String concurrentConsumerProperty(final String systemName) { + return DERIVATIVE_PREFIX + "." + systemName + "." + DERIVATIVE_CONCURRENT_PROPERTY; + } + + /** + * Return the expected max-concurrent consumers property. + * @param systemName the derivative system name + * @return the property + */ + private String maxConcurrentConsumerProperty(final String systemName) { + return DERIVATIVE_PREFIX + "." + systemName + "." + DERIVATIVE_MAX_CONCURRENT_PROPERTY; + } + + /** + * Return the expected async-consumer property. + * @param systemName the derivative system name + * @return the property + */ + private String asyncConsumerProperty(final String systemName) { + return DERIVATIVE_PREFIX + "." + systemName + "." + DERIVATIVE_ASYNC_CONSUMER; + } + +} diff --git a/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/RequestConfigConfigurer.java b/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/RequestConfigConfigurer.java deleted file mode 100644 index 566f8ccb..00000000 --- a/islandora-connector-derivative/src/main/java/ca/islandora/alpaca/connector/derivative/RequestConfigConfigurer.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to Islandora Foundation under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional - * information regarding copyright ownership. - * - * The Islandora Foundation licenses this file to you under the MIT License. - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ca.islandora.alpaca.connector.derivative; - -import org.apache.camel.component.http4.HttpClientConfigurer; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.HttpClientBuilder; - -/** - * Provides a default HttpClient {@link RequestConfig} with custom values for connection request, connect, and socket - * timeout values. Custom values must be set on this class prior to invoking - * {@link #configureHttpClient(HttpClientBuilder)}. - * - * @author Elliot Metsger (emetsger@jhu.edu) - */ -public class RequestConfigConfigurer implements HttpClientConfigurer { - - /** - * The RequestConfig instance that is built by this configurer; exposed for testing purposes. - */ - RequestConfig built; - - private int connectionRequestTimeoutMs = RequestConfig.DEFAULT.getConnectionRequestTimeout(); - - private int connectTimeoutMs = RequestConfig.DEFAULT.getConnectTimeout(); - - private int socketTimeoutMs = RequestConfig.DEFAULT.getSocketTimeout(); - - /** - * Creates a {@link RequestConfig} using custom values for {@code connectionRequestTimeout}, {@code connectTimeout}, - * and {@code socketTimeout}. If custom values are not provided by this class, then the default values from - * {@link RequestConfig#DEFAULT} are used. - * - * @param clientBuilder the HttpClientBuilder - * @see #setConnectionRequestTimeoutMs(int) - * @see #setConnectTimeoutMs(int) - * @see #setSocketTimeoutMs(int) - */ - @Override - public void configureHttpClient(final HttpClientBuilder clientBuilder) { - final RequestConfig.Builder builder = RequestConfig.copy(RequestConfig.DEFAULT); - final RequestConfig config = buildConfig(builder); - clientBuilder.setDefaultRequestConfig(config); - } - - /** - * Package-private to support testing. - * - * @param builder the RequestConfig builder - * @return the RequestConfig - */ - RequestConfig buildConfig(final RequestConfig.Builder builder) { - builder.setConnectionRequestTimeout(connectionRequestTimeoutMs) - .setSocketTimeout(socketTimeoutMs) - .setConnectTimeout(connectTimeoutMs); - built = builder.build(); - return built; - } - - /** - * Get the value of the connection request timeout - * - * @return the value - */ - public int getConnectionRequestTimeoutMs() { - return connectionRequestTimeoutMs; - } - - /** - * Set the value of the connection request timeout - * - * @param connectionRequestTimeoutMs the value - */ - public void setConnectionRequestTimeoutMs(final int connectionRequestTimeoutMs) { - this.connectionRequestTimeoutMs = connectionRequestTimeoutMs; - } - - /** - * Get the value of the connect timeout - * - * @return the value - */ - public int getConnectTimeoutMs() { - return connectTimeoutMs; - } - - /** - * Set the value of the connect timeout - * - * @param connectTimeoutMs the value - */ - public void setConnectTimeoutMs(final int connectTimeoutMs) { - this.connectTimeoutMs = connectTimeoutMs; - } - - /** - * Get the value of the socket timeout - * - * @return the value - */ - public int getSocketTimeoutMs() { - return socketTimeoutMs; - } - - /** - * Set the value of the socket timeout - * - * @param socketTimeoutMs the value - */ - public void setSocketTimeoutMs(final int socketTimeoutMs) { - this.socketTimeoutMs = socketTimeoutMs; - } -} diff --git a/islandora-connector-derivative/src/main/resources/logback.xml b/islandora-connector-derivative/src/main/resources/logback.xml new file mode 100644 index 00000000..f03dc45c --- /dev/null +++ b/islandora-connector-derivative/src/main/resources/logback.xml @@ -0,0 +1,22 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/DerivativeConnectorTest.java b/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/DerivativeConnectorTest.java index 50094f16..139f7d3f 100644 --- a/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/DerivativeConnectorTest.java +++ b/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/DerivativeConnectorTest.java @@ -18,68 +18,79 @@ package ca.islandora.alpaca.connector.derivative; -import org.apache.camel.EndpointInject; +import static org.apache.camel.Exchange.CONTENT_TYPE; +import static org.apache.camel.util.ObjectHelper.loadResourceAsStream; +import static org.slf4j.LoggerFactory.getLogger; + +import org.apache.camel.CamelContext; import org.apache.camel.Exchange; import org.apache.camel.Produce; import org.apache.camel.ProducerTemplate; -import org.apache.camel.builder.AdviceWithRouteBuilder; +import org.apache.camel.builder.AdviceWith; import org.apache.camel.component.mock.MockEndpoint; -import org.apache.camel.test.blueprint.CamelBlueprintTestSupport; +import org.apache.camel.model.ModelCamelContext; +import org.apache.camel.spring.javaconfig.CamelConfiguration; +import org.apache.camel.test.spring.UseAdviceWith; import org.apache.commons.io.IOUtils; +import org.junit.BeforeClass; import org.junit.Test; - -import static org.apache.camel.Exchange.CONTENT_TYPE; -import static org.apache.camel.util.ObjectHelper.loadResourceAsStream; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import ca.islandora.alpaca.support.config.ActivemqConfig; /** * @author dannylamb + * @author whikloj */ -public class DerivativeConnectorTest extends CamelBlueprintTestSupport { +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@UseAdviceWith +@ContextConfiguration(classes = DerivativeConnectorTest.ContextConfig.class, + loader = AnnotationConfigContextLoader.class) +@RunWith(SpringJUnit4ClassRunner.class) +public class DerivativeConnectorTest { - @EndpointInject(uri = "mock:result") - protected MockEndpoint resultEndpoint; + private static final Logger LOGGER = getLogger(DerivativeConnectorTest.class); - @Produce(uri = "direct:start") + @Produce("direct:start") protected ProducerTemplate template; - @Override - public boolean isUseAdviceWith() { - return true; - } - - @Override - public boolean isUseRouteBuilder() { - return false; - } - - @Override - protected String getBlueprintDescriptor() { - return "/OSGI-INF/blueprint/blueprint-test.xml"; - } + @Autowired + CamelContext camelContext; @Test public void testDerivativeConnector() throws Exception { - final String route = "IslandoraConnectorDerivative"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - - // Rig Drupal REST endpoint to return canned jsonld - interceptSendToEndpoint("http://example.org/derivative/convert?connectionClose=true") - .skipSendToOriginalEndpoint() - .process(exchange -> { - exchange.getIn().removeHeaders("*", "Authorization"); - exchange.getIn().setHeader("Content-Type", "image/jpeg"); - exchange.getIn().setBody("SOME DERIVATIVE", String.class); - }); - - mockEndpointsAndSkip("http://localhost:8000/node/2/media/image/3?connectionClose=true"); - } + final String route = "IslandoraConnectorDerivative-testRoutes"; + + final var context = camelContext.adapt(ModelCamelContext.class); + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + + // Rig Drupal REST endpoint to return canned jsonld + a.interceptSendToEndpoint("http://example.org/derivative/convert?connectionClose=true&" + + "disableStreamCache=true") + .skipSendToOriginalEndpoint() + .process(exchange -> { + exchange.getIn().removeHeaders("*", "Authorization"); + exchange.getIn().setHeader("Content-Type", "image/jpeg"); + exchange.getIn().setBody("SOME DERIVATIVE", String.class); + }); + + a.mockEndpointsAndSkip("http://localhost:8000/node/2/media/image/3?connectionClose=true&" + + "disableStreamCache=true"); }); context.start(); - final MockEndpoint endpoint = getMockEndpoint("mock:http:localhost:8000/node/2/media/image/3"); + final MockEndpoint endpoint = (MockEndpoint) context + .getEndpoint("mock:http:localhost:8000/node/2/media/image/3"); endpoint.expectedMessageCount(1); endpoint.expectedHeaderReceived(Exchange.HTTP_METHOD, "PUT"); @@ -92,7 +103,24 @@ public void configure() throws Exception { exchange.getIn().setHeader("Authorization", "Bearer islandora"); }); - assertMockEndpointsSatisfied(); + endpoint.assertIsSatisfied(); } + @BeforeClass + public static void beforeClass() { + System.setProperty("derivative.systems.installed", "testRoutes"); + System.setProperty("derivative.testRoutes.enabled", "true"); + System.setProperty("derivative.testRoutes.in.stream", "topic:input"); + System.setProperty("derivative.testRoutes.service.url", "http://example.org/derivative/convert"); + System.setProperty("error.maxRedeliveries", "1"); + } + + @Configuration + @ComponentScan(basePackageClasses = {DerivativeOptions.class, ActivemqConfig.class}, + useDefaultFilters = false, + includeFilters = @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, + classes = {DerivativeOptions.class, ActivemqConfig.class})) + static class ContextConfig extends CamelConfiguration { + + } } diff --git a/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/HttpConfigurerTest.java b/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/HttpConfigurerTest.java deleted file mode 100644 index 461f3cb2..00000000 --- a/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/HttpConfigurerTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to Islandora Foundation under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional - * information regarding copyright ownership. - * - * The Islandora Foundation licenses this file to you under the MIT License. - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ca.islandora.alpaca.connector.derivative; - -import org.apache.camel.component.http4.HttpComponent; -import org.apache.camel.test.blueprint.CamelBlueprintTestSupport; -import org.junit.Test; - -/** - * Insures that the Camel HTTP component as configured by Blueprint is properly configured. - * - * @author Elliot Metsger (emetsger@jhu.edu) - */ -public class HttpConfigurerTest extends CamelBlueprintTestSupport { - - @Override - protected String getBlueprintDescriptor() { - return "/OSGI-INF/blueprint/blueprint-httpconfigurer-test.xml"; - } - - /** - * Insure that the default RequestConfig for the HttpComponent carries the timeout values specified in the - * blueprint xml. - * - * Note that the RequestConfig and RequestConfigConfigurer are difficult to test with mocking frameworks such as - * Mockito due to the presence of final methods in the relevant HttpClient classes. - * - * @throws Exception - */ - @Test - public void testRequestConfig() throws Exception { - context.start(); - final HttpComponent http = (HttpComponent) context.getComponent("http"); - - final RequestConfigConfigurer configurer = (RequestConfigConfigurer) http.getHttpClientConfigurer(); - - assertEquals(10000, configurer.built.getSocketTimeout()); - assertEquals(10000, configurer.built.getConnectTimeout()); - assertEquals(10000, configurer.built.getConnectionRequestTimeout()); - } - -} diff --git a/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/RequestConfigConfigurerTest.java b/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/RequestConfigConfigurerTest.java deleted file mode 100644 index c4b98fcf..00000000 --- a/islandora-connector-derivative/src/test/java/ca/islandora/alpaca/connector/derivative/RequestConfigConfigurerTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Islandora Foundation under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional - * information regarding copyright ownership. - * - * The Islandora Foundation licenses this file to you under the MIT License. - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ca.islandora.alpaca.connector.derivative; - -import org.apache.http.client.config.RequestConfig; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -/** - * Verifies the behaviors and state of the RequestConfigConfigurer. - * - * @author Elliot Metsger (emetsger@jhu.edu) - */ -public class RequestConfigConfigurerTest { - - /** - * The default state of the RequestConfigConfigurer should provide the same timeout values as the default - * RequestConfig. - */ - @Test - public void testDefaultValues() { - final RequestConfigConfigurer underTest = new RequestConfigConfigurer(); - - assertEquals(RequestConfig.DEFAULT.getConnectionRequestTimeout(), underTest.getConnectionRequestTimeoutMs()); - assertEquals(RequestConfig.DEFAULT.getConnectTimeout(), underTest.getConnectTimeoutMs()); - assertEquals(RequestConfig.DEFAULT.getSocketTimeout(), underTest.getSocketTimeoutMs()); - } - - /** - * Insure state is properly maintained by the RequestConfigConfigurer. - */ - @Test - public void testCustomValues() { - final RequestConfigConfigurer underTest = new RequestConfigConfigurer(); - underTest.setConnectionRequestTimeoutMs(12345); - underTest.setConnectTimeoutMs(1111111); - underTest.setSocketTimeoutMs(9999999); - - assertEquals(12345, underTest.getConnectionRequestTimeoutMs()); - assertEquals(1111111, underTest.getConnectTimeoutMs()); - assertEquals(9999999, underTest.getSocketTimeoutMs()); - } - - /** - * Insure state from the RequestConfigConfigurer is properly communicated to the built RequestConfig. - */ - @Test - public void testBuild() { - final RequestConfigConfigurer underTest = new RequestConfigConfigurer(); - underTest.setConnectionRequestTimeoutMs(12345); - underTest.setConnectTimeoutMs(1111111); - underTest.setSocketTimeoutMs(9999999); - - underTest.buildConfig(RequestConfig.custom()); - - assertNotNull(underTest.built); - - assertEquals(12345, underTest.built.getConnectionRequestTimeout()); - assertEquals(1111111, underTest.built.getConnectTimeout()); - assertEquals(9999999, underTest.built.getSocketTimeout()); - } -} \ No newline at end of file diff --git a/islandora-connector-derivative/src/test/resources/OSGI-INF/blueprint/blueprint-httpconfigurer-test.xml b/islandora-connector-derivative/src/test/resources/OSGI-INF/blueprint/blueprint-httpconfigurer-test.xml deleted file mode 100644 index 028e859c..00000000 --- a/islandora-connector-derivative/src/test/resources/OSGI-INF/blueprint/blueprint-httpconfigurer-test.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - ca.islandora.alpaca.connector.derivative - - - diff --git a/islandora-connector-derivative/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml b/islandora-connector-derivative/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml deleted file mode 100644 index 7e5dc64e..00000000 --- a/islandora-connector-derivative/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - ca.islandora.alpaca.connector.derivative - - - diff --git a/islandora-connector-derivative/src/test/resources/logback-test.xml b/islandora-connector-derivative/src/test/resources/logback-test.xml new file mode 100644 index 00000000..c2fe05e2 --- /dev/null +++ b/islandora-connector-derivative/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + \ No newline at end of file diff --git a/islandora-connector-derivative/src/test/resources/simplelogger.properties b/islandora-connector-derivative/src/test/resources/simplelogger.properties deleted file mode 100644 index 9c32bc7d..00000000 --- a/islandora-connector-derivative/src/test/resources/simplelogger.properties +++ /dev/null @@ -1,34 +0,0 @@ -# SLF4J's SimpleLogger configuration file -# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. - -# Default logging detail level for all instances of SimpleLogger. -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, defaults to "info". -#org.slf4j.simpleLogger.defaultLogLevel=debug - -# Logging detail level for a SimpleLogger instance named "xxxxx". -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, the default logging detail level is used. -#org.slf4j.simpleLogger.log.xxxxx= - -# Set to true if you want the current date and time to be included in output messages. -# Default is false, and will output the number of milliseconds elapsed since startup. -#org.slf4j.simpleLogger.showDateTime=false - -# The date and time format to be used in the output messages. -# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. -# If the format is not specified or is invalid, the default format is used. -# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. -#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z - -# Set to true if you want to output the current thread name. -# Defaults to true. -#org.slf4j.simpleLogger.showThreadName=true - -# Set to true if you want the Logger instance name to be included in output messages. -# Defaults to true. -#org.slf4j.simpleLogger.showLogName=true - -# Set to true if you want the last component of the name to be included in output messages. -# Defaults to false. -#org.slf4j.simpleLogger.showShortLogName=false diff --git a/islandora-event-support/build.gradle b/islandora-event-support/build.gradle deleted file mode 100644 index 21d1de16..00000000 --- a/islandora-event-support/build.gradle +++ /dev/null @@ -1,19 +0,0 @@ -apply plugin: 'osgi' - -description = 'Islandora 8 Supporting Libraries' - -dependencies { - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: versions.jackson - testImplementation group: 'junit', name: 'junit', version: versions.junit -} - -jar { - manifest { - description project.description - docURL project.docURL - vendor project.vendor - license project.license - instruction 'Import-Package', defaultOsgiImports - instruction 'Export-Package', "ca.islandora.alpaca.support.*;version=\"${project.version}\"" - } -} diff --git a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Object.java b/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Object.java deleted file mode 100644 index fb08246b..00000000 --- a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Object.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Islandora Foundation under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional - * information regarding copyright ownership. - * - * The Islandora Foundation licenses this file to you under the MIT License. - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ca.islandora.alpaca.support.event; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * POJO for a user performing an action. Part of a AS2Event. - * - * @author Danny Lamb - */ -public class AS2Object { - - /** - * The object type if applicable. - */ - private String type; - /** - * The object UUID. - */ - private String id; - /** - * The URLs passed with the event. - */ - private AS2Url[] url; - - /** - * Are we creating a new revision? - */ - private Boolean isNewVersion; - - /** - * @return Type of object - */ - public String getType() { - return type; - } - - /** - * @param type Type of object - */ - public void setType(final String type) { - this.type = type; - } - - /** - * @return URN of object - */ - public String getId() { - return id; - } - - /** - * @param id URN of object - */ - public void setId(final String id) { - this.id = id; - } - - /** - * @return URLs for object - */ - public AS2Url[] getUrl() { - return url; - } - - /** - * @param url URLs for object - */ - public void setUrl(final AS2Url[] url) { - this.url = url.clone(); - } - - /** - * @return true or false - */ - @JsonProperty(value = "isNewVersion") - public Boolean getIsNewVersion() { - return isNewVersion; - } - - /** - * @param isNewVersion true or false - */ - public void setIsNewVersion(final Boolean isNewVersion) { - this.isNewVersion = isNewVersion; - } -} diff --git a/islandora-http-client/build.gradle b/islandora-http-client/build.gradle index 36d2272c..d084aaf9 100644 --- a/islandora-http-client/build.gradle +++ b/islandora-http-client/build.gradle @@ -1,33 +1,12 @@ -apply plugin: 'osgi' - description = 'Islandora 8 HTTP Client' dependencies { - implementation group: 'org.apache.httpcomponents', name: 'httpclient-osgi', version: versions.httpClientOsgi - testImplementation group: 'junit', name: 'junit', version: versions.junit -} + implementation "org.apache.httpcomponents:httpclient:${versions.httpClient}" + implementation "org.springframework:spring-context:${versions.spring}" -jar { - manifest { - description project.description - docURL project.docURL - vendor project.vendor - license project.license - - instruction 'Import-Package', - 'org.apache.http,' + - 'org.apache.http.protocol,' + - 'org.apache.http.message,' + - 'org.apache.http.impl.client,' + - 'org.apache.http.client,' + - defaultOsgiImports - instruction 'Export-Package', 'ca.islandora.alpaca.http.client' - } + testImplementation "junit:junit:${versions.junit4}" } -artifacts { - archives (file('build/cfg/main/ca.islandora.alpaca.http.client.cfg')) { - classifier 'configuration' - type 'cfg' - } +jar { + enabled = true } diff --git a/islandora-http-client/src/main/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptor.java b/islandora-http-client/src/main/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptor.java index 2654b6a0..469170c1 100644 --- a/islandora-http-client/src/main/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptor.java +++ b/islandora-http-client/src/main/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptor.java @@ -28,6 +28,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; import org.apache.http.protocol.HttpContext; +import org.springframework.stereotype.Component; /** * Adds a single authentication header to any request that does not @@ -36,6 +37,7 @@ * @author ajs6f * */ +@Component public class StaticTokenRequestInterceptor implements HttpRequestInterceptor { /** diff --git a/islandora-http-client/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/islandora-http-client/src/main/resources/OSGI-INF/blueprint/blueprint.xml deleted file mode 100644 index ee156b74..00000000 --- a/islandora-http-client/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/islandora-http-client/src/test/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptorTest.java b/islandora-http-client/src/test/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptorTest.java index d5556c12..0f9c2aec 100644 --- a/islandora-http-client/src/test/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptorTest.java +++ b/islandora-http-client/src/test/java/ca/islandora/alpaca/http/client/StaticTokenRequestInterceptorTest.java @@ -19,18 +19,19 @@ package ca.islandora.alpaca.http.client; import static ca.islandora.alpaca.http.client.StaticTokenRequestInterceptor.AUTH_HEADER; +import static org.junit.Assert.assertEquals; import org.apache.http.Header; import org.apache.http.HttpRequest; import org.apache.http.client.methods.HttpGet; -import org.junit.Assert; import org.junit.Test; + /** * @author ajs6f * */ -public class StaticTokenRequestInterceptorTest extends Assert { +public class StaticTokenRequestInterceptorTest { @Test public void shouldInjectHeaderWhenNoAuthHeadersPresent() { @@ -39,7 +40,7 @@ public void shouldInjectHeaderWhenNoAuthHeadersPresent() { testInterceptor.process(request, null); final Header[] authHeaders = request.getHeaders(AUTH_HEADER); assertEquals("Should only be one auth header!", 1, authHeaders.length); - assertEquals("Wrong value for header!", "Bearer testToken", authHeaders[0].getValue()); + assertEquals( "Wrong value for header!", "Bearer testToken", authHeaders[0].getValue()); } @Test @@ -51,6 +52,6 @@ public void shouldNotInjectHeaderWhenAuthHeadersPresent() { testInterceptor.process(request, null); final Header[] authHeaders = request.getHeaders(AUTH_HEADER); assertEquals("Should only be one auth header!", 1, authHeaders.length); - assertEquals("Wrong value for header!", "fake header", authHeaders[0].getValue()); + assertEquals( "Wrong value for header!", "fake header", authHeaders[0].getValue()); } } diff --git a/islandora-indexing-fcrepo/build.gradle b/islandora-indexing-fcrepo/build.gradle index 1f15c95d..ab1e3a9b 100644 --- a/islandora-indexing-fcrepo/build.gradle +++ b/islandora-indexing-fcrepo/build.gradle @@ -1,55 +1,27 @@ -apply plugin: 'osgi' -description = 'Islandora 8 Fcrepo Indexer' +description = 'Islandora Fcrepo Indexer' dependencies { - implementation group: 'org.apache.camel', name: 'camel-core', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-blueprint', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-http4', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-jackson', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-jsonpath', version: versions.camel - implementation(project(':islandora-event-support')) - implementation group: 'org.slf4j', name: 'slf4j-api', version: versions.slf4j - implementation group: 'commons-io', name: 'commons-io', version: versions.commonsIo - testImplementation group: 'org.apache.camel', name: 'camel-test-blueprint', version: versions.camel - testImplementation group: 'org.apache.servicemix.bundles', name: 'org.apache.servicemix.bundles.xerces', version: versions.xercesServiceMix - testImplementation group: 'org.ow2.asm', name: 'asm-commons', version: versions.asmCommons - testImplementation group: 'com.googlecode.junit-toolbox', name: 'junit-toolbox', version: versions.junitToolbox - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: versions.slf4j -} + implementation "ch.qos.logback:logback-core:${versions.logback}" + implementation "commons-io:commons-io:${versions.commonsIo}" + implementation "javax.inject:javax.inject:${versions.javaxInject}" + implementation "org.apache.camel:camel-activemq:${versions.camel}" + implementation "org.apache.camel:camel-core:${versions.camel}" + implementation "org.apache.camel:camel-http:${versions.camel}" + implementation "org.apache.camel:camel-jackson:${versions.camel}" + implementation "org.apache.camel:camel-jsonpath:${versions.camel}" + implementation "org.apache.camel:camel-spring-javaconfig:${versions.camel}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation project(':islandora-support') -test { - maxParallelForks = Runtime.getRuntime().availableProcessors() -} + runtimeOnly "ch.qos.logback:logback-classic:${versions.logback}" -jar { - manifest { - description project.description - docURL project.docURL - vendor project.vendor - license project.license + testImplementation "org.apache.camel:camel-test-spring:${versions.camel}" + testImplementation "org.ow2.asm:asm-commons:${versions.asmCommons}" + testImplementation "junit:junit:${versions.junit4}" - instruction 'Import-Package', 'org.apache.camel.component.http4,' + - "org.apache.camel.jsonpath," + - "org.apache.camel;version=\"${versions.camel}\"," + - "ca.islandora.alpaca.support.event;version=\"${project.version}\"," + - defaultOsgiImports - instruction 'Export-Package', 'ca.islandora.alpaca.indexing.fcrepo' - } -} - -artifacts { - archives (file('build/cfg/main/ca.islandora.alpaca.indexing.fcrepo.cfg')) { - classifier 'configuration' - type 'cfg' - } } test { - testLogging { - // Uncomment the below line while debugging. - //events 'standard_out', 'standard_error' - exceptionFormat = 'full' - displayGranularity = 0 - } + maxParallelForks = Runtime.getRuntime().availableProcessors() } diff --git a/islandora-indexing-fcrepo/src/main/cfg/ca.islandora.alpaca.indexing.fcrepo.cfg b/islandora-indexing-fcrepo/src/main/cfg/ca.islandora.alpaca.indexing.fcrepo.cfg deleted file mode 100644 index f27351a5..00000000 --- a/islandora-indexing-fcrepo/src/main/cfg/ca.islandora.alpaca.indexing.fcrepo.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Number of times to retry -error.maxRedeliveries=5 - -# Input queues -node.stream=activemq:queue:islandora-indexing-fcrepo-content -node.delete.stream=activemq:queue:islandora-indexing-fcrepo-delete -media.stream=activemq:queue:islandora-indexing-fcrepo-media -file.external.stream=activemq:queue:islandora-indexing-fcrepo-file-external - -# Base url for microservices -milliner.baseUrl=http://localhost:8000/milliner/ diff --git a/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/CommonProcessor.java b/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/CommonProcessor.java new file mode 100644 index 00000000..e925a8e5 --- /dev/null +++ b/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/CommonProcessor.java @@ -0,0 +1,53 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.fcrepo; + +import org.apache.camel.Exchange; +import org.apache.camel.Processor; + +import ca.islandora.alpaca.support.event.AS2Event; + +/** + * A processor to perform some common actions on the Exchange + * @author whikloj + */ +public class CommonProcessor implements Processor { + + private FcrepoIndexerOptions config; + + /** + * Basic constructor. + * @param fcrepoConfig + * The FcrepoIndexerOptions we are using. + */ + public CommonProcessor(final FcrepoIndexerOptions fcrepoConfig) { + config = fcrepoConfig; + } + + @Override + public void process(final Exchange exchange) { + final var msg = exchange.getIn(); + exchange.setProperty("event", msg.getBody()); + final AS2Event json = msg.getBody(AS2Event.class); + exchange.setProperty("fedoraBaseUrl", json.getTarget()); + msg.removeHeaders("*", "Authorization"); + msg.setHeader(config.getFedoraUriHeader(), exchange.getProperty("fedoraBaseUrl")); + msg.setBody(null); + exchange.setMessage(msg); + } +} diff --git a/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexer.java b/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexer.java index e1d05437..a44dd307 100644 --- a/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexer.java +++ b/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexer.java @@ -18,44 +18,38 @@ package ca.islandora.alpaca.indexing.fcrepo; +import static org.apache.camel.LoggingLevel.DEBUG; import static org.apache.camel.LoggingLevel.ERROR; import static org.apache.camel.LoggingLevel.INFO; -import static org.apache.camel.LoggingLevel.DEBUG; +import static org.apache.camel.LoggingLevel.TRACE; +import static org.apache.camel.LoggingLevel.WARN; import static org.slf4j.LoggerFactory.getLogger; -import ca.islandora.alpaca.support.event.AS2Event; import org.apache.camel.Exchange; import org.apache.camel.Predicate; -import org.apache.camel.PropertyInject; -import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.Processor; import org.apache.camel.builder.PredicateBuilder; -import org.apache.camel.http.common.HttpOperationFailedException; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.http.base.HttpOperationFailedException; import org.apache.camel.model.dataformat.JsonLibrary; import org.slf4j.Logger; -// import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.springframework.beans.factory.annotation.Autowired; + +import ca.islandora.alpaca.support.event.AS2Event; +import ca.islandora.alpaca.support.exceptions.MissingCanonicalUrlException; +import ca.islandora.alpaca.support.exceptions.MissingJsonUrlException; +import ca.islandora.alpaca.support.exceptions.MissingJsonldUrlException; /** + * Camel Route to index Drupal nodes into Fedora. + * * @author Danny Lamb + * @author whikloj */ -// @JsonIgnoreProperties(ignoreUnknown = true) public class FcrepoIndexer extends RouteBuilder { - /** - * Header to use to pass the Fedora Base URI on. - */ - public final static String FEDORA_HEADER = "X-Islandora-Fedora-Endpoint"; - - /** - * Maximum attempts to deliver a message. - */ - @PropertyInject("error.maxRedeliveries") - private int maxRedeliveries; - - /** - * Base URI of the milliner web service. - */ - @PropertyInject("milliner.baseUrl") - private String millinerBaseUrl; + @Autowired + private FcrepoIndexerOptions config; /** * The Logger. @@ -63,44 +57,17 @@ public class FcrepoIndexer extends RouteBuilder { private static final Logger LOGGER = getLogger(FcrepoIndexer.class); /** - * @return Number of times to retry + * PMD likes short methods (less than 100 lines) but that would make this RouteBuilder less clear. + * So we are ignoring rule. */ - public int getMaxRedeliveries() { - return maxRedeliveries; - } - - /** - * @param maxRedeliveries Number of times to retry - */ - public void setMaxRedeliveries(final int maxRedeliveries) { - this.maxRedeliveries = maxRedeliveries; - } - - /** - * @return Milliner base url - */ - public String getMillinerBaseUrl() { - return enforceTrailingSlash(millinerBaseUrl); - } - - /** - * @param millinerBaseUrl Milliner base url - */ - public void setMillinerBaseUrl(final String millinerBaseUrl) { - this.millinerBaseUrl = millinerBaseUrl; - } - - private String enforceTrailingSlash(final String baseUrl) { - final String trimmed = baseUrl.trim(); - return trimmed.endsWith("/") ? trimmed : trimmed + "/"; - } - - + @SuppressWarnings("PMD.ExcessiveMethodLength") @Override public void configure() { - + LOGGER.info("FcrepoIndexer routes starting"); final Predicate is412 = PredicateBuilder.toPredicate(simple("${exception.statusCode} == 412")); final Predicate is404 = PredicateBuilder.toPredicate(simple("${exception.statusCode} == 404")); + final Predicate is410 = PredicateBuilder.toPredicate(simple("${exception.statusCode} == 410")); + final Processor commonProcessor = new CommonProcessor(config); onException(HttpOperationFailedException.class) .onWhen(is412) @@ -111,49 +78,61 @@ public void configure() { LOGGER, "Received 412 from Milliner, skipping indexing." ); - + onException(HttpOperationFailedException.class) + .onWhen(is410) + .useOriginalMessage() + .handled(true) + .log( + WARN, + LOGGER, + "Received 410 from Milliner (object has already been deleted), skipping processing." + ); + onException(MissingJsonldUrlException.class) + .useOriginalMessage() + .handled(true) + .log( + WARN, + LOGGER, + "Could not locate the Json Url for the object, skipping processing." + ); onException(Exception.class) - .maximumRedeliveries(maxRedeliveries) + .maximumRedeliveries(config.getMaxRedeliveries()) .log( ERROR, LOGGER, "Error indexing resource in fcrepo: ${exception.message}\n\n${exception.stacktrace}" ); - from("{{node.stream}}") - .routeId("FcrepoIndexerNode") + from(config.getNodeIndex()) + .routeId("FcrepoIndexerNode") // Parse the event into a POJO. .unmarshal().json(JsonLibrary.Jackson, AS2Event.class) - // Extract relevant data from the event. - .setProperty("event").simple("${body}") + .process(commonProcessor) .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}") - .setProperty("jsonldUrl").simple("${exchangeProperty.event.object.url[2].href}") - .setProperty("fedoraBaseUrl").simple("${exchangeProperty.event.target}") + .setProperty("jsonldUrl").simple("${exchangeProperty.event.object.getJsonldUrl().href}") .log(DEBUG, LOGGER, "Received Node event for UUID (${exchangeProperty.uuid}), jsonld URL (" + "${exchangeProperty.jsonldUrl}), fedora base URL (${exchangeProperty.fedoraBaseUrl})") - // Prepare the message. - .removeHeaders("*", "Authorization") .setHeader(Exchange.HTTP_METHOD, constant("POST")) .setHeader("Content-Location", simple("${exchangeProperty.jsonldUrl}")) - .setHeader(FEDORA_HEADER, exchangeProperty("fedoraBaseUrl")) - .setBody(simple("${null}")) .multicast().parallelProcessing() - //pass it to milliner - .toD(getMillinerBaseUrl() + "node/${exchangeProperty.uuid}?connectionClose=true") - .choice() - .when() - .simple("${exchangeProperty.event.object.isNewVersion}") - //pass it to milliner - .toD( - getMillinerBaseUrl() + - "node/${exchangeProperty.uuid}/version?connectionClose=true" - ).endChoice(); - - - - from("{{node.delete.stream}}") + .to("seda:nodeIndex", "seda:nodeVersionIndex") + .end(); + + from("seda:nodeIndex") + .routeId("FcrepoIndexerNodeIndex") + .toD(makeMillinerUri("node/${exchangeProperty.uuid}")); + + from("seda:nodeVersionIndex") + .routeId("FcrepoIndexerNodeVersion") + .log(TRACE, LOGGER, "Node indexer version endpoint, isNewVersion is " + + "(${exchangeProperty.event.object.isNewVersion}") + .filter(simple("${exchangeProperty.event.object.isNewVersion}")) + .toD(makeMillinerUri("node/${exchangeProperty.uuid}/version")) + .end(); + + from(config.getNodeDelete()) .routeId("FcrepoIndexerDeleteNode") .onException(HttpOperationFailedException.class) .onWhen(is404) @@ -167,81 +146,89 @@ public void configure() { .end() // Parse the event into a POJO. .unmarshal().json(JsonLibrary.Jackson, AS2Event.class) - // Extract relevant data from the event. - .setProperty("event").simple("${body}") + .process(commonProcessor) .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}") - .setProperty("fedoraBaseUrl").simple("${exchangeProperty.event.target}") .log(DEBUG, LOGGER, "Received Node delete event for UUID (${exchangeProperty.uuid}), fedora base URL" + " (${exchangeProperty.fedoraBaseUrl})") - // Prepare the message. - .removeHeaders("*", "Authorization") .setHeader(Exchange.HTTP_METHOD, constant("DELETE")) - .setHeader(FEDORA_HEADER, exchangeProperty("fedoraBaseUrl")) - .setBody(simple("${null}")) - // Remove the file from Drupal. - .toD(getMillinerBaseUrl() + "node/${exchangeProperty.uuid}?connectionClose=true"); + .toD(makeMillinerUri("node/${exchangeProperty.uuid}")); - from("{{media.stream}}") + from(config.getMediaIndex()) .routeId("FcrepoIndexerMedia") - + .onException(MissingJsonUrlException.class) + .useOriginalMessage() + .handled(true) + .log( + WARN, + LOGGER, + "Could not locate the Json Url for the media, event could be pre-upload. Skipping processing." + ) + .end() // Parse the event into a POJO. .unmarshal().json(JsonLibrary.Jackson, AS2Event.class) - // Extract relevant data from the event. - .setProperty("event").simple("${body}") + .process(commonProcessor) .setProperty("sourceField").simple("${exchangeProperty.event.attachment.content.sourceField}") - .setProperty("jsonUrl").simple("${exchangeProperty.event.object.url[1].href}") - .setProperty("fedoraBaseUrl").simple("${exchangeProperty.event.target}") + .setProperty("jsonUrl").simple("${exchangeProperty.event.object.getJsonUrl().href}") .log(DEBUG, LOGGER, "Received Media event for sourceField (${exchangeProperty.sourceField}), jsonld" + " URL (${exchangeProperty.jsonUrl}), fedora Base URL (${exchangeProperty.fedoraBaseUrl})") - // Prepare the message. - .removeHeaders("*", "Authorization") .setHeader(Exchange.HTTP_METHOD, constant("POST")) .setHeader("Content-Location", simple("${exchangeProperty.jsonUrl}")) - .setHeader(FEDORA_HEADER, exchangeProperty("fedoraBaseUrl")) - .setBody(simple("${null}")) - - // Pass it to milliner. - .toD(getMillinerBaseUrl() + "media/${exchangeProperty.sourceField}?connectionClose=true") - .choice() - .when() - .simple("${exchangeProperty.event.object.isNewVersion}") - .setHeader("Content-Location", simple( - "${exchangeProperty.jsonUrl}")) - - //pass it to milliner - .toD( - getMillinerBaseUrl() + - "media/${exchangeProperty.sourceField}/version?connectionClose=true" - ).endChoice(); - - from("{{file.external.stream}}") + .multicast().parallelProcessing() + .to("seda:mediaIndex", "seda:mediaVersionIndex") + .end(); + + from("seda:mediaIndex") + .routeId("FcrepoIndexerMediaIndex") + .toD(makeMillinerUri("media/${exchangeProperty.sourceField}")); + + from("seda:mediaVersionIndex") + .routeId("FcrepoIndexerMediaIndexVersion") + .log(TRACE, LOGGER, "Media indexer version endpoint, isNewVersion is " + + "(${exchangeProperty.event.object.isNewVersion}") + .filter(simple("${exchangeProperty.event.object.isNewVersion}")) + //pass it to milliner + .toD(makeMillinerUri("media/${exchangeProperty.sourceField}/version")) + .end(); + + from(config.getExternalIndex()) .routeId("FcrepoIndexerExternalFile") - + .onException(MissingCanonicalUrlException.class) + .useOriginalMessage() + .handled(true) + .log( + ERROR, + LOGGER, + "Unable to index external file to Fedora, missing the Drupal URL." + ) + .end() // Parse the event into a POJO. .unmarshal().json(JsonLibrary.Jackson, AS2Event.class) - // Extract relevant data from the event. - .setProperty("event").simple("${body}") + .process(commonProcessor) .setProperty("uuid").simple("${exchangeProperty.event.object.id.replaceAll(\"urn:uuid:\",\"\")}") - .setProperty("drupal").simple("${exchangeProperty.event.object.url[0].href}") - .setProperty("fedoraBaseUrl").simple("${exchangeProperty.event.target}") + .setProperty("drupal").simple("${exchangeProperty.event.object.getCanonicalUrl().href}") .log(DEBUG, LOGGER, "Received File external event for UUID (${exchangeProperty.uuid}), drupal URL " + "(${exchangeProperty.drupal}), fedora base URL (${exchangeProperty.fedoraBaseUrl})") - // Prepare the message. - .removeHeaders("*", "Authorization") .setHeader(Exchange.HTTP_METHOD, constant("POST")) .setHeader("Content-Location", simple("${exchangeProperty.drupal}")) - .setHeader(FEDORA_HEADER, exchangeProperty("fedoraBaseUrl")) - .setBody(simple("${null}")) - // Pass it to milliner. - .toD(getMillinerBaseUrl() + "external/${exchangeProperty.uuid}?connectionClose=true"); + .toD(makeMillinerUri("external/${exchangeProperty.uuid}")); + } + /** + * Utility to build a milliner URI. + * @param uriPart + * The part of the uri after the milliner base uri. + * @return + * The full URI. + */ + private String makeMillinerUri(final String uriPart) { + return config.addHttpOptions(config.getMillinerBaseUrl() + uriPart); } } diff --git a/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerOptions.java b/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerOptions.java new file mode 100644 index 00000000..d8c254ef --- /dev/null +++ b/islandora-indexing-fcrepo/src/main/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerOptions.java @@ -0,0 +1,144 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.fcrepo; + +import org.apache.camel.builder.RouteBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +import ca.islandora.alpaca.support.config.ConditionOnPropertyTrue; +import ca.islandora.alpaca.support.config.PropertyConfig; + +/** + * Property configuration class. + * @author whikloj + */ +@Configuration +@Conditional(FcrepoIndexerOptions.FcrepoIndexerEnabled.class) +public class FcrepoIndexerOptions extends PropertyConfig { + + private static final String FCREPO_INDEXER_ENABLED = "fcrepo.indexer.enabled"; + private static final String FCREPO_INDEXER_NODE_INDEX = "fcrepo.indexer.node"; + private static final String FCREPO_INDEXER_NODE_DELETE = "fcrepo.indexer.delete"; + private static final String FCREPO_INDEXER_MEDIA_INDEX = "fcrepo.indexer.media"; + private static final String FCREPO_INDEXER_EXTERNAL_INDEX = "fcrepo.indexer.external"; + private static final String FCREPO_INDEXER_MILLINER = "fcrepo.indexer.milliner.baseUrl"; + private static final String FCREPO_BASE_URI_HEADER_PROPERTY = "fcrepo.indexer.fedoraHeader"; + private static final String FCREPO_INDEXER_CONCURRENT = "fcrepo.indexer.concurrent-consumers"; + private static final String FCREPO_INDEXER_MAX_CONCURRENT = "fcrepo.indexer.max-concurrent-consumers"; + private static final String FCREPO_INDEXER_ASYNC_CONSUMER = "fcrepo.indexer.async-consumer"; + + @Value("${" + FCREPO_INDEXER_NODE_INDEX + ":}") + private String fcrepoNodeIndex; + + @Value("${" + FCREPO_INDEXER_NODE_DELETE + ":}") + private String fcrepoNodeDelete; + + @Value("${" + FCREPO_INDEXER_MEDIA_INDEX + ":}") + private String fcrepoMediaIndex; + + @Value("${" + FCREPO_INDEXER_EXTERNAL_INDEX + ":}") + private String fcrepoExternalIndex; + + @Value("${" + FCREPO_INDEXER_MILLINER + ":}") + private String fcrepoMillinerBaseUrl; + + @Value("${" + FCREPO_BASE_URI_HEADER_PROPERTY + ":X-Islandora-Fedora-Endpoint}") + private String fcrepoFedoraUriHeader; + + @Value("${" + FCREPO_INDEXER_CONCURRENT + ":-1}") + private int fcrepoConcurrentConsumers; + + @Value("${" + FCREPO_INDEXER_MAX_CONCURRENT + ":-1}") + private int fcrepoMaxConcurrentConsumers; + + @Value("${" + FCREPO_INDEXER_ASYNC_CONSUMER + ":false}") + private boolean fcrepoAsyncConsumers; + + /** + * Defines that Fedora indexer is only enabled if the appropriate property is set to "true". + */ + static class FcrepoIndexerEnabled extends ConditionOnPropertyTrue { + FcrepoIndexerEnabled() { + super(FcrepoIndexerOptions.FCREPO_INDEXER_ENABLED, false); + } + } + + /** + * @return the node index endpoint. + */ + public String getNodeIndex() { + return addConcurrent(JMS_ENDPOINT_NAME + ":" + fcrepoNodeIndex); + } + + /** + * @return the node delete endpoint. + */ + public String getNodeDelete() { + return addConcurrent(JMS_ENDPOINT_NAME + ":" + fcrepoNodeDelete); + } + + /** + * @return the media index endpoint. + */ + public String getMediaIndex() { + return addConcurrent(JMS_ENDPOINT_NAME + ":" + fcrepoMediaIndex); + } + + /** + * @return the external content index endpoint. + */ + public String getExternalIndex() { + return addConcurrent(JMS_ENDPOINT_NAME + ":" + fcrepoExternalIndex); + } + + /** + * Utility to avoid passing variables each time. + * @param queueString + * The topic/queue string to alter. + * @return + * The altered topic/queue string. + */ + private String addConcurrent(final String queueString) { + return addJmsOptions(queueString, fcrepoConcurrentConsumers, fcrepoMaxConcurrentConsumers, fcrepoAsyncConsumers); + } + + /** + * @return the milliner base url. + */ + public String getMillinerBaseUrl() { + return fcrepoMillinerBaseUrl; + } + + /** + * @return the header containing the fedora base uri. + */ + public String getFedoraUriHeader() { + return fcrepoFedoraUriHeader; + } + + /** + * @return bean for the fcrepo indexer camel route. + */ + @Bean + public RouteBuilder fcrepoIndexer() { + return new FcrepoIndexer(); + } +} diff --git a/islandora-indexing-fcrepo/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/islandora-indexing-fcrepo/src/main/resources/OSGI-INF/blueprint/blueprint.xml deleted file mode 100644 index e392f497..00000000 --- a/islandora-indexing-fcrepo/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - ca.islandora.alpaca.indexing.fcrepo - - - diff --git a/islandora-indexing-fcrepo/src/main/resources/logback.xml b/islandora-indexing-fcrepo/src/main/resources/logback.xml new file mode 100644 index 00000000..f03dc45c --- /dev/null +++ b/islandora-indexing-fcrepo/src/main/resources/logback.xml @@ -0,0 +1,22 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/islandora-indexing-fcrepo/src/test/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerTest.java b/islandora-indexing-fcrepo/src/test/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerTest.java index 775c0e9d..e58b1aec 100644 --- a/islandora-indexing-fcrepo/src/test/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerTest.java +++ b/islandora-indexing-fcrepo/src/test/java/ca/islandora/alpaca/indexing/fcrepo/FcrepoIndexerTest.java @@ -18,23 +18,40 @@ package ca.islandora.alpaca.indexing.fcrepo; +import static org.apache.camel.util.ObjectHelper.loadResourceAsStream; + import org.apache.camel.Exchange; -import org.apache.camel.builder.AdviceWithRouteBuilder; -import org.apache.camel.component.mock.MockEndpoint; import org.apache.camel.Produce; import org.apache.camel.ProducerTemplate; -import org.apache.camel.test.blueprint.CamelBlueprintTestSupport; +import org.apache.camel.builder.AdviceWith; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.model.ToDynamicDefinition; +import org.apache.camel.spring.javaconfig.CamelConfiguration; +import org.apache.camel.test.spring.CamelSpringRunner; +import org.apache.camel.test.spring.CamelSpringTestSupport; import org.apache.commons.io.IOUtils; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.test.annotation.DirtiesContext; -import static org.apache.camel.util.ObjectHelper.loadResourceAsStream; +import ca.islandora.alpaca.support.config.ActivemqConfig; /** * @author dannylamb */ -public class FcrepoIndexerTest extends CamelBlueprintTestSupport { +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@RunWith(CamelSpringRunner.class) +public class FcrepoIndexerTest extends CamelSpringTestSupport { - @Produce(uri = "direct:start") + @Produce("direct:start") protected ProducerTemplate template; @Override @@ -47,34 +64,30 @@ public boolean isUseRouteBuilder() { return false; } - @Override - protected String getBlueprintDescriptor() { - return "/OSGI-INF/blueprint/blueprint-test.xml"; - } - @Test public void testNode() throws Exception { final String route = "FcrepoIndexerNode"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip( - "http://localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b1d?connectionClose=true" - ); - } - }); + final String nodeSubRoute = "FcrepoIndexerNodeIndex"; + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> + a.replaceFromWith("direct:start") + ); + AdviceWith.adviceWith(context, nodeSubRoute, a -> + a.weaveByType(ToDynamicDefinition.class).selectIndex(0).replace().toD("mock:localhost:8000" + + "/milliner/node/${exchangeProperty.uuid}") + ); context.start(); // Assert we POST to milliner with creds. final MockEndpoint milliner = getMockEndpoint( - "mock:http:localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b1d" + "mock:localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b1d" ); milliner.expectedMessageCount(1); milliner.expectedHeaderReceived("Authorization", "Bearer islandora"); milliner.expectedHeaderReceived("Content-Location", "http://localhost:8000/node/2?_format=jsonld"); milliner.expectedHeaderReceived(Exchange.HTTP_METHOD, "POST"); - milliner.expectedHeaderReceived(FcrepoIndexer.FEDORA_HEADER, "http://localhost:8080/fcrepo/rest/node"); + milliner.expectedHeaderReceived("X-ISLANDORA-FEDORA-HEADER", "http://localhost:8080/fcrepo/rest/node"); // Send an event. template.send(exchange -> { @@ -85,28 +98,27 @@ public void configure() throws Exception { ); }); - assertMockEndpointsSatisfied(); + milliner.assertIsSatisfied(); } @Test public void testNodeVersion() throws Exception { final String route = "FcrepoIndexerNode"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip( - "http://localhost:8000/milliner/node/72358916-51e9-" - + "4712-b756-4b0404c91b/version?connectionClose=true" - ); - } - }); + final String versionSubRoute = "FcrepoIndexerNodeVersion"; + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> + a.replaceFromWith("direct:start") + ); + AdviceWith.adviceWith(context, versionSubRoute, a -> + a.weaveByType(ToDynamicDefinition.class).selectIndex(0).replace().toD("mock:localhost:8000" + + "/milliner/node/${exchangeProperty.uuid}/version") + ); context.start(); // Assert we POST to milliner with creds. final MockEndpoint milliner = getMockEndpoint( - "mock:http:localhost:8000/milliner/" - + "node/72358916-51e9-4712-b756-4b0404c91b/version" + "mock:localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b/version" ); milliner.expectedMessageCount(1); milliner.expectedHeaderReceived("Authorization", "Bearer islandora"); @@ -121,31 +133,29 @@ public void configure() throws Exception { String.class); }); - assertMockEndpointsSatisfied(); + milliner.assertIsSatisfied(); } @Test public void testNodeDelete() throws Exception { final String route = "FcrepoIndexerDeleteNode"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip( - "http://localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b1d?connectionClose=true" - ); - } + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + a.weaveByType(ToDynamicDefinition.class).selectIndex(0).replace().toD("mock:localhost:8000/milliner/node" + + "/${exchangeProperty.uuid}"); }); context.start(); // Assert we DELETE to milliner with creds. final MockEndpoint milliner = getMockEndpoint( - "mock:http:localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b1d" + "mock:localhost:8000/milliner/node/72358916-51e9-4712-b756-4b0404c91b1d" ); milliner.expectedMessageCount(1); milliner.expectedHeaderReceived("Authorization", "Bearer islandora"); milliner.expectedHeaderReceived(Exchange.HTTP_METHOD, "DELETE"); - milliner.expectedHeaderReceived(FcrepoIndexer.FEDORA_HEADER, "http://localhost:8080/fcrepo/rest/node"); + milliner.expectedHeaderReceived("X-ISLANDORA-FEDORA-HEADER", "http://localhost:8080/fcrepo/rest/node"); // Send an event. template.send(exchange -> { @@ -156,26 +166,24 @@ public void configure() throws Exception { ); }); - assertMockEndpointsSatisfied(); + milliner.assertIsSatisfied(); } @Test public void testExternalFile() throws Exception { final String route = "FcrepoIndexerExternalFile"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip( - "http://localhost:8000/milliner/external/148dfe8f-9711-4263-97e7-3ef3fb15864f?connectionClose=true" - ); - } + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + a.weaveByType(ToDynamicDefinition.class).selectIndex(0).replace().toD("mock:localhost:8000/milliner/" + + "external/${exchangeProperty.uuid}"); }); context.start(); // Assert we POST to Milliner with creds. final MockEndpoint milliner = getMockEndpoint( - "mock:http:localhost:8000/milliner/external/148dfe8f-9711-4263-97e7-3ef3fb15864f" + "mock:localhost:8000/milliner/external/148dfe8f-9711-4263-97e7-3ef3fb15864f" ); milliner.expectedMessageCount(1); milliner.expectedHeaderReceived("Authorization", "Bearer islandora"); @@ -184,7 +192,7 @@ public void configure() throws Exception { "http://localhost:8000/sites/default/files/2018-08/Voltaire-Records1.jpg" ); milliner.expectedHeaderReceived(Exchange.HTTP_METHOD, "POST"); - milliner.expectedHeaderReceived(FcrepoIndexer.FEDORA_HEADER, "http://localhost:8080/fcrepo/rest/externalFile"); + milliner.expectedHeaderReceived("X-ISLANDORA-FEDORA-HEADER", "http://localhost:8080/fcrepo/rest/externalFile"); // Send an event. template.send(exchange -> { @@ -195,28 +203,34 @@ public void configure() throws Exception { ); }); - assertMockEndpointsSatisfied(); + milliner.assertIsSatisfied(); } @Test public void testMedia() throws Exception { final String route = "FcrepoIndexerMedia"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip("http://localhost:8000/milliner/media/field_media_image?connectionClose=true"); - } - }); + final String mediaSubRoute = "FcrepoIndexerMediaIndex"; + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> + a.replaceFromWith("direct:start") + ); + AdviceWith.adviceWith(context, mediaSubRoute, a-> + a.weaveByType(ToDynamicDefinition.class).selectIndex(0).replace().toD("mock:localhost:8000/milliner/" + + "media/${exchangeProperty.sourceField}") + ); + context.start(); // Assert we POST the event to milliner with creds. - final MockEndpoint milliner = getMockEndpoint("mock:http:localhost:8000/milliner/media/field_media_image"); + final MockEndpoint milliner = getMockEndpoint( + "mock:localhost:8000/milliner/media/field_media_image" + ); milliner.expectedMessageCount(1); milliner.expectedHeaderReceived("Authorization", "Bearer islandora"); milliner.expectedHeaderReceived("Content-Location", "http://localhost:8000/media/6?_format=json"); milliner.expectedHeaderReceived(Exchange.HTTP_METHOD, "POST"); - milliner.expectedHeaderReceived(FcrepoIndexer.FEDORA_HEADER, "http://localhost:8080/fcrepo/rest/media"); + milliner.expectedHeaderReceived("X-ISLANDORA-FEDORA-HEADER", "http://localhost:8080/fcrepo/rest/media"); // Send an event. template.send(exchange -> { @@ -227,25 +241,26 @@ public void configure() throws Exception { ); }); - assertMockEndpointsSatisfied(); + milliner.assertIsSatisfied(); } @Test public void testVersionMedia() throws Exception { final String route = "FcrepoIndexerMedia"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip("http://localhost:8000/milliner/media/field_media_image/" - + "version?connectionClose=true"); - } - }); + final String versionSubRoute = "FcrepoIndexerMediaIndexVersion"; + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> a.replaceFromWith("direct:start")); + AdviceWith.adviceWith(context, versionSubRoute, a -> + a.weaveByType(ToDynamicDefinition.class).selectIndex(0).replace().toD("mock:localhost:8000/milliner/" + + "media/${exchangeProperty.sourceField}/version") + ); context.start(); // Assert we POST the event to milliner with creds. - final MockEndpoint milliner = getMockEndpoint("mock:http:localhost:8000/milliner/media/" - + "field_media_image/version"); + final MockEndpoint milliner = getMockEndpoint( + "mock:localhost:8000/milliner/media/field_media_image/version" + ); milliner.expectedHeaderReceived("Authorization", "Bearer islandora"); milliner.expectedHeaderReceived("Content-Location", "http://localhost:8000/media/7?_format=json"); milliner.expectedHeaderReceived(Exchange.HTTP_METHOD, "POST"); @@ -257,7 +272,38 @@ public void configure() throws Exception { String.class); }); - assertMockEndpointsSatisfied(); + milliner.assertIsSatisfied(); + } + + @BeforeClass + public static void setProperties() { + System.setProperty("error.maxRedeliveries", "1"); + System.setProperty("fcrepo.indexer.enabled", "true"); + System.setProperty("fcrepo.indexer.node", "topic:islandora-indexing-fcrepo-content"); + System.setProperty("fcrepo.indexer.delete", "topic:islandora-indexing-fcrepo-delete"); + System.setProperty("fcrepo.indexer.external", "topic:islandora-indexing-fcrepo-file-external"); + System.setProperty("fcrepo.indexer.media", "topic:islandora-indexing-fcrepo-media"); + System.setProperty("fcrepo.indexer.milliner.baseUrl", "http://localhost:8000/milliner/"); + System.setProperty("fcrepo.indexer.fedoraHeader", "X-ISLANDORA-FEDORA-HEADER"); } + @Override + protected AbstractApplicationContext createApplicationContext() { + final var context = new AnnotationConfigApplicationContext(); + context.register(FcrepoIndexerTest.ContextConfig.class); + return context; + } + + @Configuration + @ComponentScan(basePackageClasses = {FcrepoIndexerOptions.class, ActivemqConfig.class}, + useDefaultFilters = false, + includeFilters = @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, + classes = {FcrepoIndexerOptions.class, ActivemqConfig.class})) + static class ContextConfig extends CamelConfiguration { + + @Bean + public RouteBuilder route() { + return new FcrepoIndexer(); + } + } } diff --git a/islandora-indexing-fcrepo/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml b/islandora-indexing-fcrepo/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml deleted file mode 100644 index 07627978..00000000 --- a/islandora-indexing-fcrepo/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - ca.islandora.alpaca.indexing.fcrepo - - - diff --git a/islandora-indexing-fcrepo/src/test/resources/logback-test.xml b/islandora-indexing-fcrepo/src/test/resources/logback-test.xml new file mode 100644 index 00000000..c2fe05e2 --- /dev/null +++ b/islandora-indexing-fcrepo/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + \ No newline at end of file diff --git a/islandora-indexing-fcrepo/src/test/resources/simplelogger.properties b/islandora-indexing-fcrepo/src/test/resources/simplelogger.properties deleted file mode 100644 index 9c32bc7d..00000000 --- a/islandora-indexing-fcrepo/src/test/resources/simplelogger.properties +++ /dev/null @@ -1,34 +0,0 @@ -# SLF4J's SimpleLogger configuration file -# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. - -# Default logging detail level for all instances of SimpleLogger. -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, defaults to "info". -#org.slf4j.simpleLogger.defaultLogLevel=debug - -# Logging detail level for a SimpleLogger instance named "xxxxx". -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, the default logging detail level is used. -#org.slf4j.simpleLogger.log.xxxxx= - -# Set to true if you want the current date and time to be included in output messages. -# Default is false, and will output the number of milliseconds elapsed since startup. -#org.slf4j.simpleLogger.showDateTime=false - -# The date and time format to be used in the output messages. -# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. -# If the format is not specified or is invalid, the default format is used. -# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. -#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z - -# Set to true if you want to output the current thread name. -# Defaults to true. -#org.slf4j.simpleLogger.showThreadName=true - -# Set to true if you want the Logger instance name to be included in output messages. -# Defaults to true. -#org.slf4j.simpleLogger.showLogName=true - -# Set to true if you want the last component of the name to be included in output messages. -# Defaults to false. -#org.slf4j.simpleLogger.showShortLogName=false diff --git a/islandora-indexing-triplestore/build.gradle b/islandora-indexing-triplestore/build.gradle index b4290fb8..a6116bce 100644 --- a/islandora-indexing-triplestore/build.gradle +++ b/islandora-indexing-triplestore/build.gradle @@ -1,38 +1,25 @@ -apply plugin: 'osgi' -description = 'Islandora 8 Triplestore Indexer' +description = 'Islandora Triplestore Indexer' dependencies { - implementation group: 'org.apache.camel', name: 'camel-core', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-blueprint', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-http4', version: versions.camel - implementation group: 'org.apache.camel', name: 'camel-jsonpath', version: versions.camel - implementation group: 'org.fcrepo.camel', name: 'fcrepo-camel', version: versions.fcrepoCamel - implementation group: 'org.slf4j', name: 'slf4j-api', version: versions.slf4j - testImplementation group: 'org.apache.camel', name: 'camel-test-blueprint', version: versions.camel - testImplementation group: 'org.apache.servicemix.bundles', name: 'org.apache.servicemix.bundles.xerces', version: versions.xercesServiceMix - testImplementation group: 'org.ow2.asm', name: 'asm-commons', version: versions.asmCommons - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: versions.slf4j -} - -jar { - manifest { - description project.description - docURL project.docURL - vendor project.vendor - license project.license + implementation "ch.qos.logback:logback-core:${versions.logback}" + implementation "commons-io:commons-io:${versions.commonsIo}" + implementation "javax.inject:javax.inject:${versions.javaxInject}" + implementation "org.apache.camel:camel-activemq:${versions.camel}" + implementation "org.apache.camel:camel-core:${versions.camel}" + implementation "org.apache.camel:camel-jackson:${versions.camel}" + implementation "org.apache.camel:camel-jsonpath:${versions.camel}" + implementation "org.apache.camel:camel-spring-javaconfig:${versions.camel}" + implementation "org.apache.jena:jena-core:${versions.jena}" + implementation "org.apache.jena:jena-arq:${versions.jena}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + implementation "net.minidev:json-smart:${versions.jsonSmart}" + implementation project(':islandora-support') - instruction 'Import-Package', 'org.apache.camel.component.http4,' + - "org.apache.camel;version=\"${versions.camel}\"," + - "org.fcrepo.camel.processor;version=\"${versions.fcrepoCamel}\"," + - defaultOsgiImports - instruction 'Export-Package', 'ca.islandora.alpaca.indexing.triplestore' - } -} + runtimeOnly "ch.qos.logback:logback-classic:${versions.logback}" + runtimeOnly "xerces:xerces:${versions.xerces}" -artifacts { - archives (file('build/cfg/main/ca.islandora.alpaca.indexing.triplestore.cfg')) { - classifier 'configuration' - type 'cfg' - } + testImplementation "org.apache.camel:camel-test-spring:${versions.camel}" + testImplementation "org.ow2.asm:asm-commons:${versions.asmCommons}" + testImplementation "junit:junit:${versions.junit4}" } diff --git a/islandora-indexing-triplestore/src/main/cfg/ca.islandora.alpaca.indexing.triplestore.cfg b/islandora-indexing-triplestore/src/main/cfg/ca.islandora.alpaca.indexing.triplestore.cfg deleted file mode 100644 index 8bb98327..00000000 --- a/islandora-indexing-triplestore/src/main/cfg/ca.islandora.alpaca.indexing.triplestore.cfg +++ /dev/null @@ -1,9 +0,0 @@ -# In the event of failure, the maximum number of times a redelivery will be attempted. -error.maxRedeliveries=10 - -# The camel URI for the incoming message stream. -index.stream=activemq:queue:islandora-indexing-triplestore-index -delete.stream=activemq:queue:islandora-indexing-triplestore-delete - -# The base URL of the triplestore being used. -triplestore.baseUrl=http://localhost:8080/bigdata/namespace/kb/sparql diff --git a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexer.java b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexer.java index 0eb97e17..4f4f16ed 100644 --- a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexer.java +++ b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexer.java @@ -18,21 +18,27 @@ package ca.islandora.alpaca.indexing.triplestore; +import static ca.islandora.alpaca.indexing.triplestore.processors.FcrepoHeaders.FCREPO_URI; import static org.apache.camel.LoggingLevel.ERROR; import static org.apache.camel.LoggingLevel.INFO; -import static org.fcrepo.camel.FcrepoHeaders.FCREPO_URI; +import static org.apache.camel.LoggingLevel.TRACE; import static org.slf4j.LoggerFactory.getLogger; -import com.jayway.jsonpath.JsonPathException; - -import net.minidev.json.JSONArray; -import org.apache.camel.builder.RouteBuilder; import org.apache.camel.Exchange; -import org.fcrepo.camel.processor.SparqlUpdateProcessor; -import org.fcrepo.camel.processor.SparqlDeleteProcessor; +import org.apache.camel.builder.RouteBuilder; import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; -import java.util.LinkedHashMap; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.JsonPathException; + +import ca.islandora.alpaca.indexing.triplestore.processors.SparqlDeleteProcessor; +import ca.islandora.alpaca.indexing.triplestore.processors.SparqlUpdateProcessor; +import ca.islandora.alpaca.support.event.AS2Event; +import ca.islandora.alpaca.support.event.AS2Url; +import ca.islandora.alpaca.support.exceptions.MissingDescribesUrlException; +import ca.islandora.alpaca.support.exceptions.MissingJsonldUrlException; +import ca.islandora.alpaca.support.exceptions.MissingPropertyException; /** * @author dhlamb @@ -44,37 +50,44 @@ public class TriplestoreIndexer extends RouteBuilder { */ private static final Logger LOGGER = getLogger(TriplestoreIndexer.class); + @Autowired + private TriplestoreIndexerOptions config; + + private ObjectMapper objectMapper = new ObjectMapper(); + @Override public void configure() { + LOGGER.info("TriplestoreIndexer routes starting"); // Global exception handler for the indexer. // Just logs after retrying X number of times. onException(Exception.class) - .maximumRedeliveries("{{error.maxRedeliveries}}") + .maximumRedeliveries(config.getMaxRedeliveries()) .log( ERROR, LOGGER, - "Error indexing ${property.uri} in triplestore: ${exception.message}\n\n${exception.stacktrace}" + "Error indexing ${exchangeProperty.uri} in triplestore: ${exception.message}\n\n${exception.stacktrace}" ); - from("{{index.stream}}") + from(config.getJmsIndexStream()) .routeId("IslandoraTriplestoreIndexer") + .log(TRACE, LOGGER, "Received message on IslandoraTriplestoreIndexer") .to("direct:parse.url") .removeHeaders("*", "Authorization") .setHeader(Exchange.HTTP_METHOD, constant("GET")) .setBody(simple("${null}")) - .toD("${exchangeProperty.jsonld_url}&connectionClose=true") + .toD(config.addHttpOptions("${exchangeProperty.jsonld_url}", true)) .setHeader(FCREPO_URI, simple("${exchangeProperty.subject_url}")) .process(new SparqlUpdateProcessor()) .log(INFO, LOGGER, "Indexing ${exchangeProperty.subject_url} in triplestore") - .to("{{triplestore.baseUrl}}?connectionClose=true"); + .to(config.getTriplestoreBaseUrl()); - from("{{delete.stream}}") + from(config.getJmsDeleteStream()) .routeId("IslandoraTriplestoreIndexerDelete") .to("direct:parse.url") .setHeader(FCREPO_URI, simple("${exchangeProperty.subject_url}")) .process(new SparqlDeleteProcessor()) .log(INFO, LOGGER, "Deleting ${exchangeProperty.subject_url} in triplestore") - .to("{{triplestore.baseUrl}}?connectionClose=true"); + .to(config.getTriplestoreBaseUrl()); // Extracts the JSONLD URL from the event message and stores it on the exchange. from("direct:parse.url") @@ -88,7 +101,7 @@ public void configure() { "Error extracting properties from event: ${exception.message}\n\n${exception.stacktrace}" ) .end() - .onException(RuntimeException.class) + .onException(MissingPropertyException.class) .maximumRedeliveries(0) .log( ERROR, @@ -96,35 +109,34 @@ public void configure() { "Error extracting properties from event: ${exception.message}\n\n${exception.stacktrace}" ) .end() - .transform().jsonpath("$.object.url") + .onException(MissingJsonldUrlException.class) + .maximumRedeliveries(0) + .log( + INFO, + LOGGER, + "Unable to find JsonLD Url, this is an error or happens when a file is pre-uploaded to Drupal" + ) + .end() .process(ex -> { // Parse the event message. - final JSONArray message = ex.getIn().getBody(JSONArray.class); + final String message = ex.getIn().getBody(String.class); + + LOGGER.trace("Triplestore ParseUrl incoming message is \n{}", message); + + final AS2Event object = objectMapper.readValue(message, AS2Event.class); + + final AS2Url jsonldUrl = object.getObject().getJsonldUrl(); - // Get the JSONLD url. - final LinkedHashMap jsonldUrl = message.stream() - .map(LinkedHashMap.class::cast) - .filter(elem -> "application/ld+json".equals(elem.get("mediaType"))) - .findFirst() - .orElseThrow(() -> new RuntimeException("Cannot find JSONLD URL in event message.")); - ex.setProperty("jsonld_url", jsonldUrl.get("href")); + ex.setProperty("jsonld_url", jsonldUrl.getHref()); // Attempt to get the 'describes' url first, but if it fails, fall back to the canonical. + AS2Url subjectUrl; try { - final LinkedHashMap describesUrl = message.stream() - .map(LinkedHashMap.class::cast) - .filter(elem -> "describes".equals(elem.get("rel"))) - .findFirst() - .orElseThrow(() -> new RuntimeException("Cannot find describes URL in event message.")); - ex.setProperty("subject_url", describesUrl.get("href")); - } catch (RuntimeException e) { - final LinkedHashMap canonicalUrl = message.stream() - .map(LinkedHashMap.class::cast) - .filter(elem -> "canonical".equals(elem.get("rel"))) - .findFirst() - .orElseThrow(() -> new RuntimeException("Cannot find canonical URL in event message.")); - ex.setProperty("subject_url", canonicalUrl.get("href")); + subjectUrl = object.getObject().getDescribesUrl(); + } catch (final MissingDescribesUrlException e) { + subjectUrl = object.getObject().getCanonicalUrl(); } - }); + ex.setProperty("subject_url", subjectUrl.getHref()); + }).transform().jsonpath("$.object.url"); } } diff --git a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerOptions.java b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerOptions.java new file mode 100644 index 00000000..4d3bcf39 --- /dev/null +++ b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerOptions.java @@ -0,0 +1,116 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.triplestore; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +import ca.islandora.alpaca.support.config.ConditionOnPropertyTrue; +import ca.islandora.alpaca.support.config.PropertyConfig; + +/** + * Triplestore indexer configuration class. + * @author whikloj + */ +@Configuration +@Conditional(TriplestoreIndexerOptions.TriplestoreIndexerEnabled.class) +public class TriplestoreIndexerOptions extends PropertyConfig { + + /** + * Name of property to enable the triplestore indexer service. + */ + private static final String TRIPLESTORE_INDEXER_ENABLED = "triplestore.indexer.enabled"; + + private static final String BASE_URL_PROPERTY = "triplestore.baseUrl"; + private static final String TRIPLESTORE_INDEX_QUEUE = "triplestore.index.stream"; + private static final String TRIPLESTORE_DELETE_QUEUE = "triplestore.delete.stream"; + private static final String TRIPLESTORE_CONCURRENT = "triplestore.indexer.concurrent-consumers"; + private static final String TRIPLESTORE_MAX_CONCURRENT = "triplestore.indexer.max-concurrent-consumers"; + private static final String TRIPLESTORE_ASYNC_CONSUMER = "triplestore.indexer.async-consumer"; + + @Value("${" + TRIPLESTORE_INDEX_QUEUE + ":}") + private String jmsIndexStream; + + @Value("${" + TRIPLESTORE_DELETE_QUEUE + ":}") + private String jmsDeleteStream; + + @Value("${" + BASE_URL_PROPERTY + "}") + private String triplestoreBaseUrl; + + @Value("${" + TRIPLESTORE_CONCURRENT + ":-1}") + private int triplestoreConcurrent; + + @Value("${" + TRIPLESTORE_MAX_CONCURRENT + ":-1}") + private int triplestoreMaxConcurrent; + + @Value("${" + TRIPLESTORE_ASYNC_CONSUMER + ":false}") + private boolean triplestoreAsyncConsumer; + + /** + * Defines that triplestore indexer is only enabled if the appropriate property is set to "true". + */ + static class TriplestoreIndexerEnabled extends ConditionOnPropertyTrue { + TriplestoreIndexerEnabled() { + super(TriplestoreIndexerOptions.TRIPLESTORE_INDEXER_ENABLED, false); + } + } + + /** + * @return the jms index stream endpoint. + */ + public String getJmsIndexStream() { + // Prepend the current broker name + return addConcurrent(JMS_ENDPOINT_NAME + ":" + jmsIndexStream); + } + + /** + * @return the jms delete stream endpoint. + */ + public String getJmsDeleteStream() { + // Prepend the current broker name + return addConcurrent(JMS_ENDPOINT_NAME + ":" + jmsDeleteStream); + } + + /** + * @return the triplestore base url. + */ + public String getTriplestoreBaseUrl() { + return addHttpOptions(triplestoreBaseUrl); + } + + /** + * Utility to avoid passing variables each time. + * @param queueString + * The topic/queue string to alter. + * @return + * The altered topic/queue string. + */ + private String addConcurrent(final String queueString) { + return addJmsOptions(queueString, triplestoreConcurrent, triplestoreMaxConcurrent, triplestoreAsyncConsumer); + } + + /** + * @return Triplestore indexer bean. + */ + @Bean + public TriplestoreIndexer triplestoreRoute() { + return new TriplestoreIndexer(); + } +} diff --git a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/FcrepoHeaders.java b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/FcrepoHeaders.java new file mode 100644 index 00000000..8fe53411 --- /dev/null +++ b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/FcrepoHeaders.java @@ -0,0 +1,39 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.triplestore.processors; + +/** + * @author acoburn + * + * @author whikloj + * Copied from fcrepo-camel and modified for use in Alpaca - 2021-10-13 + */ +public final class FcrepoHeaders { + + public static final String FCREPO_BASE_URL = "CamelFcrepoBaseUrl"; + + public static final String FCREPO_IDENTIFIER = "CamelFcrepoIdentifier"; + + public static final String FCREPO_NAMED_GRAPH = "CamelFcrepoNamedGraph"; + + public static final String FCREPO_URI = "CamelFcrepoUri"; + + private FcrepoHeaders() { + // prevent instantiation + } +} diff --git a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/ProcessorUtils.java b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/ProcessorUtils.java new file mode 100644 index 00000000..17d31997 --- /dev/null +++ b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/ProcessorUtils.java @@ -0,0 +1,151 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.triplestore.processors; + +import static ca.islandora.alpaca.indexing.triplestore.processors.FcrepoHeaders.FCREPO_BASE_URL; +import static ca.islandora.alpaca.indexing.triplestore.processors.FcrepoHeaders.FCREPO_IDENTIFIER; +import static ca.islandora.alpaca.indexing.triplestore.processors.FcrepoHeaders.FCREPO_URI; +import static java.util.Arrays.stream; +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.apache.camel.support.ExchangeHelper.getMandatoryHeader; +import static org.apache.jena.util.URIref.encode; +import static org.slf4j.LoggerFactory.getLogger; + +import java.util.List; + +import org.apache.camel.CamelContext; +import org.apache.camel.Exchange; +import org.apache.camel.NoSuchHeaderException; + +import org.slf4j.Logger; + +/** + * Utility functions for fcrepo processor classes + * @author Aaron Coburn + * @since November 14, 2014 + * + * @author whikloj + * Copied from fcrepo-camel and modified for use in Alpaca - 2021-10-13 + */ + +public final class ProcessorUtils { + + private static final Logger LOGGER = getLogger(ProcessorUtils.class); + + /** + * This is a utility class; the constructor is off-limits. + */ + private ProcessorUtils() { + } + + private static String trimTrailingSlash(final String path) { + if (path.endsWith("/")) { + return path.substring(0, path.length() - 1); + } + return path; + } + + /** + * Extract the subject URI from the incoming exchange. + * @param exchange the incoming Exchange + * @return the subject URI + * @throws NoSuchHeaderException when the CamelFcrepoBaseUrl header is not present + */ + public static String getSubjectUri(final Exchange exchange) throws NoSuchHeaderException { + final String uri = exchange.getIn().getHeader(FCREPO_URI, "", String.class); + if (uri.isEmpty()) { + LOGGER.trace("uri isEmpty"); + final String base = getMandatoryHeader(exchange, FCREPO_BASE_URL, String.class); + final String path = exchange.getIn().getHeader(FCREPO_IDENTIFIER, "", String.class); + LOGGER.trace("base is {}, path is {}", base, path); + return trimTrailingSlash(base) + path; + } + LOGGER.trace("Returning {}", uri); + return uri; + } + + /** + * Create a DELETE WHERE { ... } statement from the provided subject + * + * @param subject the subject of the triples to delete. + * @param namedGraph an optional named graph + * @return the delete statement + */ + public static String deleteWhere(final String subject, final String namedGraph) { + final StringBuilder stmt = new StringBuilder("DELETE WHERE { "); + + if (!namedGraph.isEmpty()) { + stmt.append("GRAPH <").append(encode(namedGraph)).append("> { "); + } + + stmt.append('<').append(encode(subject)).append("> ?p ?o "); + + if (!namedGraph.isEmpty()) { + stmt.append("} "); + } + + stmt.append('}'); + return stmt.toString(); + } + + /** + * Create an INSERT DATA { ... } update query with the provided ntriples + * + * @param serializedGraph the triples to insert + * @param namedGraph an optional named graph + * @return the insert statement + */ + public static String insertData(final String serializedGraph, final String namedGraph) { + final StringBuilder query = new StringBuilder("INSERT DATA { "); + + if (!namedGraph.isEmpty()) { + query.append("GRAPH <"); + query.append(encode(namedGraph)); + query.append("> { "); + } + + query.append(serializedGraph); + + if (!namedGraph.isEmpty()) { + query.append("} "); + } + + query.append('}'); + return query.toString(); + } + + /** + * Tokenize a property placeholder value + * + * @param context the camel context + * @param property the name of the property placeholder + * @param token the token used for splitting the value + * @return a list of values + */ + public static List tokenizePropertyPlaceholder(final CamelContext context, final String property, + final String token) { + try { + return stream(context.resolvePropertyPlaceholders(property).split(token)).map(String::trim) + .filter(val -> !val.isEmpty()).collect(toList()); + } catch (final IllegalArgumentException ex) { + LOGGER.debug("No property value found for {}", property); + return emptyList(); + } + } +} diff --git a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/SparqlDeleteProcessor.java b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/SparqlDeleteProcessor.java new file mode 100644 index 00000000..1ecaa843 --- /dev/null +++ b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/SparqlDeleteProcessor.java @@ -0,0 +1,62 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.triplestore.processors; + +import static ca.islandora.alpaca.indexing.triplestore.processors.FcrepoHeaders.FCREPO_NAMED_GRAPH; +import static ca.islandora.alpaca.indexing.triplestore.processors.ProcessorUtils.deleteWhere; +import static ca.islandora.alpaca.indexing.triplestore.processors.ProcessorUtils.getSubjectUri; +import static java.net.URLEncoder.encode; +import static org.apache.camel.Exchange.CONTENT_TYPE; +import static org.apache.camel.Exchange.HTTP_METHOD; + +import java.io.IOException; + +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.NoSuchHeaderException; +import org.apache.camel.Processor; + +/** + * Represents a message processor that deletes objects from an + * external triplestore. + * + * @author Aaron Coburn + * + * @author whikloj + * Copied from fcrepo-camel and modified for use in Alpaca - 2021-10-13 + */ +public class SparqlDeleteProcessor implements Processor { + + /** + * Define how the message should be processed. + * + * @param exchange the current camel message exchange + */ + @Override + public void process(final Exchange exchange) throws IOException, NoSuchHeaderException { + + final Message in = exchange.getIn(); + final String namedGraph = in.getHeader(FCREPO_NAMED_GRAPH, "", String.class); + final String subject = getSubjectUri(exchange); + + in.setBody("update=" + encode(deleteWhere(subject, namedGraph), "UTF-8")); + in.setHeader(HTTP_METHOD, "POST"); + in.setHeader(CONTENT_TYPE, "application/x-www-form-urlencoded; charset=utf-8"); + } + +} diff --git a/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/SparqlUpdateProcessor.java b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/SparqlUpdateProcessor.java new file mode 100644 index 00000000..4dc9fc15 --- /dev/null +++ b/islandora-indexing-triplestore/src/main/java/ca/islandora/alpaca/indexing/triplestore/processors/SparqlUpdateProcessor.java @@ -0,0 +1,79 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.indexing.triplestore.processors; + +import static ca.islandora.alpaca.indexing.triplestore.processors.FcrepoHeaders.FCREPO_NAMED_GRAPH; +import static ca.islandora.alpaca.indexing.triplestore.processors.ProcessorUtils.deleteWhere; +import static ca.islandora.alpaca.indexing.triplestore.processors.ProcessorUtils.getSubjectUri; +import static ca.islandora.alpaca.indexing.triplestore.processors.ProcessorUtils.insertData; +import static java.net.URLEncoder.encode; +import static org.apache.http.entity.ContentType.parse; +import static org.apache.jena.rdf.model.ModelFactory.createDefaultModel; +import static org.apache.jena.riot.RDFDataMgr.read; +import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; +import static org.apache.camel.Exchange.CONTENT_TYPE; +import static org.apache.camel.Exchange.HTTP_METHOD; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.IOException; + +import org.apache.jena.rdf.model.Model; +import org.apache.camel.Exchange; +import org.apache.camel.Message; +import org.apache.camel.NoSuchHeaderException; +import org.apache.camel.Processor; + +/** + * Represents a processor for creating the sparql-update message to + * be passed to an external triplestore. + * + * @author Aaron Coburn + * + * @author whikloj + * Copied from fcrepo-camel and modified for use in Alpaca - 2021-10-13 + */ +public class SparqlUpdateProcessor implements Processor { + + /** + * Define how the message is processed. + * + * @param exchange the current camel message exchange + */ + @Override + public void process(final Exchange exchange) throws IOException, NoSuchHeaderException { + + final Message in = exchange.getIn(); + + final ByteArrayOutputStream serializedGraph = new ByteArrayOutputStream(); + final String namedGraph = in.getHeader(FCREPO_NAMED_GRAPH, "", String.class); + final Model model = createDefaultModel(); + final String subject = getSubjectUri(exchange); + + read(model, in.getBody(InputStream.class), + contentTypeToLang(parse(in.getHeader(CONTENT_TYPE, String.class)).getMimeType())); + + model.write(serializedGraph, "N-TRIPLE"); + + in.setBody("update=" + encode(deleteWhere(subject, namedGraph) + ";\n" + + insertData(serializedGraph.toString("UTF-8"), namedGraph), "UTF-8")); + + in.setHeader(HTTP_METHOD, "POST"); + in.setHeader(CONTENT_TYPE, "application/x-www-form-urlencoded; charset=utf-8"); + } +} diff --git a/islandora-indexing-triplestore/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/islandora-indexing-triplestore/src/main/resources/OSGI-INF/blueprint/blueprint.xml deleted file mode 100644 index ecb8cbc1..00000000 --- a/islandora-indexing-triplestore/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - ca.islandora.alpaca.indexing.triplestore - - - diff --git a/islandora-indexing-triplestore/src/main/resources/logback.xml b/islandora-indexing-triplestore/src/main/resources/logback.xml new file mode 100644 index 00000000..f03dc45c --- /dev/null +++ b/islandora-indexing-triplestore/src/main/resources/logback.xml @@ -0,0 +1,22 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/islandora-indexing-triplestore/src/test/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerTest.java b/islandora-indexing-triplestore/src/test/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerTest.java index b02a4fb1..3c12c6e2 100644 --- a/islandora-indexing-triplestore/src/test/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerTest.java +++ b/islandora-indexing-triplestore/src/test/java/ca/islandora/alpaca/indexing/triplestore/TriplestoreIndexerTest.java @@ -18,34 +18,52 @@ package ca.islandora.alpaca.indexing.triplestore; +import static java.net.URLEncoder.encode; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.camel.Exchange.CONTENT_TYPE; +import static org.apache.camel.util.ObjectHelper.loadResourceAsStream; + import java.util.Arrays; import java.util.List; -import com.jayway.jsonpath.JsonPathException; import org.apache.camel.CamelExecutionException; import org.apache.camel.EndpointInject; import org.apache.camel.Exchange; import org.apache.camel.Produce; import org.apache.camel.ProducerTemplate; -import org.apache.camel.builder.AdviceWithRouteBuilder; +import org.apache.camel.builder.AdviceWith; +import org.apache.camel.builder.RouteBuilder; import org.apache.camel.component.mock.MockEndpoint; -import org.apache.camel.test.blueprint.CamelBlueprintTestSupport; +import org.apache.camel.spring.javaconfig.CamelConfiguration; +import org.apache.camel.test.spring.CamelSpringRunner; +import org.apache.camel.test.spring.CamelSpringTestSupport; import org.apache.commons.io.IOUtils; +import org.junit.BeforeClass; import org.junit.Test; - -import static java.net.URLEncoder.encode; -import static org.apache.camel.Exchange.CONTENT_TYPE; -import static org.apache.camel.util.ObjectHelper.loadResourceAsStream; +import org.junit.runner.RunWith; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.test.annotation.DirtiesContext; + +import ca.islandora.alpaca.support.config.ActivemqConfig; +import ca.islandora.alpaca.support.exceptions.MissingCanonicalUrlException; +import ca.islandora.alpaca.support.exceptions.MissingJsonldUrlException; /** * @author dannylamb */ -public class TriplestoreIndexerTest extends CamelBlueprintTestSupport { +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@RunWith(CamelSpringRunner.class) +public class TriplestoreIndexerTest extends CamelSpringTestSupport { - @EndpointInject(uri = "mock:result") + @EndpointInject("mock:result") protected MockEndpoint resultEndpoint; - @Produce(uri = "direct:start") + @Produce("direct:start") protected ProducerTemplate template; @Override @@ -58,20 +76,13 @@ public boolean isUseRouteBuilder() { return false; } - @Override - protected String getBlueprintDescriptor() { - return "/OSGI-INF/blueprint/blueprint-test.xml"; - } - @Test public void testParseUrl() throws Exception { final String route = "IslandoraTriplestoreIndexerParseUrl"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - weaveAddLast().to(resultEndpoint); - } + + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + a.weaveAddLast().to(resultEndpoint); }); context.start(); @@ -81,75 +92,89 @@ public void configure() throws Exception { xchange.getIn().setBody(IOUtils.toString(loadResourceAsStream("AS2Event.jsonld"), "UTF-8")) ); - this.assertPredicate( + + assertPredicate( exchangeProperty("jsonld_url").isEqualTo("http://localhost:8000/node/1?_format=jsonld"), exchange, true ); - this.assertPredicate( + assertPredicate( exchangeProperty("subject_url").isEqualTo("http://localhost:8000/node/1"), exchange, true ); - assertMockEndpointsSatisfied(); + resultEndpoint.assertIsSatisfied(); } @Test - public void testParseUrlDiesOnMalformed() throws Exception { + public void testParseUrlDiesOnNoJsonld() throws Exception { final String route = "IslandoraTriplestoreIndexerParseUrl"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - weaveAddLast().to(resultEndpoint); - } + + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + a.weaveAddLast().to(resultEndpoint); }); context.start(); resultEndpoint.expectedMessageCount(0); - // Make sure it dies if the jsonpath fails. + // Make sure it dies if you can't extract the jsonld url from the event. try { template.sendBody( - IOUtils.toString(loadResourceAsStream("AS2EventNoUrls.jsonld"), "UTF-8")); - } catch (CamelExecutionException e) { - assertIsInstanceOf(JsonPathException.class, e.getCause().getCause()); + IOUtils.toString(loadResourceAsStream("AS2EventNoJsonldUrl.jsonld"), "UTF-8")); + } catch (final CamelExecutionException e) { + assertIsInstanceOf(MissingJsonldUrlException.class, e.getCause()); } + resultEndpoint.assertIsSatisfied(); + } + + @Test + public void testParseUrlDiesOnNoCanonical() throws Exception { + final String route = "IslandoraTriplestoreIndexerParseUrl"; + + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + a.weaveAddLast().to(resultEndpoint); + }); + context.start(); + + resultEndpoint.expectedMessageCount(0); + // Make sure it dies if you can't extract the jsonld url from the event. try { template.sendBody( - IOUtils.toString(loadResourceAsStream("AS2EventNoJsonldUrl.jsonld"), "UTF-8")); - } catch (CamelExecutionException e) { - assertIsInstanceOf(RuntimeException.class, e.getCause()); + IOUtils.toString(loadResourceAsStream("AS2EventNoCanonicalUrl.jsonld"), UTF_8)); + } catch (final CamelExecutionException e) { + assertIsInstanceOf(MissingCanonicalUrlException.class, e.getCause()); } - assertMockEndpointsSatisfied(); + resultEndpoint.assertIsSatisfied(); } @Test public void testIndex() throws Exception { final String route = "IslandoraTriplestoreIndexer"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - - // Rig Drupal REST endpoint to return canned jsonld - interceptSendToEndpoint("http://localhost:8000/node/1?_format=jsonld&connectionClose=true") - .skipSendToOriginalEndpoint() - .process(exchange -> { - exchange.getIn().removeHeaders("*"); - exchange.getIn().setHeader("Content-Type", "application/ld+json"); - exchange.getIn().setBody( - IOUtils.toString(loadResourceAsStream("node.jsonld"), "UTF-8"), - String.class); - }); - - mockEndpointsAndSkip( - "http://localhost:8080/bigdata/namespace/islandora/sparql?connectionClose=true" - ); - } + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + + // Rig Drupal REST endpoint to return canned jsonld + a.interceptSendToEndpoint("http://localhost:8000/node/1?_format=jsonld&connectionClose=true" + + "&disableStreamCache=true") + .skipSendToOriginalEndpoint() + .process(exchange -> { + exchange.getIn().removeHeaders("*"); + exchange.getIn().setHeader("Content-Type", "application/ld+json"); + exchange.getIn().setBody( + IOUtils.toString(loadResourceAsStream("node.jsonld"), UTF_8), + String.class); + }); + + a.mockEndpointsAndSkip( + "http://localhost:8080/bigdata/namespace/islandora/sparql?connectionClose=true&disableStreamCache=true" + ); }); context.start(); @@ -168,28 +193,29 @@ public void configure() throws Exception { endpoint.expectedMessageCount(1); endpoint.expectedHeaderReceived(Exchange.HTTP_METHOD, "POST"); endpoint.expectedHeaderReceived(CONTENT_TYPE, "application/x-www-form-urlencoded; charset=utf-8"); - endpoint.allMessages().body().startsWith("update=" + encode(responsePrefix, "UTF-8")); - endpoint.allMessages().body().endsWith(encode("\n}", "UTF-8")); + endpoint.allMessages().body().startsWith("update=" + encode(responsePrefix, UTF_8)); + endpoint.allMessages().body().endsWith(encode("\n}", UTF_8)); for (final String triple : triples) { - endpoint.expectedBodyReceived().body().contains(encode(triple, "UTF-8")); + endpoint.expectedBodyReceived().body().contains(encode(triple, UTF_8)); } template.send(exchange -> { - exchange.getIn().setBody(IOUtils.toString(loadResourceAsStream("AS2Event.jsonld"), "UTF-8")); + exchange.getIn().setBody(IOUtils.toString(loadResourceAsStream("AS2Event.jsonld"), UTF_8)); }); - assertMockEndpointsSatisfied(); + endpoint.assertIsSatisfied(); } @Test public void testDelete() throws Exception { final String route = "IslandoraTriplestoreIndexerDelete"; - context.getRouteDefinition(route).adviceWith(context, new AdviceWithRouteBuilder() { - @Override - public void configure() throws Exception { - replaceFromWith("direct:start"); - mockEndpointsAndSkip("http://localhost:8080/bigdata/namespace/islandora/sparql?connectionClose=true"); - } + + context.disableJMX(); + AdviceWith.adviceWith(context, route, a -> { + a.replaceFromWith("direct:start"); + a.mockEndpoints("broker:*"); + a.mockEndpointsAndSkip("http://localhost:8080/bigdata/namespace/islandora/sparql?" + + "connectionClose=true&disableStreamCache=true"); }); context.start(); @@ -199,14 +225,42 @@ public void configure() throws Exception { endpoint.expectedHeaderReceived(Exchange.HTTP_METHOD, "POST"); endpoint.expectedHeaderReceived(CONTENT_TYPE, "application/x-www-form-urlencoded; charset=utf-8"); endpoint.allMessages().body().startsWith( - "update=" + encode("DELETE WHERE { ?p ?o }", "UTF-8") + "update=" + encode("DELETE WHERE { ?p ?o }", UTF_8) ); template.send(exchange -> { - exchange.getIn().setBody(IOUtils.toString(loadResourceAsStream("AS2Event.jsonld"), "UTF-8")); + exchange.getIn().setBody(IOUtils.toString(loadResourceAsStream("AS2Event.jsonld"), UTF_8)); }); - assertMockEndpointsSatisfied(); + endpoint.assertIsSatisfied(); + } + + @BeforeClass + public static void setProperties() { + System.setProperty("error.maxRedeliveries", "1"); + System.setProperty("triplestore.indexer.enabled", "true"); + System.setProperty("triplestore.index.stream", "topic:islandora-indexing-triplestore-index"); + System.setProperty("triplestore.delete.stream", "topic:islandora-indexing-triplestore-delete"); + System.setProperty("triplestore.baseUrl", "http://localhost:8080/bigdata/namespace/islandora/sparql"); } + @Override + protected AbstractApplicationContext createApplicationContext() { + final var context = new AnnotationConfigApplicationContext(); + context.register(TriplestoreIndexerTest.ContextConfig.class); + return context; + } + + @Configuration + @ComponentScan(basePackageClasses = {TriplestoreIndexerOptions.class, ActivemqConfig.class}, + useDefaultFilters = false, + includeFilters = @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, + classes = {TriplestoreIndexerOptions.class, ActivemqConfig.class})) + static class ContextConfig extends CamelConfiguration { + + @Bean + public RouteBuilder route() { + return new TriplestoreIndexer(); + } + } } diff --git a/islandora-indexing-triplestore/src/test/resources/AS2EventNoUrls.jsonld b/islandora-indexing-triplestore/src/test/resources/AS2EventNoCanonicalUrl.jsonld similarity index 63% rename from islandora-indexing-triplestore/src/test/resources/AS2EventNoUrls.jsonld rename to islandora-indexing-triplestore/src/test/resources/AS2EventNoCanonicalUrl.jsonld index 7c3b5b78..cf9b6d5c 100644 --- a/islandora-indexing-triplestore/src/test/resources/AS2EventNoUrls.jsonld +++ b/islandora-indexing-triplestore/src/test/resources/AS2EventNoCanonicalUrl.jsonld @@ -26,7 +26,21 @@ ] }, "object":{ - "id":"urn:uuid:9541c0c1-5bee-4973-a9d0-e55c1658bc81" + "id":"urn:uuid:9541c0c1-5bee-4973-a9d0-e55c1658bc81", + "url":[ + { + "name":"Drupal JSONLD", + "type":"Link", + "href":"http:\/\/localhost:8000\/node\/1?_format=jsonld", + "mediaType":"application\/ld+json" + }, + { + "name":"Drupal JSON", + "type":"Link", + "href":"http:\/\/localhost:8000\/node\/1?_format=json", + "mediaType":"application\/json" + } + ] }, "type":"Update" -} \ No newline at end of file +} diff --git a/islandora-indexing-triplestore/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml b/islandora-indexing-triplestore/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml deleted file mode 100644 index 9fca69c4..00000000 --- a/islandora-indexing-triplestore/src/test/resources/OSGI-INF/blueprint/blueprint-test.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - ca.islandora.alpaca.indexing.triplestore - - - diff --git a/islandora-indexing-triplestore/src/test/resources/log4j.properties b/islandora-indexing-triplestore/src/test/resources/log4j.properties deleted file mode 100644 index 88663293..00000000 --- a/islandora-indexing-triplestore/src/test/resources/log4j.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# The logging properties used for testing -# -log4j.rootLogger=INFO, out - -#log4j.logger.org.apache.camel=DEBUG - -# CONSOLE appender not used by default -log4j.appender.out=org.apache.log4j.ConsoleAppender -log4j.appender.out.layout=org.apache.log4j.PatternLayout -log4j.appender.out.layout.ConversionPattern=[%30.30t] %-30.30c{1} %-5p %m%n -#log4j.appender.out.layout.ConversionPattern=%d [%-15.15t] %-5p %-30.30c{1} - %m%n - -# File appender -log4j.appender.file=org.apache.log4j.FileAppender -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d %-5p %c{1} - %m %n -log4j.appender.file.file=target/camel-test.log diff --git a/islandora-indexing-triplestore/src/test/resources/logback-test.xml b/islandora-indexing-triplestore/src/test/resources/logback-test.xml new file mode 100644 index 00000000..c2fe05e2 --- /dev/null +++ b/islandora-indexing-triplestore/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + + + %p %d{HH:mm:ss.SSS} \(%c{0}\) %m%n + + + + + + + + + + \ No newline at end of file diff --git a/islandora-indexing-triplestore/src/test/resources/simplelogger.properties b/islandora-indexing-triplestore/src/test/resources/simplelogger.properties deleted file mode 100644 index 9c32bc7d..00000000 --- a/islandora-indexing-triplestore/src/test/resources/simplelogger.properties +++ /dev/null @@ -1,34 +0,0 @@ -# SLF4J's SimpleLogger configuration file -# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. - -# Default logging detail level for all instances of SimpleLogger. -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, defaults to "info". -#org.slf4j.simpleLogger.defaultLogLevel=debug - -# Logging detail level for a SimpleLogger instance named "xxxxx". -# Must be one of ("trace", "debug", "info", "warn", or "error"). -# If not specified, the default logging detail level is used. -#org.slf4j.simpleLogger.log.xxxxx= - -# Set to true if you want the current date and time to be included in output messages. -# Default is false, and will output the number of milliseconds elapsed since startup. -#org.slf4j.simpleLogger.showDateTime=false - -# The date and time format to be used in the output messages. -# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. -# If the format is not specified or is invalid, the default format is used. -# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. -#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z - -# Set to true if you want to output the current thread name. -# Defaults to true. -#org.slf4j.simpleLogger.showThreadName=true - -# Set to true if you want the Logger instance name to be included in output messages. -# Defaults to true. -#org.slf4j.simpleLogger.showLogName=true - -# Set to true if you want the last component of the name to be included in output messages. -# Defaults to false. -#org.slf4j.simpleLogger.showShortLogName=false diff --git a/islandora-support/build.gradle b/islandora-support/build.gradle new file mode 100644 index 00000000..c3683c7b --- /dev/null +++ b/islandora-support/build.gradle @@ -0,0 +1,20 @@ +description = 'Islandora Alpaca Supporting Libraries' + +dependencies { + implementation "javax.inject:javax.inject:${versions.javaxInject}" + implementation "javax.jms:javax.jms-api:${versions.javaxJms}" + implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + implementation "org.apache.camel:camel-activemq:${versions.camel}" + implementation "org.apache.camel:camel-core:${versions.camel}" + implementation "org.apache.camel:camel-http:${versions.camel}" + implementation "org.apache.camel:camel-spring-javaconfig:${versions.camel}" + implementation "org.springframework:spring-context:${versions.spring}" + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + + testImplementation "org.apache.camel:camel-test-spring:${versions.camel}" + testImplementation "junit:junit:${versions.junit4}" +} + +jar { + enabled = true +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ActivemqConfig.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ActivemqConfig.java new file mode 100644 index 00000000..311aefd1 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ActivemqConfig.java @@ -0,0 +1,150 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.config; + +import static org.slf4j.LoggerFactory.getLogger; + +import javax.jms.ConnectionFactory; +import javax.jms.JMSException; + +import org.apache.activemq.ActiveMQConnectionFactory; +import org.apache.activemq.pool.PooledConnectionFactory; +import org.apache.camel.component.activemq.ActiveMQComponent; +import org.apache.camel.component.jms.JmsConfiguration; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * ActiveMQ configuration class + * + * @author whikloj + */ +@Configuration +public class ActivemqConfig extends PropertyConfig { + + private static final Logger LOGGER = getLogger(ActivemqConfig.class); + + public static final String JMS_BROKER_URL = "jms.brokerUrl"; + public static final String JMS_USERNAME = "jms.username"; + public static final String JMS_PASSWORD = "jms.password"; + public static final String CONNECTIONS = "jms.connections"; + + @Value("${" + JMS_BROKER_URL + ":tcp://localhost:61616}") + private String jmsBrokerUrl; + + @Value("${" + JMS_USERNAME + ":#{null}}") + private String jmsUsername; + + @Value("${" + JMS_PASSWORD + ":#{null}}") + private String jmsPassword; + + @Value("${" + CONNECTIONS + ":10}") + private int jmsConnections; + + /** + * @return the jms broker url + */ + public String getJmsBrokerUrl() { + if (jmsBrokerUrl == null) { + return ""; + } + return jmsBrokerUrl; + } + + /** + * @return the jms broker username (if applicable). + */ + public String getJmsUsername() { + if (jmsUsername == null) { + return ""; + } + return jmsUsername; + } + + /** + * @return the jms broker password (if applicable). + */ + public String getJmsPassword() { + if (jmsPassword == null) { + return ""; + } + return jmsPassword; + } + + /** + * @return the size of the connection pool. + */ + public int getJmsConnections() { + return jmsConnections; + } + + /** + * @return JMS Connection factory bean. + * @throws JMSException on failure to create new connection. + */ + @Bean + public ConnectionFactory jmsConnectionFactory() throws JMSException { + final var factory = new ActiveMQConnectionFactory(); + LOGGER.debug("jmsConnectionFactory: brokerUrl is {}", getJmsBrokerUrl()); + if (!getJmsBrokerUrl().isBlank()) { + factory.setBrokerURL(getJmsBrokerUrl()); + LOGGER.debug("jms username is {}", getJmsUsername()); + if (!getJmsUsername().isBlank() && !getJmsPassword().isBlank()) { + factory.createConnection(getJmsUsername(), getJmsPassword()); + } + } + return factory; + } + + /** + * @param connectionFactory the JMS connection factory. + * @return A pooled connection factory. + */ + @Bean + public PooledConnectionFactory pooledConnectionFactory(final ConnectionFactory connectionFactory) { + final var pooledConnectionFactory = new PooledConnectionFactory(); + pooledConnectionFactory.setMaxConnections(getJmsConnections()); + pooledConnectionFactory.setConnectionFactory(connectionFactory); + return pooledConnectionFactory; + } + + /** + * @param connectionFactory the pooled connection factory. + * @return the JMS configuration + */ + @Bean + public JmsConfiguration jmsConfiguration(final PooledConnectionFactory connectionFactory) { + final var configuration = new JmsConfiguration(); + configuration.setConnectionFactory(connectionFactory); + return configuration; + } + + /** + * @param jmsConfiguration the JMS configuration + * @return the ActiveMQ endpoint. + */ + @Bean(JMS_ENDPOINT_NAME) + public ActiveMQComponent activeMQComponent(final JmsConfiguration jmsConfiguration) { + final var component = new ActiveMQComponent(); + component.setConfiguration(jmsConfiguration); + return component; + } + +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ConditionOnProperty.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ConditionOnProperty.java new file mode 100644 index 00000000..c2b7f229 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ConditionOnProperty.java @@ -0,0 +1,70 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.config; + +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.context.annotation.ConfigurationCondition; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * This condition enables a bean/configuration when the specified property matches the expected value + * + * Implementations must provide a no-arg constructor. + * + * @author pwinckles + */ +public abstract class ConditionOnProperty implements ConfigurationCondition { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConditionOnProperty.class); + + private final String name; + private final T expected; + private final T defaultValue; + private final Class clazz; + + /** + * Basic constructor + * @param name the property name + * @param expected the expected value for the condition + * @param defaultValue the default value + * @param clazz the class to control. + */ + public ConditionOnProperty(final String name, final T expected, final T defaultValue, final Class clazz) { + this.name = name; + this.expected = expected; + this.defaultValue = defaultValue; + this.clazz = clazz; + } + + @Override + public boolean matches(final ConditionContext context, final AnnotatedTypeMetadata metadata) { + LOGGER.debug("Prop {}: {}", name, context.getEnvironment().getProperty(name)); + return Objects.equals(expected, context.getEnvironment().getProperty(name, clazz, defaultValue)); + } + + @Override + public ConfigurationPhase getConfigurationPhase() { + // This forces spring to not evaluate these conditions until after it has loaded other @Configuration classes, + // ensuring that the properties have been loaded. + return ConfigurationPhase.REGISTER_BEAN; + } +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ConditionOnPropertyTrue.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ConditionOnPropertyTrue.java new file mode 100644 index 00000000..840545b9 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/ConditionOnPropertyTrue.java @@ -0,0 +1,38 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.config; + +/** + * This condition enables a bean/configuration when the specified property is true + * + * Implementations must provide a no-arg constructor. + * + * @author pwinckles + */ +public abstract class ConditionOnPropertyTrue extends ConditionOnProperty { + + /** + * Basic constructor + * @param name the name of the property to check. + * @param defaultValue the value to pass in and test. + */ + public ConditionOnPropertyTrue(final String name, final boolean defaultValue) { + super(name, true, defaultValue, Boolean.class); + } + +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/config/PropertyConfig.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/PropertyConfig.java new file mode 100644 index 00000000..44fdec1e --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/PropertyConfig.java @@ -0,0 +1,134 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.PropertySources; + +/** + * Abstract class of common properties + * + * @author whikloj + */ +@PropertySources({ + @PropertySource(value = PropertyConfig.ALPACA_DEFAULT_CONFIG_FILE, ignoreResourceNotFound = true), + @PropertySource(value = PropertyConfig.ALPACA_CONFIG_FILE, ignoreResourceNotFound = true) +}) +public abstract class PropertyConfig { + + public static final String ALPACA_CONFIG_PROPERTY = "alpaca.config"; + public static final String ALPACA_HOME_PROPERTY = "alpaca.home"; + public static final String ALPACA_DEFAULT_HOME = "alpaca-home-directory"; + public static final String ALPACA_DEFAULT_CONFIG_FILE = "file:${" + + ALPACA_HOME_PROPERTY + ":" + ALPACA_DEFAULT_HOME + "}/config/alpaca.properties"; + public static final String ALPACA_CONFIG_FILE = "file:${" + ALPACA_CONFIG_PROPERTY + "}"; + // static endpoint name for activemq connection + protected static final String JMS_ENDPOINT_NAME = "broker"; + protected static final String MAX_REDELIVERIES_PROPERTY = "error.maxRedeliveries"; + + @Value("${" + MAX_REDELIVERIES_PROPERTY + ":5}") + private int maxRedeliveries; + + /** + * @return the error.maxRedeliveries amount. + */ + public int getMaxRedeliveries() { + return maxRedeliveries; + } + + /** + * Utility function to append various JMS options like concurrentConsumer variables. + * @param queueString + * The original topic/queue string + * @param concurrentConsumers + * The number of concurrent consumers. -1 means no setting. + * @param maxConcurrentConsumers + * The max number of concurrent consumers. -1 means no setting. + * @param asyncConsumers + * Indicate if the queue should be processed strictly queue-wise (false; + * more for dealing with overhead?); otherwise, allow multiple items to be + * processed at the same time. + * @return + * The modified topic/queue string. + */ + public static String addJmsOptions(final String queueString, final int concurrentConsumers, + final int maxConcurrentConsumers, final boolean asyncConsumers) { + final StringBuilder builder = new StringBuilder(); + if (concurrentConsumers > 0) { + builder.append("concurrentConsumers="); + builder.append(concurrentConsumers); + } + if (maxConcurrentConsumers > 0) { + if (builder.length() > 0) { + builder.append('&'); + } + builder.append("maxConcurrentConsumers="); + builder.append(maxConcurrentConsumers); + } + if (asyncConsumers) { + if (builder.length() > 0) { + builder.append('&'); + } + builder.append("asyncConsumer=") + .append(asyncConsumers); + } + if (builder.length() > 0) { + return queueString + (queueString.contains("?") ? '&' : '?') + builder; + } + return queueString; + } + + /** + * Utility to add common endpoint options to HTTP endpoints. + * @param httpEndpoint + * The http endpoint string. + * @param forceAmpersand + * If you want to use this function with a dynamic endpoint, this is whether to force an ampersand to start. + * @return + * The modified http endpoint string. + */ + public String addHttpOptions(final String httpEndpoint, final boolean forceAmpersand) { + final String commonElements = "connectionClose=true&disableStreamCache=true"; + final int bestGuessAtFinalLength = httpEndpoint.length() + commonElements.length() + 1; + final StringBuilder builder = new StringBuilder(bestGuessAtFinalLength); + builder.append(httpEndpoint); + // Only append ? or & if there is an endpoint. + if (builder.length() > 0) { + if (httpEndpoint.contains("?") || forceAmpersand) { + builder.append('&'); + } else { + builder.append('?'); + } + } + // Append the common elements. + builder.append(commonElements); + return builder.toString(); + } + + /** + * Assumes not to forceAmpersand + * @param httpEndpoint + * The http endpoint string. + * @return + * The modified http endpoint string. + */ + public String addHttpOptions(final String httpEndpoint) { + return addHttpOptions(httpEndpoint, false); + } +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/config/RequestConfigurerConfig.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/RequestConfigurerConfig.java new file mode 100644 index 00000000..bfc6f48d --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/config/RequestConfigurerConfig.java @@ -0,0 +1,85 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.config; + +import static org.slf4j.LoggerFactory.getLogger; + +import org.apache.camel.component.http.HttpComponent; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Class to enable the HTTP client configurer. + * + * @author whikloj + */ +@Configuration +public class RequestConfigurerConfig { + + private static final Logger LOGGER = getLogger(RequestConfigurerConfig.class); + + public static final String REQUEST_CONFIGURER_ENABLED_PROPERTY = "request.configurer.enabled"; + public static final String REQUEST_TIMEOUT_PROPERTY = "request.timeout"; + public static final String CONNECT_TIMEOUT_PROPERTY = "connection.timeout"; + public static final String SOCKET_TIMEOUT_PROPERTY = "socket.timeout"; + + @Value("${" + REQUEST_CONFIGURER_ENABLED_PROPERTY + ":false}") + private boolean enabled; + + @Value("${" + REQUEST_TIMEOUT_PROPERTY + ":-1}") + private int requestTimeout; + + @Value("${" + CONNECT_TIMEOUT_PROPERTY + ":-1}") + private int connectTimeout; + + @Value("${" + SOCKET_TIMEOUT_PROPERTY + ":-1}") + private int socketTimeout; + + /** + * Customize the connection setting if necessary. + * @return the http component + */ + private HttpComponent configComponent(final HttpComponent component) { + if (enabled) { + LOGGER.debug("Request configurer enabled, setting request.timeout {}, connection.timeout {}, socket" + + ".timeout {}", requestTimeout, connectTimeout, socketTimeout); + component.setConnectionRequestTimeout(requestTimeout); + component.setConnectTimeout(connectTimeout); + component.setSocketTimeout(socketTimeout); + } + return component; + } + + /** + * @return bean for the http endpoint. + */ + @Bean(name = "http") + public HttpComponent http() { + return configComponent(new HttpComponent()); + } + + /** + * @return bean for the https endpoint. + */ + @Bean(name = "https") + public HttpComponent https() { + return configComponent(new HttpComponent()); + } +} diff --git a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Actor.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Actor.java similarity index 100% rename from islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Actor.java rename to islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Actor.java diff --git a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Attachment.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Attachment.java similarity index 100% rename from islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Attachment.java rename to islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Attachment.java diff --git a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2AttachmentContent.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2AttachmentContent.java similarity index 100% rename from islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2AttachmentContent.java rename to islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2AttachmentContent.java diff --git a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Event.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Event.java similarity index 100% rename from islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Event.java rename to islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Event.java diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Object.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Object.java new file mode 100644 index 00000000..a84d0227 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Object.java @@ -0,0 +1,181 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ca.islandora.alpaca.support.event; + +import java.util.Arrays; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import ca.islandora.alpaca.support.exceptions.MissingCanonicalUrlException; +import ca.islandora.alpaca.support.exceptions.MissingDescribesUrlException; +import ca.islandora.alpaca.support.exceptions.MissingJsonUrlException; +import ca.islandora.alpaca.support.exceptions.MissingJsonldUrlException; +import ca.islandora.alpaca.support.exceptions.MissingPropertyException; + +/** + * POJO for a user performing an action. Part of a AS2Event. + * + * @author Danny Lamb + */ +public class AS2Object { + + /** + * The object type if applicable. + */ + private String type; + /** + * The object UUID. + */ + private String id; + /** + * The URLs passed with the event. + */ + private AS2Url[] url; + + /** + * Are we creating a new revision? + */ + private Boolean isNewVersion; + + /** + * @return Type of object + */ + public String getType() { + return type; + } + + /** + * @param type Type of object + */ + public void setType(final String type) { + this.type = type; + } + + /** + * @return URN of object + */ + public String getId() { + return id; + } + + /** + * @param id URN of object + */ + public void setId(final String id) { + this.id = id; + } + + /** + * @return URLs for object + */ + public AS2Url[] getUrl() { + return url; + } + + /** + * @param url URLs for object + */ + public void setUrl(final AS2Url[] url) { + this.url = url.clone(); + } + + /** + * @return true or false + */ + @JsonProperty("isNewVersion") + public Boolean getIsNewVersion() { + return isNewVersion; + } + + /** + * @param isNewVersion true or false + */ + public void setIsNewVersion(final Boolean isNewVersion) { + this.isNewVersion = isNewVersion; + } + + /** + * @return the Json-ld Url + * @throws MissingJsonldUrlException + * When there is no url with application/ld+json mimetype + */ + public AS2Url getJsonldUrl() throws MissingPropertyException { + return getObjectUrl("application/ld+json", null, new MissingJsonldUrlException()); + } + + /** + * @return the Canonical Url + * @throws MissingJsonUrlException + * When there is no url with rel = canonical and text/html mimetype + */ + public AS2Url getJsonUrl() throws MissingPropertyException { + return getObjectUrl("application/json", null, new MissingJsonUrlException()); + } + + /** + * @return the Canonical Url + * @throws MissingCanonicalUrlException + * When there is no url with rel = canonical and text/html mimetype + */ + public AS2Url getCanonicalUrl() throws MissingPropertyException { + return getObjectUrl(null, "canonical", new MissingCanonicalUrlException()); + } + + /** + * @return the Canonical Url + * @throws MissingDescribesUrlException + * When there is no url with rel = describes + */ + public AS2Url getDescribesUrl() throws MissingPropertyException { + return getObjectUrl(null, "describes", new MissingDescribesUrlException()); + } + + /** + * Utility to filter AS2Urls, filters by mimetype or rel or both + * @param mimetype + * The mimetype to filter on or null for none. + * @param rel + * The rel to filter on or null for none. + * @param e + * The exception to throw if we can't find a matching url + * @return + * The first matching AS2Url. + * @throws MissingPropertyException + * If no matching url can be found. + */ + private AS2Url getObjectUrl(final String mimetype, final String rel, final MissingPropertyException e) throws + MissingPropertyException { + if (url == null) { + throw e; + } + final var filterUrl = Arrays.stream(url).filter(a -> { + if (mimetype != null && (a.getMediaType() == null || !a.getMediaType().equalsIgnoreCase(mimetype))) { + return false; + } + if (rel != null) { + return a.getRel() != null && a.getRel().equalsIgnoreCase(rel); + } + return true; + }).findFirst().orElse(null); + if (filterUrl == null) { + throw e; + } + return filterUrl; + } +} diff --git a/islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Url.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Url.java similarity index 100% rename from islandora-event-support/src/main/java/ca/islandora/alpaca/support/event/AS2Url.java rename to islandora-support/src/main/java/ca/islandora/alpaca/support/event/AS2Url.java diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingCanonicalUrlException.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingCanonicalUrlException.java new file mode 100644 index 00000000..7757978c --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingCanonicalUrlException.java @@ -0,0 +1,31 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.exceptions; + +/** + * Exception if there is no canonical URL in the message. + * @author whikloj + */ +public class MissingCanonicalUrlException extends MissingPropertyException { + /** + * Basic constructor + */ + public MissingCanonicalUrlException() { + super("Cannot find canonical URL in event message."); + } +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingDescribesUrlException.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingDescribesUrlException.java new file mode 100644 index 00000000..f2b53833 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingDescribesUrlException.java @@ -0,0 +1,31 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.exceptions; + +/** + * Exception if there is no describes URL. + * @author whikloj + */ +public class MissingDescribesUrlException extends MissingPropertyException { + /** + * Basic constructor + */ + public MissingDescribesUrlException() { + super("Cannot find describes URL in event message."); + } +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingJsonUrlException.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingJsonUrlException.java new file mode 100644 index 00000000..66d18364 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingJsonUrlException.java @@ -0,0 +1,32 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.exceptions; + +/** + * Exception when json url cannot be found. + * @author whikloj + */ +public class MissingJsonUrlException extends MissingPropertyException { + + /** + * Basic constructor. + */ + public MissingJsonUrlException() { + super("Cannot find JSON URL in event message."); + } +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingJsonldUrlException.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingJsonldUrlException.java new file mode 100644 index 00000000..ac3a37b7 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingJsonldUrlException.java @@ -0,0 +1,31 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.exceptions; + +/** + * Exception if there is no JSON-LD URL in the message. + * @author whikloj + */ +public class MissingJsonldUrlException extends MissingPropertyException { + /** + * Basic constructor. + */ + public MissingJsonldUrlException() { + super("Cannot find JSONLD URL in event message."); + } +} diff --git a/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingPropertyException.java b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingPropertyException.java new file mode 100644 index 00000000..ea6a3df1 --- /dev/null +++ b/islandora-support/src/main/java/ca/islandora/alpaca/support/exceptions/MissingPropertyException.java @@ -0,0 +1,35 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ca.islandora.alpaca.support.exceptions; + +/** + * Parent class for all missing property problems + * + * @author whikloj + */ +public class MissingPropertyException extends Exception { + + /** + * Basic constructor + * @param message + * The exception message. + */ + public MissingPropertyException(final String message) { + super(message); + } +} diff --git a/islandora-support/src/test/java/ca/islandora/alpaca/support/config/HttpConfigurerTest.java b/islandora-support/src/test/java/ca/islandora/alpaca/support/config/HttpConfigurerTest.java new file mode 100644 index 00000000..a5cdfa95 --- /dev/null +++ b/islandora-support/src/test/java/ca/islandora/alpaca/support/config/HttpConfigurerTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to Islandora Foundation under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional + * information regarding copyright ownership. + * + * The Islandora Foundation licenses this file to you under the MIT License. + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ca.islandora.alpaca.support.config; + +import static ca.islandora.alpaca.support.config.RequestConfigurerConfig.CONNECT_TIMEOUT_PROPERTY; +import static ca.islandora.alpaca.support.config.RequestConfigurerConfig.REQUEST_CONFIGURER_ENABLED_PROPERTY; +import static ca.islandora.alpaca.support.config.RequestConfigurerConfig.REQUEST_TIMEOUT_PROPERTY; +import static ca.islandora.alpaca.support.config.RequestConfigurerConfig.SOCKET_TIMEOUT_PROPERTY; +import static org.junit.Assert.assertEquals; + +import org.apache.camel.CamelContext; +import org.apache.camel.component.http.HttpComponent; +import org.apache.camel.spring.javaconfig.CamelConfiguration; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + + +/** + * Insures that the Camel HTTP component as configured by Blueprint is properly configured. + * + * @author Elliot Metsger (emetsger@jhu.edu) + * @author whikloj + */ +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) +@ContextConfiguration(classes = HttpConfigurerTest.ContextConfig.class, + loader = AnnotationConfigContextLoader.class) +@RunWith(SpringJUnit4ClassRunner.class) +public class HttpConfigurerTest { + + @Autowired + private CamelContext context; + + /** + * Insure that the default RequestConfig for the HttpComponent carries the timeout values specified in the + * blueprint xml. + * + * Note that the RequestConfig and RequestConfigConfigurer are difficult to test with mocking frameworks such as + * Mockito due to the presence of final methods in the relevant HttpClient classes. + * + * @throws Exception + */ + @Test + public void testRequestConfig() throws Exception { + final HttpComponent http = (HttpComponent) context.getComponent("http"); + + assertEquals(11111, http.getSocketTimeout()); + assertEquals(22222, http.getConnectTimeout()); + assertEquals(33333, http.getConnectionRequestTimeout()); + } + + @BeforeClass + public static void setProperties() { + System.setProperty(REQUEST_CONFIGURER_ENABLED_PROPERTY, "true"); + System.setProperty(REQUEST_TIMEOUT_PROPERTY, "33333"); + System.setProperty(CONNECT_TIMEOUT_PROPERTY, "22222"); + System.setProperty(SOCKET_TIMEOUT_PROPERTY, "11111"); + } + + @Configuration + @ComponentScan(basePackageClasses = {RequestConfigurerConfig.class, ActivemqConfig.class}, + useDefaultFilters = false, + includeFilters = @ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, + classes = {RequestConfigurerConfig.class, ActivemqConfig.class})) + static class ContextConfig extends CamelConfiguration { + + } +} diff --git a/karaf/build.gradle b/karaf/build.gradle deleted file mode 100644 index 7a5a15b2..00000000 --- a/karaf/build.gradle +++ /dev/null @@ -1,102 +0,0 @@ -description = 'Islandora 8 Karaf Provisioning' - -dependencies { - testImplementation "javax.inject:javax.inject:${versions.javaxInject}" - testImplementation "org.apache.karaf:apache-karaf:${versions.karaf}" - testImplementation "org.ops4j.pax.exam:pax-exam-container-karaf:${versions.paxExam}" - testImplementation "org.ops4j.pax.exam:pax-exam-junit4:${versions.paxExam}" - testImplementation "org.osgi:org.osgi.core:${versions.osgiCore}" - - testRuntimeOnly "org.apache.activemq:activemq-karaf:${versions.activemq}" - testRuntimeOnly "org.apache.camel.karaf:apache-camel:${versions.camel}" - testRuntimeOnly "org.fcrepo.camel:fcrepo-camel:${versions.fcrepoCamel}" - testRuntimeOnly "org.fcrepo.camel:toolbox-features:${versions.fcrepoCamelToolbox}" - testRuntimeOnly "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - testRuntimeOnly "org.apache.httpcomponents:httpcore-osgi:${versions.httpCoreOsgi}" - testRuntimeOnly "org.apache.servicemix.bundles:org.apache.servicemix.bundles.xerces:${versions.xercesServiceMix}" - testRuntimeOnly "org.ow2.asm:asm-commons:${versions.asmCommons}" - testRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" - -} - -processResources { - outputs.upToDateWhen { false } - expand project.properties -} - -artifacts { - archives (file('build/resources/main/features.xml')) { - classifier 'features' - type 'xml' - } -} - -task generateDependsFile { - // In order to fully use org.ops4j.pax.exam.CoreOptions.maven() stuff - // we need to generate a META-INF/maven/dependencies.properties file - // just like the generate-depends-file Maven goal from ServiceMix/Karaf - - // Stolen from https://github.com/sebersole/hibernate-core/blob/HHH-9699/hibernate-osgi/hibernate-osgi.gradle - // then modified to work. - - File outputFileDir = project.file( 'build/resources/test/META-INF/maven/' ) - File outputFile = new File( outputFileDir, 'dependencies.properties' ) - - outputs.file outputFile - - doFirst { - outputFileDir.mkdirs() - - Properties properties = new Properties(); - - // first we add our GAV info - properties.setProperty( "groupId", "${project.group}" ); - properties.setProperty( "artifactId", project.name as String ); - properties.setProperty( "version", "${project.version}" ); - properties.setProperty( "${project.group}/${project.name}/version", "${project.version}" ); - - // then for all our root deps - project.configurations.testRuntimeClasspath.allDependencies.each { - final String keyBase = it.getGroup() + '/' + it.getName(); - properties.setProperty( "${keyBase}/scope", "compile" ) - properties.setProperty( "${keyBase}/version", it.getVersion() as String ) - } - // for all our transitive dependencies - project.configurations.testRuntimeClasspath.resolvedConfiguration.resolvedArtifacts.each { - final String keyBase = it.moduleVersion.id.group + '/' + it.moduleVersion.id.name; - properties.setProperty( "${keyBase}/scope", "compile" ) - properties.setProperty( "${keyBase}/type", it.extension as String ) - properties.setProperty( "${keyBase}/version", it.moduleVersion.id.version as String ) - } - - FileOutputStream outputStream = new FileOutputStream( outputFile ); - try { - properties.store( outputStream, "Generated from Gradle for PaxExam testing" ) - } - finally { - outputStream.close() - } - } -} - -tasks.test.dependsOn tasks.generateDependsFile - -test { - systemProperty "project.version", "${project.version}" - - systemProperty "org.ops4j.pax.url.mvn.useFallbackRepositories" ,"false" - systemProperty "org.ops4j.pax.url.mvn.repositories", "https://repo1.maven.org/maven2@id=central" - - // Uncomment to enable remote debugging of internal Karaf container on port 5005 - //systemProperty "debug.remote", "true" - - // Uncomment to keep the deployed karaf container inside build/exam/ for debugging. - //systemProperty "debug.keepExam", "true" - - testLogging { - // Uncomment the below line while debugging. - //events 'standard_out', 'standard_error' - exceptionFormat = 'full' - displayGranularity = 0 - } -} diff --git a/karaf/src/main/resources/features.xml b/karaf/src/main/resources/features.xml deleted file mode 100644 index 0dff9de5..00000000 --- a/karaf/src/main/resources/features.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - mvn:org.fcrepo.camel/fcrepo-camel/${versions.fcrepoCamel}/xml/features - mvn:org.fcrepo.camel/toolbox-features/${versions.fcrepoCamelToolbox}/xml/features - mvn:org.apache.camel.karaf/apache-camel/${versions.camel}/xml/features - - -
Indexes the triplestore from Drupal nodes
- - fcrepo-service-camel - camel-http4 - camel-jsonpath - - mvn:ca.islandora.alpaca/islandora-indexing-triplestore/${project.version} - - mvn:ca.islandora.alpaca/islandora-indexing-triplestore/${project.version}/cfg/configuration - - - - -
Indexes the Fedora repository from Drupal nodes
- - camel-http4 - camel-jsonpath - mvn:ca.islandora.alpaca/islandora-event-support/${project.version} - - mvn:ca.islandora.alpaca/islandora-indexing-fcrepo/${project.version} - - mvn:ca.islandora.alpaca/islandora-indexing-fcrepo/${project.version}/cfg/configuration - - - - - - mvn:org.apache.httpcomponents/httpcore-osgi/${versions.httpCoreOsgi} - mvn:org.apache.httpcomponents/httpclient-osgi/${versions.httpClientOsgi} - mvn:ca.islandora.alpaca/islandora-http-client/${project.version} - - mvn:ca.islandora.alpaca/islandora-http-client/${project.version}/cfg/configuration - - - - -
Re-usable route for connecting repository resources to derivative generation services
- - camel-http4 - camel-jsonpath - mvn:ca.islandora.alpaca/islandora-event-support/${project.version} - - mvn:ca.islandora.alpaca/islandora-connector-derivative/${project.version} - - - diff --git a/karaf/src/test/java/ca/islandora/alpaca/karaf/KarafIT.java b/karaf/src/test/java/ca/islandora/alpaca/karaf/KarafIT.java deleted file mode 100644 index 4b6ae9f5..00000000 --- a/karaf/src/test/java/ca/islandora/alpaca/karaf/KarafIT.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to Islandora Foundation under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional - * information regarding copyright ownership. - * - * The Islandora Foundation licenses this file to you under the MIT License. - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://opensource.org/licenses/MIT - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ca.islandora.alpaca.karaf; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.ops4j.pax.exam.CoreOptions.bundle; -import static org.ops4j.pax.exam.CoreOptions.maven; -import static org.ops4j.pax.exam.CoreOptions.mavenBundle; -import static org.ops4j.pax.exam.CoreOptions.options; -import static org.ops4j.pax.exam.CoreOptions.systemProperty; -import static org.ops4j.pax.exam.CoreOptions.when; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.configureConsole; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.debugConfiguration; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.editConfigurationFilePut; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.features; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.karafDistributionConfiguration; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.logLevel; -import static org.ops4j.pax.exam.karaf.options.KarafDistributionOption.keepRuntimeFolder; -import static org.ops4j.pax.exam.util.PathUtils.getBaseDir; -import static org.osgi.framework.Bundle.ACTIVE; -import static org.slf4j.LoggerFactory.getLogger; - -import java.io.File; - -import javax.inject.Inject; - -import org.apache.karaf.features.FeaturesService; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.ops4j.pax.exam.Configuration; -import org.ops4j.pax.exam.ConfigurationManager; -import org.ops4j.pax.exam.CoreOptions; -import org.ops4j.pax.exam.MavenUtils; -import org.ops4j.pax.exam.Option; -import org.ops4j.pax.exam.junit.PaxExam; -import org.ops4j.pax.exam.karaf.options.LogLevelOption.LogLevel; -import org.ops4j.pax.exam.spi.reactors.ExamReactorStrategy; -import org.ops4j.pax.exam.spi.reactors.PerClass; -import org.osgi.framework.BundleContext; -import org.slf4j.Logger; - -/** - * Test deployment in Karaf container. - * @author whikloj - * @since 2019-10-30 - */ -@RunWith(PaxExam.class) -@ExamReactorStrategy(PerClass.class) -public class KarafIT { - - private static Logger LOGGER = getLogger(KarafIT.class); - - @Inject - protected FeaturesService featuresService; - - @Inject - protected BundleContext bundleContext; - - @Configuration - public Option[] config() throws Exception { - - final ConfigurationManager cm = new ConfigurationManager(); - - final String version = cm.getProperty("project.version"); - final boolean debugRemote = Boolean.parseBoolean(cm.getProperty("debug.remote", "false")); - final boolean debugExam = Boolean.parseBoolean(cm.getProperty("debug.keepExam", "false")); - - final String islandoraHttp = getBundleUri("islandora-http-client", version); - final String islandoraEvent = getBundleUri("islandora-event-support", version); - final String islandoraIndexFcrepo = getBundleUri("islandora-indexing-fcrepo", version); - final String islandoraIndexTriple = getBundleUri("islandora-indexing-triplestore", version); - final String islandoraConnectDeriv = getBundleUri("islandora-connector-derivative", version); - - final String karafVersion = MavenUtils.getArtifactVersion("org.apache.karaf", "apache-karaf"); - final String fcrepoCamelVersion = MavenUtils.getArtifactVersion("org.fcrepo.camel", - "fcrepo-camel"); - final String fcrepoCamelToolboxVersion = MavenUtils.getArtifactVersion("org.fcrepo.camel", - "toolbox-features"); - final String activemqVersion = MavenUtils.getArtifactVersion("org.apache.activemq", "activemq-karaf"); - - - return options( - when( debugRemote ).useOptions( - debugConfiguration( "5005", true ) - ), - karafDistributionConfiguration() - .frameworkUrl( - CoreOptions.maven().groupId("org.apache.karaf").artifactId("apache-karaf") - .version(karafVersion).type("zip") - ) - .name("Apache Karaf") - .unpackDirectory(new File("build/exam")) - .useDeployFolder(false), - when( debugExam ).useOptions( - keepRuntimeFolder() - ), - logLevel(LogLevel.INFO), - configureConsole() - .ignoreLocalConsole(), - editConfigurationFilePut( - "etc/org.apache.karaf.features.repos.cfg", "fcrepo-camel", - "mvn:org.fcrepo.camel/fcrepo-camel/" + fcrepoCamelVersion + "/xml/features"), - editConfigurationFilePut( - "etc/org.apache.karaf.features.repos.cfg", "fcrepo-camel-toolbox", - "mvn:org.fcrepo.camel/toolbox-features/" + fcrepoCamelToolboxVersion + "/xml/features"), - editConfigurationFilePut( - "etc/org.apache.karaf.features.repos.cfg", "activemq", - "mvn:org.apache.activemq/activemq-karaf/" + activemqVersion + "/xml/features"), - editConfigurationFilePut( - "etc/org.apache.karaf.features.cfg", "featuresBoot", "standard" - ), - editConfigurationFilePut( - "etc/org.ops4j.pax.url.mvn.cfg", "org.ops4j.pax.url.mvn.repositories", - "https://repo1.maven.org/maven2@id=central" - ), - editConfigurationFilePut( - "etc/org.ops4j.pax.url.mvn.cfg", - "org.ops4j.pax.url.mvn.useFallbackRepositories", - "false" - ), - features(maven().groupId("org.apache.karaf.features").artifactId("standard") - .versionAsInProject().classifier("features").type("xml"), "scr"), - features(maven().groupId("org.apache.camel.karaf").artifactId("apache-camel") - .type("xml").classifier("features").versionAsInProject(), "camel-blueprint", - "camel-http4", "camel-jackson", "camel-jsonpath", "camel-jackson", "camel-spring"), - features(maven().groupId("org.apache.activemq").artifactId("activemq-karaf") - .type("xml").classifier("features").versionAsInProject(), "activemq-camel"), - features(maven().groupId("org.fcrepo.camel").artifactId("fcrepo-camel") - .type("xml").classifier("features").versionAsInProject(), "fcrepo-camel"), - mavenBundle().groupId("com.fasterxml.jackson.core").artifactId("jackson-annotations") - .versionAsInProject().start(), - mavenBundle().groupId("org.apache.httpcomponents").artifactId("httpcore-osgi") - .versionAsInProject().start(), - - systemProperty("c.i.a.http-bundle").value(islandoraHttp), - systemProperty("c.i.a.derivative-bundle").value(islandoraConnectDeriv), - systemProperty("c.i.a.fcrepo-bundle").value(islandoraIndexFcrepo), - systemProperty("c.i.a.triplestore-bundle").value(islandoraIndexTriple), - - bundle(islandoraHttp).start(), - bundle(islandoraEvent).start(), - bundle(islandoraConnectDeriv).start(), - bundle(islandoraIndexFcrepo).start(), - bundle(islandoraIndexTriple).start(), - editConfigurationFilePut("etc/org.ops4j.pax.logging.cfg", - "log4j.logger.org.apache.camel.impl.converter", "ERROR, stdout") - ); - - } - - @Test - public void testInstallation() throws Exception { - assertTrue(featuresService.isInstalled(featuresService.getFeature("camel-core"))); - assertTrue(featuresService.isInstalled(featuresService.getFeature("fcrepo-camel"))); - assertTrue(featuresService.isInstalled(featuresService.getFeature("activemq-camel"))); - assertTrue(featuresService.isInstalled(featuresService.getFeature("camel-blueprint"))); - assertTrue(featuresService.isInstalled(featuresService.getFeature("camel-http4"))); - assertTrue(featuresService.isInstalled(featuresService.getFeature("camel-jackson"))); - assertTrue(featuresService.isInstalled(featuresService.getFeature("camel-jsonpath"))); - assertNotNull(bundleContext); - - assertEquals(ACTIVE, bundleContext.getBundle(System.getProperty("c.i.a.http-bundle")).getState()); - assertEquals(ACTIVE, bundleContext.getBundle(System.getProperty("c.i.a.derivative-bundle")).getState()); - assertEquals(ACTIVE, bundleContext.getBundle(System.getProperty("c.i.a.fcrepo-bundle")).getState()); - assertEquals(ACTIVE, bundleContext.getBundle(System.getProperty("c.i.a.triplestore-bundle")).getState()); - } - - private static String getBundleUri(final String artifactId, final String version) { - final File artifact = new File(getBaseDir() + "/../" + artifactId + "/build/libs/" + - artifactId + "-" + version + ".jar"); - if (artifact.exists()) { - return artifact.toURI().toString(); - } - return "mvn:ca.islandora.alpaca/" + artifactId + "/" + version; - } - - private String getFeaturesXml() throws Exception { - return getClass().getClassLoader().getResource("features.xml").toURI().toString(); - } -} diff --git a/settings.gradle b/settings.gradle index 0d454531..9d1d8202 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,13 +1,13 @@ -include ':islandora-karaf' -include ':islandora-event-support' +include ':islandora-support' include ':islandora-indexing-triplestore' include ':islandora-indexing-fcrepo' include ':islandora-connector-derivative' include ':islandora-http-client' +include ':islandora-alpaca-app' -project(':islandora-karaf').projectDir = "$rootDir/karaf" as File -project(':islandora-event-support').projectDir = "$rootDir/islandora-event-support" as File -project(':islandora-indexing-triplestore').projectDir = "$rootDir/islandora-indexing-triplestore" as File -project(':islandora-indexing-fcrepo').projectDir = "$rootDir/islandora-indexing-fcrepo" as File -project(':islandora-connector-derivative').projectDir = "$rootDir/islandora-connector-derivative" as File -project(':islandora-http-client').projectDir = "$rootDir/islandora-http-client" as File +project(':islandora-alpaca-app').setProjectDir("$rootDir/islandora-alpaca-app" as File) +project(':islandora-support').setProjectDir("$rootDir/islandora-support" as File) +project(':islandora-indexing-triplestore').setProjectDir("$rootDir/islandora-indexing-triplestore" as File) +project(':islandora-indexing-fcrepo').setProjectDir("$rootDir/islandora-indexing-fcrepo" as File) +project(':islandora-connector-derivative').setProjectDir("$rootDir/islandora-connector-derivative" as File) +project(':islandora-http-client').setProjectDir("$rootDir/islandora-http-client" as File)