Skip to content

Commit

Permalink
Merge pull request #2 from odysseusinc/temp-vulnerability-fixes
Browse files Browse the repository at this point in the history
Updating to 2.2
  • Loading branch information
pavgra authored Oct 13, 2017
2 parents e160cdc + 2826627 commit fa524e4
Show file tree
Hide file tree
Showing 93 changed files with 4,029 additions and 1,027 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
*application.properties
.idea/
.metadata/
*.iml
sandbox/
/target/
Expand Down
31 changes: 27 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
<version>1.0.0-SNAPSHOT</version>
<name>WebAPI</name>
<properties>
<webapi.version>2.1.0</webapi.version>
<webapi.version>2.2.0</webapi.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jersey.version>2.14</jersey.version>
<!-- Spring Boot manages spring.version as well -->
<spring.boot.version>1.5.2.RELEASE</spring.boot.version>
<spring.batch.version>3.0.7.RELEASE</spring.batch.version>
<spring.batch.version>3.0.8.RELEASE</spring.batch.version>
<flyway.version>3.1</flyway.version>
<waffle.version>1.7.3</waffle.version>
<start-class>org.ohdsi.webapi.WebApi</start-class>
Expand Down Expand Up @@ -227,6 +227,10 @@
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>redshift</id>
<url>http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release</url>
</repository>
</repositories>

<pluginRepositories>
Expand All @@ -247,6 +251,11 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.amazon.redshift</groupId>
<artifactId>redshift-jdbc4-no-awssdk</artifactId>
<version>1.2.8.1005</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
Expand Down Expand Up @@ -303,7 +312,7 @@
<dependency>
<groupId>org.ohdsi.sql</groupId>
<artifactId>SqlRender</artifactId>
<version>1.3.0-SNAPSHOT</version>
<version>1.4.3</version>
</dependency>
<dependency>
<groupId>commons-dbutils</groupId>
Expand Down Expand Up @@ -432,7 +441,21 @@
<dependency>
<groupId>org.ohdsi</groupId>
<artifactId>circe</artifactId>
<version>1.1.1</version>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20170516</version>
</dependency>
<dependency>
<groupId>org.ohdsi</groupId>
<artifactId>featureExtraction</artifactId>
<version>2.0.0-BETA</version>
</dependency>
<dependency>
<groupId>com.amazon.redshift</groupId>
<artifactId>redshift-jdbc4-no-awssdk</artifactId>
</dependency>
</dependencies>
<profiles>
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/ohdsi/webapi/DataAccessConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ public DataSource primaryDataSource() {
//note autocommit defaults vary across vendors. use provided @Autowired TransactionTemplate

String[] supportedDrivers;
supportedDrivers = new String[]{"org.postgresql.Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "oracle.jdbc.driver.OracleDriver", "com.amazon.redshift.jdbc41.Driver", "com.cloudera.impala.jdbc4.Driver", "net.starschema.clouddb.jdbc.BQDriver"};

supportedDrivers = new String[]{"org.postgresql.Driver", "com.microsoft.sqlserver.jdbc.SQLServerDriver", "oracle.jdbc.driver.OracleDriver", "com.amazon.redshift.jdbc.Driver", "com.cloudera.impala.jdbc4.Driver", "net.starschema.clouddb.jdbc.BQDriver"};
for (String driverName : supportedDrivers) {
try {
Class.forName(driverName);
Expand Down
8 changes: 6 additions & 2 deletions src/main/java/org/ohdsi/webapi/WebApi.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.ohdsi.webapi;

import javax.annotation.PostConstruct;
import org.apache.catalina.webresources.TomcatURLStreamHandlerFactory;
import org.ohdsi.webapi.service.CDMResultsService;
import org.ohdsi.webapi.service.SourceService;
import org.ohdsi.webapi.source.SourceDaimon;
Expand All @@ -22,11 +23,14 @@ public class WebApi extends SpringBootServletInitializer {
protected SpringApplicationBuilder configure(final SpringApplicationBuilder application) {
return application.sources(WebApi.class);
}

public static void main(final String[] args) throws Exception {

public static void main(final String[] args) throws Exception
{
TomcatURLStreamHandlerFactory.disable();
new SpringApplicationBuilder(WebApi.class).run(args);
}


@Autowired
private CDMResultsService resultsService;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,18 @@ public CohortGenerationInfo(CohortDefinition definition, Integer sourceId)

@Column(name="is_valid")
private boolean isValid;

@Column(name="include_features", nullable = true)
private boolean includeFeatures = false;

@Column(name="fail_message")
private String failMessage;

@Column(name="person_count")
private Long personCount;

@Column(name="record_count")
private Long recordCount;

public CohortGenerationInfoId getId() {
return id;
Expand Down Expand Up @@ -104,4 +116,41 @@ public CohortGenerationInfo setIsValid(boolean isValid) {
this.isValid = isValid;
return this;
}

public boolean isIncludeFeatures() {
return includeFeatures;
}

public CohortGenerationInfo setIncludeFeatures(boolean includeFeatures) {
this.includeFeatures = includeFeatures;
return this;
}

public String getFailMessage() {
return failMessage;
}

public CohortGenerationInfo setFailMessage(String failMessage) {
this.failMessage = failMessage;
return this;
}

public Long getPersonCount() {
return personCount;
}

public CohortGenerationInfo setPersonCount(Long personCount) {
this.personCount = personCount;
return this;
}

public Long getRecordCount() {
return recordCount;
}

public CohortGenerationInfo setRecordCount(Long recordCount) {
this.recordCount = recordCount;
return this;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,9 @@
*/
package org.ohdsi.webapi.cohortdefinition;

import static org.ohdsi.webapi.util.SecurityUtils.whitelist;

import org.ohdsi.circe.cohortdefinition.CohortExpression;
import org.ohdsi.circe.cohortdefinition.CohortExpressionQueryBuilder;
import org.ohdsi.circe.cohortdefinition.InclusionRule;
import org.ohdsi.webapi.GenerationStatus;
import java.util.Calendar;
import java.util.Date;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Expand All @@ -42,12 +37,9 @@
import org.springframework.transaction.support.TransactionTemplate;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.ohdsi.sql.SqlRender;
import org.springframework.transaction.TransactionException;

/**
*
Expand Down Expand Up @@ -127,60 +119,15 @@ private int[] doTask(ChunkContext chunkContext) {
return result;
}

private CohortGenerationInfo findBySourceId(Collection<CohortGenerationInfo> infoList, Integer sourceId)
{
for (CohortGenerationInfo info : infoList) {
if (info.getId().getSourceId().equals(sourceId))
return info;
}
return null;
}
@Override
public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) throws Exception {
Date startTime = Calendar.getInstance().getTime();
Map<String, Object> jobParams = chunkContext.getStepContext().getJobParameters();
Integer defId = Integer.valueOf(jobParams.get("cohort_definition_id").toString());
Integer sourceId = Integer.valueOf(jobParams.get("source_id").toString());
boolean isValid = false;

DefaultTransactionDefinition initTx = new DefaultTransactionDefinition();
initTx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
TransactionStatus initStatus = this.transactionTemplate.getTransactionManager().getTransaction(initTx);
CohortDefinition df = this.cohortDefinitionRepository.findOne(defId);
CohortGenerationInfo info = findBySourceId(df.getGenerationInfoList(), sourceId);
info.setIsValid(isValid);
info.setStartTime(startTime);
info.setStatus(GenerationStatus.RUNNING);
df = this.cohortDefinitionRepository.save(df);
this.transactionTemplate.getTransactionManager().commit(initStatus);

try {
final int[] ret = this.transactionTemplate.execute(new TransactionCallback<int[]>() {

@Override
public int[] doInTransaction(final TransactionStatus status) {
return doTask(chunkContext);
}
});
log.debug("Update count: " + ret.length);
isValid = true;
} catch (final TransactionException e) {
log.error(whitelist(e));
throw e;//FAIL job status
}
finally {
DefaultTransactionDefinition completeTx = new DefaultTransactionDefinition();
completeTx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
TransactionStatus completeStatus = this.transactionTemplate.getTransactionManager().getTransaction(completeTx);
df = this.cohortDefinitionRepository.findOne(defId);
info = findBySourceId(df.getGenerationInfoList(), sourceId);
Date endTime = Calendar.getInstance().getTime();
info.setExecutionDuration((int) (endTime.getTime() - startTime.getTime()));
info.setIsValid(isValid);
info.setStatus(GenerationStatus.COMPLETE);
this.cohortDefinitionRepository.save(df);
this.transactionTemplate.getTransactionManager().commit(completeStatus);
}
final int[] ret = this.transactionTemplate.execute(new TransactionCallback<int[]>() {
@Override
public int[] doInTransaction(final TransactionStatus status) {
return doTask(chunkContext);
}
});

return RepeatStatus.FINISHED;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
/*
* Copyright 2017 cknoll1.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.ohdsi.webapi.cohortdefinition;

import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.ohdsi.sql.SqlRender;
import org.ohdsi.sql.SqlTranslate;
import org.ohdsi.webapi.GenerationStatus;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobExecutionListener;
import org.springframework.batch.core.JobParameters;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.DefaultTransactionDefinition;
import org.springframework.transaction.support.TransactionTemplate;

/**
*
* @author cknoll1
*/
public class GenerationJobExecutionListener implements JobExecutionListener {

private final CohortDefinitionRepository cohortDefinitionRepository;
private final TransactionTemplate transactionTemplate;
private final JdbcTemplate sourceTemplate;

public GenerationJobExecutionListener(CohortDefinitionRepository cohortDefinitionRepository,
TransactionTemplate transactionTemplate,
JdbcTemplate sourceTemplate)
{
this.cohortDefinitionRepository = cohortDefinitionRepository;
this.transactionTemplate = transactionTemplate;
this.sourceTemplate = sourceTemplate;
}

private CohortGenerationInfo findBySourceId(Collection<CohortGenerationInfo> infoList, Integer sourceId)
{
for (CohortGenerationInfo info : infoList) {
if (info.getId().getSourceId().equals(sourceId))
return info;
}
return null;
}

@Override
public void afterJob(JobExecution je) {

JobParameters jobParams = je.getJobParameters();
Integer defId = Integer.valueOf(jobParams.getString("cohort_definition_id"));
Integer sourceId = Integer.valueOf(jobParams.getString("source_id"));
String targetDialect = jobParams.getString("target_dialect");
String cohortTable = jobParams.getString("target_database_schema") + "." + jobParams.getString("target_table");

DefaultTransactionDefinition completeTx = new DefaultTransactionDefinition();
completeTx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
TransactionStatus completeStatus = this.transactionTemplate.getTransactionManager().getTransaction(completeTx);
CohortDefinition df = this.cohortDefinitionRepository.findOne(defId);
CohortGenerationInfo info = findBySourceId(df.getGenerationInfoList(), sourceId);
info.setExecutionDuration((int)(je.getEndTime().getTime() - je.getStartTime().getTime()));
info.setStatus(GenerationStatus.COMPLETE);

if (je.getStatus() == BatchStatus.FAILED) {
info.setIsValid(false);
info.setRecordCount(null);
info.setPersonCount(null);
info.setFailMessage(StringUtils.left(je.getAllFailureExceptions().get(0).getMessage(),2000));
} else {
info.setIsValid(true);
info.setFailMessage(null);

// query summary results from source
String statsQuery = "SELECT count(distinct subject_id) as person_count, count(*) as record_count from @cohort_table where cohort_definition_id = @cohort_definition_id";
String statsSql = SqlTranslate.translateSql(statsQuery, targetDialect, null, null);
String renderedSql = SqlRender.renderSql(statsSql, new String[]{"cohort_table", "cohort_definition_id"}, new String[]{cohortTable, defId.toString()});
Map<String, Object> stats = this.sourceTemplate.queryForMap(renderedSql);
info.setPersonCount(Long.parseLong(stats.get("person_count").toString()));
info.setRecordCount(Long.parseLong(stats.get("record_count").toString()));
}

this.cohortDefinitionRepository.save(df);
this.transactionTemplate.getTransactionManager().commit(completeStatus);

}

@Override
public void beforeJob(JobExecution je) {
Date startTime = Calendar.getInstance().getTime();
JobParameters jobParams = je.getJobParameters();
Integer defId = Integer.valueOf(jobParams.getString("cohort_definition_id"));
Integer sourceId = Integer.valueOf(jobParams.getString("source_id"));

DefaultTransactionDefinition initTx = new DefaultTransactionDefinition();
initTx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
TransactionStatus initStatus = this.transactionTemplate.getTransactionManager().getTransaction(initTx);
CohortDefinition df = this.cohortDefinitionRepository.findOne(defId);
CohortGenerationInfo info = findBySourceId(df.getGenerationInfoList(), sourceId);
info.setIsValid(false);
info.setStartTime(startTime);
info.setStatus(GenerationStatus.RUNNING);
this.cohortDefinitionRepository.save(df);
this.transactionTemplate.getTransactionManager().commit(initStatus);
}
}
Loading

0 comments on commit fa524e4

Please sign in to comment.