Skip to content

Commit

Permalink
Pivotal #ID 174275642: Submission loading strategy (#254)
Browse files Browse the repository at this point in the history
* added custom loading strategy
  • Loading branch information
Juan-EBI authored Aug 17, 2020
1 parent f130dba commit f9ddbc1
Show file tree
Hide file tree
Showing 24 changed files with 214 additions and 122 deletions.
10 changes: 8 additions & 2 deletions submission/persistence/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,14 @@ import TestDependencies.BaseTestCompileDependencies
import TestDependencies.BaseTestRuntimeDependencies

plugins {
id("org.jetbrains.kotlin.plugin.jpa") version "1.3.10"
id("org.jetbrains.kotlin.plugin.allopen") version "1.3.10"
id("org.jetbrains.kotlin.plugin.jpa") version "1.3.72"
id("org.jetbrains.kotlin.plugin.allopen") version "1.3.72"
}

allOpen {
annotation("javax.persistence.Entity")
annotation("javax.persistence.Embeddable")
annotation("javax.persistence.MappedSuperclass")
}

dependencies {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package ac.uk.ebi.biostd.persistence.model

internal const val ATTRS = "attributes"
internal const val FILES = "files"
internal const val LINKS = "links"
internal const val SECTS = "sections"
internal const val ACC_TAGS = "accessTags"
internal const val TAGS = "tags"
internal const val OWNER = "owner"
internal const val SUBMITTER = "submitter"
internal const val ROOT_SECTION = "rootSection"
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,30 @@ import javax.persistence.Id
import javax.persistence.JoinColumn
import javax.persistence.ManyToOne
import javax.persistence.NamedEntityGraph
import javax.persistence.NamedEntityGraphs
import javax.persistence.OneToMany
import javax.persistence.OneToOne
import javax.persistence.OrderBy
import javax.persistence.Table

internal const val SECTION_GRAPH = "Section.fullGraph"
internal const val SECTION_SIMPLE_GRAPH = "Section.simpleGraph"
private const val ATTRIBUTES_GRAPH = "Object.attributesGraph"

@Entity
@Table(name = "Section")
@NamedEntityGraph(
name = SECTION_GRAPH,
attributeNodes = [
Node(LINKS, subgraph = "attrs"),
Node(ATTRS),
Node(FILES, subgraph = "attrs"),
Node(SECTS, subgraph = SECTION_GRAPH)
])
@NamedEntityGraphs(value = [
NamedEntityGraph(
name = SECTION_SIMPLE_GRAPH,
attributeNodes = [
Node(ATTRS),
Node(SECTS),
Node(LINKS, subgraph = ATTRIBUTES_GRAPH),
Node(FILES, subgraph = ATTRIBUTES_GRAPH)],
subgraphs = [
Graph(name = ATTRIBUTES_GRAPH, attributeNodes = [Node(ATTRS)])
]
)
])
class DbSection(
@Column
var accNo: String?,
Expand Down Expand Up @@ -59,15 +66,15 @@ class DbSection(
@JoinColumn(name = "fileListId")
var fileList: ReferencedFileList? = null

@OneToMany(cascade = [CascadeType.ALL])
@OneToMany(cascade = [CascadeType.ALL], fetch = LAZY)
@JoinColumn(name = "section_id")
@OrderBy("order ASC")
var attributes: SortedSet<DbSectionAttribute> = sortedSetOf()
var links: SortedSet<DbLink> = sortedSetOf()

@OneToMany(cascade = [CascadeType.ALL])
@JoinColumn(name = "section_id")
@OrderBy("order ASC")
var links: SortedSet<DbLink> = sortedSetOf()
var attributes: SortedSet<DbSectionAttribute> = sortedSetOf()

@OneToMany(cascade = [CascadeType.ALL])
@JoinColumn(name = "sectionId")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import javax.persistence.Convert
import javax.persistence.Entity
import javax.persistence.EnumType
import javax.persistence.Enumerated
import javax.persistence.FetchType
import javax.persistence.FetchType.LAZY
import javax.persistence.GeneratedValue
import javax.persistence.Id
import javax.persistence.JoinColumn
Expand All @@ -26,52 +26,45 @@ import javax.persistence.NamedEntityGraph
import javax.persistence.NamedEntityGraphs
import javax.persistence.NamedSubgraph
import javax.persistence.OneToMany
import javax.persistence.OneToOne
import javax.persistence.OrderBy
import javax.persistence.Table
import ac.uk.ebi.biostd.persistence.model.DbSection as SectionDb

internal const val FULL_DATA_GRAPH = "Submission.fullData"
internal const val SIMPLE_QUERY_GRAPH = "Submission.simpleGraph"

internal const val ATTRS = "attributes"
internal const val FILES = "files"
internal const val LINKS = "links"
internal const val SECTS = "sections"

typealias Node = NamedAttributeNode
typealias Graph = NamedSubgraph

internal const val SIMPLE_QUERY_GRAPH = "Submission.simpleGraph"
internal const val SUBMISSION_FULL_GRAPH = "Submission.simpleFullGraph"
internal const val SUBMISSION_AND_ROOT_SECTION_FULL_GRAPH = "Submission.fullGraph"

private const val ROOT_SECTION_GRAPH = "Submission.rootSectionGraph"
private const val ATTRIBUTES_GRAPH = "SubmissionObject.attributesGraph"

@Entity
@NamedEntityGraphs(value = [
NamedEntityGraph(name = FULL_DATA_GRAPH,
attributeNodes = [
Node(value = "rootSection", subgraph = "root"),
Node("accessTags"),
Node("tags"),
NamedEntityGraph(name = SIMPLE_QUERY_GRAPH, attributeNodes = [Node(value = ROOT_SECTION)]),
NamedEntityGraph(name = SUBMISSION_FULL_GRAPH, attributeNodes = [
Node(ATTRS),
Node(ACC_TAGS),
Node(TAGS),
Node(OWNER),
Node(SUBMITTER)
]),
NamedEntityGraph(name = SUBMISSION_AND_ROOT_SECTION_FULL_GRAPH, attributeNodes = [
Node(ATTRS),
Node(ACC_TAGS),
Node(TAGS),
Node(OWNER),
Node(SUBMITTER),
Node(value = ROOT_SECTION, subgraph = ROOT_SECTION_GRAPH)
], subgraphs = [
Graph(name = ROOT_SECTION_GRAPH, attributeNodes = [
Node(LINKS, subgraph = ATTRIBUTES_GRAPH),
Node(ATTRS),
Node("owner"),
Node("submitter")
],
subgraphs = [
Graph(name = "root", attributeNodes = [
Node(LINKS, subgraph = "attrs"),
Node(ATTRS),
Node(FILES, subgraph = "attrs"),
Node(SECTS, subgraph = "l1")]),
Graph(name = "l1", attributeNodes = [
Node(LINKS, subgraph = "attrs"),
Node(ATTRS),
Node(FILES, subgraph = "attrs"),
Node(SECTS, subgraph = "l2")]),
Graph(name = "l2", attributeNodes = [
Node(LINKS, subgraph = "attrs"),
Node(ATTRS),
Node(FILES, subgraph = "attrs"),
Node(SECTS, subgraph = SECTION_GRAPH)]),
Graph(name = "attrs", attributeNodes = [Node(ATTRS)])
]),
NamedEntityGraph(name = SIMPLE_QUERY_GRAPH, attributeNodes = [Node(value = "rootSection")])
Node(SECTS),
Node(FILES, subgraph = ATTRIBUTES_GRAPH)]),
Graph(name = ATTRIBUTES_GRAPH, attributeNodes = [Node(ATTRS)])
])
])
@Table(name = "Submission")
class DbSubmission(
Expand Down Expand Up @@ -120,15 +113,18 @@ class DbSubmission(
@Convert(converter = ProcessingStatusConverter::class)
var status: ProcessingStatus = PROCESSING

@OneToOne(cascade = [CascadeType.ALL], fetch = FetchType.LAZY)
@ManyToOne(cascade = [CascadeType.ALL], fetch = LAZY)
@JoinColumn(name = "rootSection_id")
lateinit var rootSection: SectionDb

@ManyToOne
@Column(name = "rootSection_id", updatable = false, insertable = false)
var rootSectionId: Long = -1

@ManyToOne(fetch = LAZY)
@JoinColumn(name = "owner_id")
lateinit var owner: DbUser

@ManyToOne
@ManyToOne(fetch = LAZY)
@JoinColumn(name = "submitter_id")
lateinit var submitter: DbUser

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ class JdbcLockExecutor(private val template: NamedParameterJdbcTemplate) : LockE
logger.info { "acquiring lock $lockName in ${Thread.currentThread()}" }
val params = mapOf(NAME_PARAM to lockName, TIME_PARAM to timeout)
val lock = template.queryForObject<Int>(LOCK_QUERY, params, Int::class.java)
return ObjectUtils.compare(lock, 1) == 0
val acquired = ObjectUtils.compare(lock, 1) == 0
logger.info { "acquired lock $lockName in ${Thread.currentThread()}" }
return acquired
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package ac.uk.ebi.biostd.persistence.repositories
import ac.uk.ebi.biostd.persistence.model.AccessPermission
import ac.uk.ebi.biostd.persistence.model.AccessType
import ac.uk.ebi.biostd.persistence.model.DbAccessTag
import ac.uk.ebi.biostd.persistence.model.DbSection
import ac.uk.ebi.biostd.persistence.model.DbSubmission
import ac.uk.ebi.biostd.persistence.model.DbTag
import ac.uk.ebi.biostd.persistence.model.DbUser
import ac.uk.ebi.biostd.persistence.model.DbUserData
import ac.uk.ebi.biostd.persistence.model.FULL_DATA_GRAPH
import ac.uk.ebi.biostd.persistence.model.SECTION_SIMPLE_GRAPH
import ac.uk.ebi.biostd.persistence.model.SUBMISSION_AND_ROOT_SECTION_FULL_GRAPH
import ac.uk.ebi.biostd.persistence.model.SUBMISSION_FULL_GRAPH
import ac.uk.ebi.biostd.persistence.model.SecurityToken
import ac.uk.ebi.biostd.persistence.model.Sequence
import ac.uk.ebi.biostd.persistence.model.USER_DATA_GRAPH
Expand All @@ -27,13 +30,9 @@ import java.util.Optional
import javax.persistence.LockModeType
import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraph as GraphSpecification

@Suppress("TooManyFunctions")
interface SubmissionDataRepository :
EntityGraphJpaRepository<DbSubmission, Long>, EntityGraphJpaSpecificationExecutor<DbSubmission> {
@EntityGraph(value = FULL_DATA_GRAPH, type = LOAD)
fun getByAccNoAndVersionGreaterThan(id: String, version: Int = 0): DbSubmission?

@EntityGraph(value = FULL_DATA_GRAPH, type = LOAD)
fun getByAccNo(id: String): DbSubmission?

fun findByAccNoAndVersionGreaterThan(id: String, version: Int = 0): DbSubmission?

Expand All @@ -46,6 +45,16 @@ interface SubmissionDataRepository :
@Query("from DbSubmission s inner join s.owner where s.accNo = :accNo and s.version > 0")
fun findBasic(@Param("accNo") accNo: String): DbSubmission?

@EntityGraph(value = SUBMISSION_FULL_GRAPH, type = LOAD)
fun getByAccNoAndVersionGreaterThan(accNo: String, version: Int = 0): DbSubmission?

@EntityGraph(value = SUBMISSION_FULL_GRAPH, type = LOAD)
fun getByAccNoAndVersion(accNo: String, version: Int): DbSubmission?

@EntityGraph(value = SUBMISSION_AND_ROOT_SECTION_FULL_GRAPH, type = LOAD)
@Deprecated("should use SubmissionRepository#getExtByAccNo")
fun readByAccNoAndVersionGreaterThan(accNo: String, version: Int = 0): DbSubmission?

@Query("Select max(s.version) from DbSubmission s where s.accNo=?1")
fun getLastVersion(accNo: String): Int?

Expand All @@ -63,6 +72,14 @@ interface SubmissionDataRepository :
): List<DbSubmission>
}

interface SectionDataRepository : JpaRepository<DbSection, Long> {
@EntityGraph(value = SECTION_SIMPLE_GRAPH, type = LOAD)
fun getById(sectionId: Long): DbSection

@Query("select s.id from DbSection s where s.submission.id = :id and s.id <> :rootSectionId")
fun sections(@Param("id") id: Long, @Param("rootSectionId") rootSectionId: Long): List<Long>
}

interface AccessTagDataRepo : JpaRepository<DbAccessTag, Long> {
fun findByName(name: String): DbAccessTag
fun existsByName(name: String): Boolean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,46 @@ import ac.uk.ebi.biostd.persistence.exception.SubmissionNotFoundException
import ac.uk.ebi.biostd.persistence.filter.SubmissionFilter
import ac.uk.ebi.biostd.persistence.filter.SubmissionFilterSpecification
import ac.uk.ebi.biostd.persistence.mapping.extended.to.ToExtSubmissionMapper
import ac.uk.ebi.biostd.persistence.model.DbSubmission
import ac.uk.ebi.biostd.persistence.projections.SimpleSubmission
import ac.uk.ebi.biostd.persistence.projections.SimpleSubmission.Companion.asSimpleSubmission
import ac.uk.ebi.biostd.persistence.repositories.SectionDataRepository
import ac.uk.ebi.biostd.persistence.repositories.SubmissionDataRepository
import com.cosium.spring.data.jpa.entity.graph.domain.EntityGraphs
import ebi.ac.uk.extended.mapping.to.toSimpleSubmission
import ebi.ac.uk.extended.model.ExtSubmission
import ebi.ac.uk.model.Submission
import mu.KotlinLogging
import org.springframework.data.domain.PageRequest
import org.springframework.data.domain.Sort
import org.springframework.transaction.annotation.Transactional

// TODO move to repositories
// TODO: Remove ExtendedSubmission usage
class SubmissionRepository(
private val logger = KotlinLogging.logger {}

@Suppress("TooManyFunctions")
open class SubmissionRepository(
private val submissionRepository: SubmissionDataRepository,
private val sectionRepository: SectionDataRepository,
private var submissionMapper: ToExtSubmissionMapper
) {
fun getByAccNo(accNo: String): Submission = getActiveExtByAccNo(accNo).toSimpleSubmission()
@Transactional(readOnly = true)
open fun getSimpleByAccNo(accNo: String): Submission = getExtByAccNo(accNo).toSimpleSubmission()

fun getActiveExtByAccNo(accNo: String): ExtSubmission = submissionMapper.toExtSubmission(getActiveSubmission(accNo))
@Transactional(readOnly = true)
open fun getExtByAccNo(accNo: String) = submissionMapper.toExtSubmission(lodSubmission(accNo))

fun getExtByAccNo(accNo: String): ExtSubmission = submissionMapper.toExtSubmission(getSubmission(accNo))
@Transactional(readOnly = true)
open fun getExtByAccAndVersion(accNo: String, version: Int) =
submissionMapper.toExtSubmission(lodSubmission(accNo, version))

fun expireSubmission(accNo: String) {
submissionRepository.findByAccNoAndVersionGreaterThan(accNo)?.let {
it.version = -it.version
submissionRepository.save(it)
open fun expireSubmission(accNo: String) {
val submission = submissionRepository.findByAccNoAndVersionGreaterThan(accNo)
if (submission != null) {
submission.version = -submission.version
submissionRepository.save(submission)
}
}

fun getSubmissionsByUser(userId: Long, filter: SubmissionFilter): List<SimpleSubmission> {
open fun getSubmissionsByUser(userId: Long, filter: SubmissionFilter): List<SimpleSubmission> {
val filterSpecs = SubmissionFilterSpecification(userId, filter)
val pageable = PageRequest.of(filter.pageNumber, filter.limit, Sort.by("releaseTime").descending())
return submissionRepository
Expand All @@ -42,9 +52,30 @@ class SubmissionRepository(
.map { it.asSimpleSubmission() }
}

private fun getSubmission(accNo: String) =
submissionRepository.getByAccNo(accNo) ?: throw SubmissionNotFoundException(accNo)
/**
* Load submission information strategy used is basically first load submission and then load each section and its
* subsections recursively starting from the root section.
*/
private fun lodSubmission(accNo: String, version: Int? = null): DbSubmission {
logger.info { "loading submission $accNo" }

val submission = findSubmission(accNo, version)
submission ?: throw SubmissionNotFoundException(accNo)
loadSection(submission.rootSectionId)

logger.info { "Loaded submission $accNo" }
return submission
}

private fun findSubmission(accNo: String, version: Int?): DbSubmission? {
return when (version) {
null -> submissionRepository.getByAccNoAndVersionGreaterThan(accNo, 0)
else -> submissionRepository.getByAccNoAndVersion(accNo, version)
}
}

private fun getActiveSubmission(accNo: String) =
submissionRepository.getByAccNoAndVersionGreaterThan(accNo) ?: throw SubmissionNotFoundException(accNo)
private fun loadSection(sectionId: Long) {
val section = sectionRepository.getById(sectionId)
section.sections.forEach { loadSection(it.id) }
}
}
Loading

0 comments on commit f9ddbc1

Please sign in to comment.