Skip to content

Commit

Permalink
Use migration schema for norm abbreviation (#791)
Browse files Browse the repository at this point in the history
* Use migration schema for norm abbreviation
* Use migration schema for norm element
* Make norm abbreviation test reliable
* Extend migration schema local doc
* Remove reactive from most of norm abbr
* Add legacy doc_unit_id column in norm reference
---
Co-authored-by: Ralph von Petersdorff-Campen <ralph.vonpetersdorffcampen@digitalservice.bund.de>
  • Loading branch information
HPrinz authored Oct 23, 2023
1 parent 1eec53c commit ce10d4f
Show file tree
Hide file tree
Showing 89 changed files with 1,359 additions and 1,601 deletions.
2 changes: 1 addition & 1 deletion .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ fileignoreconfig:
- filename: backend/src/test/kotlin/unit/application/service/GetFileServiceTest.kt
checksum: 8133265c6f388df84a3e0f61050428e8628853e0ba80b432e4723750a34b781e
- filename: compose.yaml
checksum: 0cf11aa3c7711e07ad50cc83cad9b4c5a2081e7400d949aa84a43b1c5354058e
checksum: cd08d156c78274d39b8db8f42af9696da21b9f6815c09fbbededa937709ff945
- filename: doc/norms/backend-api.yaml
checksum: df4fa1f7e6d7023f9a7a98d1052c2b88c1785366962004b3165453edcd5bf4db
- filename: doc/structurizr/workspace.json
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,21 +25,6 @@ public LookupTableImporterController(LookupTableImporterService service) {
this.service = service;
}

// In Postman go to "Body", select "raw" and "XML" and paste the XML-contents.
// Can we use @RequestBody @Valid DocumentTypesXML directly instead of ByteBuffer?

@PutMapping(value = "doktyp")
@PreAuthorize("isAuthenticated()")
public Mono<ResponseEntity<String>> importDocumentTypeLookupTable(
@RequestBody ByteBuffer byteBuffer) {
return service
.importDocumentTypeLookupTable(byteBuffer)
.map(resultString -> ResponseEntity.status(HttpStatus.OK).body(resultString))
.onErrorReturn(
ResponseEntity.internalServerError()
.body("Could not import the document type lookup table"));
}

@PutMapping(value = "gerichtdata")
@PreAuthorize("isAuthenticated()")
public Mono<ResponseEntity<String>> importCourtLookupTable(@RequestBody ByteBuffer byteBuffer) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package de.bund.digitalservice.ris.caselaw.adapter;

import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import de.bund.digitalservice.ris.caselaw.adapter.database.jpa.JPADocumentTypeDTO;
import de.bund.digitalservice.ris.caselaw.adapter.database.jpa.JPADocumentTypeRepository;
import de.bund.digitalservice.ris.caselaw.adapter.database.jpa.JPAFieldOfLawDTO;
import de.bund.digitalservice.ris.caselaw.adapter.database.jpa.JPAFieldOfLawLinkDTO;
import de.bund.digitalservice.ris.caselaw.adapter.database.jpa.JPAFieldOfLawLinkRepository;
Expand All @@ -17,7 +15,6 @@
import de.bund.digitalservice.ris.caselaw.domain.ServiceUtils;
import de.bund.digitalservice.ris.caselaw.domain.lookuptable.citation.CitationsStyleXML;
import de.bund.digitalservice.ris.caselaw.domain.lookuptable.court.CourtsXML;
import de.bund.digitalservice.ris.caselaw.domain.lookuptable.documenttype.DocumentTypesXML;
import de.bund.digitalservice.ris.caselaw.domain.lookuptable.fieldoflaw.FieldOfLawXml;
import de.bund.digitalservice.ris.caselaw.domain.lookuptable.fieldoflaw.FieldsOfLawXml;
import de.bund.digitalservice.ris.caselaw.domain.lookuptable.state.StatesXML;
Expand All @@ -44,8 +41,6 @@
@Service
@Slf4j
public class LookupTableImporterService {

private final JPADocumentTypeRepository jpaDocumentTypeRepository;
private final DatabaseCourtRepository databaseCourtRepository;
private final StateRepository stateRepository;
private final DatabaseCitationStyleRepository databaseCitationStyleRepository;
Expand All @@ -55,59 +50,18 @@ public class LookupTableImporterService {
Pattern.compile("\\p{Lu}{2}(-\\d{2})+(?![\\p{L}\\d-])");

public LookupTableImporterService(
JPADocumentTypeRepository jpaDocumentTypeRepository,
DatabaseCourtRepository databaseCourtRepository,
StateRepository stateRepository,
DatabaseCitationStyleRepository databaseCitationStyleRepository,
JPAFieldOfLawRepository jpaFieldOfLawRepository,
JPAFieldOfLawLinkRepository jpaFieldOfLawLinkRepository) {
this.jpaDocumentTypeRepository = jpaDocumentTypeRepository;
this.databaseCourtRepository = databaseCourtRepository;
this.stateRepository = stateRepository;
this.databaseCitationStyleRepository = databaseCitationStyleRepository;
this.jpaFieldOfLawRepository = jpaFieldOfLawRepository;
this.jpaFieldOfLawLinkRepository = jpaFieldOfLawLinkRepository;
}

@Transactional(transactionManager = "jpaTransactionManager")
public Mono<String> importDocumentTypeLookupTable(ByteBuffer byteBuffer) {
XmlMapper mapper = new XmlMapper();
DocumentTypesXML documentTypesXML;
try {
documentTypesXML =
mapper.readValue(ServiceUtils.byteBufferToArray(byteBuffer), DocumentTypesXML.class);
} catch (IOException e) {
throw new ResponseStatusException(
HttpStatus.NOT_ACCEPTABLE, "Could not map ByteBuffer-content to DocumentTypesXML", e);
}

importDocumentTypeJPA(documentTypesXML);

return Mono.just("Successfully imported the document type lookup table");
}

public void importDocumentTypeJPA(DocumentTypesXML documentTypesXML) {
List<JPADocumentTypeDTO> documentTypeDTOS =
documentTypesXML.getList().stream()
.map(
documentTypeXML ->
JPADocumentTypeDTO.builder()
.id(documentTypeXML.getId())
.changeDateClient(documentTypeXML.getChangeDateClient())
.changeIndicator(documentTypeXML.getChangeIndicator())
.version(documentTypeXML.getVersion())
.jurisShortcut(documentTypeXML.getJurisShortcut())
.documentType(documentTypeXML.getDocumentType())
.multiple(documentTypeXML.getMultiple())
.label(documentTypeXML.getLabel())
.superlabel1(documentTypeXML.getSuperlabel1())
.superlabel2(documentTypeXML.getSuperlabel2())
.build())
.toList();

jpaDocumentTypeRepository.saveAll(documentTypeDTOS);
}

public Mono<String> importCourtLookupTable(ByteBuffer byteBuffer) {
XmlMapper mapper = new XmlMapper();
CourtsXML courtsXML;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,36 +28,34 @@ public NormAbbreviationController(NormAbbreviationService service) {

@GetMapping
@PreAuthorize("isAuthenticated()")
public Flux<NormAbbreviation> getAllNormAbbreviationsBySearchQuery(
@RequestParam(value = "q", required = false, defaultValue = "") String query,
@RequestParam(value = "sz", required = false) Integer size,
@RequestParam(value = "pg", required = false) Integer page) {

return service.getNormAbbreviationBySearchQuery(query, size, page);
public Flux<NormAbbreviation> getAllNormAbbreviationsStartingWithExact(
@RequestParam(value = "q", required = false, defaultValue = "") String normAbbreviation,
@RequestParam(value = "sz", required = false, defaultValue = "30") Integer size,
@RequestParam(value = "pg", required = false, defaultValue = "0") Integer page) {
return Flux.fromIterable(
service.getNormAbbreviationsStartingWithExact(normAbbreviation, size, page));
}

@GetMapping("/{uuid}")
@PreAuthorize("isAuthenticated()")
public Mono<NormAbbreviation> getNormAbbreviationController(@PathVariable("uuid") UUID uuid) {
return service.getNormAbbreviationById(uuid);
public Mono<NormAbbreviation> getNormAbbreviationById(@PathVariable("uuid") UUID uuid) {
return Mono.just(service.getNormAbbreviationById(uuid));
}

@GetMapping("/search")
@PreAuthorize("isAuthenticated()")
public Mono<List<NormAbbreviation>> getAllNormAbbreviationsByAwesomeSearchQuery(
public Mono<List<NormAbbreviation>> getAllNormAbbreviationsContaining(
@RequestParam(value = "q", required = false, defaultValue = "") String query,
@RequestParam(value = "sz", required = false) Integer size,
@RequestParam(value = "pg", required = false) Integer page) {

return service.getNormAbbreviationByAwesomeSearchQuery(query, size, page);
@RequestParam(value = "sz", required = false, defaultValue = "30") Integer size,
@RequestParam(value = "pg", required = false, defaultValue = "0") Integer page) {
return Mono.just(service.findAllNormAbbreviationsContaining(query, size, page));
}

@PutMapping("/refreshMaterializedViews")
@PreAuthorize("isAuthenticated()")
public Mono<ResponseEntity<String>> refreshMaterializedViews() {
return service
.refreshMaterializedViews()
.thenReturn(
ResponseEntity.ok("Refreshed the materialized view 'norm_abbreviation_search'"));
service.refreshMaterializedViews();
return Mono.just(
ResponseEntity.ok("Refreshed the materialized view 'norm_abbreviation_search_migration'"));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
import java.util.List;
import java.util.UUID;
import org.springframework.stereotype.Service;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

@Service
public class NormAbbreviationService {
Expand All @@ -16,33 +14,21 @@ public NormAbbreviationService(NormAbbreviationRepository repository) {
this.repository = repository;
}

public Mono<NormAbbreviation> getNormAbbreviationById(UUID uuid) {
public NormAbbreviation getNormAbbreviationById(UUID uuid) {
return repository.findById(uuid);
}

public Flux<NormAbbreviation> getNormAbbreviationBySearchQuery(
public List<NormAbbreviation> getNormAbbreviationsStartingWithExact(
String query, Integer size, Integer page) {

Integer pageOffset = null;
if (page != null && size != null) {
pageOffset = page * size;
}

return repository.findBySearchQuery(query, size, pageOffset);
return repository.getNormAbbreviationsStartingWithExact(query, size, page);
}

public Mono<List<NormAbbreviation>> getNormAbbreviationByAwesomeSearchQuery(
public List<NormAbbreviation> findAllNormAbbreviationsContaining(
String query, Integer size, Integer page) {

Integer pageOffset = null;
if (page != null && size != null) {
pageOffset = page * size;
}

return repository.findByAwesomeSearchQuery(query, size, pageOffset);
return repository.findAllContainingOrderByAccuracy(query, size, page);
}

public Mono<Void> refreshMaterializedViews() {
return repository.refreshMaterializedViews();
public void refreshMaterializedViews() {
repository.refreshMaterializedViews();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package de.bund.digitalservice.ris.caselaw.adapter.database.jpa;

import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface DatabaseDocumentCategoryRepository
extends JpaRepository<DocumentCategoryDTO, UUID> {

DocumentCategoryDTO findFirstByLabel(String label);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package de.bund.digitalservice.ris.caselaw.adapter.database.jpa;

import java.util.List;
import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;

/** */
@Repository
public interface DatabaseDocumentTypeRepository extends JpaRepository<DocumentTypeDTO, UUID> {

DocumentTypeDTO findFirstByAbbreviationAndCategory(
String jurisShortcut, DocumentCategoryDTO category);

List<DocumentTypeDTO> findAllByCategoryOrderByAbbreviationAscLabelAsc(
DocumentCategoryDTO category);

// see query explanation in CourtRepository, it's almost the same
@Query(
nativeQuery = true,
value =
"WITH label_added AS (SELECT *, "
+ " UPPER(CONCAT(abbreviation, ' ', label)) AS concat"
+ " from incremental_migration.document_type) "
+ "SELECT *,"
+ " concat, "
+ " CASE "
+ " WHEN concat LIKE UPPER(:searchStr||'%') THEN 1 "
+ " WHEN concat LIKE UPPER('% '||:searchStr||'%') THEN 2 "
+ " WHEN concat LIKE UPPER('%-'||:searchStr||'%') THEN 2 "
+ " ELSE 3 "
+ " END AS weight "
+ "FROM label_added "
+ "WHERE concat LIKE UPPER('%'||:searchStr||'%') AND document_category_id = :category "
+ "ORDER BY weight, concat")
List<DocumentTypeDTO> findCaselawBySearchStrAndCategory(String searchStr, UUID category);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package de.bund.digitalservice.ris.caselaw.adapter.database.jpa;

import jakarta.transaction.Transactional;
import java.util.List;
import java.util.UUID;
import org.springframework.context.annotation.Primary;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;

@Repository
@Primary
public interface DatabaseNormAbbreviationRepository
extends JpaRepository<NormAbbreviationDTO, UUID> {

List<NormAbbreviationDTO> findByAbbreviationIgnoreCase(String abbreviation, PageRequest of);

List<NormAbbreviationDTO> findByAbbreviationStartsWithOrderByAbbreviation(
String query, PageRequest of);

List<NormAbbreviationDTO> findByOfficialLetterAbbreviationIgnoreCase(
String officialLetterAbbreviation, PageRequest of);

List<NormAbbreviationDTO> findByAbbreviationStartsWithIgnoreCase(
String abbreviation, PageRequest of);

List<NormAbbreviationDTO> findByOfficialLetterAbbreviationStartsWithIgnoreCase(
String officialLetterAbbreviation, PageRequest size);

@Query(
value =
"select"
+ " id,"
+ " abbreviation,"
+ " decision_date,"
+ " document_id,"
+ " document_number,"
+ " official_letter_abbreviation,"
+ " official_long_title,"
+ " official_short_title,"
+ " source,"
+ " region_id,"
+ " ts_rank_cd(weighted_vector, to_tsquery('german', '' || :tsQuery || '')) rank"
+ " from norm_abbreviation_search_migration"
+ " where weighted_vector @@ to_tsquery('german', '' || :tsQuery || '')"
+ " order by rank desc"
+ " limit :size"
+ " offset :offset",
nativeQuery = true)
List<NormAbbreviationDTO> findByRankWeightedVector(String tsQuery, Integer size, Integer offset);

@Transactional
@Modifying
@Query(value = "REFRESH MATERIALIZED VIEW norm_abbreviation_search_migration", nativeQuery = true)
void refreshMaterializedViews();
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
import org.springframework.stereotype.Repository;

@Repository
public interface JPADatabaseNormElementRepository extends JpaRepository<JPANormElementDTO, UUID> {
public interface DatabaseNormElementRepository extends JpaRepository<NormElementDTO, UUID> {

@Query(
value =
"SELECT ne.id, ne.label, ne.has_number_designation, ne.norm_code "
+ "FROM norm_element ne "
+ "INNER JOIN document_category dc ON ne.document_category_id=dc.id "
+ "FROM incremental_migration.norm_element ne "
+ "INNER JOIN incremental_migration.document_category dc ON ne.document_category_id=dc.id "
+ "WHERE dc.label = 'R'",
nativeQuery = true)
List<JPANormElementDTO> findAllByDocumentCategoryLabelR();
List<NormElementDTO> findAllByDocumentCategoryLabelR();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package de.bund.digitalservice.ris.caselaw.adapter.database.jpa;

import java.util.List;
import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface DatabaseNormReferenceRepository extends JpaRepository<NormReferenceDTO, UUID> {
List<NormReferenceDTO> findAllByLegacyDocUnitIdOrderById(UUID documentationUnitId);
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package de.bund.digitalservice.ris.caselaw.adapter.database.jpa;

import java.util.UUID;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface JPADocumentTypeRepository extends JpaRepository<JPADocumentTypeDTO, Long> {}
public interface DatabaseRegionRepository extends JpaRepository<RegionDTO, UUID> {}
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package de.bund.digitalservice.ris.caselaw.adapter.database.jpa;

import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.Id;
import jakarta.persistence.Table;
import java.util.UUID;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;

@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Builder
@Entity
@Table(name = "document_category", schema = "incremental_migration")
public class DocumentCategoryDTO {
@Id @GeneratedValue private UUID id;

@Column private String label;
}
Loading

0 comments on commit ce10d4f

Please sign in to comment.