-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #194 from jembi/dev_delete_indexs
Updating indexes before linking, and after linking
- Loading branch information
Showing
19 changed files
with
554 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
198 changes: 198 additions & 0 deletions
198
JeMPI_Apps/JeMPI_Configuration/src/main/scala/configuration/CustomDgraphIndexes.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| package configuration | ||
|
|
||
| import java.io.{File, PrintWriter} | ||
|
|
||
| private object CustomDgraphIndexes { | ||
|
|
||
| private val classLocation = | ||
| "../JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/dgraph" | ||
| private val custom_className = "CustomDgraphIndexes" | ||
| private val packageText = "org.jembi.jempi.libmpi.dgraph" | ||
|
|
||
| def generate(config: Config): Unit = { | ||
| val classFile: String = | ||
| classLocation + File.separator + custom_className + ".java" | ||
| println("Creating " + classFile) | ||
| val file: File = new File(classFile) | ||
| val writer: PrintWriter = new PrintWriter(file) | ||
| writer.println( | ||
| s"""package $packageText; | ||
| | | ||
| | | ||
| |final class $custom_className { | ||
| | | ||
| | private $custom_className() { | ||
| | } | ||
| | | ||
| | public static Boolean shouldUpdateLinkingIndexes() { | ||
| | return ${get_should_update_linking_indexes()}; | ||
| | } | ||
| |""".stripMargin); | ||
|
|
||
| get_default_indexes() | ||
| get_remove_all_indexes() | ||
| get_linking_indexes() | ||
|
|
||
| writer.println( | ||
| s""" | ||
| |} | ||
| |""".stripMargin) | ||
|
|
||
| writer.flush() | ||
| writer.close() | ||
|
|
||
|
|
||
| def get_should_update_linking_indexes(): String = { | ||
| if (config.rules.link.isDefined){ | ||
| val linkRules = config.rules.link.get | ||
| if (linkRules.deterministic.isDefined && !linkRules.probabilistic.isDefined){ | ||
| return "true" | ||
| } | ||
| } | ||
|
|
||
| return "false" | ||
| } | ||
|
|
||
| def get_default_indexes(): Unit = { | ||
| write_out_indexs_fields("LOAD_DEFAULT_INDEXES", write_field_with_index) | ||
| } | ||
|
|
||
| def get_remove_all_indexes(): Unit = { | ||
| write_out_indexs_fields("REMOVE_ALL_INDEXES", write_field_without_index) | ||
| } | ||
|
|
||
| def get_linking_indexes(): Unit = { | ||
| write_out_linking_fields("LOAD_LINKING_INDEXES", "linking") | ||
| } | ||
|
|
||
| def write_out_linking_fields(prop_name: String, index_type:String): Unit = { | ||
| writer.println( | ||
| s""" static final String $prop_name = | ||
| | \"\"\" | ||
| |""".stripMargin) | ||
|
|
||
| var indexsObjOption:Option[Map[String, IndexProps]] = null; | ||
|
|
||
| if (index_type == "linking") { | ||
| indexsObjOption = config.indexes.linking | ||
| } else if (index_type == "matching") { | ||
| indexsObjOption = config.indexes.matching | ||
| } | ||
|
|
||
| if (indexsObjOption.isDefined){ | ||
| val indexsObj = indexsObjOption.get | ||
|
|
||
| val index_names = indexsObj.keySet | ||
| val valid_index_fields = config.demographicFields.filter(f => index_names.contains(f.fieldName)) | ||
|
|
||
| if (valid_index_fields.size != indexsObj.size) { | ||
| throw new IllegalArgumentException("Some fields defined in the index property do not exist. Make sure you have defined them in the demographicFields property") | ||
| } | ||
|
|
||
| valid_index_fields.foreach(field => { | ||
| val indexPropsOpt = indexsObj.get(field.fieldName) | ||
|
|
||
| if (indexPropsOpt.isDefined){ | ||
| write_field_with_index(field, "GoldenRecord", indexPropsOpt.get.props, 29) | ||
| write_field_with_index(field, "Interaction", indexPropsOpt.get.props, 30) | ||
| } | ||
| }) | ||
|
|
||
| val index_to_remove = config.demographicFields.filter(f => !index_names.contains(f.fieldName)) | ||
| index_to_remove.foreach(field => { | ||
| val indexGoldenRecord = field.indexGoldenRecord.getOrElse("") | ||
| val indexInteraction = field.indexInteraction.getOrElse("") | ||
|
|
||
| if (indexGoldenRecord != "") { | ||
| write_field_without_index(field, "GoldenRecord", indexGoldenRecord, 29) | ||
| } | ||
|
|
||
| if (indexInteraction != "") { | ||
| write_field_without_index(field, "Interaction", indexInteraction, 30) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| writer.println( | ||
| s""" | ||
| | \"\"\"; | ||
| |""".stripMargin) | ||
|
|
||
| } | ||
|
|
||
| def write_out_indexs_fields(prop_name: String, write_func: (field: DemographicField | ||
| , recordType: String | ||
| , index: String | ||
| , spacing: Int) => Unit): Unit = { | ||
| writer.println( | ||
| s""" static final String $prop_name = | ||
| | \"\"\" | ||
| |""".stripMargin) | ||
|
|
||
| config.demographicFields | ||
| .foreach(field => { | ||
| val indexGoldenRecord = field.indexGoldenRecord.getOrElse("") | ||
| val indexInteraction = field.indexInteraction.getOrElse("") | ||
|
|
||
| if (indexGoldenRecord != "") { | ||
| write_func(field, "GoldenRecord", indexGoldenRecord, 29) | ||
| } | ||
|
|
||
| if (indexInteraction != "") { | ||
| write_func(field, "Interaction", indexInteraction, 30) | ||
| } | ||
| }) | ||
|
|
||
| removeUniqueIndexes() | ||
|
|
||
| writer.println( | ||
| s""" | ||
| | \"\"\"; | ||
| |""".stripMargin) | ||
| } | ||
|
|
||
| def write_field_with_index(field: DemographicField, recordType: String, index: String, spacing: Int): Unit = { | ||
| val name = field.fieldName | ||
|
|
||
| val fieldType = | ||
| (if field.isList.isDefined && field.isList.get then "[" | ||
| else "") + field.fieldType.toLowerCase + (if field.isList.isDefined && field.isList.get | ||
| then "]" | ||
| else "") | ||
| writer.println( | ||
| s"""${" " * 9}$recordType.$name:${" " * (spacing - name.length)}$fieldType${" " * (10 - fieldType.length)}$index${" " * (35 - index.length)}.""".stripMargin | ||
| ) | ||
| } | ||
|
|
||
| def write_field_without_index(field: DemographicField, recordType: String, index: String, spacing: Int): Unit = { | ||
| val name = field.fieldName | ||
|
|
||
| val fieldType = | ||
| (if field.isList.isDefined && field.isList.get then "[" | ||
| else "") + field.fieldType.toLowerCase + (if field.isList.isDefined && field.isList.get | ||
| then "]" | ||
| else "") | ||
| writer.println( | ||
| s"""${" " * 9}$recordType.$name:${" " * (spacing - name.length)}$fieldType${" " * (10 - fieldType.length)}${" " * 35}.""".stripMargin | ||
| ) | ||
| } | ||
|
|
||
| def removeUniqueIndexes(): Unit = { | ||
| List(config.indexes.linking, config.indexes.matching).foreach(indexsObjOption => { | ||
| if (indexsObjOption.isDefined) { | ||
| val indexsObj = indexsObjOption.get | ||
|
|
||
| val index_names = indexsObj.keySet | ||
| val uniqueIndexes = config.demographicFields.filter(f => index_names.contains(f.fieldName) && f.indexGoldenRecord.getOrElse("") == "" && f.indexInteraction.getOrElse("") == "") | ||
|
|
||
| uniqueIndexes.foreach(field => { | ||
| write_field_without_index(field, "GoldenRecord", "", 29) | ||
| write_field_without_index(field, "Interaction", "", 30) | ||
| }) | ||
|
|
||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -52,5 +52,4 @@ record CandidatesNotFoundError( | |
| String error, | ||
| String interactionID) implements MpiServiceError { | ||
| } | ||
|
|
||
| } | ||
60 changes: 60 additions & 0 deletions
60
JeMPI_Apps/JeMPI_LibMPI/src/main/java/org/jembi/jempi/libmpi/dgraph/CustomDgraphIndexes.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| package org.jembi.jempi.libmpi.dgraph; | ||
|
|
||
|
|
||
| final class CustomDgraphIndexes { | ||
|
|
||
| private CustomDgraphIndexes() { | ||
| } | ||
|
|
||
| public static Boolean shouldUpdateLinkingIndexes() { | ||
| return false; | ||
| } | ||
|
|
||
| static final String LOAD_DEFAULT_INDEXES = | ||
| """ | ||
| GoldenRecord.given_name: string @index(exact,trigram) . | ||
| Interaction.given_name: string @index(exact,trigram) . | ||
| GoldenRecord.family_name: string @index(exact,trigram) . | ||
| Interaction.family_name: string @index(exact,trigram) . | ||
| GoldenRecord.gender: string @index(exact,trigram) . | ||
| GoldenRecord.city: string @index(trigram) . | ||
| GoldenRecord.phone_number: string @index(exact,trigram) . | ||
| GoldenRecord.national_id: string @index(exact,trigram) . | ||
| Interaction.national_id: string @index(exact,trigram) . | ||
| """; | ||
|
|
||
| static final String REMOVE_ALL_INDEXES = | ||
| """ | ||
| GoldenRecord.given_name: string . | ||
| Interaction.given_name: string . | ||
| GoldenRecord.family_name: string . | ||
| Interaction.family_name: string . | ||
| GoldenRecord.gender: string . | ||
| GoldenRecord.city: string . | ||
| GoldenRecord.phone_number: string . | ||
| GoldenRecord.national_id: string . | ||
| Interaction.national_id: string . | ||
| """; | ||
|
|
||
| static final String LOAD_LINKING_INDEXES = | ||
| """ | ||
| GoldenRecord.national_id: string @index(exact,trigram) . | ||
| Interaction.national_id: string @index(exact,trigram) . | ||
| GoldenRecord.given_name: string . | ||
| Interaction.given_name: string . | ||
| GoldenRecord.family_name: string . | ||
| Interaction.family_name: string . | ||
| GoldenRecord.gender: string . | ||
| GoldenRecord.city: string . | ||
| GoldenRecord.phone_number: string . | ||
| """; | ||
|
|
||
|
|
||
| } | ||
|
|
Oops, something went wrong.