[WIP] update filer with run mode

common/src/main/scala/hmda/util/Filer.scala

@@ -7,18 +7,59 @@ import java.time.temporal.{ ChronoField, TemporalAccessor }
 import com.typesafe.config.{ Config, ConfigFactory }
 import org.slf4j.LoggerFactory
 
+import scala.util.matching.Regex
 import scala.util.Try
 
 object Filer {
   private val log = LoggerFactory.getLogger(getClass)
 
   private val dateFormatter = new DateTimeFormatterBuilder().appendPattern("MMMM dd yyyy").toFormatter
 
-  private val rtTgConfig = {
-    val tgWatch = ConfigFactory.load().getConfig("hmda.cm_watch.timed_guards")
-    val ns = Try(tgWatch.getString("ns")).getOrElse("default")
-    val cmName = Try(tgWatch.getString("name")).getOrElse("timed-guards")
-    new RealTimeConfig(cmName, ns)
+  private val config = ConfigFactory.load()
+
+  private val runMode = config.getString("hmda.runtime.mode")
+
+  private val rtTgConfigOpt: Option[RealTimeConfig]  = {
+    if (runMode == "kubernetes") {
+      val tgWatch = config.getConfig("hmda.cm_watch.timed_guards")
+      val ns = Try(tgWatch.getString("ns")).getOrElse("default")
+      val cmName = Try(tgWatch.getString("name")).getOrElse("timed-guards")
+      log.info("Using real time configmap for timed guard.")
+      Some(new RealTimeConfig(cmName, ns))
+    } else {
+      log.warn("Failed to load time guard through real time config.")
+      log.info("Using default for timed guard values.")
+      None
+    }
+  }
+
+  /*
+  get default (stored in conf file) value from config key
+   */
+  def getString(key: String): String = {
+    val QuarterReg: Regex = "q([1-3])(Start|End)".r
+    val ActionQuarterReg: Regex = "actionQ([1-3])(Start|End)".r
+    val quarterlyFiling: String = "hmda.rules.quarterly-filing"
+    val localKey: String = key match {
+      case "currentYear" => "hmda.filing.current"
+      case "yearsAllowed" => "hmda.rules.yearly-filing.years-allowed"
+      case "quarterlyYearsAllowed" => s"$quarterlyFiling.years-allowed"
+      case QuarterReg(q,t) => s"$quarterlyFiling.q$q.${t.toLowerCase()}"
+      case ActionQuarterReg(q,t) => s"$quarterlyFiling.q$q.action_date_${t.toLowerCase()}"
+      case _ => key
+    }
+    Try(config.getString(localKey)).getOrElse(localKey)
+  }
+
+  /*
+  get configuration value from a key.  if running in prod/dev, it should be using k8
+  configmap.  Else (for local/test), it should be using default config file ((stored in conf file)
+   */
+  def getConfig(key: String): String = {
+    rtTgConfigOpt match {
+      case Some(rtTgConfig) => rtTgConfig.getString(key)
+      case _ => getString(key)
+    }
   }
 
   def check(filingRulesConfig: FilingRulesConfig)(year: Int, dayOfYear: Int, quarter: Option[String]): Boolean = {
@@ -119,20 +160,20 @@ object Filer {
 
   private def getRulesFromRtConfig(): FilingRulesConfig = {
     val quarterlyFilingConfig = QuarterlyFilingConfig(
-      rtTgConfig.getString("quarterlyYearsAllowed").split(",").map(_.toInt).toList,
+      getConfig("quarterlyYearsAllowed").split(",").map(_.toInt).toList,
       getQuarterConfig(1),
       getQuarterConfig(2),
       getQuarterConfig(3)
     )
-    FilingRulesConfig(quarterlyFilingConfig, rtTgConfig.getString("yearsAllowed").split(",").map(_.toInt).toList)
+    FilingRulesConfig(quarterlyFilingConfig, getConfig("yearsAllowed").split(",").map(_.toInt).toList)
   }
 
   private def getQuarterConfig(quarter: Int): QuarterConfig = {
     val currentYear = LocalDate.now().getYear
-    val startDate = rtTgConfig.getString(s"q${quarter}Start")
-    val endDate = rtTgConfig.getString(s"q${quarter}End")
-    val actionStartDate = rtTgConfig.getString(s"actionQ${quarter}Start")
-    val actionEndDate = rtTgConfig.getString(s"actionQ${quarter}End")
+    val startDate = getConfig(s"q${quarter}Start")
+    val endDate = getConfig(s"q${quarter}End")
+    val actionStartDate = getConfig(s"actionQ${quarter}Start")
+    val actionEndDate = getConfig(s"actionQ${quarter}End")
     QuarterConfig(
       dateFormatter.parse(s"$startDate $currentYear").get(ChronoField.DAY_OF_YEAR),
       dateFormatter.parse(s"$endDate $currentYear").get(ChronoField.DAY_OF_YEAR),

common/src/test/resources/application.conf

@@ -6,4 +6,8 @@ akka {
 
 kafka {
   servers = "localhost:6001"
+}
+
+hmda {
+    runtime.mode = "dev"
 }

common/src/test/scala/hmda/util/FilerSpec.scala

@@ -0,0 +1,32 @@
+package hmda.util
+
+import hmda.util.Filer._
+import org.scalatest.{MustMatchers, PropSpec}
+import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+
+class FilerSpec extends PropSpec with ScalaCheckPropertyChecks with MustMatchers {
+
+  property("correctly get local config") {
+    // these values are from common/resource/reference.conf
+    getString("currentYear") mustBe "2021"
+    getString("yearsAllowed") mustBe "2018,2019,2020,2021,2022,2023,2024,2025"
+    getString("quarterlyYearsAllowed") mustBe "2020,2021,2022,2023,2024"
+    getString("q2Start") mustBe "July 01"
+    getString("q3End") mustBe "December 31"
+    getString("actionQ2Start") mustBe "April 01"
+    getString("actionQ3End") mustBe "September 30"
+    getString("randomKey") mustBe "randomKey"
+  }
+
+  property("correctly get config") {
+    getConfig("currentYear") mustBe "2021"
+    getConfig("yearsAllowed") mustBe "2018,2019,2020,2021,2022,2023,2024,2025"
+    getConfig("quarterlyYearsAllowed") mustBe "2020,2021,2022,2023,2024"
+    getConfig("q2Start") mustBe "July 01"
+    getConfig("q3End") mustBe "December 31"
+    getConfig("actionQ2Start") mustBe "April 01"
+    getConfig("actionQ3End") mustBe "September 30"
+    getConfig("randomKey") mustBe "randomKey"
+  }
+
+}

hmda-data-publisher/src/main/resources/application.conf

@@ -111,6 +111,8 @@ hmda {
     validation.reportingUrl = ""
     validation.reportingUrl = ${?VALIDATION_REPORTING_URL}
   }
+  runtime.mode = "dev"
+  runtime.mode = ${?HMDA_RUNTIME_MODE}
 }
 
 private-aws {

institutions-api/src/main/resources/application.conf

@@ -30,6 +30,8 @@ hmda {
       timeout = 5000
     }
   }
+  runtime.mode = "dev"
+  runtime.mode = ${?HMDA_RUNTIME_MODE}
 }
 
 institution_db {

kubernetes/hmda-data-publisher/templates/deployment.yaml

@@ -44,6 +44,8 @@ spec:
               name: {{ include "hmda-data-publisher.name" . }}-dynamic-schedule-config
               optional: true
           env:
+          - name: HMDA_RUNTIME_MODE
+            value: { { .Values.hmda.runtimeMode } }
           - name: CASSANDRA_CLUSTER_HOSTS
             valueFrom:
               configMapKeyRef:

kubernetes/hmda-data-publisher/values.yaml

@@ -23,6 +23,8 @@ grpc:
   targetPort: 60082
   name: http2-publisher
 
+hmda:
+  runtimeMode: kubernetes
 #ambassador:
 #  name: ambassador-publisher
 #  port: 80

kubernetes/institutions-api/templates/deployment.yaml

@@ -43,6 +43,8 @@ spec:
             - configMapRef:
                 name: {{ template "institutions-api.fullname" . }}-config
           env:
+            - name: HMDA_RUNTIME_MODE
+              value: {{ .Values.hmda.runtimeMode }}
             - name: KAFKA_INSTITUTIONS_TOPIC
               value: {{.Values.kafka.institutionsTopic}}
             - name: KAFKA_INSTITUTIONS_GROUP

kubernetes/institutions-api/values.yaml

@@ -23,6 +23,9 @@ kafka:
   institutionsGroup: institutions-group
   institutionsTopic: institution
 
+hmda:
+  runtimeMode: kubernetes
+
 #ambassador:
 #  name: institutions-api-ambassador
 #  service: