update hw-kafka-client (#172)

Raise hw-kafka-client to version 5
freckle · Jun 7, 2024 · cda8ac8 · cda8ac8
1 parent 8de2553
commit cda8ac8
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 41 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,8 @@
-## [_Unreleased_](https://github.com/freckle/freckle-app/compare/v1.17.0.0...main)
+## [_Unreleased_](https://github.com/freckle/freckle-app/compare/v1.17.1.0...main)
+
+## [v1.17.1.0](https://github.com/freckle/freckle-app/compare/v1.17.0.0...v1.17.1.0)
+
+- Support (and require) `hw-kafka-client` `>= 5`
 
 ## [v1.17.0.0](https://github.com/freckle/freckle-app/compare/v1.16.0.0...v1.17.0.0)
 

diff --git a/freckle-app.cabal b/freckle-app.cabal
@@ -5,7 +5,7 @@ cabal-version: 1.18
 -- see: https://github.com/sol/hpack
 
 name:           freckle-app
-version:        1.17.0.0
+version:        1.17.1.0
 synopsis:       Haskell application toolkit used at Freckle
 description:    Please see README.md
 category:       Utils
@@ -165,7 +165,7 @@ library
     , http-conduit >=2.3.5
     , http-link-header
     , http-types
-    , hw-kafka-client <5.0.0
+    , hw-kafka-client
     , immortal
     , lens
     , memcache

diff --git a/library/Freckle/App/Kafka/Producer.hs b/library/Freckle/App/Kafka/Producer.hs
@@ -28,6 +28,7 @@ import qualified Freckle.App.Env as Env
 import Freckle.App.OpenTelemetry
 import Kafka.Producer
 import qualified OpenTelemetry.Trace as Trace
+import UnliftIO (withRunInIO)
 import Yesod.Core.Lens
 import Yesod.Core.Types (HandlerData)
 
@@ -91,13 +92,13 @@ createKafkaProducerPool
   -> KafkaProducerPoolConfig
   -> IO (Pool KafkaProducer)
 createKafkaProducerPool addresses KafkaProducerPoolConfig {..} =
-  Pool.newPool $
-    Pool.setNumStripes (Just kafkaProducerPoolConfigStripes) $
-      Pool.defaultPoolConfig
-        mkProducer
-        closeProducer
-        (realToFrac kafkaProducerPoolConfigIdleTimeout)
-        kafkaProducerPoolConfigSize
+  Pool.newPool
+    $ Pool.setNumStripes (Just kafkaProducerPoolConfigStripes)
+    $ Pool.defaultPoolConfig
+      mkProducer
+      closeProducer
+      (realToFrac kafkaProducerPoolConfigIdleTimeout)
+      kafkaProducerPoolConfigSize
  where
   mkProducer =
     either
@@ -122,26 +123,21 @@ produceKeyedOn prTopic values keyF = traced $ do
   logDebugNS "kafka" $ "Producing Kafka events" :# ["events" .= values]
   view kafkaProducerPoolL >>= \case
     NullKafkaProducerPool -> pure ()
-    KafkaProducerPool producerPool -> do
-      errors <-
-        liftIO $
-          Pool.withResource producerPool $ \producer ->
-            produceMessageBatch producer $
-              toList $
-                mkProducerRecord <$> values
-      unless (null errors) $
-        logErrorNS "kafka" $
-          "Failed to send events" :# ["errors" .= fmap (tshow . snd) errors]
+    KafkaProducerPool producerPool ->
+      withRunInIO $ \run ->
+        Pool.withResource producerPool $ \producer ->
+          for_ @NonEmpty values $ \value -> do
+            mError <- liftIO $ produceMessage producer $ mkProducerRecord value
+            for_ @Maybe mError $ \e ->
+              run $ logErrorNS "kafka" $ "Failed to send event" :# ["error" .= tshow e]
  where
   mkProducerRecord value =
     ProducerRecord
       { prTopic
       , prPartition = UnassignedPartition
       , prKey = Just $ toStrict $ encode $ keyF value
-      , prValue =
-          Just $
-            toStrict $
-              encode value
+      , prValue = Just $ toStrict $ encode value
+      , prHeaders = mempty
       }
 
   traced =

diff --git a/package.yaml b/package.yaml
@@ -1,5 +1,5 @@
 name: freckle-app
-version: 1.17.0.0
+version: 1.17.1.0
 maintainer: Freckle Education
 category: Utils
 github: freckle/freckle-app
@@ -112,7 +112,7 @@ library:
     - http-conduit >= 2.3.5 # addToRequestQueryString
     - http-link-header
     - http-types
-    - hw-kafka-client < 5.0.0
+    - hw-kafka-client
     - immortal
     - lens
     - memcache

diff --git a/stack-lts-20.26.yaml b/stack-lts-20.26.yaml
@@ -8,6 +8,7 @@ extra-deps:
   - hs-opentelemetry-instrumentation-persistent-0.1.0.0
   - hs-opentelemetry-instrumentation-wai-0.1.0.0
   - hs-opentelemetry-sdk-0.0.3.6
+  - hw-kafka-client-5.0.0
   - resource-pool-0.4.0.0
 
   # for hs-opentelemetry-sdk

diff --git a/stack-lts-20.26.yaml.lock b/stack-lts-20.26.yaml.lock
@@ -4,13 +4,27 @@
 #   https://docs.haskellstack.org/en/stable/lock_files
 
 packages:
+- completed:
+    hackage: Blammo-1.2.0.0@sha256:727b02bff2ef0363501778bfc01a13d4c7b2912843b9e1a2538081f0b44484d4,4836
+    pantry-tree:
+      sha256: 35a51b2053f5b56a16458e8316bb1cd5d13490eab87b0ba30cd1dbbbdef99c0c
+      size: 1724
+  original:
+    hackage: Blammo-1.2.0.0
 - completed:
     hackage: bugsnag-1.1.0.0@sha256:9723af13b09e7aed7e5855fcbcd7f89a904df80324f21d32480f01aabd8d035b,4565
     pantry-tree:
       sha256: 1c23c30271a5b136a165f817c6db2828b95b5d37399fe052f02d80049a464575
       size: 1664
   original:
     hackage: bugsnag-1.1.0.0
+- completed:
+    hackage: fast-logger-3.2.3@sha256:41b4f1c07d5ee4a7cc785689eb7772554d29ddbbcced3cc184fe50fc63ece3f7,2176
+    pantry-tree:
+      sha256: c4a8dcfa5f5bc3bd77cfe86d904e96f90607adc1e4f3f1cf082e722673ee7230
+      size: 1302
+  original:
+    hackage: fast-logger-3.2.3
 - completed:
     hackage: monad-validate-1.3.0.0@sha256:eb6ddd5c9cf72ff0563cba604fa00291376e96138fdb4932d00ff3a99d66706e,2605
     pantry-tree:
@@ -46,6 +60,13 @@ packages:
       size: 1430
   original:
     hackage: hs-opentelemetry-sdk-0.0.3.6
+- completed:
+    hackage: hw-kafka-client-5.0.0@sha256:99ce4899a39e3be37acf013a2f8f12761e868bf9777c0135547fcfb4c6fbf882,4861
+    pantry-tree:
+      sha256: d87af59eabad9c90fdf2d778740374911dd3b15b767c8e9fe5ba33244f8952f7
+      size: 2056
+  original:
+    hackage: hw-kafka-client-5.0.0
 - completed:
     hackage: resource-pool-0.4.0.0@sha256:9c1e448a159875e21a7e68697feee2b61a4e584720974fa465a2fa1bc0776c73,1342
     pantry-tree:

diff --git a/stack-lts-21.25.yaml b/stack-lts-21.25.yaml
@@ -9,6 +9,7 @@ extra-deps:
   - hs-opentelemetry-instrumentation-wai-0.1.0.0
   - hs-opentelemetry-propagator-datadog-0.0.0.0
   - hs-opentelemetry-sdk-0.0.3.6
+  - hw-kafka-client-5.0.0
 
   # for hs-opentelemetry-sdk
   - hs-opentelemetry-exporter-otlp-0.0.1.5

diff --git a/stack-lts-21.25.yaml.lock b/stack-lts-21.25.yaml.lock
@@ -67,6 +67,13 @@ packages:
       size: 1430
   original:
     hackage: hs-opentelemetry-sdk-0.0.3.6
+- completed:
+    hackage: hw-kafka-client-5.0.0@sha256:99ce4899a39e3be37acf013a2f8f12761e868bf9777c0135547fcfb4c6fbf882,4861
+    pantry-tree:
+      sha256: d87af59eabad9c90fdf2d778740374911dd3b15b767c8e9fe5ba33244f8952f7
+      size: 2056
+  original:
+    hackage: hw-kafka-client-5.0.0
 - completed:
     hackage: hs-opentelemetry-exporter-otlp-0.0.1.5@sha256:89b0a6481096a338fa6383fbdf08ccaa0eb7bb009c4cbb340894eac33e55c5de,2214
     pantry-tree:

diff --git a/stack-nightly.yaml b/stack-nightly.yaml
@@ -18,10 +18,6 @@ extra-deps:
   - thread-utils-context-0.3.0.4
   - thread-utils-finalizers-0.1.1.0
 
-  # We're not ready to take on the breaking changes in v5
-  # @sled --exclude hw-kafka-client
-  - hw-kafka-client-4.0.3
-
   # These are me just adding what Stack told me to
   - Cabal-3.10.2.1
   - Cabal-syntax-3.10.2.0

diff --git a/stack-nightly.yaml.lock b/stack-nightly.yaml.lock
@@ -4,13 +4,27 @@
 #   https://docs.haskellstack.org/en/stable/lock_files
 
 packages:
+- completed:
+    hackage: Blammo-1.2.1.0@sha256:d319b91109e14762f06c340bbf955efcfccd6cc853a182e4c384113e236ddcb5,4836
+    pantry-tree:
+      sha256: ea00c0835cbbcfa749803647d2c0415d1991d2bd325ecb13d8de4186b6dd2c4f
+      size: 1725
+  original:
+    hackage: Blammo-1.2.1.0
 - completed:
     hackage: bcp47-0.2.0.6@sha256:9071d1f97ef249ae62e4554e3cba892cd6059ac263271fd72635157c83743a30,2949
     pantry-tree:
       sha256: 3cd17d04bc9d13c8ba7e8e390973fce7d79003b18379533c8ce702f9ad3f82b3
       size: 1498
   original:
     hackage: bcp47-0.2.0.6
+- completed:
+    hackage: fast-logger-3.2.3@sha256:41b4f1c07d5ee4a7cc785689eb7772554d29ddbbcced3cc184fe50fc63ece3f7,2176
+    pantry-tree:
+      sha256: c4a8dcfa5f5bc3bd77cfe86d904e96f90607adc1e4f3f1cf082e722673ee7230
+      size: 1302
+  original:
+    hackage: fast-logger-3.2.3
 - completed:
     hackage: monad-validate-1.3.0.0@sha256:eb6ddd5c9cf72ff0563cba604fa00291376e96138fdb4932d00ff3a99d66706e,2605
     pantry-tree:
@@ -95,13 +109,6 @@ packages:
       size: 400
   original:
     hackage: thread-utils-finalizers-0.1.1.0
-- completed:
-    hackage: hw-kafka-client-4.0.3@sha256:20e3614454d5afd2a6acfc0709b9bf00bd6ee9a9bc646c2e2e4ccec1d16831a1,4861
-    pantry-tree:
-      sha256: 709ea6818b3b3beb8eed6f79ea76b8dc13cce9ae165870c17986d5de2d695447
-      size: 2057
-  original:
-    hackage: hw-kafka-client-4.0.3
 - completed:
     hackage: Cabal-3.10.2.1@sha256:0f7cc73c7c0c18464ce249c97267a5188d796690a926d73b6e084a4612a66e32,12693
     pantry-tree:

diff --git a/stack.yaml b/stack.yaml
@@ -23,10 +23,6 @@ extra-deps:
   # for thread-utils-context
   - thread-utils-finalizers-0.1.1.0
 
-  # We're not ready to take on the breaking changes in v5
-  # @sled --exclude hw-kafka-client
-  - hw-kafka-client-4.0.3
-
 allow-newer: true
 allow-newer-deps:
   - hs-opentelemetry-propagator-datadog