Skip to content

Commit

Permalink
fix(kafka-connect): Setup connectors
Browse files Browse the repository at this point in the history
  • Loading branch information
Alexnortung committed Oct 31, 2024
1 parent 6779d30 commit c88fd43
Showing 1 changed file with 5 additions and 22 deletions.
27 changes: 5 additions & 22 deletions src/modules/services/kafka-connect.nix
Original file line number Diff line number Diff line change
Expand Up @@ -74,33 +74,16 @@ in
${lib.optionalString (lib.lists.length cfg.pluginDirectories > 0) "plugin.path=${lib.concatStringsSep "," cfg.pluginDirectories}"}
'';

# Create a json file for each connector
connectorFiles = lib.lists.map (c: pkgs.writeText "connector.json" (builtins.toJSON c)) cfg.initialConnectors;
connectorFilesConcatted = lib.concatStringsSep " " connectorFiles;

startKafkaConnect = pkgs.writeShellScriptBin "start-kafka-connect" ''
mkdir -p ${stateDir}
${pkg}/bin/connect-standalone.sh ${configFile}
'';

firstListener = if lib.length cfg.listeners > 0 then (lib.lists.head cfg.listeners) else "http://localhost:8083";
# initialConnectorNames = lib.lists.map (c: c.name) cfg.initialConnectors;

/**
* @param {string} connector config as JSON string
*/
setupConnector = pkgs.writeShellScriptBin "setup-connector" ''
${pkgs.curl}/bin/curl \
-X POST \
-H "Content-Type: application/json" \
--data $1 \
${firstListener}/connectors
'';

setupConnectorsCommands = lib.lists.map (c: "${setupConnector}/bin/setup-connector '${builtins.toJSON c}'") cfg.initialConnectors;
setupConnectors = pkgs.writeShellScriptBin "setup-connectors" ''
${lib.concatStringsSep "\n" setupConnectorsCommands}
${pkg}/bin/connect-standalone.sh ${configFile} ${connectorFilesConcatted}
'';
in
lib.mkIf cfg.enable (lib.mkIf kafkaCfg.enable {
processes.kafka-connect.exec = "${startKafkaConnect}/bin/start-kafka-connect";
# TODO: Make this process run when kafka connect is ready
processes.kafka-connect-setup.exec = "${setupConnectors}/bin/setup-connectors";
});
}

0 comments on commit c88fd43

Please sign in to comment.