Merge pull request #55 from stefanDeveloper/simplify-logserver-and-co…

…llector

Simplify LogServer, Collector and KafkaHandlers

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 *.py[cod]
 *$py.class
 *.pkl
+
 # C extensions
 *.so
 *.html
@@ -320,6 +321,7 @@ dmypy.json
 cython_debug/
 
 # End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks
-/docs/api/
+
+# Others
+docs/api/
 !/docs/api/index.rst
-/pipeline_prototype/

config.yaml

@@ -18,7 +18,6 @@ logging:
 pipeline:
   log_storage:
     logserver:
-      input_kafka_topic: "LogServer"
       input_file: "/opt/file.txt"
       max_number_of_connections: 1000
 
@@ -74,7 +73,10 @@ environment:
       port: 8098
     - hostname: 172.27.0.5
       port: 8099
-  logserver:
-    hostname: 172.27.0.8
-    port_in: 9998
-    port_out: 9999
+  kafka_topics:
+    pipeline:
+      logserver_in: "pipeline.logserver_in"
+      logserver_to_collector: "pipeline.logserver_to_collector"
+      batch_sender_to_prefilter: "pipeline.batch_sender_to_prefilter"
+      prefilter_to_inspector: "pipeline.prefilter_to_inspector"
+      inspector_to_detector: "pipeline.inspector_to_detector"

docker/docker-compose.yml

@@ -47,8 +47,6 @@ services:
         condition: service_healthy
       clickhouse-server:
         condition: service_healthy
-    ports:
-      - 9998:9998
     networks:
       heidgaf:
         ipv4_address: 172.27.0.8
@@ -168,7 +166,7 @@ services:
           devices:
             - driver: nvidia
               count: 1 # alternatively, use `count: all` for all GPUs
-              capabilities: [gpu]
+              capabilities: [ gpu ]
 
   clickhouse-server:
     image: clickhouse/clickhouse-server:24.3.12.75-alpine

docker/dockerfiles/Dockerfile.logserver

@@ -5,7 +5,7 @@ ENV PYTHONDONTWRITEBYTECODE=1
 WORKDIR /usr/src/app
 
 COPY requirements/requirements.logserver.txt ./
-RUN pip --disable-pip-version-check install --no-cache-dir --no-compile  -r requirements.logserver.txt
+RUN pip --disable-pip-version-check install --no-cache-dir --no-compile -r requirements.logserver.txt
 
 COPY src/base ./src/base
 COPY src/logserver ./src/logserver

requirements/requirements.detector.txt

@@ -3,6 +3,5 @@ scikit-learn~=1.5.2
 requests
 colorlog~=6.8.2
 PyYAML~=6.0.1
-colorlog~=6.8.2
 confluent-kafka~=2.4.0
 marshmallow_dataclass~=8.7.1

setup.cfg

@@ -6,6 +6,7 @@ omit =
     tests/*
     version.py
     src/train/*
+    *sandbox.py
 
 [coverage:report]
 show_missing = True