Merge pull request #5 from bsuryadevara/bhargav_integrated_training_716

Bhargav integrated training 716

examples/digital_fingerprinting/demo/README.md

@@ -1,7 +1,8 @@
+### GUI Setup for Submitting Control Messages
 
 #### Kafka Setup
 
-Start Kafka service.
+Start Kafka service to publish control messages to kafka topic
 
 ```
 cd ~/examples/digital_fingerprinting/production
@@ -10,30 +11,43 @@ docker-compose up kafka zookeeper
 ```
 
 ##### Create Kafka Topic
+
+Create Kafka topic `test_cm` to submit control messages from `cm_app`.
 ```
 docker exec -it kafka kafka-topics --create --topic test_cm --bootstrap-server localhost:9092
 ```
 
-Verify created topic is receiving messages. 
+Make sure the topic you created is getting messages.
 ```
 docker exec -it kafka kafka-console-consumer --topic test_cm --from-beginning --bootstrap-server localhost:9092
 ```
 
 #### Flask Server Setup
 
+Install flask python module to run the demo.
+
 ```
 pip install flask
 ```
 
+Navigate to the bin directory and execute start script.
 ```
 cd ~/examples/digital_fingerprinting/demo/bin
 
 bash start.sh
 ```
 
 #### Endpoint URL's
+Flexibility to demonstrate the range of control message creation options.
 ```
 http://localhost:3000
+```
+Generates control messages for training purposes exclusively with some user-specified parameters.
+```
 http://localhost:3000/training
 ```
 
+Submit training messages after reviewing inference results
+```
+http://localhost:3000/review/results
+```

examples/digital_fingerprinting/demo/cm_app/__init__.py

@@ -1,2 +1,3 @@
 import flask
+
 app = flask.Flask(__name__)

examples/digital_fingerprinting/demo/cm_app/helper.py

@@ -1,41 +1,50 @@
-from confluent_kafka import Producer
 import json
 import logging
 
-logging.basicConfig()
-logger = logging.getLogger("logger")
+logger = logging.getLogger(__name__)
 
-def delivery_report(err, msg):
-    """ Called once for each message produced to indicate delivery result.
-        Triggered by poll() or flush(). """
-    if err is not None:
-        print('Message delivery failed: {}'.format(err))
-    else:
-        print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition()))
 
+class KafkaWriter:
 
-def publish_message(message):
-    p = Producer({'bootstrap.servers': 'localhost:9092'})
+    def __init__(self, kafka_topic, batch_size, producer):
+        self._kafka_topic = kafka_topic
+        self._batch_size = batch_size
+        self._producer = producer
 
-    p.poll(0)
+    @property
+    def producer(self):
+        return self._producer
 
-    # Asynchronously produce a message. The delivery report callback will
-    # be triggered from the call to poll() above, or flush() below, when the
-    # message has been successfully delivered or failed permanently.
-    p.produce('test_cm', message.encode('utf-8'))
+    def write_data(self, message):
+        self.producer.produce(self._kafka_topic, message.encode('utf-8'))
+        if len(self.producer) >= self._batch_size:
+            logger.info(
+                "Batch reached, calling poll... producer unsent: %s",
+                len(self.producer),
+            )
+            self.producer.flush()
+
+    def close(self):
+        logger.info("Closing kafka writer...")
+        if self.producer is not None:
+            self.producer.flush()
+        logger.info("Closing kafka writer...Done")
 
-    # Wait for any outstanding messages to be delivered and delivery report
-    # callbacks to be triggered.
-    p.flush()
 
 def process_cm(request):
     control_messages_json = request.form.get("control-messages-json")
-    publish_message(control_messages_json)
-    logging.error(control_messages_json)
-    data = {
-        "status": "Successfully published task to kafka topic.",
+
+    logging.info("Received control message: {}".format(control_messages_json))
+
+    return control_messages_json
+
+
+def generate_success_message(control_messages_json):
+    sucess_message = {
+        "status": "Successfully published control message to kafka topic.",
         "status_code": 200,
         "control_messages": json.loads(control_messages_json)
     }
-    data = json.dumps(data, indent=4)
-    return data
+
+    sucess_message = json.dumps(sucess_message, indent=4)
+    return sucess_message

examples/digital_fingerprinting/demo/cm_app/static/review/results.css

@@ -0,0 +1,73 @@
+body {
+    font-family: Arial, sans-serif;
+    margin: 0;
+    padding: 0;
+  }
+  form {
+    display: flex;
+    justify-content: flex-start;
+    align-items: center;
+    margin-left: 20px;
+}
+  input[type="file"] {
+    margin-top: 10px;
+    display: block;
+    margin-bottom: 10px;
+  }
+
+  label {
+    display: block;
+    margin-bottom: 5px;
+    margin-left: 25px;
+  }
+  select {
+    display: block;
+    margin-bottom: -40px;
+    margin-left: 25px;
+    width: 30%;
+    height: 100px;
+    border: 1px solid #ccc;
+    padding: 5px;
+  }
+  button {
+    display: block;
+    margin: 25px;
+    padding: 10px 20px;
+    background-color: #8bb521;
+    color: black;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    display: inline-block;
+    margin-right: 0px;
+  }
+  button:hover {
+    background-color: #380e0ec1;
+  }
+  table {
+    border-collapse: collapse;
+    margin: 20px;
+    width: calc(100% - 40px);
+  }
+  th, td {
+    border: 1px solid #ccc;
+    padding: 5px;
+    text-align: left;
+  }
+  th {
+    background-color: #8bb521;
+    color: black;
+  }
+  .hidden {
+    display: none;
+  }
+  .marked {
+    background-color: rgb(243, 243, 161);
+  }
+  .alert {
+    background-color: #f8d7da;
+    border: 1px solid #f5c6cb;
+    color: #721c24;
+    padding: 10px;
+    margin-bottom: 10px;
+  }

examples/digital_fingerprinting/demo/cm_app/static/review/results.js

@@ -0,0 +1,186 @@
+let columnIndexes = [];
+
+// Function to read and display CSV file contents
+function handleFileSelect() {
+    const select = document.getElementById('columns');
+    select.innerHTML = '';
+    const file = document.getElementById('csv-file').files[0];
+    const reader = new FileReader();
+    reader.readAsText(file);
+    reader.onload = function(event) {
+        const csv = event.target.result;
+        const rows = csv.split('\n');
+        const table = document.createElement('table');
+        table.border = '1';
+        const headerRow = document.createElement('tr');
+        const headerCells = parseCSVRow(rows[0]);
+
+        // Create a new header cell for the checkboxes
+        const markHeaderCell = document.createElement('th');
+        markHeaderCell.innerHTML = 'Mark';
+        headerRow.appendChild(markHeaderCell);
+
+        for (let j = 0; j < headerCells.length; j++) {
+            const headerCell = document.createElement('th');
+            headerCell.innerHTML = headerCells[j];
+            headerRow.appendChild(headerCell);
+            const option = document.createElement('option');
+            option.value = j;
+            option.text = headerCells[j];
+            document.getElementById('columns').add(option);
+        }
+        table.appendChild(headerRow);
+        for (let i = 1; i < rows.length; i++) {
+            const cells = parseCSVRow(rows[i]);
+            // skip rows with no data
+            if (cells.length === 1 && cells[0] === '') {
+                continue;
+            }
+            const row = document.createElement('tr');
+
+            // Add checkbox as a cell in the first column
+            const checkbox = document.createElement('input');
+            checkbox.type = 'checkbox';
+            checkbox.value = i;
+            checkbox.onclick = toggleMark;
+            const markCell = document.createElement('td');
+            markCell.appendChild(checkbox);
+            row.appendChild(markCell);
+
+            for (let j = 0; j < cells.length; j++) {
+                const cell = document.createElement('td');
+                cell.innerHTML = cells[j];
+                row.appendChild(cell);
+            }
+            table.appendChild(row);
+        }
+        const csvContent = document.getElementById('csv-content');
+        csvContent.innerHTML = ''; // Clear existing content
+        csvContent.appendChild(table);
+    };
+}
+
+// Function to parse a row of CSV data, accounting for commas within double quotes
+function parseCSVRow(row) {
+    const cells = [];
+    let inQuotes = false;
+    let start = 0;
+    for (let i = 0; i < row.length; i++) {
+        if (row[i] === '"') {
+            inQuotes = !inQuotes;
+        } else if (row[i] === ',' && !inQuotes) {
+            cells.push(row.substring(start, i));
+            start = i + 1;
+        }
+    }
+    cells.push(row.substring(start));
+    return cells;
+}
+
+// Function to toggle column visibility
+function toggleColumn() {
+    const select = document.getElementById('columns');
+    columnIndexes = [];
+    for (let i = 0; i < select.options.length; i++) {
+        if (select.options[i].selected) {
+            columnIndexes.push(i);
+        }
+    }
+    const table = document.querySelector('table');
+    const cells = table.querySelectorAll('td,th');
+    for (let i = 0; i < cells.length; i++) {
+        const columnIndex = getColumnIndex(cells[i]);
+        if (columnIndexes.includes(columnIndex)) {
+            cells[i].classList.remove('hidden');
+        } else {
+            cells[i].classList.add('hidden');
+        }
+    }
+}
+// Function to get the index of the column containing a cell
+
+
+// Function to toggle row mark
+function toggleMark() {
+    const row = this.parentNode.parentNode;
+    if (this.checked) {
+        row.classList.add('marked');
+    } else {
+        row.classList.remove('marked');
+    }
+}
+
+function getColumnIndex(cell) {
+    const row = cell.parentNode;
+    const headerRow = row.parentNode.querySelector('tr:first-child');
+    const cells = row.querySelectorAll('td,th');
+    for (let i = 0; i < cells.length; i++) {
+        if (cells[i] === cell) {
+            return i - 1;
+        }
+    }
+    return -1;
+}
+
+function saveMarked() {
+    // Check if any rows have been marked
+    if ($('#csv-content tr.marked').length === 0) {
+        alert('Please select at least one row before saving.');
+        return;
+    }
+    // Ask user for filename
+    const filename = prompt("Enter filename:");
+
+    const table = document.querySelector('table');
+    const rows = table.querySelectorAll('tr:not(:first-child)');
+    const selectedRows = [];
+    for (let i = 0; i < rows.length; i++) {
+        if (rows[i].classList.contains('marked')) {
+            const cells = rows[i].querySelectorAll('td:not(:first-child),th:not(:first-child)');
+            const selectedCells = [];
+            for (let j = 0; j < cells.length; j++) {
+                const columnIndex = getColumnIndex(cells[j]);
+                if (columnIndexes.includes(columnIndex)) {
+                    selectedCells.push(cells[j].innerHTML);
+                }
+            }
+            selectedRows.push(selectedCells.join(','));
+        }
+    }
+
+    // Add header to CSV
+    const headerCells = [];
+    const select = document.getElementById('columns');
+    for (let i = 0; i < select.options.length; i++) {
+        if (select.options[i].selected) {
+            headerCells.push(select.options[i].text);
+        }
+    }
+    const header = headerCells.join(',');
+    const csvContent = 'data:text/csv;charset=utf-8,' + header + '\n' + selectedRows.join('\n');
+
+    // Create a link and click it to download the CSV file
+    const encodedUri = encodeURI(csvContent);
+    const link = document.createElement('a');
+    link.setAttribute('href', encodedUri);
+    link.setAttribute('download', `${filename}.csv`);
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+
+    // Display message to navigate to download location and copy file location manually
+    alert(`The file "${filename}.csv" has been downloaded. Please navigate to the download location and copy the file location manually.`);
+
+    $('#submit-button').prop('disabled', false);
+  }
+
+  function submitTrainingMessage() {
+    // Make HTTP GET request
+    $.ajax({
+        url: "/training",
+        method: "GET",
+        success: function(response) {
+            window.location.href = "/training";
+        }
+    });
+  }