Feature/classifier nn

api/neural_networks/product_quality_classifier/ann_app.py

@@ -0,0 +1,14 @@
+import os
+import requests
+
+from neural_networks.ann_app_base import AnnAppBase
+
+
+class AnnApp(AnnAppBase):
+
+    def process(self, path):
+        f_path = open(os.path.abspath(path), 'rb')
+        file = {'file': f_path}
+        quality = requests.post('http://localhost:8080/classify', files=file)
+
+        return quality.text, ''

api/neural_networks/product_quality_classifier/data/test_1.tsv

@@ -0,0 +1 @@
+

api/neural_networks/product_quality_classifier/data/test_2.tsv

@@ -0,0 +1 @@
+

api/neural_networks/product_quality_classifier/data/test_3.tsv

@@ -0,0 +1 @@
+

api/neural_networks/product_quality_classifier/data/test_4.tsv

@@ -0,0 +1 @@
+

api/neural_networks/product_quality_classifier/data/test_5.tsv

@@ -0,0 +1 @@
+

api/static/extensions.json

@@ -1,3 +1,4 @@
 {
-    "sequence_prediction": ["txt"]
-}
+    "sequence_prediction": ["txt"],
+    "product_quality_classifier": ["tsv"]
+}

api/static/properties.json

@@ -1,3 +1,4 @@
 {
-    "sequence_prediction": "Awesome ANN to predict your sequence future!"
-}
+    "sequence_prediction": "Awesome ANN to predict your sequence future!",
+    "product_quality_classifier": "Awesome ANN for defining quality of a product"
+}

kb/classifier/classifier_ann.scs

@@ -0,0 +1,8 @@
+product_quality_classifier_ann
+	=> nrel_main_idtf: [Product quality ANN] (* <-lang_en;; *);
+    => nrel_main_idtf: [ИНС для решения задачи классификации качества продукта] (* <-lang_ru;; *);
+    => nrel_idtf: [Product quality ANN];
+	=> nrel_api_idtf: [product_quality_classifier];
+	<- concept_neural_network_in_graphical_representation;
+    <- multilayer_neural_network;;
+

kb/classifier/classifier_concepts.scs

@@ -0,0 +1,17 @@
+sc_node_not_relation->concept_product;;
+concept_product=>nrel_main_idtf:[product](* <-lang_en;; *);;
+concept_product=>nrel_main_idtf:[продукт](* <-lang_ru;; *);;
+
+sc_node_not_relation->concept_quality_product;;
+concept_quality_product=>nrel_main_idtf:[quality product](* <-lang_en;; *);;
+concept_quality_product=>nrel_main_idtf:[качественный продукт](* <-lang_ru;; *);;
+
+sc_node_not_relation->concept_poor_quality_product;;
+concept_poor_quality_product=>nrel_main_idtf:[poor quality product](* <-lang_en;; *);;
+concept_poor_quality_product=>nrel_main_idtf:[некачественный продукт](* <-lang_ru;; *);;
+
+concept_product <= nrel_subdividing:
+    {
+  		concept_quality_product;
+		concept_poor_quality_product
+	};;

kb/classifier/classifier_relations.scs

@@ -0,0 +1,15 @@
+sc_node_norole_relation->nrel_uniformity;;
+nrel_uniformity=>nrel_main_idtf:[uniformity*](* <-lang_en;; *);;
+nrel_uniformity=>nrel_main_idtf:[однородность*](* <-lang_ru;; *);;
+
+sc_node_norole_relation->nrel_mass_fat_fraction;;
+nrel_mass_fat_fraction=>nrel_main_idtf:[mass fat fraction*](* <-lang_en;; *);;
+nrel_mass_fat_fraction=>nrel_main_idtf:[массовая доля жиров*](* <-lang_ru;; *);;
+
+sc_node_norole_relation->nrel_mass_protein_fraction;;
+nrel_mass_protein_fraction=>nrel_main_idtf:[mass protein fraction*](* <-lang_en;; *);;
+nrel_mass_protein_fraction=>nrel_main_idtf:[массовая доля белков*](* <-lang_ru;; *);;
+
+sc_node_norole_relation->nrel_has_mold;;
+nrel_has_mold=>nrel_main_idtf:[has mold*](* <-lang_en;; *);;
+nrel_has_mold=>nrel_main_idtf:[наличие плесени*](* <-lang_ru;; *);;

kb/classifier/classifier_test_files.scs

@@ -0,0 +1,53 @@
+test_1
+	=> nrel_main_idtf: [N1 Test file] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N1 Тестовый файл] (* <-lang_ru;; *);
+	=> nrel_idtf: [test_1.tsv];
+	=> nrel_file_extension: [tsv];
+    <- rrel_file: test_product_1;
+	=> nrel_file_name: [test_1];
+	<- concept_file;;
+
+test_2
+	=> nrel_main_idtf: [N2 Test file] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N2 Тестовый файл] (* <-lang_ru;; *);
+	=> nrel_idtf: [test_2.tsv];
+	=> nrel_file_extension: [tsv];
+    <- rrel_file: test_product_2;
+	=> nrel_file_name: [test_2];
+	<- concept_file;;
+
+test_3
+	=> nrel_main_idtf: [N3 Test file] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N3 Тестовый файл] (* <-lang_ru;; *);
+	=> nrel_idtf: [test_3.tsv];
+	=> nrel_file_extension: [tsv];
+    <- rrel_file: test_product_3;
+	=> nrel_file_name: [test_3];
+	<- concept_file;;
+
+test_4
+	=> nrel_main_idtf: [N4 Test file] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N4 Тестовый файл] (* <-lang_ru;; *);
+	=> nrel_idtf: [test_4.tsv];
+	=> nrel_file_extension: [tsv];
+    <- rrel_file: test_product_4;
+	=> nrel_file_name: [test_4];
+	<- concept_file;;
+
+test_5
+	=> nrel_main_idtf: [N5 Test file] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N5 Тестовый файл] (* <-lang_ru;; *);
+	=> nrel_idtf: [test_5.tsv];
+	=> nrel_file_extension: [tsv];
+    <- rrel_file: test_product_5;
+	=> nrel_file_name: [test_5];
+	<- concept_file;;
+
+test_invalid
+	=> nrel_main_idtf: [Invalid Test file] (* <-lang_en;; *);
+	=> nrel_main_idtf: [Некорректный тестовый файл] (* <-lang_ru;; *);
+	=> nrel_idtf: [test_invalid.tsv];
+	=> nrel_file_extension: [tsv];
+    <- rrel_file: test_product_invalid;
+	=> nrel_file_name: [test_invalid];
+	<- concept_file;;

kb/classifier/classifier_test_products.scs

@@ -0,0 +1,52 @@
+test_product_1
+	=> nrel_main_idtf: [N1 Test product] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N1 Тестовый продукт] (* <-lang_ru;; *);
+	=> nrel_uniformity: [0.898994959386161];
+    => nrel_mass_fat_fraction: [3.3163295173507312];
+	=> nrel_mass_protein_fraction: [6.650435535205284];
+	=> nrel_has_mold: [false];
+	<- concept_product;;
+
+test_product_2
+	=> nrel_main_idtf: [N2 Test product] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N2 Тестовый продукт] (* <-lang_ru;; *);
+	=> nrel_uniformity: [0.828086488920396];
+    => nrel_mass_fat_fraction: [5.393480385474044];
+	=> nrel_mass_protein_fraction: [36.31920921626641];
+	=> nrel_has_mold: [false];
+	<- concept_product;;
+
+test_product_3
+	=> nrel_main_idtf: [N3 Test product] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N3 Тестовый продукт] (* <-lang_ru;; *);
+	=> nrel_uniformity: [0.8094435163522983];
+    => nrel_mass_fat_fraction: [4.676179149431384];
+	=> nrel_mass_protein_fraction: [2.708277854780958];
+	=> nrel_has_mold: [false];
+	<- concept_product;;
+
+test_product_4
+	=> nrel_main_idtf: [N4 Test product] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N4 Тестовый продукт] (* <-lang_ru;; *);
+	=> nrel_uniformity: [0.8917947658875199];
+    => nrel_mass_fat_fraction: [4.736776647993627];
+	=> nrel_mass_protein_fraction: [17.966025265543564];
+	=> nrel_has_mold: [true];
+	<- concept_product;;
+
+test_product_5
+	=> nrel_main_idtf: [N5 Test product] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N5 Тестовый продукт] (* <-lang_ru;; *);
+	=> nrel_uniformity: [0.7891603438650896];
+    => nrel_mass_fat_fraction: [6.864936281507398];
+	=> nrel_mass_protein_fraction: [28.175836097673177];
+	=> nrel_has_mold: [true];
+	<- concept_product;;
+
+test_product_invalid
+	=> nrel_main_idtf: [N5 Test product] (* <-lang_en;; *);
+	=> nrel_main_idtf: [N5 Тестовый продукт] (* <-lang_ru;; *);
+    => nrel_mass_fat_fraction: [6.864936281507398];
+	=> nrel_mass_protein_fraction: [28.175836097673177];
+	=> nrel_has_mold: [true];
+	<- concept_product;;

problem-solver/CMakeLists.txt

@@ -1 +1,2 @@
 add_subdirectory(cxx)
+add_subdirectory(py)

problem-solver/py/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(ClassifierAnswerWriterAgent)
+add_subdirectory(DataProcessingAgent)

problem-solver/py/ClassifierAnswerWriterAgent/CMakeLists.txt

@@ -0,0 +1,10 @@
+file(GLOB_RECURSE SOURCES "*.cpp" "*.hpp")
+
+add_library (ClassifierAnswerWriterModule SHARED ${SOURCES})
+
+target_include_directories(ClassifierAnswerWriterModule PRIVATE ${SC_MEMORY_SRC})
+target_link_libraries(ClassifierAnswerWriterModule sc-memory)
+
+add_dependencies(ClassifierAnswerWriterModule sc-memory)
+
+sc_codegen(ClassifierAnswerWriterModule ${CMAKE_CURRENT_LIST_DIR})

problem-solver/py/ClassifierAnswerWriterAgent/cawModule.cpp

@@ -0,0 +1,23 @@
+/*
+* This source file is part of an OSTIS project. For the latest info, see http://ostis.net
+* Distributed under the MIT License
+* (See accompanying file COPYING.MIT or copy at http://opensource.org/licenses/MIT)
+*/
+
+#include "cawModule.hpp"
+
+SC_IMPLEMENT_MODULE(ClassifierAnswerWriterAgentModule)
+
+sc_result ClassifierAnswerWriterAgentModule::InitializeImpl()
+{
+  m_cawService.reset(new ClassifierAnswerWriterAgentPythonService("ClassifierAnswerWriterAgent/ClassifierAnswerWriterModule.py"));
+  m_cawService->Run();
+  return SC_RESULT_OK;
+}
+
+sc_result ClassifierAnswerWriterAgentModule::ShutdownImpl()
+{
+  m_cawService->Stop();
+  m_cawService.reset();
+  return SC_RESULT_OK;
+}

problem-solver/py/ClassifierAnswerWriterAgent/cawModule.hpp

@@ -0,0 +1,25 @@
+/*
+* This source file is part of an OSTIS project. For the latest info, see http://ostis.net
+* Distributed under the MIT License
+* (See accompanying file COPYING.MIT or copy at http://opensource.org/licenses/MIT)
+*/
+
+
+#pragma once
+
+#include "sc-memory/sc_module.hpp"
+#include "cawService.hpp"
+#include "cawModule.generated.hpp"
+
+
+class ClassifierAnswerWriterAgentModule : public ScModule
+{
+  SC_CLASS(LoadOrder(1000))
+  SC_GENERATED_BODY()
+
+  virtual sc_result InitializeImpl() override;
+  virtual sc_result ShutdownImpl() override;
+
+private:
+  std::unique_ptr<ClassifierAnswerWriterAgentPythonService> m_cawService;
+};

problem-solver/py/ClassifierAnswerWriterAgent/cawService.hpp

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "sc-memory/python/sc_python_service.hpp"
+
+PYTHON_DECLARE_SERVICE(ClassifierAnswerWriterAgent)

problem-solver/py/DataProcessingAgent/CMakeLists.txt

@@ -0,0 +1,10 @@
+file(GLOB_RECURSE SOURCES "*.cpp" "*.hpp")
+
+add_library (DataProcessingModule SHARED ${SOURCES})
+
+target_include_directories(DataProcessingModule PRIVATE ${SC_MEMORY_SRC})
+target_link_libraries(DataProcessingModule sc-memory)
+
+add_dependencies(DataProcessingModule sc-memory)
+
+sc_codegen(DataProcessingModule ${CMAKE_CURRENT_LIST_DIR})

problem-solver/py/DataProcessingAgent/dpModule.cpp

@@ -0,0 +1,23 @@
+/*
+* This source file is part of an OSTIS project. For the latest info, see http://ostis.net
+* Distributed under the MIT License
+* (See accompanying file COPYING.MIT or copy at http://opensource.org/licenses/MIT)
+*/
+
+#include "dpModule.hpp"
+
+SC_IMPLEMENT_MODULE(DataProcessingAgentModule)
+
+sc_result DataProcessingAgentModule::InitializeImpl()
+{
+  m_dpService.reset(new DataProcessingAgentPythonService("DataProcessingAgent/DataProcessingModule.py"));
+  m_dpService->Run();
+  return SC_RESULT_OK;
+}
+
+sc_result DataProcessingAgentModule::ShutdownImpl()
+{
+  m_dpService->Stop();
+  m_dpService.reset();
+  return SC_RESULT_OK;
+}

problem-solver/py/DataProcessingAgent/dpModule.hpp

@@ -0,0 +1,25 @@
+/*
+* This source file is part of an OSTIS project. For the latest info, see http://ostis.net
+* Distributed under the MIT License
+* (See accompanying file COPYING.MIT or copy at http://opensource.org/licenses/MIT)
+*/
+
+
+#pragma once
+
+#include "sc-memory/sc_module.hpp"
+#include "dpService.hpp"
+#include "dpModule.generated.hpp"
+
+
+class DataProcessingAgentModule : public ScModule
+{
+  SC_CLASS(LoadOrder(1000))
+  SC_GENERATED_BODY()
+
+  virtual sc_result InitializeImpl() override;
+  virtual sc_result ShutdownImpl() override;
+
+private:
+  std::unique_ptr<DataProcessingAgentPythonService> m_dpService;
+};

problem-solver/py/DataProcessingAgent/dpService.hpp

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "sc-memory/python/sc_python_service.hpp"
+
+PYTHON_DECLARE_SERVICE(DataProcessingAgent)

problem-solver/py/services/ClassifierAnswerWriterAgent/ClassifierAnswerWriterAgent.py

@@ -0,0 +1,55 @@
+from common import ScResult, ScAgent, ScEventParams
+from sc import *
+
+
+class ClassifierAnswerWriterAgent(ScAgent):
+
+    def CheckImpl(self, evt: ScEventParams) -> bool:
+        answer_addr = ""
+
+        src, trg = self.module.ctx.GetEdgeInfo(evt.edge_addr)
+        result_addr = self.module.ctx.HelperResolveSystemIdtf("nrel_processing_result", ScType.NodeConstNoRole)
+
+        it5 = self.module.ctx.Iterator5(
+            src, ScType.EdgeDCommonConst, ScType.Link, ScType.EdgeAccessConstPosPerm, result_addr)
+        while it5.Next():
+            answer_addr = it5.Get(2)
+
+        if answer_addr == "":
+            return False
+
+        # TODO: move back to True after bug described below will be fixed
+        return False
+
+    def RunImpl(self, evt: ScEventParams) -> ScResult:
+        answer = ""
+        test_product_addr = ""
+        edge_addr = ""
+
+        result_addr = self.module.ctx.HelperResolveSystemIdtf("nrel_processing_result", ScType.NodeConstNoRole)
+
+        it5 = self.module.ctx.Iterator5(
+            evt.addr, ScType.EdgeDCommonConst, ScType.Link, ScType.EdgeAccessConstPosPerm, result_addr)
+        while it5.Next():
+            addr2 = it5.Get(2)
+            answer = self.module.ctx.GetLinkContent(addr2).AsString()  # TODO: fix getting link content
+
+        if answer == "":
+            print("Answer is not found")
+            return ScResult.ErrorNotFound
+
+        quality_idtf = "quality_product" if "GOOD" in answer else "poor_quality_product"
+        print(quality_idtf)
+
+        it3 = self.module.ctx.Iterator3(
+            evt.addr, ScType.EdgeAccessConstPosPerm, ScType.NodeConst)
+        while it3.Next():
+            edge_addr = it3.Get(1)
+            test_product_addr = it3.Get(2)
+
+        self.module.ctx.CreateEdge(ScType.EdgeAccessConstPosPerm, quality_idtf, test_product_addr)
+        print('Answer was successfully processed')
+
+        # Clean up
+        self.module.ctx.DeleteElement(edge_addr)
+        return ScResult.Ok