From 86e5f8282669104bb3601ce4c011efa036de3889 Mon Sep 17 00:00:00 2001 From: Fabian Klopfer Date: Mon, 18 Nov 2019 14:04:32 +0100 Subject: [PATCH] Revert "started multithreadding, introduced a lot of bugs; added printing facilities to latex" This reverts commit cb35bc721c292c83c235f060a67e310e06208bc6. --- neo4j-concept/pom.xml | 128 +++-- neo4j-concept/src/main/cypher/import_ldbc.sh | 0 .../dbis/neo4j/conceptual/algos/Cobweb.java | 468 ++++++------------ .../neo4j/conceptual/algos/ConceptNode.java | 290 ++++------- .../neo4j/conceptual/algos/ConceptValue.java | 41 +- .../neo4j/conceptual/algos/NominalValue.java | 33 +- .../neo4j/conceptual/algos/NumericValue.java | 56 +-- .../conceptual/algos/PropertyGraphCobweb.java | 220 ++++---- .../dbis/neo4j/conceptual/algos/Value.java | 32 +- .../proc/PropertyGraphCobwebProc.java | 10 +- .../dbis/neo4j/conceptual/util/LockUtils.java | 57 --- .../dbis/neo4j/conceptual/util/MathUtils.java | 26 - .../neo4j/conceptual/util/PrintUtils.java | 158 ------ .../neo4j/conceptual/util/ResourceLock.java | 14 - .../dbis/neo4j/conceptual/util/TreeUtils.java | 74 --- .../PropertyGraphCobwebPerfTest.java | 11 +- .../PropertyGraphCobwebProcTest.java | 217 ++------ neo4j-eval/pom.xml | 53 +- .../dbis/neo4j/eval/datasets/CSVImporter.java | 3 + .../uni/dbis/neo4j/eval/datasets/Dataset.java | 1 + pmd_rules.xml | 56 --- pom.xml | 107 +--- 22 files changed, 565 insertions(+), 1490 deletions(-) mode change 100755 => 100644 neo4j-concept/src/main/cypher/import_ldbc.sh delete mode 100644 neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/LockUtils.java delete mode 100644 neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/MathUtils.java delete mode 100644 neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/PrintUtils.java delete mode 100644 neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/ResourceLock.java delete mode 100644 neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/TreeUtils.java delete mode 100644 pmd_rules.xml diff --git a/neo4j-concept/pom.xml b/neo4j-concept/pom.xml index bb95c2a..6b55d19 100644 --- a/neo4j-concept/pom.xml +++ b/neo4j-concept/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 @@ -20,14 +22,14 @@ - + UTF-8 8 ${java.version} ${java.version} benchmarks yyyy.MM.dd.HH.mm.ss - + 3.5.0 5.3.1 @@ -68,7 +70,7 @@ neo4j-eval 0.0.1-SNAPSHOT - + org.neo4j neo4j @@ -80,7 +82,7 @@ ${neo4j.version} - + org.neo4j graph-algorithms-algo @@ -97,7 +99,7 @@ 3.5.0.2 - + org.neo4j neo4j-io @@ -131,12 +133,7 @@ ${junit.jupiter.version} - - - com.google.guava - guava - 28.1-jre - + com.github.oshi oshi-core @@ -156,7 +153,7 @@ - compile + test maven-clean-plugin @@ -181,7 +178,7 @@ - + maven-surefire-plugin 2.22.0 @@ -226,13 +223,14 @@ ${uberjar.name} - + org.openjdk.jmh.Main - + *:* META-INF/*.SF @@ -245,6 +243,84 @@ + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.0 + + **/PropertyGraphCobwebProc.java + ${project.build.sourceDirectories} + true + false + ../checkstyle.xml + true + warning + + + + process-sources + + checkstyle + check + + + + + + org.codehaus.mojo + findbugs-maven-plugin + 3.0.4 + + max + Low + + + + process-classes + + findbugs + check + + + + + + com.github.spotbugs + spotbugs-maven-plugin + 3.1.12.2 + + max + Low + + + + process-classes + + spotbugs + check + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.12.0 + + 1.8 + true + true + + + + process-classes + + pmd + check + + + + org.apache.maven.plugins maven-jxr-plugin @@ -262,22 +338,4 @@ - - infer-capture - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - javac - true - true - /opt/infer-linux64-v0.17.0/lib/infer/infer/bin/infer - - - - - - + diff --git a/neo4j-concept/src/main/cypher/import_ldbc.sh b/neo4j-concept/src/main/cypher/import_ldbc.sh old mode 100755 new mode 100644 diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Cobweb.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Cobweb.java index f40327f..24d50d7 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Cobweb.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Cobweb.java @@ -1,80 +1,37 @@ package kn.uni.dbis.neo4j.conceptual.algos; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.logging.Logger; - -import javax.annotation.concurrent.ThreadSafe; - -import com.google.common.util.concurrent.AtomicDouble; - -import kn.uni.dbis.neo4j.conceptual.util.PrintUtils; -import kn.uni.dbis.neo4j.conceptual.util.ResourceLock; - -import static kn.uni.dbis.neo4j.conceptual.util.LockUtils.lockAll; /** * Implementation of Douglas Fishers Cobweb. * * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> + * */ -@ThreadSafe -public final class Cobweb { - /** - * Logger. - */ - static final Logger LOG = Logger.getLogger("PropertyGraphCobweb"); - - /** - * hidden default constructor. - */ +public class Cobweb { private Cobweb() { // NOOP } -/* - - * @param threadPool used to execute probing - , - final ExecutorService threadPool - try { - final Future createFuture = threadPool.submit(() -> createNewNodeCU(currentNode, newNode)); - final Result findResult = threadPool.submit(() ->findHost(currentNode, newNode)).get(); - final ConceptNode host = findResult.getNode(); - final Future mergeFuture = threadPool.submit(() -> mergeNodesCU(currentNode, host, newNode)); - final Future splitFuture = threadPool.submit(() -> splitNodesCU(host, newNode)); - final double[] results = {findResult.getCu(), createFuture.get(), mergeFuture.get(), splitFuture.get()}; - - - // By default take create new as standard action if no other is better - double best = results[1]; - int bestIdx = 1; - for (int i = 0; i < results.length; i++) { - if (results[i] > best) { - best = results[i]; - bestIdx = i; - } - } -*/ /** * run the actual cobweb algorithm. * - * @param newNode node to incorporate - * @param currentNode node currently visiting + * @param newNode node to incorporate + * @param currentNode node currently visiting */ public static void cobweb(final ConceptNode newNode, final ConceptNode currentNode) { - final Result findResult = findHost(currentNode, newNode); - final ConceptNode host = findResult.getNode(); - final double[] results = {findResult.getCu(), createNewNodeCU(currentNode, newNode), - mergeNodesCU(currentNode, host, newNode), splitNodesCU(host, newNode)}; + final double[] results = new double[4]; + + final Result result = findHost(currentNode, newNode); + results[0] = result.cu; + final ConceptNode host = result.node; + + results[1] = createNewNodeCU(currentNode, newNode); + results[2] = mergeNodesCU(currentNode, host, newNode); + results[3] = splitNodesCU(host, newNode); // By default take create new as standard action if no other is better double best = results[1]; @@ -89,7 +46,6 @@ public static void cobweb(final ConceptNode newNode, final ConceptNode currentNo switch (bestIdx) { case 1: createNewNode(currentNode, newNode, true); - LOG.info("Create"); break; case 2: final ConceptNode mergedNode = mergeNodes(currentNode, host, newNode, true); @@ -97,16 +53,13 @@ public static void cobweb(final ConceptNode newNode, final ConceptNode currentNo throw new RuntimeException("Unreachable"); } currentNode.updateCounts(newNode); - LOG.info("Merge"); cobweb(newNode, mergedNode); break; case 3: - LOG.info("Split"); splitNodes(host, currentNode, true); cobweb(newNode, currentNode); break; case 0: - LOG.info("Recurse"); currentNode.updateCounts(newNode); cobweb(newNode, host); break; @@ -118,46 +71,34 @@ public static void cobweb(final ConceptNode newNode, final ConceptNode currentNo /** * finds the child most suitable to host the new node. * - * @param currentNode node currently visiting - * @param newNode node to be hosted + * @param parent node currently visiting + * @param newNode node to be hosted * @return the best host and the cu for that */ - private static Result findHost(final ConceptNode currentNode, final ConceptNode newNode) { + private static Result findHost(final ConceptNode parent, final ConceptNode newNode) { double curCU; double maxCU = Integer.MIN_VALUE; - int count = 0; + int i = 0; ConceptNode clone; ConceptNode best = null; + final ConceptNode parentTemp = new ConceptNode(parent); + parentTemp.updateCounts(newNode); + ConceptNode parentClone; + final double parentEAP = getExpectedAttributePrediction(parent); - ArrayList locks = new ArrayList<>(); - locks.add(currentNode.getLock().readLock()); - locks.add(newNode.getLock().readLock()); - for (ConceptNode child : currentNode.getChildren()) { - locks.add(child.getLock().readLock()); - } - - try (ResourceLock ignored = lockAll(locks)) { - final ConceptNode currentNodeTemp = new ConceptNode(currentNode); - currentNodeTemp.updateCounts(newNode); - ConceptNode currentNodeClone; - final double currentNodeEAP = getExpectedAttributePrediction(currentNode); - - for (final ConceptNode child : currentNode.getChildren()) { - clone = new ConceptNode(child); - clone.updateCounts(newNode); - - currentNodeClone = new ConceptNode(currentNodeTemp); + for (ConceptNode child : parent.getChildren()) { + clone = new ConceptNode(child); + clone.updateCounts(newNode); - currentNodeClone.setChild(count, clone); + parentClone = new ConceptNode(parentTemp); + parentClone.getChildren().set(i, clone); - curCU = computeCU(currentNodeClone, currentNodeEAP); - if (maxCU < curCU) { - maxCU = curCU; - best = child; - } - count++; + curCU = computeCU(parentClone, parentEAP); + if (maxCU < curCU) { + maxCU = curCU; + best = child; } - + i++; } return new Result(maxCU, best); } @@ -170,20 +111,18 @@ private static Result findHost(final ConceptNode currentNode, final ConceptNode * @return the op result including the cu and the altered node */ private static double createNewNodeCU(final ConceptNode currentNode, final ConceptNode newNode) { - final double cu; - try (ResourceLock ignored = lockAll(currentNode.getParent().getLock().readLock(), - currentNode.getLock().readLock())) { - - final ConceptNode clone = new ConceptNode(currentNode); - if (currentNode.getId() != null) { - final ConceptNode parentClone = new ConceptNode(currentNode.getParent()); - clone.setParent(parentClone); - } - clone.updateCounts(newNode); - createNewNode(clone, newNode, false); - cu = currentNode.getId() == null ? computeCU(clone) : computeCU(clone.getParent()); + final ConceptNode clone = new ConceptNode(currentNode); + if (currentNode.getId() != null) { + final ConceptNode parentClone = new ConceptNode(currentNode.getParent()); + clone.setParent(parentClone); + } + clone.updateCounts(newNode); + createNewNode(clone, newNode, false); + if (currentNode.getId() != null) { + return computeCU(clone.getParent()); + } else { + return computeCU(clone); } - return cu; } /** @@ -191,123 +130,72 @@ private static double createNewNodeCU(final ConceptNode currentNode, final Conce * * @param currentNode not to add the child to * @param newNode to to be added - * @param setParent flag indicating weather to set the parent pointers (false when computing the cu in order not to - * alter the tree when probing an action) + * @param setParent flag indicating weather to set the parent pointers (false when computing the cu in order not to + * alter the tree when probing an action) */ - private static void createNewNode(final ConceptNode currentNode, final ConceptNode newNode, - final boolean setParent) { - final int holdCount = currentNode.getParent().getLock().getReadHoldCount(); - if (!setParent) { - for (int i = 0; i < holdCount; i++) { - currentNode.getParent().getLock().readLock().unlock(); - } - } - // FIXME create fails; CU may be /is wrong - try (ResourceLock ignored = lockAll(currentNode.getParent().getLock().writeLock(), - currentNode.getLock().writeLock(), newNode.getLock().writeLock())) { - if (currentNode.getId() != null) { - final ConceptNode parent = currentNode.getParent(); - // we are in a leaf node. leaf nodes are concrete data instances and shall stay leaves - // remove the leaf from it's current parent - parent.removeChild(currentNode); - // make a new node containing the same count and attributes - final ConceptNode conceptNode = new ConceptNode(currentNode); - // add the new node as intermediate between the current leaf and its parent - parent.addChild(conceptNode); - // set the intermediate nodes id to null as its an inner node - conceptNode.setId(null); - // update the attribute counts to incorporate the new node - conceptNode.updateCounts(newNode); - // add the leaf and the new node as children - conceptNode.addChild(newNode); - conceptNode.addChild(currentNode); - currentNode.setParent(conceptNode); - if (setParent) { - newNode.setParent(conceptNode); - } - - } else { - currentNode.updateCounts(newNode); - currentNode.addChild(newNode); - if (setParent) { - newNode.setParent(currentNode); - } + private static void createNewNode(final ConceptNode currentNode, final ConceptNode newNode, final boolean setParent) { + if (currentNode.getId() != null) { + // we are in a leaf node. leaf nodes are concrete data instances and shall stay leaves + // remove the leaf from it's current parent + currentNode.getParent().getChildren().remove(currentNode); + // make a new node containing the same count and attributes + final ConceptNode conceptNode = new ConceptNode(currentNode); + // add the new node as intermediate between the current leaf and its parent + currentNode.getParent().addChild(conceptNode); + // set the intermediate nodes id to null as its an inner node + conceptNode.setId(null); + // update the attribute counts to incorporate the new node + conceptNode.updateCounts(newNode); + // add the leaf and the new node as children + conceptNode.addChild(newNode); + conceptNode.addChild(currentNode); + currentNode.setParent(conceptNode); + if (setParent) { + newNode.setParent(conceptNode); } - } - if (!setParent) { - for (int i = 0; i < holdCount; i++) { - currentNode.getParent().getLock().readLock().lock(); + } else { + currentNode.updateCounts(newNode); + currentNode.addChild(newNode); + if (setParent) { + newNode.setParent(currentNode); } } - } /** * clones the current node, splits & computes the cu. * - * @param host node to be split - * @param currentNode node to append children of host + * @param host node to be split + * @param current node to append children of host * @return op result including altered node and the cu */ - private static double splitNodesCU(final ConceptNode host, final ConceptNode currentNode) { - if (host == null) { - return Integer.MIN_VALUE; - } - final double cu; - try (ResourceLock ignored = lockAll(currentNode.getLock().readLock(), host.getLock().readLock())) { - if (host.getChildren().size() == 0) { - return Integer.MIN_VALUE; - } - final ConceptNode clone = new ConceptNode(currentNode); - splitNodes(host, clone, false); - cu = computeCU(clone); - } - return cu; + private static double splitNodesCU(final ConceptNode host, final ConceptNode current) { + final ConceptNode clone = new ConceptNode(current); + splitNodes(host, clone, false); + return computeCU(clone); } /** * splits the host node and appends its children to the current node. * - * @param host node to be split - * @param current node to append children of host + * @param host node to be split + * @param current node to append children of host * @param setParent flag indicating weather to set the parent pointers (false when computing the cu in order not to - * alter the tree when probing an action) + * alter the tree when probing an action) */ - private static void splitNodes(final ConceptNode host, final ConceptNode current, - final boolean setParent) { + private static void splitNodes(final ConceptNode host, final ConceptNode current, final boolean setParent) { if (host == null) { return; } - - ArrayList locks = new ArrayList<>(); - locks.add(current.getLock().writeLock()); - if (setParent) { - locks.add(host.getLock().writeLock()); - for (ConceptNode child : host.getChildren()) { - locks.add(child.getLock().writeLock()); - } - } else { - locks.add(host.getLock().readLock()); - for (ConceptNode child : host.getChildren()) { - locks.add(child.getLock().readLock()); - } - } - - try (ResourceLock ignored = lockAll(locks)) { - for (final ConceptNode child : host.getChildren()) { - if (setParent) { - child.setParent(current); - current.addChild(child); - } else { - current.addChild(child); - } + for (ConceptNode child : host.getChildren()) { + if (setParent) { + child.setParent(current); } - current.removeChild(host); + current.addChild(child); } + current.getChildren().remove(host); if (setParent) { - try (ResourceLock ignored1 = lockAll(host.getLock().writeLock())) { - host.setParent(null); - } + host.setParent(null); } } @@ -319,79 +207,51 @@ private static void splitNodes(final ConceptNode host, final ConceptNode current * @param newNode node to be incorporated * @return op result including altered node and the cu */ - private static double mergeNodesCU(final ConceptNode current, final ConceptNode host, - final ConceptNode newNode) { - final double cu; - try (ResourceLock ignored = lockAll(current.getLock().readLock())) { - if (current.getChildren().size() < 2) { - return Integer.MIN_VALUE; - } - final ConceptNode clonedParent = new ConceptNode(current); - cu = (mergeNodes(clonedParent, host, newNode, false) != null) ? computeCU(clonedParent) - : Integer.MIN_VALUE; - } - return cu; + private static double mergeNodesCU(final ConceptNode current, final ConceptNode host, final ConceptNode newNode) { + final ConceptNode clonedParent = new ConceptNode(current); + return (mergeNodes(clonedParent, host, newNode, false) != null) ? computeCU(clonedParent) + : Integer.MIN_VALUE; } /** * merges the host node with the next best host and appends its children to the resulting node. * - * @param host node to be merged - * @param current parent of the host - * @param newNode node to be incorporated + * @param host node to be merged + * @param current parent of the host + * @param newNode node to be incorporated * @param setParent flag indicating weather to set the parent pointers (false when computing the cu in order not to * alter the tree when probing an action) * @return the node that results by the merge. */ - private static ConceptNode mergeNodes(final ConceptNode current, final ConceptNode host, - final ConceptNode newNode, final boolean setParent) { + private static ConceptNode mergeNodes(final ConceptNode current, final ConceptNode host, final ConceptNode newNode, + final boolean setParent) { if (host == null) { return null; } - final ConceptNode secondHost; - final ConceptNode mNode; + current.getChildren().remove(host); - ArrayList locks = new ArrayList<>(); - locks.add(current.getLock().writeLock()); + final Result secondHost = findHost(current, newNode); + if (secondHost.node == null) { + return null; + } + current.getChildren().remove(secondHost.node); if (setParent) { - locks.add(host.getLock().writeLock()); - } else { - locks.add(host.getLock().readLock()); + secondHost.node.setParent(null); + host.setParent(null); } - locks.add(newNode.getLock().readLock()); - - try (ResourceLock ignored1 = lockAll(locks)) { - - current.removeChild(host); - secondHost = findHost(current, newNode).getNode(); - if (secondHost == null) { - return null; - } - locks.clear(); - if (setParent) { - locks.add(secondHost.getLock().writeLock()); - } else { - locks.add(secondHost.getLock().readLock()); - } - try (ResourceLock ignored2 = lockAll(locks)) { - current.removeChild(secondHost); - - mNode = new ConceptNode(host); - mNode.clearChildren(); - mNode.setId(null); - mNode.updateCounts(secondHost); - mNode.addChild(host); - mNode.addChild(secondHost); - - if (setParent) { - host.setParent(mNode); - secondHost.setParent(mNode); - mNode.setParent(current); - current.addChild(mNode); - } - } + final ConceptNode mNode = new ConceptNode(host); + mNode.getChildren().clear(); + mNode.setId(null); + mNode.updateCounts(secondHost.node); + mNode.addChild(host); + mNode.addChild(secondHost.node); + if (setParent) { + host.setParent(mNode); + secondHost.node.setParent(mNode); + mNode.setParent(current); } + current.addChild(mNode); return mNode; } @@ -406,12 +266,8 @@ private static double computeCU(final ConceptNode parent) { if (parent.getChildren().size() == 0) { return 0; } - final double cu; - try (ResourceLock ignored = lockAll(parent.getLock().readLock())) { - final double parentEAP = getExpectedAttributePrediction(parent); - cu = computeCU(parent, parentEAP); - } - return cu; + final double parentEAP = getExpectedAttributePrediction(parent); + return computeCU(parent, parentEAP); } /** @@ -422,27 +278,17 @@ private static double computeCU(final ConceptNode parent) { * @return the category utility */ private static double computeCU(final ConceptNode parent, final double parentEAP) { - double cu = 0.0; final double parentChildCount = parent.getChildren().size(); if (parentChildCount == 0) { - return Integer.MIN_VALUE; + return 0; } - - ArrayList locks = new ArrayList<>(); - locks.add(parent.getLock().readLock()); + double cu = 0.0; + final double parentCount = parent.getCount(); for (ConceptNode child : parent.getChildren()) { - locks.add(child.getLock().readLock()); - } - - try (ResourceLock ignored = lockAll(locks)) { - final double parentCount = parent.getCount(); - for (final ConceptNode child : parent.getChildren()) { - cu += (double) child.getCount() / parentCount - * (getExpectedAttributePrediction(child) - parentEAP); - } - cu = cu / parentChildCount; + cu += (double) child.getCount() / parentCount + * (getExpectedAttributePrediction(child) - parentEAP); } - return cu; + return cu / parentChildCount; } /** @@ -452,34 +298,28 @@ private static double computeCU(final ConceptNode parent, final double parentEAP * @return the EAP */ private static double getExpectedAttributePrediction(final ConceptNode category) { - final double eap; - try (ResourceLock ignored = lockAll(category.getLock().readLock())) { - final double noAttributes = category.getAttributes().size(); - if (noAttributes == 0) { - return 0; - } + double exp = 0; + final double noAttributes = category.getAttributes().size(); + final double total = category.getCount(); + double intermediate; + NumericValue num; + + if (noAttributes == 0) { + return 0; + } - double exp = 0; - final double total = category.getCount(); - double intermediate; - NumericValue num; - - for (final Map.Entry> attrib : category.getAttributes().entrySet()) { - for (final Value val : attrib.getValue()) { - try (ResourceLock ignored1 = lockAll(val.getLock().readLock())) { - if (val instanceof NumericValue) { - num = (NumericValue) val; - exp += 1.0 / (num.getStd() / num.getMean() + 1) - 1; - } else { - intermediate = (double) val.getCount() / total; - exp += intermediate * intermediate; - } - } + for (Map.Entry> attrib : category.getAttributes().entrySet()) { + for (Value val : attrib.getValue()) { + if (val instanceof NumericValue) { + num = (NumericValue) val; + exp += 1.0 / (num.getStd()/num.getMean() + 1) - 1; + } else { + intermediate = (double) val.getCount() / total; + exp += intermediate * intermediate; } } - eap = exp / noAttributes; } - return eap; + return exp / noAttributes; } /** @@ -489,39 +329,21 @@ private static final class Result { /** * the cu that the operation yields. */ - private final AtomicDouble cu = new AtomicDouble(0); + private final double cu; /** * the node with the operation applied. */ - private final AtomicReference node = new AtomicReference<>(null); + private final ConceptNode node; /** * Constructor. * - * @param cu the cu that the operation yields. - * @param node the node with the operation applied. - */ - Result(final double cu, final ConceptNode node) { - this.cu.set(cu); - this.node.set(node); - } - - /** - * getter. - * - * @return the cu - */ - double getCu() { - return this.cu.get(); - } - - /** - * getter. - * - * @return the concept node + * @param cu the cu that the operation yields. + * @param node the node with the operation applied. */ - ConceptNode getNode() { - return this.node.get(); + private Result(final double cu, final ConceptNode node) { + this.cu = cu; + this.node = node; } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptNode.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptNode.java index 6737036..a5baded 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptNode.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptNode.java @@ -1,25 +1,16 @@ package kn.uni.dbis.neo4j.conceptual.algos; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.PropertyContainer; import org.neo4j.graphdb.Relationship; -import kn.uni.dbis.neo4j.conceptual.util.ResourceLock; - -import static kn.uni.dbis.neo4j.conceptual.util.LockUtils.lockAll; - /** * The basic data type for the conceptual hierarchy (tree) constructed and used by cobweb. * @@ -29,39 +20,35 @@ public class ConceptNode { /** * Attribute aggregation over all (sub-)instances of this Concept. */ - private final ConcurrentMap> attributes = new ConcurrentHashMap<>(); + private final Map> attributes; + /** * Number of Nodes under this ConceptNode. */ - private final AtomicInteger count = new AtomicInteger(1); + private int count; /** * Neo4j ID of the Incorporated node.. */ - private final AtomicReference id = new AtomicReference<>(null); - - /** - * Label of the node. - */ - private final AtomicReference label = new AtomicReference<>(null); + private String id; /** * The sub-concepts of this Concept. */ - private final List children = Collections.synchronizedList(new ArrayList<>()); + private ArrayList children; /** * The super-concept of this concept. */ - private final AtomicReference parent = new AtomicReference<>(null); + private ConceptNode parent; /** - * Lock used by the cobweb methods. - */ - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - /** - * Default constructor. + * Constructor. */ public ConceptNode() { + this.count = 1; + this.id = null; + this.attributes = new HashMap<>(); + this.children = new ArrayList<>(); + this.parent = null; } /** @@ -70,22 +57,24 @@ public ConceptNode() { * @param node ConceptNode to copy. */ public ConceptNode(final ConceptNode node) { + this.count = node.count; + this.id = node.id; + this.attributes = new HashMap<>(); + List values; String attributeName; - for (final ConcurrentMap.Entry> attribute : node.getAttributes().entrySet()) { + for (Map.Entry> attribute : node.attributes.entrySet()) { attributeName = attribute.getKey(); - values = Collections.synchronizedList(new ArrayList<>()); + values = new ArrayList<>(); this.attributes.put(attributeName, values); - for (final Value value : attribute.getValue()) { - try (ResourceLock ignored = lockAll(value.getLock().readLock())) { - values.add(value.copy()); - } + for (Value value : attribute.getValue()) { + values.add(value.copy()); } } - this.children.addAll(node.getChildren()); - - this.setParent(node.getParent()); + this.children = new ArrayList<>(); + this.children = new ArrayList<>(node.children); + this.parent = node.parent; } /** @@ -95,19 +84,23 @@ public ConceptNode(final ConceptNode node) { * @param propertyContainer The property container to parse. */ ConceptNode(final PropertyContainer propertyContainer) { - List values = Collections.synchronizedList(new ArrayList<>()); + this.count = 1; + this.attributes = new HashMap<>(); + this.children = new ArrayList<>(); + this.parent = null; + + List values = new ArrayList<>(); if (propertyContainer instanceof Relationship) { final Relationship rel = (Relationship) propertyContainer; - this.setId(Long.toString(rel.getId())); + this.id = Long.toString(rel.getId()); values.add(new NominalValue(rel.getType().name())); this.attributes.put("RelType", values); } else if (propertyContainer instanceof Node) { final Node mNode = (Node) propertyContainer; - this.setId(Long.toString(mNode.getId())); - for (final Label label : mNode.getLabels()) { + this.id = Long.toString(mNode.getId()); + for (Label label : mNode.getLabels()) { values.add(new NominalValue(label.name())); - this.attributes.put("Label", values); } } @@ -115,13 +108,13 @@ public ConceptNode(final ConceptNode node) { // loop over the properties of a N4J node and cast them to a Value Object o; Object[] arr; - for (final ConcurrentMap.Entry property : propertyContainer.getAllProperties().entrySet()) { - values = Collections.synchronizedList(new ArrayList<>()); + for (Map.Entry property : propertyContainer.getAllProperties().entrySet()) { + values = new ArrayList<>(); o = property.getValue(); if (o.getClass().isArray()) { arr = (Object[]) o; - for (final Object ob : arr) { + for (Object ob : arr) { values.add(Value.cast(ob)); } } else { @@ -131,52 +124,41 @@ public ConceptNode(final ConceptNode node) { } } - /** - * Sets the fields of a ConceptNode to be appropriate for a root. - * - * @return the transformed node - */ public ConceptNode root() { - try (ResourceLock ignored = lockAll(this.lock.writeLock())) { - this.setCount(0); - this.setParent(this); - } + this.count = 0; + this.parent = this; + return this; } /** * Aggregates the count and attributes of the nodes, hosted by this concept. * - * @param usedForUpdate the node to incorporate into the concept. + * @param node the node to incorporate into the concept. */ - public void updateCounts(final ConceptNode usedForUpdate) { + public void updateCounts(final ConceptNode node) { List thisValues; NumericValue thisNumeric; boolean matched; - try (ResourceLock ignored = lockAll(this.getLock().writeLock(), usedForUpdate.getLock().readLock())) { - - this.setCount(this.getCount() + usedForUpdate.getCount()); + this.count = this.count + node.count; // loop over the attributes of the node to incorporate - for (final ConcurrentMap.Entry> otherAttributes : usedForUpdate.getAttributes().entrySet()) { - thisValues = this.getAttributes().get(otherAttributes.getKey()); + for (Map.Entry> otherAttributes : node.getAttributes().entrySet()) { + thisValues = this.attributes.get(otherAttributes.getKey()); // If the attribute is present in this node, check against the present values if (thisValues != null) { - for (final Value otherValue : otherAttributes.getValue()) { - + for (Value otherValue : otherAttributes.getValue()) { // iterate over values if (otherValue instanceof NumericValue) { // When encountering a NumericValue, it must be matched with the one present in this node matched = false; - for (final Value thisVal : thisValues) { - try (ResourceLock ignored3 = lockAll(otherValue.getLock().readLock(), thisVal.getLock().writeLock())) { - // per attribute there is only one NumericValue that accumulates all numeric values. - if (thisVal instanceof NumericValue) { - thisNumeric = (NumericValue) thisVal; - thisNumeric.update(otherValue); - matched = true; - break; - } + for (Value thisVal : thisValues) { + // per attribute there is only one NumericValue that accumulates all numeric values. + if (thisVal instanceof NumericValue) { + thisNumeric = (NumericValue) thisVal; + thisNumeric.update(otherValue); + matched = true; + break; } } // When there is no NumericValue in this node for the attribute, add the one of the other node @@ -194,20 +176,15 @@ public void updateCounts(final ConceptNode usedForUpdate) { } } else { // Else add the attribute and it's properties of the new node as they are - final List copies = Collections.synchronizedList(new ArrayList<>()); - for (final Value value : otherAttributes.getValue()) { - try (ResourceLock ignored1 = lockAll(value.getLock().readLock())) { - copies.add(value.copy()); - } + final List copies = new ArrayList<>(); + for (Value value : otherAttributes.getValue()) { + copies.add(value.copy()); } - this.getAttributes().put(otherAttributes.getKey(), copies); + this.attributes.put(otherAttributes.getKey(), copies); } } - } } - - /** * Check if the given concept is a superconcept of the current. * @@ -215,12 +192,12 @@ public void updateCounts(final ConceptNode usedForUpdate) { * @return true if c is the parent of the node or one of it's parent. */ boolean isSuperConcept(final ConceptNode c) { - if (this.getParent() == this) { + if (this.parent == this) { return false; - } else if (this.getParent().equals(c)) { + } else if (this.parent.equals(c)) { return true; } else { - return this.getParent().isSuperConcept(c); + return this.parent.isSuperConcept(c); } } @@ -230,7 +207,7 @@ boolean isSuperConcept(final ConceptNode c) { * @return number of instances and sub-concepts hosted by this concept */ public int getCount() { - return this.count.get(); + return this.count; } /** @@ -238,8 +215,8 @@ public int getCount() { * * @param count number of instances and sub-concepts hosted by this concept */ - private void setCount(final int count) { - this.count.set(count); + void setCount(final int count) { + this.count = count; } /** @@ -247,7 +224,7 @@ private void setCount(final int count) { * * @return the map that stores the attributes as strings and possible values with counts as map */ - public ConcurrentMap> getAttributes() { + public Map> getAttributes() { return this.attributes; } @@ -258,7 +235,6 @@ public ConcurrentMap> getAttributes() { */ public List getChildren() { return this.children; - } /** @@ -276,7 +252,7 @@ void addChild(final ConceptNode node) { * @return the super concept of this node */ public ConceptNode getParent() { - return this.parent.get(); + return this.parent; } /** @@ -285,137 +261,65 @@ public ConceptNode getParent() { * @param parent the super-concept to be set */ void setParent(final ConceptNode parent) { - this.parent.set(parent); + this.parent = parent; } - - @Override - public boolean equals(final Object o) { - if (o instanceof ConceptNode) { - final ConceptNode node = (ConceptNode) o; - final boolean result; - try (ResourceLock ignored = lockAll(this.lock.readLock(), node.lock.readLock())) { - result = node.getCount() == this.getCount() && node.getAttributes().equals(this.getAttributes()); - } - return result; + /** + * Prints nodes recursively from this node downwards the tree. + * + * @param sb StringBuilder to use. + * @param depth the depth when called in order to arrange the output appropriately + * @return a String holding the representation of the tree + */ + String printRec(final StringBuilder sb, final int depth) { + if (depth == 0) { + sb.append("|__"); } else { - return false; + for (int i = 0; i < depth; i++) { + sb.append("\t"); + } + sb.append("|____"); } - } - @Override - public int hashCode() { - final int hash; - try (ResourceLock ignored = lockAll(this.lock.readLock())) { - hash = Objects.hash(this.getCount(), this.getAttributes()); + sb.append(this.toString()).append("\n"); + + for (ConceptNode child : this.children) { + child.printRec(sb, depth + 1); } - return hash; + + return sb.toString(); } @Override public String toString() { - try (ResourceLock ignored = lockAll(this.lock.readLock())) { - final String id = this.getId() != null ? "ID: " + this.getId() : ""; - return "ConceptNode " + id + " Count: " + this.getCount() + " Attributes: " - + this.getAttributes(); - } + String id = this.id != null ? "ID: " + this.id : ""; + return "ConceptNode " + id + " Count: " + this.count + " Attributes: " + + this.attributes.toString(); } /** * Getter for the ID field. - * * @return the ID of the node or null */ public String getId() { - return this.id.get(); - } + return this.id; + } /** * Setter for the Id field. - * * @param id id to be set */ public void setId(final String id) { - this.id.set(id); - } - - /** - * Returns the label of the superclass that is cutoffLevel steps away from the root. - * - * @param cutoffLevel how far the returned concept should be from the root - * @return the label of a super-concept of the current node - */ - String getCutoffLabel(final int cutoffLevel) { - return this.getLabel().substring(0, cutoffLevel); - } - - /** - * getter for the label. - * - * @return the label of this node - */ - public String getLabel() { - return this.label.get(); - } - - /** - * setter for the label. - * - * @param label the label to be set in this node - */ - public void setLabel(final String label) { - this.label.set(label); - } - - /** - * setter for children. - * - * @param idx index - * @param child to be set - */ - void setChild(final int idx, final ConceptNode child) { - this.children.set(idx, child); - } - - /** - * removes children. - * - * @param child to be removed - */ - void removeChild(final ConceptNode child) { - this.children.remove(child); - } - - /** - * clears the children. - */ - void clearChildren() { - this.children.clear(); - } - - /** - * getter for the lock. - * - * @return the lock - */ - ReentrantReadWriteLock getLock() { - return this.lock; + this.id = id; } -} - /* - * Returns the super-concept of the node cutoffLevel traversal steps away from the root. - * @param cutoffLevel how far the returned concept should be from the root - * @return A super-concept of the current node - public ConceptNode getCutoffConcept(final int cutoffLevel) { - final List trace = new ArrayList<>(); + public ConceptNode getCutoffConcept(int cutoffLevel) { + List trace = new ArrayList<>(); ConceptNode current = this; do { trace.add(current); current = current.getParent(); } while (current.getParent() != current); - // also add the root so that we aren't off by one when returning the cutoff level - trace.add(current); - return trace.get(cutoffLevel); - }*/ - + return trace.get(cutoffLevel - 1); + } +} diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptValue.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptValue.java index 809c56c..81d8aef 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptValue.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/ConceptValue.java @@ -1,21 +1,17 @@ package kn.uni.dbis.neo4j.conceptual.algos; import java.util.Objects; -import java.util.concurrent.atomic.AtomicReference; - -import javax.annotation.concurrent.ThreadSafe; /** * Holder for Concept values. * * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> */ -@ThreadSafe -public class ConceptValue extends Value { +public class ConceptValue extends Value implements Cloneable { /** * The concept to encapsulate. */ - private final AtomicReference concept; + private final ConceptNode concept; /** * Constructor. @@ -24,28 +20,28 @@ public class ConceptValue extends Value { */ public ConceptValue(final ConceptNode node) { this.setCount(1); - this.concept = new AtomicReference<>(node); + this.concept = node; } /** - * Copy Constructor. + * Copy COnstructor. * @param count to be set * @param node to be set as concept */ private ConceptValue(final int count, final ConceptNode node) { this.setCount(count); - this.concept = new AtomicReference<>(node); + this.concept = node; } @Override public Value copy() { - return new ConceptValue(this.getCount(), this.concept.get()); + return new ConceptValue(this.getCount(), this.concept); } @Override public boolean equals(final Object o) { if (o instanceof ConceptValue) { - return this.concept.get().equals(((ConceptValue) o).concept.get()); + return this.concept.equals(((ConceptValue) o).concept); } else { return false; } @@ -55,7 +51,7 @@ public boolean equals(final Object o) { public void update(final Value other) { if (other instanceof ConceptValue) { final ConceptValue c = (ConceptValue) other; - if (this.concept.get().equals(c.concept.get())) { + if (this.concept.equals(c.concept)) { this.setCount(this.getCount() + c.getCount()); } } else { @@ -65,7 +61,7 @@ public void update(final Value other) { @Override public int hashCode() { - return Objects.hash(this.concept.get()); + return Objects.hash(this.concept); } /** @@ -75,10 +71,10 @@ public int hashCode() { * @return The probability of this concept given the concept val */ double getFactor(final ConceptValue val) { - if (val.concept.get().equals(this.concept.get()) || this.concept.get().isSuperConcept(val.concept.get())) { + if (val.concept.equals(this.concept) || this.concept.isSuperConcept(val.concept)) { return 1.0; - } else if (val.concept.get().isSuperConcept(this.concept.get())) { - return (double) this.concept.get().getCount() / (double) val.concept.get().getCount(); + } else if (val.concept.isSuperConcept(this.concept)) { + return (double) this.concept.getCount() / (double) val.concept.getCount(); } else { // they're on different paths => disjoint return 0; @@ -87,16 +83,7 @@ public int hashCode() { @Override public String toString() { - return "ConceptValue: count=" + this.getCount() + " Concept=(" - + this.concept.get().toString() + ")"; - } - - /** - * Returns a string that is formatted to be used in a latex tabular environment. - * @return a sting representation of the node for letx tables - */ - @Override - public String toTexString() { - return "Concept & " + this.concept.get().getLabel() + "&"; + return "ConceptValue: " + System.identityHashCode(this) + " count=" + this.getCount() + " Concept=(" + + this.concept.toString() + ")"; } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NominalValue.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NominalValue.java index 4d59435..4e56daf 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NominalValue.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NominalValue.java @@ -1,21 +1,17 @@ package kn.uni.dbis.neo4j.conceptual.algos; import java.util.Objects; -import java.util.concurrent.atomic.AtomicReference; - -import javax.annotation.concurrent.ThreadSafe; /** * Holder for nominal values, representing Strings, characters, booleans and IDs as String. * * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> */ -@ThreadSafe public class NominalValue extends Value { /** * the nominal value as string. */ - private final AtomicReference str; + private final String str; /** * Constructor using a string. @@ -24,7 +20,7 @@ public class NominalValue extends Value { */ public NominalValue(final String value) { this.setCount(1); - this.str = new AtomicReference<>(value); + this.str = value; } /** @@ -35,7 +31,7 @@ public NominalValue(final String value) { */ private NominalValue(final int count, final String value) { this.setCount(count); - this.str = new AtomicReference<>(value); + this.str = value; } /** @@ -45,7 +41,7 @@ private NominalValue(final int count, final String value) { */ NominalValue(final boolean value) { this.setCount(1); - this.str = value ? new AtomicReference<>("true") : new AtomicReference<>("false"); + this.str = value ? "true" : "false"; } /** @@ -55,19 +51,19 @@ private NominalValue(final int count, final String value) { */ NominalValue(final char value) { this.setCount(1); - this.str = new AtomicReference<>(Character.toString(value)); + this.str = Character.toString(value); } @Override public Value copy() { - return new NominalValue(this.getCount(), this.str.get()); + return new NominalValue(this.getCount(), this.str); } @Override public boolean equals(final Object o) { if (o instanceof NominalValue) { final NominalValue n = (NominalValue) o; - return n.str.get().equals(this.str.get()); + return n.str.equals(this.str); } else { return false; } @@ -75,14 +71,14 @@ public boolean equals(final Object o) { @Override public int hashCode() { - return Objects.hash(this.str.get()); + return Objects.hash(this.str); } @Override public void update(final Value other) { if (other instanceof NominalValue) { final NominalValue n = (NominalValue) other; - if (n.str.get().equals(this.str.get())) { + if (n.str.equals(this.str)) { this.setCount(this.getCount() + n.getCount()); } } else { @@ -92,15 +88,6 @@ public void update(final Value other) { @Override public String toString() { - return "NominalValue count=" + this.getCount() + " string=" + this.str.get(); - } - - /** - * Returns a string that is formatted to be used in a latex tabular environment. - * @return a sting representation of the node for letx tables - */ - @Override - public String toTexString() { - return "Nominal & " + this.str.get() + "& "; + return "NominalValue count=" + this.getCount() + " string=" + this.str; } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NumericValue.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NumericValue.java index 0103b04..991d847 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NumericValue.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/NumericValue.java @@ -1,28 +1,25 @@ package kn.uni.dbis.neo4j.conceptual.algos; -import javax.annotation.concurrent.ThreadSafe; - -import com.google.common.util.concurrent.AtomicDouble; +import java.util.Objects; /** * Holder for numeric values, aggregates from single values a gaussian/normal distribution. * * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> */ -@ThreadSafe public class NumericValue extends Value { /** * Mean of the gaussian. */ - private AtomicDouble mean; + private double mean; /** * standard deviation of the gaussian. */ - private AtomicDouble std; + private double std; /** * Used for Welford's and Chan's methods to compute the mean and variance incrementally and based on partitons. */ - private AtomicDouble m2; + private double m2; /** * Constructor for a single Number instance. @@ -32,9 +29,9 @@ public class NumericValue extends Value { */ public NumericValue(final Number nr) { this.setCount(1); - this.mean = new AtomicDouble(nr.doubleValue()); - this.std = new AtomicDouble(0.0f); - this.m2 = new AtomicDouble(0.0f); + this.mean = nr.doubleValue(); + this.std = 0.0f; + this.m2 = 0.0f; } /** @@ -47,27 +44,25 @@ public NumericValue(final Number nr) { */ private NumericValue(final int count, final double mean, final double std, final double m2) { this.setCount(count); - this.mean = new AtomicDouble(mean); - this.std = new AtomicDouble(std); - this.m2 = new AtomicDouble(m2); + this.mean = mean; + this.std = std; + this.m2 = m2; } /** * returns the count of the updated node. * @param other node to incorporate */ - @Override public void update(final Value other) { if (other instanceof NumericValue) { final NumericValue v = (NumericValue) other; final int totalCount = this.getCount() + v.getCount(); - final double delta = v.mean.get() - this.mean.get(); - final double mean = this.mean.get() + delta * (double) v.getCount() / (double) totalCount; - final double m2x = this.m2.get() + v.m2.get() + delta * delta * (this.getCount() * v.getCount()) - / totalCount; - this.mean.set(mean); - this.std.set(Math.sqrt(m2x / totalCount)); - this.m2.set(m2x); + final double delta = v.mean - this.mean; + final double mean = this.mean + delta * (double) v.getCount() / (double) totalCount; + final double m2x = this.m2 + v.m2 + delta * delta * (this.getCount() * v.getCount()) / totalCount; + this.mean = mean; + this.std = Math.sqrt(m2x / totalCount); + this.m2 = m2x; this.setCount(totalCount); } else { throw new RuntimeException("updated with wrong type!"); @@ -80,7 +75,7 @@ public void update(final Value other) { * @return std of the gaussian representing the NumericValue. */ double getStd() { - return this.std.get(); + return this.std; } /** @@ -89,26 +84,17 @@ public void update(final Value other) { * @return std of the gaussian representing the NumericValue. */ double getMean() { - return this.mean.get(); + return this.mean; } @Override public Value copy() { - return new NumericValue(this.getCount(), this.mean.get(), this.std.get(), this.m2.get()); + return new NumericValue(this.getCount(), this.mean, this.std, this.m2); } @Override public String toString() { - return "NumericValue: count=" + this.getCount() + " mean= " + this.mean.get() - + " std=" + this.std.get(); - } - - /** - * Returns a string that is formatted to be used in a latex tabular environment. - * @return a sting representation of the node for letx tables - */ - @Override - public String toTexString() { - return "Numeric & mean= " + this.mean.get() + ", std=" + this.std.get() + " &"; + return "NumericValue: count=" + this.getCount() + " mean= " + this.mean + + " std=" + this.std; } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/PropertyGraphCobweb.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/PropertyGraphCobweb.java index 8c95c1c..4e39acb 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/PropertyGraphCobweb.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/PropertyGraphCobweb.java @@ -4,28 +4,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Logger; -import javax.annotation.concurrent.ThreadSafe; - -import kn.uni.dbis.neo4j.conceptual.util.PrintUtils; +import org.junit.jupiter.api.Assertions; import org.neo4j.graphdb.Direction; -import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.RelationshipType; -import org.neo4j.graphdb.Transaction; - -import kn.uni.dbis.neo4j.conceptual.util.MathUtils; -import kn.uni.dbis.neo4j.conceptual.util.TreeUtils; - -import static kn.uni.dbis.neo4j.conceptual.util.PrintUtils.printFullTrees; /** @@ -33,18 +17,22 @@ * * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> */ -@ThreadSafe public class PropertyGraphCobweb { - /** Logger. */ - static final Logger LOG = Logger.getLogger("PropertyGraphCobweb"); - /** tree root for the node properties. */ - private final AtomicReference nodePropertiesTree = new AtomicReference<>(new ConceptNode().root()); + private final ConceptNode nodePropertiesTree = new ConceptNode().root(); /** Cobweb tree for the relationship properties. */ - private final AtomicReference relationshipPropertiesTree = - new AtomicReference<>(new ConceptNode().root()); + private final ConceptNode relationshipPropertiesTree = new ConceptNode().root(); /** Cobweb tree for the node summary. */ - private final AtomicReference nodeSummaryTree = new AtomicReference<>(new ConceptNode().root()); + private final ConceptNode nodeSummaryTree = new ConceptNode().root(); + + /** + * convenience method for printing. + */ + public void print() { + System.out.println(this.nodePropertiesTree.printRec(new StringBuilder(), 0)); + System.out.println(this.relationshipPropertiesTree.printRec(new StringBuilder(), 0)); + System.out.println(this.nodeSummaryTree.printRec(new StringBuilder(), 0)); + } /** * Integrates a neo4j node into the cobweb trees. @@ -52,114 +40,102 @@ public class PropertyGraphCobweb { * 2. takes the results from 1, extracts structural features, creates a new node that incorporates all information * and integrates it into the tree * - * @param db The GraphDatabaseService to be used - * @param nodes the list of nodes of the graph to be incorporated - * @param relationships the relationships of the graph to be incorporated + * @param nodes the list of nodes to be incorporated */ - public void integrate(final GraphDatabaseService db, final List nodes, final List relationships) { + public void integrate(final List nodes, final List relationships) { // Static categorization according to properties, labels and relationship type - ExecutorService threadPool = Executors.newFixedThreadPool(8); - Runnable cobwebRunnable; - for (Node node : nodes) { - cobwebRunnable = () -> { - try (Transaction ignored = db.beginTx()) { - final ConceptNode properties = new ConceptNode(node); - Cobweb.cobweb(properties, this.getNodePropertiesTree()); - } - }; - threadPool.execute(cobwebRunnable); - } - - - for (Relationship rel : relationships) { - cobwebRunnable = () -> { - try (Transaction ignored = db.beginTx()) { - final ConceptNode properties = new ConceptNode(rel); - Cobweb.cobweb(properties, this.getRelationshipPropertiesTree()); - printFullTrees(this.relationshipPropertiesTree.get()); - } - }; - threadPool.execute(cobwebRunnable); - } - - threadPool.shutdown(); - try { - threadPool.awaitTermination(1, TimeUnit.HOURS); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - e.printStackTrace(); - } - threadPool = Executors.newWorkStealingPool(); - - final Future nodeCutoff = threadPool.submit(() -> MathUtils.log2(TreeUtils.deepestLevel( - this.getNodePropertiesTree()))); - final Future relCutoff = - threadPool.submit(() -> MathUtils.log2(TreeUtils.deepestLevel(this.getRelationshipPropertiesTree()))); + ConceptNode properties; + for (Node node : nodes) { + properties = new ConceptNode(node); + Cobweb.cobweb(properties, this.nodePropertiesTree); + } - threadPool.execute(() -> TreeUtils.labelTree(this.getNodePropertiesTree(), "", "n")); - threadPool.execute(() -> TreeUtils.labelTree(this.getNodePropertiesTree(), "", "r")); + for (Relationship rel : relationships) { + properties = new ConceptNode(rel); + Cobweb.cobweb(properties, this.relationshipPropertiesTree); + } + ConceptNode summarizedNode; + List co; - threadPool.shutdown(); + int deepestNodes = deepestLevel(this.nodePropertiesTree); + int deepestRels = deepestLevel(this.relationshipPropertiesTree); + System.out.println(deepestNodes); + System.out.println(deepestRels); - int cutoffLevelNodes = 0; - int cutoffLevelRelationships = 0; - try { - threadPool.awaitTermination(1, TimeUnit.HOURS); - cutoffLevelRelationships = relCutoff.get(); - cutoffLevelNodes = nodeCutoff.get(); - } catch (final InterruptedException | ExecutionException e) { - Thread.currentThread().interrupt(); - e.printStackTrace(); - } + int cutoffLevelNodes = (int) Math.log(deepestNodes + 1); + int cutoffLevelRelationships = (int) Math.log(deepestRels + 1); + System.out.println("Node cutoff " + cutoffLevelNodes); + System.out.println("Relations cutoff " + cutoffLevelRelationships); + // FIXME those are wrong - - threadPool = Executors.newWorkStealingPool(); for (Node node : nodes) { - final int finalCutoffLevelNodes = cutoffLevelNodes; - final int finalCutoffLevelRelationships = cutoffLevelRelationships; - cobwebRunnable = () -> { - try (Transaction ignored = db.beginTx()) { - final ConceptNode summarizedNode = new ConceptNode(); - final List co = new ArrayList<>(); + summarizedNode = new ConceptNode(); + co = new ArrayList<>(); + + summarizedNode.setId(Long.toString(node.getId())); + properties = findById(Long.toString(node.getId()), this.nodePropertiesTree); + assert properties != null; + properties = properties.getCutoffConcept(cutoffLevelNodes); + co.add(new ConceptValue(properties)); + summarizedNode.getAttributes().put("NodePropertiesConcept", co); + + co.clear(); + ConceptValue check; + for (Relationship rel : node.getRelationships()) { + properties = findById(Long.toString(rel.getId()), this.relationshipPropertiesTree); + assert properties != null; + properties = properties.getCutoffConcept(cutoffLevelRelationships); + check = new ConceptValue(properties); + if (co.contains(check)) { + co.get(co.indexOf(check)).update(check); + } else { + co.add(check); + } + } + summarizedNode.getAttributes().put("RelationshipConcepts", co); - summarizedNode.setId(Long.toString(node.getId())); - ConceptNode properties = TreeUtils.findById(Long.toString(node.getId()), this.getNodePropertiesTree()); - assert properties != null; - String label = properties.getCutoffLabel(finalCutoffLevelNodes); - co.add(new NominalValue(label)); - summarizedNode.getAttributes().put("NodePropertiesConcept", co); + extractStructuralFeatures(node, summarizedNode); + Cobweb.cobweb(summarizedNode, this.nodeSummaryTree); + } - co.clear(); - for (Relationship rel : node.getRelationships()) { - properties = TreeUtils.findById(Long.toString(rel.getId()), this.getNodeSummaryTree()); - assert properties != null; - label = properties.getCutoffLabel(finalCutoffLevelRelationships); - final NominalValue check = new NominalValue(label); - if (co.contains(check)) { - co.get(co.indexOf(check)).update(check); - } else { - co.add(check); - } - } - summarizedNode.getAttributes().put("RelationshipConcepts", co); + } - extractStructuralFeatures(node, summarizedNode); - Cobweb.cobweb(summarizedNode, this.getNodeSummaryTree()); - } - }; - threadPool.execute(cobwebRunnable); + private static int deepestLevel(ConceptNode node) { + if (node.getChildren().isEmpty()) { + return 0; + } else { + int deepest = 0; + int temp; + for (ConceptNode child : node.getChildren()) { + temp = deepestLevel(child); + if (temp > deepest) { + deepest = temp; + } } + return deepest + 1; + } + } - threadPool.shutdown(); - try { - threadPool.awaitTermination(1, TimeUnit.HOURS); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - e.printStackTrace(); + /** + * find and return the conceptnode for already incorporated conceptnodes, else returns null. + * @param id the id of the relationship. + * @param node the node we are currently inspecting in the relationshipPropertiesCobweb tree + * @return the corresponding conceptnode or null + */ + private static ConceptNode findById(final String id, final ConceptNode node) { + if (node.getId() != null) { + return node.getId().equals(id) ? node : null; + } else { + ConceptNode temp; + for (ConceptNode child : node.getChildren()) { + temp = findById(id, child); + if (temp != null) { + return temp; + } } - - TreeUtils.labelTree(this.getNodeSummaryTree(), "", "l"); + return null; + } } /** @@ -250,7 +226,7 @@ private static void extractStructuralFeatures(final Node node, final ConceptNode * @return the cobweb instance for node properties */ public ConceptNode getNodePropertiesTree() { - return this.nodePropertiesTree.get(); + return this.nodePropertiesTree; } /** @@ -258,7 +234,7 @@ public ConceptNode getNodePropertiesTree() { * @return the cobweb instance for relationship properties */ public ConceptNode getRelationshipPropertiesTree() { - return this.relationshipPropertiesTree.get(); + return this.relationshipPropertiesTree; } /** @@ -266,6 +242,6 @@ public ConceptNode getRelationshipPropertiesTree() { * @return the cobweb instance for node summaries. */ public ConceptNode getNodeSummaryTree() { - return this.nodeSummaryTree.get(); + return this.nodeSummaryTree; } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Value.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Value.java index 367e6d4..ddc2f48 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Value.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/algos/Value.java @@ -1,22 +1,14 @@ package kn.uni.dbis.neo4j.conceptual.algos; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import javax.annotation.concurrent.ThreadSafe; - /** * Interface for a Value to cluster using Cobweb. * I.e. it's held by the attributes map as value in the @link{ConceptNode} class. * * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> */ -@ThreadSafe public abstract class Value { /** counter for the occurrence of the value. */ - private AtomicInteger count = new AtomicInteger(); - /** Lock. */ - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private int count; /** * given a primitive datatype or a String, decide which Value type to instantiate. @@ -34,7 +26,9 @@ static Value cast(final Object o) { return new NominalValue((Character) o); } else if (o instanceof Number) { return new NumericValue((Number) o); - } else { + } else if (o instanceof ConceptNode) { + return new ConceptValue((ConceptNode) o); + } else { System.out.println("Encountered Property with unsupported type!"); return new NominalValue("Unsupported Type!"); } @@ -58,7 +52,7 @@ static Value cast(final Object o) { * @return the count of the value */ public int getCount() { - return this.count.get(); + return this.count; } /** @@ -66,20 +60,6 @@ public int getCount() { * @param count count to be set */ void setCount(final int count) { - this.count.set(count); - } - - /** - * returns a string representing in .tex. - * @return a String containing a table entry of a tex table - */ - public abstract String toTexString(); - - /** - * getter for the lock. - * @return the lock - */ - ReentrantReadWriteLock getLock() { - return this.lock; + this.count = count; } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/proc/PropertyGraphCobwebProc.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/proc/PropertyGraphCobwebProc.java index 3d5a6f6..eda0e63 100644 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/proc/PropertyGraphCobwebProc.java +++ b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/proc/PropertyGraphCobwebProc.java @@ -20,10 +20,6 @@ * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> */ public class PropertyGraphCobwebProc { - - public PropertyGraphCobwebProc(GraphDatabaseService db) { - this.db = db; - } /** * The database service to execute the procedure against. */ @@ -36,16 +32,16 @@ public PropertyGraphCobwebProc(GraphDatabaseService db) { */ @Procedure( name = "kn.uni.dbis.neo4j.conceptual.PropertyGraphCobwebStream", - mode = Mode.WRITE + mode = Mode.READ ) - public Stream integrate(@Name("nodes") final Stream nodes, + public static Stream integrate(@Name("nodes") final Stream nodes, @Name("edges") final Stream relationships) { final PropertyGraphCobweb tree = new PropertyGraphCobweb(); List nodesList = nodes.collect(Collectors.toList()); List relationshipsList = relationships.collect(Collectors.toList()); System.out.println("Number of nodes " + nodesList.size()); System.out.println("Number of Relationships " + relationshipsList.size()); - tree.integrate(db, nodesList, relationshipsList); + tree.integrate(nodesList, relationshipsList); return Stream.of(tree); } } diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/LockUtils.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/LockUtils.java deleted file mode 100644 index 1a858c3..0000000 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/LockUtils.java +++ /dev/null @@ -1,57 +0,0 @@ -package kn.uni.dbis.neo4j.conceptual.util; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * Auto-Closeable Multi Object Locking. - * - * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> - */ -public class LockUtils { - private LockUtils() { - // NOOP - } - - public static ResourceLock lockAll(List locks) { - return lockAll(locks.toArray(new Lock[0])); - } - - public static ResourceLock lockAll(Lock... locks) { - List successful = new ArrayList<>(); - boolean acquired = false; - - for (final Lock lock : locks) { - acquired = false; - try { - acquired = lock.tryLock(3, TimeUnit.SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (acquired) { - successful.add(lock); - } else { - break; - } - } - if (!acquired) { - for (Lock lock1 : successful) { - lock1.unlock(); - } - // Preempt the thread and try again - Thread.yield(); - return lockAll(locks); - } - - return () -> { - for (final Lock lock : locks) { - lock.unlock(); - } - }; - } -} - - diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/MathUtils.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/MathUtils.java deleted file mode 100644 index 7a2d04b..0000000 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/MathUtils.java +++ /dev/null @@ -1,26 +0,0 @@ -package kn.uni.dbis.neo4j.conceptual.util; - -/** - * Utilities for doing maths. - * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> - */ -public final class MathUtils { - /** - * Hidden default constructor. - */ - private MathUtils() { - // NOOP - } - - /** - * computes the logarithm to the basis 2 without numeric instability (as when dividing Math.log(x) / Match.log(2)). - * @param bits the number to take the binary logarithm of - * @return log_2(x) as integer rounded down - */ - public static int log2(final int bits) { - if (bits == 0) { - return 0; - } - return 31 - Integer.numberOfLeadingZeros(bits); - } -} diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/PrintUtils.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/PrintUtils.java deleted file mode 100644 index 97b1c4a..0000000 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/PrintUtils.java +++ /dev/null @@ -1,158 +0,0 @@ -package kn.uni.dbis.neo4j.conceptual.util; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentMap; -import java.util.logging.Logger; - -import kn.uni.dbis.neo4j.conceptual.algos.ConceptNode; -import kn.uni.dbis.neo4j.conceptual.algos.Value; - -/** - * PrintUtils. - * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> - */ -public final class PrintUtils { - /** Logger. */ - private static final Logger LOG = Logger.getLogger("PropertyGraphCobweb"); - - /** - * Hidden default constructor. - */ - private PrintUtils() { - // NOOP - } - - /** - * Prints nodes recursively from node node downwards the tree. - * - * @param node node to print - * @param sb StringBuilder to use. - * @param depth the depth when called in order to arrange the output appropriately - * @param maxDepth the maximal depth to be considered - * @return a String holding the representation of the tree - */ - private static String printRec(final ConceptNode node, final StringBuilder sb, final int depth, final int maxDepth) { - if (depth == 0) { - sb.append("|__"); - } else { - for (int i = 0; i < depth; i++) { - sb.append("\t"); - } - sb.append("|____"); - } - - sb.append(node.toString()).append("\n"); - if (depth <= maxDepth) { - final List localChildren; - synchronized (node.getChildren()) { - localChildren = new ArrayList<>(node.getChildren()); - } - for (ConceptNode child : localChildren) { - printRec(child, sb, depth + 1, maxDepth); - } - } - return sb.toString(); - } - - /** - * Returns a Latex table representation of the ConceptNode. - * @param node stupid - * @return a String containing a latex table representation of the ConceptNode - */ - public String toTexTable(final ConceptNode node) { - final StringBuilder sb = new StringBuilder(); - sb.append("ConceptNode \\hspace{1cm} P(node) = ") - .append(Double.toString((double) node.getCount() / (double) node.getParent().getCount()), 0, 5) - .append("\\\\ Attributes: \\\\ \\begin{tabular}{|c|c|c|} \\hline"); - for (ConcurrentMap.Entry> attribute : node.getAttributes().entrySet()) { - sb.append("\\multirow{4}{*}{").append(attribute.getKey()).append("} "); - synchronized (attribute.getValue()) { - for (Value value : attribute.getValue()) { - sb.append(value.toTexString()).append((double) value.getCount() / (double) node.getCount()) - .append("\\\\ \\hline"); - } - } - } - sb.append("\\end{tabular}"); - return sb.toString(); - } - - /** - * Prints a tikz tree representing the labeled Concept Hierarchy. - * @param node currently visited node - * @param sb the stringbuilder that collects the String representation - * @param depth the current depth - * @param maxDepth the maximal depth - */ - private static void printRecTexTree(final ConceptNode node, final StringBuilder sb, final int depth, - final int maxDepth) { - if (depth == 0) { - sb.append("\\node {Root}\n"); - } - - if (depth <= maxDepth) { - for (ConceptNode child : node.getChildren()) { - for (int i = 0; i <= depth; i++) { - sb.append("\t"); - } - sb.append("child { node {").append(child.getLabel()).append("} "); - if (child.getChildren().size() > 0) { - sb.append("\n"); - printRecTexTree(child, sb, depth + 1, maxDepth); - } - sb.append("}"); - } - } - } - - /** - * Constructs a tikzpicture containing a tree representation of the concept hierarchy. - * @param root the root of the tree to be visualized - * @param maxDepth the maximal depth to be visualized - * @return a String containing a tkizpicture - */ - private static String getTexTree(final ConceptNode root, final int maxDepth) { - TreeUtils.labelTree(root, "", "l"); - final StringBuilder sb = new StringBuilder(); - sb.append("\\begin{tikzpicture}[sibling distance=10em, " - + "every node/.style = {shape=rectangle, rounded corners, " - + "draw, align=center," - + "top color=white, bottom color=blue!20}]]"); - printRecTexTree(root, sb, 0, maxDepth); - sb.append(";\n").append("\\end{tikzpicture}"); - return sb.toString(); - } - - /** - * convenience method for printing. - * @param roots stupid - */ - public static void printFullTrees(final ConceptNode... roots) { - for (final ConceptNode root : roots) { - LOG.info("\n" + PrintUtils.printRec(root, new StringBuilder(), 0, - TreeUtils.deepestLevel(root))); - } - } - - /** - * convenience method for printing. - * @param roots stupid - */ - public static void printCutoffTrees(final ConceptNode... roots) { - for (final ConceptNode root : roots) { - LOG.info(PrintUtils.printRec(root, new StringBuilder(), 0, - MathUtils.log2(TreeUtils.deepestLevel(root)))); - } - } - - /** - * convenience method for printing. - * @param root stupid - */ - public static void prettyPrint(final ConceptNode root) { - final int cut = MathUtils.log2(TreeUtils.deepestLevel(root)); - LOG.info(PrintUtils.printRec(root, new StringBuilder(), 0, cut)); - LOG.info(getTexTree(root, cut)); - } -} diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/ResourceLock.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/ResourceLock.java deleted file mode 100644 index ff23290..0000000 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/ResourceLock.java +++ /dev/null @@ -1,14 +0,0 @@ -package kn.uni.dbis.neo4j.conceptual.util; - -/** - * Helper Interface for an AutoClosable lock. - * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> - */ -public interface ResourceLock extends AutoCloseable { - - /** - * Unlocking doesn't throw any checked exception. - */ - @Override - void close(); -} diff --git a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/TreeUtils.java b/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/TreeUtils.java deleted file mode 100644 index 724ef01..0000000 --- a/neo4j-concept/src/main/java/kn/uni/dbis/neo4j/conceptual/util/TreeUtils.java +++ /dev/null @@ -1,74 +0,0 @@ -package kn.uni.dbis.neo4j.conceptual.util; - -import kn.uni.dbis.neo4j.conceptual.algos.ConceptNode; - -/** - * Utilities for trees. - * @author Fabian Klopfer <fabian.klopfer@uni-konstanz.de> - */ -public final class TreeUtils { - /** - * Hidden default constructor. - */ - private TreeUtils() { - // NOOP - } - - /** - * computes the maximal depth of the tree. - * @param node the currently visited node - * @return an integer representing the depth of the tree - */ - public static int deepestLevel(final ConceptNode node) { - if (node.getChildren().isEmpty()) { - return 0; - } else { - int deepest = 0; - int temp; - for (ConceptNode child : node.getChildren()) { - temp = deepestLevel(child); - if (temp > deepest) { - deepest = temp; - } - } - return deepest + 1; - } - } - - /** - * find and return the conceptnode for already incorporated conceptnodes, else returns null. - * @param id the id of the relationship. - * @param node the node we are currently inspecting in the relationshipPropertiesCobweb tree - * @return the corresponding conceptnode or null - */ - public static ConceptNode findById(final String id, final ConceptNode node) { - if (node.getId() != null) { - return node.getId().equals(id) ? node : null; - } else { - ConceptNode temp; - for (ConceptNode child : node.getChildren()) { - temp = findById(id, child); - if (temp != null) { - return temp; - } - } - return null; - } - } - - /** - * Assigns a label to each node in the tree. - * @param node currently visited node - * @param parentLabel prefix of the current label - * @param num postfix of the current label - */ - public static void labelTree(final ConceptNode node, final String parentLabel, final String num) { - node.setLabel(parentLabel + num); - - int i = 0; - for (ConceptNode child : node.getChildren()) { - labelTree(child, parentLabel + num, Integer.toString(i)); - i++; - } - } -} diff --git a/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebPerfTest.java b/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebPerfTest.java index b04d8bf..eb2b82c 100644 --- a/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebPerfTest.java +++ b/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebPerfTest.java @@ -7,17 +7,17 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.internal.kernel.api.exceptions.KernelException; import org.neo4j.server.CommunityBootstrapper; - import kn.uni.dbis.neo4j.conceptual.algos.PropertyGraphCobweb; import kn.uni.dbis.neo4j.conceptual.proc.PropertyGraphCobwebProc; -import kn.uni.dbis.neo4j.conceptual.util.PrintUtils; import kn.uni.dbis.neo4j.eval.DefaultPaths; import kn.uni.dbis.neo4j.eval.TestDatabaseFactory; import kn.uni.dbis.neo4j.eval.util.FileUtils; @@ -76,10 +76,11 @@ public static void main(final String[] args) throws IOException { } try (Transaction tx = db.beginTx()) { - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), + final Stream nodes = db.getAllNodes().stream().limit(100); + System.out.println(nodes.count()); + final PropertyGraphCobweb tree = PropertyGraphCobwebProc.integrate(db.getAllNodes().stream(), db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); - PrintUtils.prettyPrint(tree.getNodeSummaryTree()); + tree.print(); } } } diff --git a/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebProcTest.java b/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebProcTest.java index f593140..d502a6d 100644 --- a/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebProcTest.java +++ b/neo4j-concept/src/test/java/kn.uni.dbis.neo4j.conceptual/PropertyGraphCobwebProcTest.java @@ -1,30 +1,31 @@ package kn.uni.dbis.neo4j.conceptual; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import kn.uni.dbis.neo4j.conceptual.algos.ConceptValue; +import kn.uni.dbis.neo4j.eval.annotations.Preprocessing; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import kn.uni.dbis.neo4j.conceptual.algos.Cobweb; import kn.uni.dbis.neo4j.conceptual.algos.ConceptNode; -import kn.uni.dbis.neo4j.conceptual.algos.ConceptValue; import kn.uni.dbis.neo4j.conceptual.algos.NominalValue; import kn.uni.dbis.neo4j.conceptual.algos.NumericValue; import kn.uni.dbis.neo4j.conceptual.algos.PropertyGraphCobweb; import kn.uni.dbis.neo4j.conceptual.algos.Value; import kn.uni.dbis.neo4j.conceptual.proc.PropertyGraphCobwebProc; -import kn.uni.dbis.neo4j.conceptual.util.PrintUtils; import kn.uni.dbis.neo4j.eval.annotations.GraphDBConfig; import kn.uni.dbis.neo4j.eval.annotations.GraphDBSetup; import kn.uni.dbis.neo4j.eval.annotations.GraphSource; -import kn.uni.dbis.neo4j.eval.annotations.Preprocessing; import kn.uni.dbis.neo4j.eval.annotations.Procedures; import kn.uni.dbis.neo4j.eval.datasets.Dataset; @@ -36,7 +37,6 @@ @ExtendWith(GraphDBSetup.class) @GraphDBConfig() @GraphSource() -@Preprocessing(preprocessing = "MATCH (n) REMOVE n.name RETURN n") @Procedures(procedures = PropertyGraphCobweb.class) class PropertyGraphCobwebProcTest { @@ -48,18 +48,16 @@ class PropertyGraphCobwebProcTest { */ @Test void testCobwebSmall(final GraphDatabaseService db, final Dataset dataset) { - try (Transaction ignored = db.beginTx()) { - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), + try (Transaction tx = db.beginTx()) { + final Stream nodes = db.getAllNodes().stream(); + System.out.println(nodes.count()); + final PropertyGraphCobweb tree = PropertyGraphCobwebProc.integrate(db.getAllNodes().stream(), db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); Assertions.assertNotNull(tree); - final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), - tree.getNodeSummaryTree()}; - - PrintUtils.printCutoffTrees(subtrees); - PrintUtils.prettyPrint(subtrees[2]); + tree.print(); - final List ids = new ArrayList<>(); + final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), + tree.getNodeSummaryTree()}; final List ids = new ArrayList<>(); for (ConceptNode root : subtrees) { ids.clear(); this.checkIds(root, ids); @@ -83,124 +81,15 @@ void testCobwebSmall(final GraphDatabaseService db, final Dataset dataset) { @Preprocessing(preprocessing = "MATCH (n) REMOVE n.nodeId RETURN n") @GraphSource(getDataset = Dataset.Rome99) void testCobwebMedium(final GraphDatabaseService db, final Dataset dataset) { - try (Transaction ignored = db.beginTx()) { - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), - db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); - Assertions.assertNotNull(tree); - - final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), - tree.getNodeSummaryTree()}; - - PrintUtils.printCutoffTrees(subtrees); - PrintUtils.prettyPrint(subtrees[2]); - final List ids = new ArrayList<>(); - for (ConceptNode root : subtrees) { - ids.clear(); - this.checkIds(root, ids); - this.checkPartitionCounts(root); - this.checkParent(root); - this.checkLeafType(root); - } - Assertions.assertEquals(this.leafCount(subtrees[0]), dataset.getNodes()); - Assertions.assertEquals(this.leafCount(subtrees[1]), dataset.getArcs()); - Assertions.assertEquals(this.leafCount(subtrees[2]), dataset.getNodes()); - } - } - - /** - * stupid. - * @param db stupid - * @param dataset stupid - */ - @Disabled - //@Preprocessing(preprocessing = "MATCH (n) REMOVE n.nodeId RETURN n") - @GraphSource(getDataset = Dataset.RoadNetNY) - @Test - void testCobwebMediumLarge(final GraphDatabaseService db, final Dataset dataset) { - try (Transaction ignored = db.beginTx()) { - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), - db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); - Assertions.assertNotNull(tree); - - final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), - tree.getNodeSummaryTree()}; - - PrintUtils.printCutoffTrees(subtrees); - PrintUtils.prettyPrint(subtrees[2]); - - final List ids = new ArrayList<>(); - for (ConceptNode root : subtrees) { - ids.clear(); - this.checkIds(root, ids); - this.checkPartitionCounts(root); - this.checkParent(root); - this.checkLeafType(root); - } - Assertions.assertEquals(this.leafCount(subtrees[0]), dataset.getNodes()); - Assertions.assertEquals(this.leafCount(subtrees[1]), dataset.getArcs()); - Assertions.assertEquals(this.leafCount(subtrees[2]), dataset.getNodes()); - } - } - - /** - * stupid. - * @param db stupid - * @param dataset stupid - */ - @Disabled - //@Preprocessing(preprocessing = "MATCH (n) REMOVE n.nodeId RETURN n") - @GraphSource(getDataset = Dataset.InternetTopology) - @Test - void testCobwebLarge(final GraphDatabaseService db, final Dataset dataset) { - try (Transaction ignored = db.beginTx()) { - - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), + try (Transaction tx = db.beginTx()) { + final PropertyGraphCobweb tree = PropertyGraphCobwebProc.integrate(db.getAllNodes().stream(), db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); Assertions.assertNotNull(tree); - final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), - tree.getNodeSummaryTree()}; - - PrintUtils.printCutoffTrees(subtrees); - PrintUtils.prettyPrint(subtrees[2]); - final List ids = new ArrayList<>(); - for (ConceptNode root : subtrees) { - ids.clear(); - this.checkIds(root, ids); - this.checkPartitionCounts(root); - this.checkParent(root); - this.checkLeafType(root); - } - Assertions.assertEquals(this.leafCount(subtrees[0]), dataset.getNodes()); - Assertions.assertEquals(this.leafCount(subtrees[1]), dataset.getArcs()); - Assertions.assertEquals(this.leafCount(subtrees[2]), dataset.getNodes()); - } - } - - /** - * stupid. - * @param db stupid - * @param dataset stupid - */ - @Disabled - //@Preprocessing(preprocessing = "MATCH (n) REMOVE n.nodeId RETURN n") - @GraphSource(getDataset = Dataset.RoadNetUSA) - @Test - void testCobwebVeryLarge(final GraphDatabaseService db, final Dataset dataset) { - try (Transaction ignored = db.beginTx()) { - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), - db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); - Assertions.assertNotNull(tree); + tree.print(); final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), - tree.getNodeSummaryTree()}; - - PrintUtils.printCutoffTrees(subtrees); - PrintUtils.prettyPrint(subtrees[2]); + tree.getNodeSummaryTree()}; final List ids = new ArrayList<>(); for (ConceptNode root : subtrees) { ids.clear(); @@ -215,53 +104,13 @@ void testCobwebVeryLarge(final GraphDatabaseService db, final Dataset dataset) { } } - /** - * stupid. - * @param db stupid - * @param dataset stupid - */ - @Disabled - //@Preprocessing(preprocessing = "MATCH (n) REMOVE n.nodeId RETURN n") - @GraphSource(getDataset = Dataset.Orkut) - @Test - void testCobwebHuge(final GraphDatabaseService db, final Dataset dataset) { - try (Transaction ignored = db.beginTx()) { - final PropertyGraphCobwebProc proc = new PropertyGraphCobwebProc(db); - final PropertyGraphCobweb tree = proc.integrate(db.getAllNodes().stream(), - db.getAllRelationships().stream()).findFirst().orElseThrow(() -> new RuntimeException("Unreachable")); - Assertions.assertNotNull(tree); - - final ConceptNode[] subtrees = {tree.getNodePropertiesTree(), tree.getRelationshipPropertiesTree(), - tree.getNodeSummaryTree()}; - - PrintUtils.printCutoffTrees(subtrees); - PrintUtils.prettyPrint(subtrees[2]); - final List ids = new ArrayList<>(); - for (ConceptNode root : subtrees) { - ids.clear(); - this.checkIds(root, ids); - this.checkPartitionCounts(root); - this.checkParent(root); - this.checkLeafType(root); - } - Assertions.assertEquals(this.leafCount(subtrees[0]), dataset.getNodes()); - Assertions.assertEquals(this.leafCount(subtrees[1]), dataset.getArcs()); - Assertions.assertEquals(this.leafCount(subtrees[2]), dataset.getNodes()); - } - } - - /** - * Stupid test comment. - * @param node stupid - * @param ids params - */ - void checkIds(final ConceptNode node, final List ids) { + void checkIds(final ConceptNode node, List ids) { if (node.getId() != null) { Assertions.assertFalse(ids.contains(node.getId())); ids.add(node.getId()); } else { for (ConceptNode child : node.getChildren()) { - this.checkIds(child, ids); + checkIds(child, ids); } } } @@ -359,7 +208,7 @@ void testUpdateCounts() { List vals = c1.getAttributes().get("A"); Assertions.assertEquals(2, vals.get(vals.indexOf(new NominalValue("a"))).getCount()); vals = c1.getAttributes().get("B"); - Assertions.assertEquals(2, vals.get(0).getCount()); + Assertions.assertEquals(2, vals.get(vals.indexOf(new NumericValue(1))).getCount()); vals = c1.getAttributes().get("C"); Assertions.assertEquals(2, vals.get(vals.indexOf(cv)).getCount()); } @@ -376,6 +225,13 @@ void testEquals() { Assertions.assertEquals(n1, n3); Assertions.assertNotEquals(n1, n2); + final NumericValue n4 = new NumericValue(1); + final NumericValue n5 = new NumericValue(2); + final NumericValue n6 = new NumericValue(1); + + Assertions.assertEquals(n4, n6); + Assertions.assertNotEquals(n4, n5); + final ConceptNode n7 = new ConceptNode(); final ConceptNode n8 = new ConceptNode(); final ConceptNode n9 = new ConceptNode(); @@ -420,11 +276,10 @@ void testCreate() { conceptNode.getAttributes().put("name", val); final ConceptNode clone = new ConceptNode(conceptNode); final ConceptNode clone1 = new ConceptNode(conceptNode); - final ExecutorService threadPool = Executors.newWorkStealingPool(); - //Cobweb.cobweb(conceptNode, root, threadPool); - //Cobweb.cobweb(clone, root, threadPool); - //Cobweb.cobweb(clone1, root, threadPool); + Cobweb.cobweb(conceptNode, root); + Cobweb.cobweb(clone, root); + Cobweb.cobweb(clone1, root); Assertions.assertEquals(3, root.getChildren().size()); for (int i = 0; i < 3; ++i) { @@ -463,13 +318,11 @@ void testMerge() { otherConcept.setId("b"); final ConceptNode other1 = new ConceptNode(otherConcept); - final ExecutorService threadPool = Executors.newWorkStealingPool(); -/* - Cobweb.cobweb(conceptNode, root, threadPool); - Cobweb.cobweb(clone, root, threadPool); - Cobweb.cobweb(otherConcept, root, threadPool); - Cobweb.cobweb(clone1, root, threadPool); - Cobweb.cobweb(other1, root, threadPool);*/ + Cobweb.cobweb(conceptNode, root); + Cobweb.cobweb(clone, root); + Cobweb.cobweb(otherConcept, root); + Cobweb.cobweb(clone1, root); + Cobweb.cobweb(other1, root); Assertions.assertEquals(2, root.getChildren().size()); Assertions.assertEquals(3, root.getChildren().get(0).getChildren().size()); diff --git a/neo4j-eval/pom.xml b/neo4j-eval/pom.xml index 009b2cc..d97eec5 100644 --- a/neo4j-eval/pom.xml +++ b/neo4j-eval/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 @@ -19,19 +21,26 @@ - + UTF-8 - + 8 ${java.version} ${java.version} benchmarks yyyy.MM.dd.HH.mm.ss - + 3.5.0 5.3.1 - - + + -Xms2048m -Xmx2048m @@ -67,7 +76,7 @@ - + org.neo4j neo4j @@ -80,7 +89,7 @@ ${neo4j.version} - + org.neo4j graph-algorithms-algo @@ -103,7 +112,7 @@ 1.2 - + org.neo4j neo4j-io @@ -142,7 +151,7 @@ test - + com.github.oshi oshi-core @@ -174,7 +183,7 @@ - + maven-surefire-plugin 2.22.2 @@ -255,7 +264,7 @@ 3.12.0 1.8 - true + true true @@ -279,22 +288,4 @@ - - infer-capture - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - javac - true - true - /opt/infer-linux64-v0.17.0/lib/infer/infer/bin/infer - - - - - - + diff --git a/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/CSVImporter.java b/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/CSVImporter.java index 673fb0b..3c595c6 100644 --- a/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/CSVImporter.java +++ b/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/CSVImporter.java @@ -71,6 +71,7 @@ void csvImport(final Dataset dataset) throws IOException { + "--id-type=STRING " + " --max-memory=2G " + "--nodes " + String.format("%s ", nodesData) + "--relationships:" + relationshipName + String.format(" %s", arcsData)).split(" "); try { + System.gc(); System.out.println("Starting import"); System.out.println(Arrays.toString(command)); final ProcessBuilder pb = new ProcessBuilder(command); @@ -101,6 +102,8 @@ void csvImport(final Dataset dataset) throws IOException { ebr.close(); ibr.close(); + + System.gc(); } catch (final InterruptedException | IOException e) { e.printStackTrace(); } diff --git a/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/Dataset.java b/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/Dataset.java index 0a87766..1e078be 100644 --- a/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/Dataset.java +++ b/neo4j-eval/src/main/java/kn/uni/dbis/neo4j/eval/datasets/Dataset.java @@ -280,6 +280,7 @@ public void fetchAndImport() { default: throw new IllegalStateException("Unreachable"); } + System.gc(); } /** diff --git a/pmd_rules.xml b/pmd_rules.xml deleted file mode 100644 index 2d03b7c..0000000 --- a/pmd_rules.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - My custom rules - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml index cd70760..7af0536 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 kn.uni.dbis @@ -20,14 +22,14 @@ - + UTF-8 8 ${java.version} ${java.version} benchmarks yyyy.MM.dd.HH.mm.ss - + 3.5.0 5.3.1 @@ -63,79 +65,9 @@ - compile + test - - org.apache.maven.plugins - maven-checkstyle-plugin - 3.1.0 - - **/PropertyGraphCobwebProc.java - ${project.build.sourceDirectories} - true - false - checkstyle.xml - true - warning - - - - process-sources - - checkstyle - check - - - - - - com.github.spotbugs - spotbugs-maven-plugin - 3.1.12.2 - - - - com.github.spotbugs - spotbugs - 4.0.0-beta3 - - - - max - - - - process-classes - - spotbugs - check - - - - - - + maven-surefire-plugin 2.22.0 @@ -180,13 +112,14 @@ ${uberjar.name} - + org.openjdk.jmh.Main - + *:* META-INF/*.SF @@ -216,22 +149,4 @@ - - infer-capture - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - javac - true - true - /opt/infer-linux64-v0.17.0/lib/infer/infer/bin/infer - - - - - - + \ No newline at end of file