diff --git a/CHANGELOG.md b/CHANGELOG.md index 3955c188e4b..6fd30b26ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Minor improvements & Bug Fixes +* [#1335](https://github.com/osmosis-labs/osmosis/pull/1335) Add utility for deriving total orderings from partial orderings. * [#1308](https://github.com/osmosis-labs/osmosis/pull/1308) Make panics inside of epochs no longer chain halt by default. * [#1286](https://github.com/osmosis-labs/osmosis/pull/1286) Fix release build scripts. * [#1203](https://github.com/osmosis-labs/osmosis/pull/1203) cleanup Makefile and ci workflows diff --git a/osmoutils/partialord/internal/dag/dag.go b/osmoutils/partialord/internal/dag/dag.go new file mode 100644 index 00000000000..d33a0a63a1a --- /dev/null +++ b/osmoutils/partialord/internal/dag/dag.go @@ -0,0 +1,320 @@ +package dag + +import ( + "fmt" + "sort" +) + +// DAG struct maintains a directed acyclic graph, using adjacency lists to track edges. +type DAG struct { + // there is a directed edge from u -> v, if directedEdgeList[u][v] = 1 + // there is a directed edge from v -> u, if directedEdgeList[u][v] = -1 + directedEdgeList []map[int]int8 + nodeNameToId map[string]int + idToNodeNames map[int]string +} + +func NewDAG(nodes []string) DAG { + nodeNameToId := make(map[string]int, len(nodes)) + idToNodeNames := make(map[int]string, len(nodes)) + directedEdgeList := make([]map[int]int8, len(nodes)) + for i, node := range nodes { + nodeNameToId[node] = i + idToNodeNames[i] = node + directedEdgeList[i] = map[int]int8{} + } + if len(nodeNameToId) != len(nodes) { + panic("provided multiple nodes with the same name") + } + return DAG{ + directedEdgeList: directedEdgeList, + nodeNameToId: nodeNameToId, + idToNodeNames: idToNodeNames, + } +} + +// Copy returns a new dag struct that is a copy of the original dag. +// Edges can be mutated in the copy, without altering the original. +func (dag DAG) Copy() DAG { + directedEdgeList := make([]map[int]int8, len(dag.nodeNameToId)) + for i := 0; i < len(dag.nodeNameToId); i++ { + originalEdgeList := dag.directedEdgeList[i] + directedEdgeList[i] = make(map[int]int8, len(originalEdgeList)) + for k, v := range originalEdgeList { + directedEdgeList[i][k] = v + } + } + // we re-use nodeNameToId and idToNodeNames as these are fixed at dag creation. + return DAG{ + directedEdgeList: directedEdgeList, + nodeNameToId: dag.nodeNameToId, + idToNodeNames: dag.idToNodeNames, + } +} + +func (dag DAG) hasDirectedEdge(u, v int) bool { + uAdjacencyList := dag.directedEdgeList[u] + _, exists := uAdjacencyList[v] + return exists +} + +// addEdge adds a directed edge from u -> v. +func (dag *DAG) addEdge(u, v int) error { + if u == v { + return fmt.Errorf("can't make self-edge") + } + if dag.hasDirectedEdge(v, u) { + return fmt.Errorf("dag has conflicting edge") + } + dag.directedEdgeList[u][v] = 1 + dag.directedEdgeList[v][u] = -1 + return nil +} + +// replaceEdge adds a directed edge from u -> v. +// it removes any edge that may already exist between the two. +func (dag *DAG) replaceEdge(u, v int) error { + if u == v { + return fmt.Errorf("can't make self-edge") + } + + dag.directedEdgeList[u][v] = 1 + dag.directedEdgeList[v][u] = -1 + return nil +} + +// resetEdges deletes all edges directed to or from node `u` +func (dag *DAG) resetEdges(u int) { + edges := dag.directedEdgeList[u] + for v := range edges { + delete(dag.directedEdgeList[v], u) + } + dag.directedEdgeList[u] = map[int]int8{} +} + +// deleteEdge deletes edges between u and v. +func (dag *DAG) deleteEdge(u, v int) { + delete(dag.directedEdgeList[u], v) + delete(dag.directedEdgeList[v], u) +} + +// AddEdge checks if either edge between u and v exists and adds a directed edge from u -> v +func (dag *DAG) AddEdge(u, v string) error { + uIndex, uExists := dag.nodeNameToId[u] + vIndex, vExists := dag.nodeNameToId[v] + if !uExists || !vExists { + return fmt.Errorf("one of %s, %s does not exist in dag", u, v) + } + return dag.addEdge(uIndex, vIndex) +} + +// ReplaceEdge adds a directed edge from u -> v. +// it removes any edge that may already exist between the two. +func (dag *DAG) ReplaceEdge(u, v string) error { + uIndex, uExists := dag.nodeNameToId[u] + vIndex, vExists := dag.nodeNameToId[v] + if !uExists || !vExists { + return fmt.Errorf("one of %s, %s does not exist in dag", u, v) + } + return dag.replaceEdge(uIndex, vIndex) +} + +// AddFirstElements sets the provided elements to be first in all orderings. +// So if were making an ordering over {A, B, C, D, E}, and elems provided is {D, B, A} +// then we are guaranteed that the total ordering will begin with {D, B, A} +func (dag *DAG) AddFirstElements(nodes ...string) error { + nodeIds, err := dag.namesToIds(nodes) + if err != nil { + return err + } + + return dag.addFirst(nodeIds) +} + +func (dag *DAG) addFirst(nodes []int) error { + nodeMap := map[int]bool{} + for i := 0; i < len(nodes); i++ { + nodeMap[nodes[i]] = true + } + // First we add an edge from nodes[-1] to every node in the graph aside from the provided first nodes. + // then we make nodes[-1] depend on nodes[-2], etc. + // We also clear all other edges from nodes[-2], to override previous settings. + lastOfFirstNodes := nodes[len(nodes)-1] + for i := 0; i < len(dag.nodeNameToId); i++ { + // skip any node in the 'first set' + _, inMap := nodeMap[i] + if inMap { + continue + } + // We make everything on `lastOfFirstNodes`, and therefore have an edge from `lastOfFirstNodes` to it + err := dag.replaceEdge(lastOfFirstNodes, i) + // can't happen by above check + if err != nil { + return err + } + } + + // Make nodes[i+1] depend on nodes[i] + for i := len(nodes) - 2; i >= 0; i-- { + dag.resetEdges(nodes[i]) + err := dag.replaceEdge(nodes[i], nodes[i+1]) + // can't happen by above check + if err != nil { + return err + } + } + return nil +} + +// AddLastElements sets the provided elements to be last in all orderings. +// So if were making an ordering over {A, B, C, D, E}, and elems provided is {D, B, A} +// then we are guaranteed that the total ordering will end with {D, B, A} +func (dag *DAG) AddLastElements(nodes ...string) error { + nodeIds, err := dag.namesToIds(nodes) + if err != nil { + return err + } + + return dag.addLast(nodeIds) +} + +func (dag *DAG) addLast(nodes []int) error { + nodeMap := map[int]bool{} + for i := 0; i < len(nodes); i++ { + nodeMap[nodes[i]] = true + } + // First we add an edge from every node in the graph aside from the provided last nodes, to nodes[0] + // then we make nodes[1] depend on nodes[0], etc. + // We also clear all other edges from nodes[1], to override previous settings. + firstOfLastNodes := nodes[0] + for i := 0; i < len(dag.nodeNameToId); i++ { + // skip any node in the 'last set' + _, inMap := nodeMap[i] + if inMap { + continue + } + // We make `firstOfLastNodes` depend on every node, and therefore have an edge from each node to `firstOfLastNodes` + err := dag.replaceEdge(i, firstOfLastNodes) + // can't happen by above check + if err != nil { + return err + } + } + + // Make nodes[i] depend on nodes[i-1], and clear all other edges from nodes[i] + for i := 1; i < len(nodes); i++ { + dag.resetEdges(nodes[i]) + err := dag.replaceEdge(nodes[i-1], nodes[i]) + // can't happen by above check + if err != nil { + return err + } + } + return nil +} + +func (dag DAG) hasEdges() bool { + for _, m := range dag.directedEdgeList { + if len(m) > 0 { + return true + } + } + return false +} + +func (dag *DAG) namesToIds(names []string) ([]int, error) { + nodeIds := []int{} + for _, name := range names { + nodeIndex, nodeExists := dag.nodeNameToId[name] + if !nodeExists { + return []int{}, fmt.Errorf("%s does not exist in dag", name) + } + nodeIds = append(nodeIds, nodeIndex) + } + return nodeIds, nil +} + +func (dag DAG) idsToNames(ids []int) []string { + sortedNames := make([]string, 0, len(ids)) + for i := 0; i < len(dag.nodeNameToId); i++ { + id := ids[i] + sortedNames = append(sortedNames, dag.idToNodeNames[id]) + } + return sortedNames +} + +func (dag DAG) hasIncomingEdge(u int) bool { + adjacencyList := dag.directedEdgeList[u] + for _, v := range adjacencyList { + if v == -1 { + return true + } + } + return false +} + +// returns nodes with no incoming edges. +func (dag *DAG) topologicalTopLevelNodes() []int { + topLevelNodes := []int{} + + for i := 0; i < len(dag.nodeNameToId); i++ { + if !dag.hasIncomingEdge(i) { + topLevelNodes = append(topLevelNodes, i) + } + } + + return topLevelNodes +} + +// Returns a Topological Sort of the DAG, using Kahn's algorithm. +// https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm +func (dag DAG) TopologicalSort() []string { + // G is the mutable graph we work on, which we remove edges from. + G := dag.Copy() + // L in pseudocode + sortedIDs := make([]int, 0, len(dag.nodeNameToId)) + topLevelNodes := dag.topologicalTopLevelNodes() + + // while len(topLevelNodes) != 0 + for { + if len(topLevelNodes) == 0 { + break + } + // pop a node `n`` off of topLevelNodes + n := topLevelNodes[0] + topLevelNodes = topLevelNodes[1:] + // add it to the sorted list + sortedIDs = append(sortedIDs, n) + nEdgeList := G.directedEdgeList[n] + + // normally we'd do map iteration, but because we need cross-machine determinism, + // we gather all the nodes M for which there is an edge n -> m, sort that list, + // and then iterate over it. + nodesM := make([]int, 0, len(nEdgeList)) + for m, direction := range nEdgeList { + if direction != 1 { + panic("dag: topological sort correctness error. " + + "Popped node n was expected to have no incoming edges") + } + nodesM = append(nodesM, m) + } + + sort.Ints(nodesM) + + for _, m := range nodesM { + // remove edge from n -> m + G.deleteEdge(n, m) + // if m has no incomingEdges, add to topLevelNodes + if !G.hasIncomingEdge(m) { + topLevelNodes = append(topLevelNodes, m) + } + } + } + + if G.hasEdges() { + fmt.Println(G) + panic("dag: invalid construction, attempted to topologically sort a tree that is not a dag. A cycle exists") + } + + return dag.idsToNames(sortedIDs) +} diff --git a/osmoutils/partialord/internal/dag/dag_test.go b/osmoutils/partialord/internal/dag/dag_test.go new file mode 100644 index 00000000000..ec99d230307 --- /dev/null +++ b/osmoutils/partialord/internal/dag/dag_test.go @@ -0,0 +1,154 @@ +package dag_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/osmosis-labs/osmosis/v7/osmoutils/partialord/internal/dag" +) + +type edge struct { + start, end string +} + +func TestTopologicalSort(t *testing.T) { + // Tests that topological sort works for various inputs. + // We hardcode the satisfying solution in the tests, even though it suffices + // to check that the partial ordering is sufficient. (and thats the only guarantee given externally) + // This is to ensure we catch differences in order between changes, and across machines. + simpleNodes := []string{"dog", "cat", "banana", "apple"} + simpleNodesRev := []string{"apple", "banana", "cat", "dog"} + tests := []struct { + nodes []string + edges []edge + expectedTopologicalOrder []string + }{ + { + // alphabetical ordering of simple nodes + nodes: simpleNodes, + edges: []edge{{"banana", "apple"}, {"cat", "banana"}, {"dog", "cat"}}, + expectedTopologicalOrder: simpleNodes, + }, + { + // apple > dog + nodes: simpleNodes, + edges: []edge{{"apple", "dog"}}, + expectedTopologicalOrder: []string{"cat", "banana", "apple", "dog"}, + }, + { + // apple > everything + nodes: simpleNodes, + edges: []edge{{"apple", "banana"}, {"apple", "cat"}, {"apple", "dog"}}, + expectedTopologicalOrder: []string{"apple", "dog", "cat", "banana"}, + }, + { + // apple > everything, on list with reversed initial order + nodes: simpleNodesRev, + edges: []edge{{"apple", "banana"}, {"apple", "cat"}, {"apple", "dog"}}, + expectedTopologicalOrder: []string{"apple", "banana", "cat", "dog"}, + }, + } + for _, tc := range tests { + dag := dag.NewDAG(tc.nodes) + for _, edge := range tc.edges { + err := dag.AddEdge(edge.start, edge.end) + require.NoError(t, err) + } + order := dag.TopologicalSort() + require.Equal(t, tc.expectedTopologicalOrder, order) + } +} + +func TestAddFirst(t *testing.T) { + simpleNodes := []string{"frog", "elephant", "dog", "cat", "banana", "apple"} + tests := []struct { + nodes []string + first []string + replaceEdges []edge + expectedTopologicalOrder []string + }{ + { + nodes: simpleNodes, + first: []string{"frog"}, + replaceEdges: []edge{{"banana", "apple"}, {"cat", "banana"}, {"dog", "cat"}}, + expectedTopologicalOrder: simpleNodes, + }, + { + nodes: simpleNodes, + first: []string{"elephant"}, + replaceEdges: []edge{{"banana", "apple"}, {"apple", "frog"}, {"dog", "cat"}}, + expectedTopologicalOrder: []string{"elephant", "dog", "banana", "cat", "apple", "frog"}, + }, + { + nodes: simpleNodes, + first: []string{"elephant", "frog"}, + replaceEdges: []edge{}, + expectedTopologicalOrder: []string{"elephant", "frog", "dog", "cat", "banana", "apple"}, + }, + { + // add three items in first, if implemented incorrectly could cause a cycle + nodes: simpleNodes, + first: []string{"dog", "elephant", "frog"}, + replaceEdges: []edge{}, + expectedTopologicalOrder: []string{"dog", "elephant", "frog", "cat", "banana", "apple"}, + }, + } + for _, tc := range tests { + dag := dag.NewDAG(tc.nodes) + dag.AddFirstElements(tc.first...) + for _, edge := range tc.replaceEdges { + err := dag.ReplaceEdge(edge.start, edge.end) + require.NoError(t, err) + } + order := dag.TopologicalSort() + require.Equal(t, tc.expectedTopologicalOrder, order) + } +} + +func TestAddLast(t *testing.T) { + simpleNodes := []string{"frog", "elephant", "dog", "cat", "banana", "apple"} + tests := []struct { + nodes []string + last []string + replaceEdges []edge + expectedTopologicalOrder []string + }{ + { + // causes no order change + nodes: simpleNodes, + last: []string{"apple"}, + replaceEdges: []edge{{"banana", "apple"}, {"cat", "banana"}, {"dog", "cat"}}, + expectedTopologicalOrder: simpleNodes, + }, + { + nodes: simpleNodes, + last: []string{"elephant"}, + replaceEdges: []edge{{"banana", "apple"}, {"apple", "frog"}, {"dog", "cat"}}, + expectedTopologicalOrder: []string{"dog", "banana", "cat", "apple", "frog", "elephant"}, + }, + { + nodes: simpleNodes, + last: []string{"elephant", "frog"}, + replaceEdges: []edge{}, + expectedTopologicalOrder: []string{"dog", "cat", "banana", "apple", "elephant", "frog"}, + }, + { + // add three items in last, if implemented incorrectly could cause a cycle + nodes: simpleNodes, + last: []string{"dog", "elephant", "frog"}, + replaceEdges: []edge{}, + expectedTopologicalOrder: []string{"cat", "banana", "apple", "dog", "elephant", "frog"}, + }, + } + for _, tc := range tests { + dag := dag.NewDAG(tc.nodes) + dag.AddLastElements(tc.last...) + for _, edge := range tc.replaceEdges { + err := dag.ReplaceEdge(edge.start, edge.end) + require.NoError(t, err) + } + order := dag.TopologicalSort() + require.Equal(t, tc.expectedTopologicalOrder, order) + } +} diff --git a/osmoutils/partialord/internal/dag/module.go b/osmoutils/partialord/internal/dag/module.go new file mode 100644 index 00000000000..4b4bcd971c4 --- /dev/null +++ b/osmoutils/partialord/internal/dag/module.go @@ -0,0 +1,9 @@ +// Package dag implements a simple Directed Acyclical Graph (DAG) for deterministic topological sorts +// +// It should not be externally exposed, and is intended to be a very simple dag implementation +// utilizing adjacency lists to store edges. +// +// This package is intended to be used for small scales, where performance of the algorithms is not critical. +// (e.g. sub 10k entries) +// Thus none of the algorithms in here are benchmarked, and just have correctness checks. +package dag diff --git a/osmoutils/partialord/module.go b/osmoutils/partialord/module.go new file mode 100644 index 00000000000..1d02c8665bc --- /dev/null +++ b/osmoutils/partialord/module.go @@ -0,0 +1,2 @@ +// package partialord allows one to define partial orderings, and derive a total ordering +package partialord diff --git a/osmoutils/partialord/partialord.go b/osmoutils/partialord/partialord.go new file mode 100644 index 00000000000..d97501b7a01 --- /dev/null +++ b/osmoutils/partialord/partialord.go @@ -0,0 +1,85 @@ +package partialord + +import ( + "sort" + + "github.com/osmosis-labs/osmosis/v7/osmoutils/partialord/internal/dag" +) + +type PartialOrdering struct { + // underlying dag, the partial ordering is stored via a dag + // https://en.wikipedia.org/wiki/Topological_sorting#Relation_to_partial_orders + dag dag.DAG + // bools for sealing, to prevent repeated invocation of first or last methods. + firstSealed bool + lastSealed bool +} + +// NewPartialOrdering creates a new partial ordering over the set of provided elements. +func NewPartialOrdering(elements []string) PartialOrdering { + elementsCopy := make([]string, len(elements)) + copy(elementsCopy, elements) + sort.Strings(elementsCopy) + return PartialOrdering{ + dag: dag.NewDAG(elements), + firstSealed: false, + lastSealed: false, + } +} + +func handleDAGErr(err error) { + // all dag errors are logic errors that the intended users of this package should not make. + if err != nil { + panic(err) + } +} + +// After marks that A should come after B +func (ord *PartialOrdering) After(A string, B string) { + // Set that A depends on B / an edge from B -> A + err := ord.dag.AddEdge(B, A) + handleDAGErr(err) +} + +// After marks that A should come before B +func (ord *PartialOrdering) Before(A string, B string) { + // Set that B depends on A / an edge from A -> B + err := ord.dag.AddEdge(A, B) + handleDAGErr(err) +} + +// Sets elems to be the first elements in the ordering. +// So if were making an ordering over {A, B, C, D, E}, and elems provided is {D, B, A} +// then we are guaranteed that the total ordering will begin with {D, B, A} +func (ord *PartialOrdering) FirstElements(elems ...string) { + if ord.firstSealed { + panic("FirstElements has already been called") + } + // We make every node in the dag have a dependency on elems[-1] + // then we change elems[-1] to depend on elems[-2], and so forth. + err := ord.dag.AddFirstElements(elems...) + handleDAGErr(err) + ord.firstSealed = true +} + +// Sets elems to be the last elements in the ordering. +// So if were making an ordering over {A, B, C, D, E}, and elems provided is {D, B, A} +// then we are guaranteed that the total ordering will end with {D, B, A} +func (ord *PartialOrdering) LastElements(elems ...string) { + if ord.lastSealed { + panic("FirstElements has already been called") + } + // We make every node in the dag have a dependency on elems[0] + // then we make elems[1] depend on elems[0], and so forth. + err := ord.dag.AddLastElements(elems...) + handleDAGErr(err) + ord.lastSealed = true +} + +// TotalOrdering returns a deterministically chosen total ordering that satisfies all specified +// partial ordering constraints. +// +// Panics if no total ordering exists. +func (ord *PartialOrdering) TotalOrdering() []string { + return ord.dag.TopologicalSort() +} diff --git a/osmoutils/partialord/partialord_test.go b/osmoutils/partialord/partialord_test.go new file mode 100644 index 00000000000..6973b5fda3d --- /dev/null +++ b/osmoutils/partialord/partialord_test.go @@ -0,0 +1,52 @@ +package partialord_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/osmosis-labs/osmosis/v7/osmoutils/partialord" +) + +func TestAPI(t *testing.T) { + // begin block use case, we have a dozen modules, but only care about a couple orders. + // In practice this will be gotten from some API, e.g. app.AllModuleNames() + moduleNames := []string{"auth", "authz", "bank", "capabilities", + "staking", "distribution", "epochs", "mint", "upgrades", "wasm", "ibc", + "ibctransfers", "bech32ibc"} + beginBlockOrd := partialord.NewPartialOrdering(moduleNames) + beginBlockOrd.FirstElements("upgrades", "epochs", "capabilities") + beginBlockOrd.After("ibctransfers", "ibc") + beginBlockOrd.After("bech32ibc", "ibctransfers") + beginBlockOrd.Before("mint", "distribution") + // This is purely just to test last functionality, doesn't make sense in context + beginBlockOrd.LastElements("auth", "authz", "wasm") + + totalOrd := beginBlockOrd.TotalOrdering() + expTotalOrd := []string{"upgrades", "epochs", "capabilities", + "bank", "staking", "mint", "ibc", "distribution", "ibctransfers", "bech32ibc", + "auth", "authz", "wasm"} + require.Equal(t, expTotalOrd, totalOrd) +} + +func TestNonStandardAPIOrder(t *testing.T) { + // This test uses direct ordering before First, and after Last + names := []string{"A", "B", "C", "D", "E", "F", "G"} + ord := partialord.NewPartialOrdering(names) + ord.After("A", "C") + ord.After("A", "D") + ord.After("E", "B") + // overrides the "A" after "C" & "A" after "D" constraints + ord.FirstElements("A", "B", "C") + expOrdering := []string{"A", "B", "C", "D", "E", "F", "G"} + require.Equal(t, expOrdering, ord.TotalOrdering()) + + ord.After("E", "D") + expOrdering = []string{"A", "B", "C", "D", "F", "G", "E"} + require.Equal(t, expOrdering, ord.TotalOrdering()) + + ord.LastElements("G") + ord.After("F", "E") + expOrdering = []string{"A", "B", "C", "D", "E", "F", "G"} + require.Equal(t, expOrdering, ord.TotalOrdering()) +}