Vector pkg

.github/workflows/bins.yaml

@@ -36,12 +36,6 @@ jobs:
       package: virtwhat
     secrets:
       token: ${{ secrets.HUB_JWT }}
-  logtrunc:
-    uses: ./.github/workflows/bin-package.yaml
-    with:
-      package: logtrunc
-    secrets:
-      token: ${{ secrets.HUB_JWT }}
   yggdrasil:
     uses: ./.github/workflows/bin-package.yaml
     with:
@@ -96,10 +90,16 @@ jobs:
       package: virtiofsd
     secrets:
       token: ${{ secrets.HUB_JWT }}
-  node-exporter:
+  # node-exporter:
+  #   uses: ./.github/workflows/bin-package.yaml
+  #   with:
+  #     package: node-exporter
+  #   secrets:
+  #     token: ${{ secrets.HUB_JWT }}
+  vector:
     uses: ./.github/workflows/bin-package.yaml
     with:
-      package: node-exporter
+      package: vector
     secrets:
       token: ${{ secrets.HUB_JWT }}
   tpm:

bins/packages/node-exporter/root/etc/zinit/node-exporter.yaml

@@ -1 +1 @@
-exec: sh -c "if grep -E '(runmode=test|runmode=dev|runmode=qa)' /proc/cmdline ; then node_exporter '--collector.netdev.device-exclude=(b|br)-.*' ; else sleep infinity ; fi"
+exec: node_exporter '--collector.netdev.device-exclude=(b|br)-.*'

bins/packages/vector/files/vector.yaml

@@ -0,0 +1,96 @@
+---
+sources:
+  zinit:
+    type: exec
+    mode: streaming
+    command:
+      - zinit
+      - log
+  metrics:
+    type: host_metrics
+    network:
+      devices:
+        excludes:
+          - b-*
+          - br-*
+          - dumdum
+          - dummy*
+          - tozos*
+    filesystem:
+      mountpoints:
+        excludes:
+          - /var/cache
+          - /var/cache/*
+          - /var/run/*
+          - /dev
+          - /dev/*
+          - /sys
+          - /sys/*
+          - /proc
+          - /proc/*
+transforms:
+  parsed:
+    type: remap
+    inputs:
+      - zinit
+    source: |-
+      structured, err = parse_regex(.message, r'\[(?P<output>\+|\-)\] (?P<module>[^:]+):')
+      if err == null {
+        . = merge(., structured)
+      }
+
+      level, err = parse_regex(.message, r'(?P<level>debug|info|error|warn|fatal|panic)')
+      if err != null {
+        .level = "info"
+      } else {
+        . = merge(., level)
+      }
+
+      if .output == "+" {
+        .stream = "stdout"
+      } else {
+        .stream = "stderr"
+      }
+
+      .node = get_env_var("NODE") ?? "unknown"
+      .network = get_env_var("NETWORK") ?? "unknown"
+      .farm = get_env_var("FARM") ?? "unknown"
+
+      del(.output)
+      del(.command)
+      del(.host)
+      del(.source_type)
+      del(.pid)
+
+sinks:
+  # out:
+  #   inputs:
+  #     - parsed
+  #   type: console
+  #   encoding:
+  #     codec: "text"
+  # TODO: enable once we have vector installed
+  # vector:
+  #   inputs:
+  #     - metrics
+  #   type: console
+  #   encoding:
+  #     codec: "json"
+
+  loki:
+    inputs:
+      - parsed
+    type: loki
+    labels:
+      node: "{{ node }}"
+      level: "{{ level }}"
+      module: "{{ module }}"
+      network: "{{ network }}"
+      stream: "{{ stream }}"
+      farm: "{{ farm }}"
+    endpoint: http://loki.grid.tf:3100
+    encoding:
+      codec: text
+    compression: snappy
+    healthcheck:
+      enabled: false

bins/packages/vector/files/zinit-vector.yaml

@@ -0,0 +1,14 @@
+# pkill -HUP is for older version of zinit that doesn't start process
+# in its own process group
+exec: |
+  sh -c '
+  pkill vector
+
+  export NODE=$(identityd -address)
+  export FARM=$(identityd -farm)
+  export NETWORK=$(identityd -net)
+  exec vector -c /etc/vector/vector.yaml
+  '
+log: stdout
+after:
+  - identityd

bins/packages/vector/vector.sh

@@ -0,0 +1,52 @@
+VECTOR_VERSION="0.25.1"
+VECTOR_CHECKSUM="07bcae774d8f6dc5f34a5f4f7bafd313"
+VECTOR_LINK="https://github.com/vectordotdev/vector/releases/download/v${VECTOR_VERSION}/vector-${VECTOR_VERSION}-x86_64-unknown-linux-musl.tar.gz"
+VECTOR_PACKAGE="vector.tar.gz"
+
+download_vector() {
+    download_file ${VECTOR_LINK} ${VECTOR_CHECKSUM} ${VECTOR_PACKAGE}
+}
+
+extract_vector() {
+    tar -xf  ${DISTDIR}/${VECTOR_PACKAGE} -C ${WORKDIR}
+}
+
+prepare_vector() {
+    echo "[+] prepare vector"
+    github_name "vector-${VECTOR_VERSION}"
+}
+
+compile_vector() {
+    echo "[+] compile vector"
+}
+
+install_vector() {
+    echo "[+] install vector"
+
+    mkdir -p "${ROOTDIR}/usr/bin"
+    mkdir -p "${ROOTDIR}/etc/zinit"
+    mkdir -p "${ROOTDIR}/etc/vector"
+
+    cp ${WORKDIR}/vector-x86_64-unknown-linux-musl/bin/vector ${ROOTDIR}/usr/bin/vector
+
+    cp ${FILESDIR}/zinit-vector.yaml ${ROOTDIR}/etc/zinit/vector.yaml
+    cp ${FILESDIR}/vector.yaml ${ROOTDIR}/etc/vector/
+
+    chmod +x ${ROOTDIR}/usr/bin/*
+}
+
+build_vector() {
+    pushd "${DISTDIR}"
+
+    download_vector
+    extract_vector
+
+    popd
+    pushd ${WORKDIR}
+
+    prepare_vector
+    compile_vector
+    install_vector
+
+    popd
+}

cmds/identityd/main.go

@@ -64,30 +64,53 @@ func main() {
 		interval int
 		ver      bool
 		debug    bool
-		id       bool
+
+		id      bool
+		net     bool
+		farm    bool
+		address bool
 	)
 
 	flag.StringVar(&root, "root", "/var/cache/modules/identityd", "root working directory of the module")
 	flag.StringVar(&broker, "broker", redisSocket, "connection string to broker")
 	flag.IntVar(&interval, "interval", 600, "interval in seconds between update checks, default to 600")
 	flag.BoolVar(&ver, "v", false, "show version and exit")
-	flag.BoolVar(&debug, "d", false, "when set, no self update is done before upgradeing")
-	flag.BoolVar(&id, "id", false, "prints the node ID and exits")
+	flag.BoolVar(&debug, "d", false, "when set, no self update is done before upgrading")
+	flag.BoolVar(&id, "id", false, "[deprecated] prints the node ID and exits")
+	flag.BoolVar(&net, "net", false, "prints the node network and exits")
+	flag.BoolVar(&farm, "farm", false, "prints the node farm id and exits")
+	flag.BoolVar(&address, "address", false, "prints the node ss58 address and exits")
 
 	flag.Parse()
 	if ver {
 		version.ShowAndExit(false)
 	}
 
-	if id {
+	if farm {
+		env := environment.MustGet()
+		fmt.Println(env.FarmerID)
+		os.Exit(0)
+	} else if net {
+		env := environment.MustGet()
+		fmt.Println(env.RunningMode.String())
+		os.Exit(0)
+	} else if id || address {
 		ctx := context.Background()
 		client, err := zbus.NewRedisClient(broker)
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed to connect to zbus")
 		}
 		stub := stubs.NewIdentityManagerStub(client)
-		nodeID := stub.NodeID(ctx)
-		fmt.Println(nodeID)
+
+		if id {
+			fmt.Println(stub.NodeID(ctx))
+		} else { // address
+			add, err := stub.Address(ctx)
+			if err != nil {
+				log.Fatal().Err(err).Msg("failed to get node address")
+			}
+			fmt.Println(add)
+		}
 		os.Exit(0)
 	}
 

pkg/identity.go

@@ -18,6 +18,12 @@ func (s StrIdentifier) Identity() string {
 	return string(s)
 }
 
+type Address string
+
+func (s Address) String() string {
+	return string(s)
+}
+
 // IdentityManager interface.
 type IdentityManager interface {
 	// Store returns the key store kind
@@ -26,8 +32,8 @@ type IdentityManager interface {
 	// NodeID returns the node id (public key)
 	NodeID() StrIdentifier
 
-	// NodeIDNumeric returns the node registered ID.
-	NodeIDNumeric() (uint32, error)
+	// Address return the node address (SS58Address address)
+	Address() (Address, error)
 
 	// FarmID return the farm id this node is part of. this is usually a configuration
 	// that the node is booted with. An error is returned if the farmer id is not configured

pkg/identity/identityd.go

@@ -20,7 +20,6 @@ type identityManager struct {
 	env  environment.Environment
 
 	farm string
-	node uint32
 }
 
 // NewManager creates an identity daemon from seed
@@ -83,31 +82,13 @@ func (d *identityManager) NodeID() pkg.StrIdentifier {
 	return pkg.StrIdentifier(d.key.Identity())
 }
 
-func (d *identityManager) NodeIDNumeric() (uint32, error) {
-	if d.node != 0 {
-		return d.node, nil
-	}
+// NodeID returns the node identity
+func (d *identityManager) Address() (pkg.Address, error) {
 	id, err := substrate.NewIdentityFromEd25519Key(d.key.PrivateKey)
 	if err != nil {
-		return 0, err
-	}
-	cl, err := d.sub.Substrate()
-	if err != nil {
-		return 0, err
-	}
-	defer cl.Close()
-
-	twin, err := cl.GetTwinByPubKey(id.PublicKey())
-	if err != nil {
-		return 0, errors.Wrap(err, "failed to get node twin")
-	}
-
-	node, err := cl.GetNodeByTwinID(twin)
-	if err != nil {
-		return 0, err
+		return "", err
 	}
-	d.node = node
-	return node, nil
+	return pkg.Address(id.Address()), nil
 }
 
 func (d *identityManager) Farm() (string, error) {

pkg/network/nr/net_resource_test.go

@@ -28,8 +28,8 @@ func (t *testIdentityManager) NodeID() pkg.StrIdentifier {
 	return pkg.StrIdentifier(t.id)
 }
 
-func (t *testIdentityManager) NodeIDNumeric() (uint32, error) {
-	return 1, nil
+func (t *testIdentityManager) Address() (pkg.Address, error) {
+	return pkg.Address(t.id), nil
 }
 
 func (t *testIdentityManager) Farm() (string, error) {