Add params for delay in k8s

examples/0004-big-network-w-delay.toml

@@ -0,0 +1,29 @@
+[settings]
+global_delay = "yes"
+timeout = 666
+
+[relaychain]
+default_image = "docker.io/parity/polkadot:latest"
+default_command = "polkadot"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+  [[relaychain.node_groups]]
+  name = "a"
+  args = [ "-lparachain=debug", "--database=paritydb-experimental" ]
+  count = 5
+
+  [[relaychain.node_groups]]
+  name = "b"
+  count = 5
+
+[[parachains]]
+id = 100
+
+  [[parachains.collator_groups]]
+    count = 2
+    [parachains.collator_groups.collator]
+    name = "collator"
+    command = "polkadot-parachain"
+    image = "docker.io/parity/polkadot-parachain:latest"

examples/0004-big-network-w-delay.zndsl

@@ -0,0 +1,11 @@
+Description: Big Network test
+Network: ./0004-big-network-w-delay.toml
+Creds: config
+
+# metrics
+a: reports node_roles is 4
+b: reports sub_libp2p_is_major_syncing is 0
+
+## test the block height - (or minus) finalised block
+a: reports block height minus finalised block is lower than 10 within 20 seconds
+b: reports block height - finalised block is lower than 10 within 20 seconds

javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts

@@ -3,8 +3,13 @@ import { getUniqueName } from "../../configGenerator";
 import { TMP_DONE, WAIT_UNTIL_SCRIPT_SUFIX } from "../../constants";
 import { Network } from "../../network";
 import { Node, ZombieRole } from "../../types";
-import { BootNodeResource, NodeResource, ServiceResource } from "./resources";
-import { PodSpec, ServiceSpec } from "./resources/types";
+import {
+  BootNodeResource,
+  ChaosResource,
+  NodeResource,
+  ServiceResource,
+} from "./resources";
+import { ChaosSpec, PodSpec, ServiceSpec } from "./resources/types";
 
 export async function genBootnodeDef(
   namespace: string,
@@ -22,9 +27,14 @@ export async function genNodeDef(
   return nodeResource.generateSpec();
 }
 
+export function genChaosDef(podSpec: PodSpec): ChaosSpec {
+  const resource = new ChaosResource(podSpec);
+  return resource.generateSpec();
+}
+
 export function genServiceDef(podSpec: PodSpec): ServiceSpec {
-  const serviceResource = new ServiceResource(podSpec);
-  return serviceResource.generateSpec();
+  const resource = new ServiceResource(podSpec);
+  return resource.generateSpec();
 }
 
 export function replaceNetworkRef(podDef: any, network: Network) {

javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts

@@ -25,7 +25,7 @@ import {
   RunCommandResponse,
   setClient,
 } from "../client";
-import { genServiceDef } from "./dynResourceDefinition";
+import { genChaosDef, genServiceDef } from "./dynResourceDefinition";
 const fs = require("fs").promises;
 
 const debug = require("debug")("zombie::kube::client");
@@ -141,8 +141,8 @@ export class KubeClient extends Client {
 
     await this.createResource(podDef, true);
     if (podDef.metadata.labels["zombie-role"] !== ZombieRole.Temp) {
-      const serviceDef = genServiceDef(podDef);
-      await this.createResource(serviceDef, true);
+      const genFn = podDef.spec.delay ? genChaosDef : genServiceDef;
+      await this.createResource(genFn(podDef));
     }
 
     await this.waitTransferContainerReady(name);

javascript/packages/orchestrator/src/providers/k8s/resources/chaosResource.ts

@@ -0,0 +1,119 @@
+import {
+  JAEGER_AGENT_SERVE_CONFIGS_PORT,
+  JAEGER_AGENT_THRIFT_BINARY_PORT,
+  JAEGER_AGENT_THRIFT_COMPACT_PORT,
+  JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+  P2P_PORT,
+  PROMETHEUS_PORT,
+  RPC_HTTP_PORT,
+  RPC_WS_PORT,
+} from "../../../constants";
+import { ChaosSpec, PodSpec, ServiceSpec } from "./types";
+
+import { BadNetworkSettings } from "../../../types";
+
+export class ChaosResource {
+  constructor(private readonly podSpec: PodSpec) {}
+
+  public generateSpec() {
+    const ports = this.generatePorts();
+    const name = this.podSpec.metadata.name;
+    const delay = this.podSpec.spec.delay!;
+
+    if (delay.latency.slice(-2) !== "ms") {
+      throw Error(
+        "Latency value should include the 'ms' indicator (e.g. '100ms')",
+      );
+    }
+
+    if (delay?.jitter.slice(-2) !== "ms") {
+      throw Error(
+        "Jitter value should include the 'ms' indicator (e.g. '100ms')",
+      );
+    }
+    return this.generateChaosSpec(name, ports, delay);
+  }
+
+  private shouldExposeJaegerPorts(): boolean {
+    return this.podSpec.spec.containers.some(
+      (container) => container.name === "jaeger-agent",
+    );
+  }
+
+  private generatePorts(): ServiceSpec["spec"]["ports"] {
+    let ports: ServiceSpec["spec"]["ports"] = [
+      {
+        name: "prometheus",
+        protocol: "TCP",
+        port: PROMETHEUS_PORT,
+        targetPort: PROMETHEUS_PORT,
+      },
+      {
+        name: "rpc-http",
+        protocol: "TCP",
+        port: RPC_HTTP_PORT,
+        targetPort: RPC_HTTP_PORT,
+      },
+      {
+        name: "rpc-ws",
+        protocol: "TCP",
+        port: RPC_WS_PORT,
+        targetPort: RPC_WS_PORT,
+      },
+      {
+        name: "p2p",
+        protocol: "TCP",
+        port: P2P_PORT,
+        targetPort: P2P_PORT,
+      },
+    ];
+
+    if (this.shouldExposeJaegerPorts()) {
+      ports = ports.concat([
+        {
+          name: "jaeger-agent-zipkin-compact",
+          protocol: "UDP",
+          port: JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+          targetPort: JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+        },
+        {
+          name: "jaeger-agent-serve-configs",
+          protocol: "TCP",
+          port: JAEGER_AGENT_SERVE_CONFIGS_PORT,
+          targetPort: JAEGER_AGENT_SERVE_CONFIGS_PORT,
+        },
+        {
+          name: "jaeger-agent-thrift-compact",
+          protocol: "UDP",
+          port: JAEGER_AGENT_THRIFT_COMPACT_PORT,
+          targetPort: JAEGER_AGENT_THRIFT_COMPACT_PORT,
+        },
+        {
+          name: "jaeger-agent-thrift-binary",
+          protocol: "UDP",
+          port: JAEGER_AGENT_THRIFT_BINARY_PORT,
+          targetPort: JAEGER_AGENT_THRIFT_BINARY_PORT,
+        },
+      ]);
+    }
+
+    return ports;
+  }
+
+  private generateChaosSpec(
+    name: string,
+    ports: ChaosSpec["spec"]["ports"],
+    delay: BadNetworkSettings,
+  ): ChaosSpec {
+    return {
+      apiVersion: "chaos-mesh.org/v1alpha1",
+      kind: "NetworkChaos",
+      metadata: { name },
+      spec: {
+        selector: { pods: name },
+        ports,
+        delay,
+      },
+    };
+  }
+}

javascript/packages/orchestrator/src/providers/k8s/resources/index.ts

@@ -1,3 +1,4 @@
 export { BootNodeResource } from "./bootNodeResource";
+export { ChaosResource } from "./chaosResource";
 export { NodeResource } from "./nodeResource";
 export { ServiceResource } from "./serviceResource";

javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts

@@ -16,7 +16,6 @@ export class ServiceResource {
   public generateSpec() {
     const ports = this.generatePorts();
     const name = this.podSpec.metadata.name;
-
     return this.generateServiceSpec(name, ports);
   }
 

javascript/packages/orchestrator/src/providers/k8s/resources/types.ts

@@ -1,4 +1,9 @@
-import { envVars, Resources, ZombieRoleLabel } from "../../../types";
+import {
+  BadNetworkSettings,
+  envVars,
+  Resources,
+  ZombieRoleLabel,
+} from "../../../types";
 
 type ContainerResource = Resources["resources"];
 
@@ -53,6 +58,7 @@ export interface InnerPodSpec {
     runAsUser: number;
     runAsGroup: number;
   };
+  delay?: BadNetworkSettings;
 }
 
 export interface PodSpec {
@@ -82,3 +88,27 @@ export interface ServiceSpec {
     }[];
   };
 }
+
+export interface ChaosSpec {
+  apiVersion: "chaos-mesh.org/v1alpha1";
+  kind: "NetworkChaos";
+  metadata: { name: string };
+  spec: {
+    selector: SelectorTypeNS | SelectorTypePods;
+    ports: {
+      name: string;
+      protocol: "TCP" | "UDP";
+      port: number;
+      targetPort: number;
+    }[];
+    delay: BadNetworkSettings;
+  };
+}
+
+interface SelectorTypeNS {
+  namespaces: string;
+}
+
+interface SelectorTypePods {
+  pods: string;
+}

javascript/packages/orchestrator/src/providers/native/chainSpec.ts

@@ -21,6 +21,8 @@ export async function setupChainSpec(
   // 1: User provide the file (we DON'T expect the raw file)
   // 2: User provide the chainSpecCommand (without the --raw option)
   const client = getClient();
+
+  console.log("0000000000000 ", chainConfig);
   if (chainConfig.chainSpecPath) {
     // copy file to temp to use
     await fsPromises.copyFile(chainConfig.chainSpecPath, chainFullPath);

javascript/packages/orchestrator/src/test-runner/index.ts

@@ -49,6 +49,8 @@ export async function run(
     ? testDef.network
     : path.resolve(configBasePath, testDef.network);
 
+  console.log("networkConfigFilePath", networkConfigFilePath);
+
   const config: LaunchConfig = readNetworkConfig(networkConfigFilePath);
 
   // set the provider
@@ -83,6 +85,8 @@ export async function run(
   const suite = Suite.create(mocha.suite, suiteName);
 
   suite.beforeAll("launching", async function () {
+    console.log("config: ", config);
+
     const launchTimeout = config.settings?.timeout || 500;
     this.timeout(launchTimeout * 1000);
     try {

javascript/packages/orchestrator/src/types.ts

@@ -37,6 +37,7 @@ export interface Settings {
   backchannel?: boolean; // only used in k8s at the moment, spawn a backchannel instance
   image_pull_policy?: "IfNotPresent" | "Never" | "Always";
   local_ip?: string; // ip used for expose local services (rpc/metrics/monitors)
+  bad_network_global_settings?: BadNetworkSettings;
 }
 
 export interface RelayChainConfig {
@@ -57,6 +58,7 @@ export interface RelayChainConfig {
   node_groups?: NodeGroupConfig[];
   total_node_in_groups?: number;
   genesis?: JSON | ObjectJSON;
+  bad_network_settings?: BadNetworkSettings;
 }
 
 export interface NodeConfig {
@@ -117,6 +119,7 @@ export interface ParachainConfig {
   collators?: NodeConfig[];
   collator_groups?: NodeGroupConfig[];
   genesis?: JSON | ObjectJSON;
+  bad_network_settings?: BadNetworkSettings;
 }
 
 export interface HrmpChannelsConfig {
@@ -358,3 +361,9 @@ export enum SubstrateCliArgsVersion {
   V0 = 0,
   V1 = 1,
 }
+
+export interface BadNetworkSettings {
+  latency: string;
+  correlation: string;
+  jitter: string;
+}