Merge branch 'main' into bad-name

docs/metatype.dev/docs/reference/typegate/synchronization/index.mdx

@@ -69,6 +69,7 @@ Synchronization variable names start with `SYNC_`.
 | SYNC\__S3_SECRET_KEY (\_Required_)  | Access key secret for the S3 store credentials;              |
 | SYNC\__S3_PATH_STYLE (\_Optional_)  | `true` or `false`, force path style if `true`.               |
 | SYNC\__S3_BUCKET (\_Required_)      | The bucket to be used for the system (dedicated).            |
+| SYNC\__FORCE_REMOVE (\_Optional_)   | `true` or `false`, Undeploy cached typegraphs at boot        |
 
 ## Synchronized mode features
 

src/common/src/typegraph/runtimes/deno.rs

@@ -1,6 +1,7 @@
 // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
 // SPDX-License-Identifier: MPL-2.0
 
+use anyhow::{anyhow, Context, Result};
 use indexmap::IndexMap;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -11,6 +12,47 @@ pub struct FunctionMatData {
     pub script: String,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, Hash, PartialEq, Eq)]
+#[serde(tag = "type", content = "value")]
+#[serde(rename_all = "snake_case")]
+pub enum ContextCheckX {
+    NotNull,
+    Value(String),
+    Pattern(String),
+}
+
+#[derive(PartialEq, Eq, Hash, Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "snake_case")]
+#[serde(tag = "name", content = "param")]
+pub enum PredefinedFunctionMatData {
+    Identity,
+    True,
+    False,
+    Allow,
+    Deny,
+    Pass,
+    InternalPolicy,
+    ContextCheck { key: String, value: ContextCheckX },
+}
+
+#[derive(Serialize)]
+struct PredefinedFunctionMatDataRaw {
+    name: String,
+    param: Option<Value>,
+}
+
+impl PredefinedFunctionMatData {
+    pub fn from_raw(name: String, param: Option<String>) -> Result<Self> {
+        let param = param
+            .map(|p| serde_json::from_str(&p))
+            .transpose()
+            .context("invalid predefined function materializer parameter")?;
+        let value = serde_json::to_value(&PredefinedFunctionMatDataRaw { name, param })?;
+        serde_json::from_value(value)
+            .map_err(|e| anyhow!("invalid predefined function materializer: {e:?}"))
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct ModuleMatData {

src/typegate/src/config.ts

@@ -117,6 +117,7 @@ export function transformSyncConfig(raw: SyncConfig): SyncConfigX {
     redis,
     s3,
     s3Bucket: raw.s3_bucket,
+    forceRemove: raw.force_remove
   };
 }
 

src/typegate/src/config/types.ts

@@ -115,12 +115,14 @@ export const syncConfigSchema = z.object({
   s3_access_key: refineEnvVar("SYNC_S3_ACCESS_KEY"),
   s3_secret_key: refineEnvVar("SYNC_S3_SECRET_KEY"),
   s3_path_style: zBooleanString.default(false),
+  force_remove: zBooleanString.default(false),
 });
 export type SyncConfig = z.infer<typeof syncConfigSchema>;
 export type SyncConfigX = {
   redis: RedisConnectOptions;
   s3: S3ClientConfig;
   s3Bucket: string;
+  forceRemove?: boolean
 };
 
 export type TypegateConfig = {

src/typegate/src/main.ts

@@ -53,7 +53,6 @@ try {
     base: defaultTypegateConfigBase,
   });
   const typegate = await Typegate.init(config);
-
   await SystemTypegraph.loadAll(typegate, !globalConfig.packaged);
 
   const server = Deno.serve(

src/typegate/src/runtimes/deno/deno.ts

@@ -18,7 +18,6 @@ import type {
 } from "../../typegraph/types.ts";
 import * as ast from "graphql/ast";
 import { InternalAuth } from "../../services/auth/protocols/internal.ts";
-import { DenoMessenger } from "./deno_messenger.ts";
 import type { Task } from "./shared_types.ts";
 import { path } from "compress/deps.ts";
 import { globalConfig as config } from "../../config.ts";
@@ -27,17 +26,46 @@ import { PolicyResolverOutput } from "../../engine/planner/policies.ts";
 import { getInjectionValues } from "../../engine/planner/injection_utils.ts";
 import DynamicInjection from "../../engine/injection/dynamic.ts";
 import { getLogger } from "../../log.ts";
+import { WorkerManager } from "./worker_manager.ts";
 
 const logger = getLogger(import.meta);
 
-const predefinedFuncs: Record<string, Resolver<Record<string, unknown>>> = {
-  identity: ({ _, ...args }) => args,
-  true: () => true,
-  false: () => false,
-  allow: () => "ALLOW" as PolicyResolverOutput,
-  deny: () => "DENY" as PolicyResolverOutput,
-  pass: () => "PASS" as PolicyResolverOutput,
-  internal_policy: ({ _: { context } }) => context.provider === "internal" ? "ALLOW" : "PASS" as PolicyResolverOutput,
+const predefinedFuncs: Record<
+  string,
+  (param: any) => Resolver<Record<string, unknown>>
+> = {
+  identity: () => ({ _, ...args }) => args,
+  true: () => () => true,
+  false: () => () => false,
+  allow: () => () => "ALLOW" as PolicyResolverOutput,
+  deny: () => () => "DENY" as PolicyResolverOutput,
+  pass: () => () => "PASS" as PolicyResolverOutput,
+  internal_policy: () => ({ _: { context } }) =>
+    context.provider === "internal" ? "ALLOW" : "PASS" as PolicyResolverOutput,
+  context_check: ({ key, value }) => {
+    let check: (value: any) => boolean;
+    switch (value.type) {
+      case "not_null":
+        check = (v) => v != null;
+        break;
+      case "value":
+        check = (v) => v === value.value;
+        break;
+      case "pattern":
+        check = (v) => v != null && new RegExp(value.value).test(v);
+        break;
+      default:
+        throw new Error("unreachable");
+    }
+    const path = key.split(".");
+    return ({ _: { context } }) => {
+      let value: any = context;
+      for (const segment of path) {
+        value = value?.[segment];
+      }
+      return check(value) ? "PASS" : "DENY" as PolicyResolverOutput;
+    };
+  },
 };
 
 export class DenoRuntime extends Runtime {
@@ -46,8 +74,7 @@ export class DenoRuntime extends Runtime {
     uuid: string,
     private tg: TypeGraphDS,
     private typegate: Typegate,
-    private w: DenoMessenger,
-    private registry: Map<string, number>,
+    private workerManager: WorkerManager,
     private secrets: Record<string, string>,
   ) {
     super(typegraphName, uuid);
@@ -138,36 +165,24 @@ export class DenoRuntime extends Runtime {
       }
     }
 
-    const w = new DenoMessenger(
-      name,
-      {
-        ...(args.permissions ?? {}),
-        read: [basePath],
-      } as Deno.PermissionOptionsObject,
-      false,
-      ops,
-      typegate.config.base,
-    );
-
-    if (Deno.env.get("DENO_TESTING") === "true") {
-      w.disableLazyness();
-    }
+    const workerManager = new WorkerManager({
+      timeout_ms: typegate.config.base.timer_max_timeout_ms,
+    });
 
     const rt = new DenoRuntime(
       typegraphName,
       uuid,
       tg,
       typegate,
-      w,
-      registry,
+      workerManager,
       secrets,
     );
 
     return rt;
   }
 
   async deinit(): Promise<void> {
-    await this.w.terminate();
+    // await this.workerManager.deinit();
   }
 
   materialize(
@@ -241,7 +256,7 @@ export class DenoRuntime extends Runtime {
       if (!func) {
         throw new Error(`predefined function ${mat.data.name} not found`);
       }
-      return func;
+      return func(mat.data.param);
     }
 
     if (mat.name === "static") {
@@ -257,7 +272,10 @@ export class DenoRuntime extends Runtime {
       const modMat = this.tg.materializers[mat.data.mod as number];
       const entryPoint =
         this.tg.meta.artifacts[modMat.data.entryPoint as string];
-      const op = this.registry.get(entryPoint.hash)!;
+      const depMetas = (modMat.data.deps as string[]).map((dep) =>
+        createArtifactMeta(this.typegraphName, this.tg.meta.artifacts[dep])
+      );
+      const moduleMeta = createArtifactMeta(this.typegraphName, entryPoint);
 
       return async ({
         _: {
@@ -269,33 +287,33 @@ export class DenoRuntime extends Runtime {
       }) => {
         const token = await InternalAuth.emit(this.typegate.cryptoKeys);
 
-        return await this.w.execute(
-          op,
+        // TODO cache??
+        const entryModulePath = await this.typegate.artifactStore.getLocalPath(
+          moduleMeta,
+          depMetas,
+        );
+
+        return await this.workerManager.callFunction(
+          mat.data.name as string,
+          entryModulePath,
+          entryPoint.path,
+          args,
           {
-            type: "import_func",
-            args,
-            internals: {
-              parent,
-              context,
-              secrets,
-              effect: mat.effect.effect ?? null,
-              meta: {
-                url: `${url.protocol}//${url.host}/${this.typegraphName}`,
-                token,
-              },
-              headers,
+            parent,
+            context,
+            secrets,
+            effect: mat.effect.effect ?? null,
+            meta: {
+              url: `${url.protocol}//${url.host}/${this.typegraphName}`,
+              token,
             },
-            name: mat.data.name as string,
-            verbose,
+            headers,
           },
-          [],
-          pulseCount,
         );
       };
     }
 
     if (mat.name === "function") {
-      const op = this.registry.get(mat.data.script as string)!;
       return async ({
         _: {
           context,
@@ -306,26 +324,29 @@ export class DenoRuntime extends Runtime {
       }) => {
         const token = await InternalAuth.emit(this.typegate.cryptoKeys);
 
-        return await this.w.execute(
-          op,
+        const modulePath = await this.typegate.artifactStore.getInlineArtifact(
+          this.typegraphName,
+          mat.data.script as string,
+          ".ts",
+          exportInlineFunction("inlineFunction"),
+        );
+
+        return await this.workerManager.callFunction(
+          "inlineFunction",
+          modulePath,
+          "tg",
+          args,
           {
-            type: "func",
-            args,
-            internals: {
-              parent,
-              context,
-              secrets,
-              effect: mat.effect.effect ?? null,
-              meta: {
-                url: `${url.protocol}//${url.host}/${this.typegraphName}`,
-                token,
-              },
-              headers,
+            parent,
+            context,
+            secrets,
+            effect: mat.effect.effect ?? null,
+            meta: {
+              url: `${url.protocol}//${url.host}/${this.typegraphName}`,
+              token,
             },
-            verbose,
+            headers,
           },
-          [],
-          pulseCount,
         );
       };
     }
@@ -365,6 +386,18 @@ export class DenoRuntime extends Runtime {
   }
 }
 
+function exportInlineFunction(name = "fn", symbol = "_my_lambda") {
+  if (!name.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
+    throw new Error(`Invalid identifier: ${name}`);
+  }
+  if (!symbol.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
+    throw new Error(`Invalid identifier: ${symbol}`);
+  }
+  return (code: string) => {
+    return `${code}\nexport const ${name} = ${symbol};`;
+  };
+}
+
 function getInjectionData(d: InjectionData) {
   if ("value" in d) {
     return d.value;