Modules support relative path 2

CHANGELOG.md

@@ -14,6 +14,9 @@ Main (unreleased)
 
 - Add the function `path_join` to the stdlib. (@wildum)
 
+- Add support for relative paths to `import.file`. This new functionality allows users to use `import.file` blocks in modules
+  imported via `import.git` and other `import.file`. (@wildum)
+
 ### Bugfixes
 
 - Update yet-another-cloudwatch-exporter from v0.60.0 vo v0.61.0: (@morremeyer)

docs/sources/reference/config-blocks/import.file.md

@@ -17,6 +17,9 @@ Imported directories are treated as single modules to support composability.
 That means that you can define a custom component in one file and use it in another custom component in another file
 in the same directory.
 
+You can use the keyword `module_path` in combination with the `stdlib` function [file.path_join][] to import a module relative to the current module's path.
+The `module_path` keyword works for modules that are imported via `import.file`, `import.git` and `import.string`.
+
 ## Usage
 
 ```alloy
@@ -41,6 +44,21 @@ The following arguments are supported:
 
 This example imports a module from a file and instantiates a custom component from the import that adds two numbers:
 
+{{< collapse title="main.alloy" >}}
+
+```alloy
+import.file "math" {
+  filename = "module.alloy"
+}
+
+math.add "default" {
+  a = 15
+  b = 45
+}
+```
+
+{{< /collapse >}}
+
 {{< collapse title="module.alloy" >}}
 
 ```alloy
@@ -56,11 +74,71 @@ declare "add" {
 
 {{< /collapse >}}
 
-{{< collapse title="importer.alloy" >}}
+
+This example imports a module from a file inside of a module that is imported via [import.git][]:
+
+{{< collapse title="main.alloy" >}}
+
+```alloy
+import.git "math" {
+  repository = "https://github.com/wildum/module.git"
+  path       = "relative_math.alloy"
+}
+
+math.add "default" {
+  a = 15
+  b = 45
+}
+```
+
+{{< /collapse >}}
+
+{{< collapse title="relative_math.alloy" >}}
+
+```alloy
+import.file "lib" {
+  filename = file.path_join(module_path, "lib.alloy")
+}
+
+declare "add" {
+  argument "a" {}
+  argument "b" {}
+
+  lib.plus "default" {
+    a = argument.a.value
+    b = argument.b.value
+  }
+
+  export "output" {
+    value = lib.plus.default.sum
+  }
+}
+```
+
+{{< /collapse >}}
+
+{{< collapse title="lib.alloy" >}}
+
+```alloy
+declare "plus" {
+  argument "a" {}
+  argument "b" {}
+
+  export "sum" {
+    value = argument.a.value + argument.b.value
+  }
+}
+```
+
+{{< /collapse >}}
+
+This example imports a module from a file inside of a module that is imported via another `import.file`:
+
+{{< collapse title="main.alloy" >}}
 
 ```alloy
 import.file "math" {
-  filename = "module.alloy"
+  filename = "path/to/module/relative_math.alloy"
 }
 
 math.add "default" {
@@ -70,3 +148,47 @@ math.add "default" {
 ```
 
 {{< /collapse >}}
+
+{{< collapse title="relative_math.alloy" >}}
+
+```alloy
+import.file "lib" {
+  filename = file.path_join(module_path, "lib.alloy")
+}
+
+declare "add" {
+  argument "a" {}
+  argument "b" {}
+
+  lib.plus "default" {
+    a = argument.a.value
+    b = argument.b.value
+  }
+
+  export "output" {
+    value = lib.plus.default.sum
+  }
+}
+```
+
+{{< /collapse >}}
+
+{{< collapse title="lib.alloy" >}}
+
+```alloy
+declare "plus" {
+  argument "a" {}
+  argument "b" {}
+
+  export "sum" {
+    value = argument.a.value + argument.b.value
+  }
+}
+```
+
+{{< /collapse >}}
+
+
+
+[file.path_join]: ../../stdlib/file/
+[import.git]: ../import.git/

internal/alloycli/cmd_run.go

@@ -326,7 +326,7 @@ func (fr *alloyRun) Run(configPath string) error {
 		if err != nil {
 			return nil, fmt.Errorf("reading config path %q: %w", configPath, err)
 		}
-		if err := f.LoadSource(alloySource, nil); err != nil {
+		if err := f.LoadSource(alloySource, nil, configPath); err != nil {
 			return alloySource, fmt.Errorf("error during the initial load: %w", err)
 		}
 

internal/converter/internal/test_common/testing.go

@@ -217,7 +217,7 @@ func attemptLoadingAlloyConfig(t *testing.T, bb []byte) {
 		},
 		EnableCommunityComps: true,
 	})
-	err = f.LoadSource(cfg, nil)
+	err = f.LoadSource(cfg, nil, "")
 
 	// Many components will fail to build as e.g. the cert files are missing, so we ignore these errors.
 	// This is not ideal, but we still validate for other potential issues.

internal/runtime/alloy.go

@@ -61,6 +61,8 @@ import (
 	"github.com/grafana/alloy/internal/runtime/logging/level"
 	"github.com/grafana/alloy/internal/runtime/tracing"
 	"github.com/grafana/alloy/internal/service"
+	"github.com/grafana/alloy/internal/util"
+	"github.com/grafana/alloy/syntax/vm"
 )
 
 // Options holds static options for an Alloy controller.
@@ -296,22 +298,36 @@ func (f *Runtime) Run(ctx context.Context) {
 // The controller will only start running components after Load is called once
 // without any configuration errors.
 // LoadSource uses default loader configuration.
-func (f *Runtime) LoadSource(source *Source, args map[string]any) error {
-	return f.loadSource(source, args, nil)
+func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath string) error {
+	return f.applyLoaderConfig(controller.ApplyOptions{
+		Args:            args,
+		ComponentBlocks: source.components,
+		ConfigBlocks:    source.configBlocks,
+		DeclareBlocks:   source.declareBlocks,
+		ArgScope: &vm.Scope{
+			Parent: nil,
+			Variables: map[string]interface{}{
+				"module_path": util.ExtractDirPath(configPath),
+			},
+		},
+	})
 }
 
 // Same as above but with a customComponentRegistry that provides custom component definitions.
 func (f *Runtime) loadSource(source *Source, args map[string]any, customComponentRegistry *controller.CustomComponentRegistry) error {
-	f.loadMut.Lock()
-	defer f.loadMut.Unlock()
-
-	applyOptions := controller.ApplyOptions{
+	return f.applyLoaderConfig(controller.ApplyOptions{
 		Args:                    args,
 		ComponentBlocks:         source.components,
 		ConfigBlocks:            source.configBlocks,
 		DeclareBlocks:           source.declareBlocks,
 		CustomComponentRegistry: customComponentRegistry,
-	}
+		ArgScope:                customComponentRegistry.Scope(),
+	})
+}
+
+func (f *Runtime) applyLoaderConfig(applyOptions controller.ApplyOptions) error {
+	f.loadMut.Lock()
+	defer f.loadMut.Unlock()
 
 	diags := f.loader.Apply(applyOptions)
 	if !f.loadedOnce.Load() && diags.HasErrors() {

internal/runtime/alloy_services.go

@@ -98,7 +98,8 @@ func (sc ServiceController) LoadSource(b []byte, args map[string]any) error {
 	if err != nil {
 		return err
 	}
-	return sc.f.LoadSource(source, args)
+	// The config is loaded via a service, the config path is left empty.
+	return sc.f.LoadSource(source, args, "")
 }
 func (sc ServiceController) Ready() bool { return sc.f.Ready() }
 

internal/runtime/alloy_services_test.go

@@ -38,7 +38,7 @@ func TestServices(t *testing.T) {
 	opts.Services = append(opts.Services, svc)
 
 	ctrl := New(opts)
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)
@@ -90,7 +90,7 @@ func TestServices_Configurable(t *testing.T) {
 
 	ctrl := New(opts)
 
-	require.NoError(t, ctrl.LoadSource(f, nil))
+	require.NoError(t, ctrl.LoadSource(f, nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)
@@ -137,7 +137,7 @@ func TestServices_Configurable_Optional(t *testing.T) {
 
 	ctrl := New(opts)
 
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)
@@ -171,7 +171,7 @@ func TestAlloy_GetServiceConsumers(t *testing.T) {
 
 	ctrl := New(opts)
 	defer cleanUpController(ctrl)
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	expectConsumers := []service.Consumer{{
 		Type:  service.ConsumerTypeService,
@@ -253,7 +253,7 @@ func TestComponents_Using_Services(t *testing.T) {
 		ComponentRegistry: registry,
 		ModuleRegistry:    newModuleRegistry(),
 	})
-	require.NoError(t, ctrl.LoadSource(f, nil))
+	require.NoError(t, ctrl.LoadSource(f, nil, ""))
 	go ctrl.Run(ctx)
 
 	require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built")
@@ -332,7 +332,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) {
 		ComponentRegistry: registry,
 		ModuleRegistry:    newModuleRegistry(),
 	})
-	require.NoError(t, ctrl.LoadSource(f, nil))
+	require.NoError(t, ctrl.LoadSource(f, nil, ""))
 	go ctrl.Run(ctx)
 
 	require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built")
@@ -360,7 +360,7 @@ func TestNewControllerNoLeak(t *testing.T) {
 	opts.Services = append(opts.Services, svc)
 
 	ctrl := New(opts)
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)

internal/runtime/alloy_test.go

@@ -43,7 +43,7 @@ func TestController_LoadSource_Evaluation(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, f)
 
-	err = ctrl.LoadSource(f, nil)
+	err = ctrl.LoadSource(f, nil, "")
 	require.NoError(t, err)
 	require.Len(t, ctrl.loader.Components(), 4)
 
@@ -54,6 +54,41 @@ func TestController_LoadSource_Evaluation(t *testing.T) {
 	require.Equal(t, "hello, world!", out.(testcomponents.PassthroughExports).Output)
 }
 
+var modulePathTestFile = `
+	testcomponents.tick "ticker" {
+		frequency = "1s"
+	}
+	testcomponents.passthrough "static" {
+		input = module_path
+	}
+	testcomponents.passthrough "ticker" {
+		input = testcomponents.tick.ticker.tick_time
+	}
+	testcomponents.passthrough "forwarded" {
+		input = testcomponents.passthrough.ticker.output
+	}
+`
+
+func TestController_LoadSource_WithModulePath_Evaluation(t *testing.T) {
+	defer verifyNoGoroutineLeaks(t)
+	ctrl := New(testOptions(t))
+	defer cleanUpController(ctrl)
+
+	f, err := ParseSource(t.Name(), []byte(modulePathTestFile))
+	require.NoError(t, err)
+	require.NotNil(t, f)
+
+	err = ctrl.LoadSource(f, nil, "path/to/config/main.alloy")
+	require.NoError(t, err)
+	require.Len(t, ctrl.loader.Components(), 4)
+
+	// Check the inputs and outputs of things that should be immediately resolved
+	// without having to run the components.
+	in, out := getFields(t, ctrl.loader.Graph(), "testcomponents.passthrough.static")
+	require.Equal(t, "path/to/config", in.(testcomponents.PassthroughConfig).Input)
+	require.Equal(t, "path/to/config", out.(testcomponents.PassthroughExports).Output)
+}
+
 func getFields(t *testing.T, g *dag.Graph, nodeID string) (component.Arguments, component.Exports) {
 	t.Helper()