Composite Actions: Support Env Flow

src/Runner.Worker/ActionManager.cs

@@ -395,6 +395,12 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio
                             Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
                         }
                     }
+                    else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                    {
+                        var compositeAction = definition.Data.Execution as CompositeActionExecutionData;
+                        Trace.Info($"Action steps: {compositeAction.Steps}.");
+                        Trace.Info($"Action environment: {compositeAction.Environment}.");
+                    }
                     else
                     {
                         throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
@@ -1101,6 +1107,11 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution
                     Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
                     return null;
                 }
+                else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                {
+                    Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation.");
+                    return null;
+                }
                 else
                 {
                     throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
@@ -1211,6 +1222,7 @@ public enum ActionExecutionType
         NodeJS,
         Plugin,
         Script,
+        Composite
     }
 
     public sealed class ContainerActionExecutionData : ActionExecutionData
@@ -1267,6 +1279,15 @@ public sealed class ScriptActionExecutionData : ActionExecutionData
         public override bool HasPost => false;
     }
 
+    public sealed class CompositeActionExecutionData : ActionExecutionData
+    {
+        public override ActionExecutionType ExecutionType => ActionExecutionType.Composite;
+        public override bool HasPre => false;
+        public override bool HasPost => false;
+        public List<Pipelines.ActionStep> Steps { get; set; }
+        public MappingToken Environment { get; set; }
+    }
+
     public abstract class ActionExecutionData
     {
         private string _initCondition = $"{Constants.Expressions.Always}()";

src/Runner.Worker/ActionManifestManager.cs

@@ -14,6 +14,7 @@
 using YamlDotNet.Core.Events;
 using System.Globalization;
 using System.Linq;
+using Pipelines = GitHub.DistributedTask.Pipelines;
 
 namespace GitHub.Runner.Worker
 {
@@ -26,6 +27,7 @@ public interface IActionManifestManager : IRunnerService
 
         Dictionary<string, string> EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
 
+        public Dictionary<string, string> EvaluateCompositeActionEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
         string EvaluateDefaultInput(IExecutionContext executionContext, string inputName, TemplateToken token);
     }
 
@@ -92,7 +94,7 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani
                             break;
 
                         case "runs":
-                            actionDefinition.Execution = ConvertRuns(context, actionPair.Value);
+                            actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value);
                             break;
                         default:
                             Trace.Info($"Ignore action property {propertyName}.");
@@ -181,7 +183,7 @@ public Dictionary<string, string> EvaluateContainerEnvironment(
                 var context = CreateContext(executionContext, extraExpressionValues);
                 try
                 {
-                    var evaluateResult = TemplateEvaluator.Evaluate(context, "container-runs-env", token, 0, null, omitHeader: true);
+                    var evaluateResult = TemplateEvaluator.Evaluate(context, "runs-env", token, 0, null, omitHeader: true);
                     context.Errors.Check();
 
                     Trace.Info($"Environments evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
@@ -213,6 +215,48 @@ public Dictionary<string, string> EvaluateContainerEnvironment(
             return result;
         }
 
+        public Dictionary<string, string> EvaluateCompositeActionEnvironment(
+            IExecutionContext executionContext,
+            MappingToken token,
+            IDictionary<string, PipelineContextData> extraExpressionValues)
+        {
+            var result = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+            if (token != null)
+            {
+                var context = CreateContext(executionContext, extraExpressionValues);
+                try
+                {
+                    var evaluateResult = TemplateEvaluator.Evaluate(context, "runs-env", token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+
+                    // Mapping
+                    var mapping = evaluateResult.AssertMapping("composite env");
+
+                    foreach (var pair in mapping)
+                    {
+                        // Literal key
+                        var key = pair.Key.AssertString("composite env key");
+
+                        // Literal value
+                        var value = pair.Value.AssertString("composite env value");
+                        result[key.Value] = value.Value;
+
+                        Trace.Info($"Add env {key} = {value}");
+                    }
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    Trace.Error(ex);
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result;
+
+        }
         public string EvaluateDefaultInput(
             IExecutionContext executionContext,
             string inputName,
@@ -284,7 +328,7 @@ private TemplateContext CreateContext(
             // Add the file table
             if (_fileTable?.Count > 0)
             {
-                for (var i = 0 ; i < _fileTable.Count ; i++)
+                for (var i = 0; i < _fileTable.Count; i++)
                 {
                     result.GetFileId(_fileTable[i]);
                 }
@@ -294,6 +338,7 @@ private TemplateContext CreateContext(
         }
 
         private ActionExecutionData ConvertRuns(
+            IExecutionContext executionContext,
             TemplateContext context,
             TemplateToken inputsToken)
         {
@@ -311,6 +356,8 @@ private ActionExecutionData ConvertRuns(
             var postToken = default(StringToken);
             var postEntrypointToken = default(StringToken);
             var postIfToken = default(StringToken);
+            var stepsLoaded = default(List<Pipelines.ActionStep>);
+
             foreach (var run in runsMapping)
             {
                 var runsKey = run.Key.AssertString("runs key").Value;
@@ -355,6 +402,15 @@ private ActionExecutionData ConvertRuns(
                     case "pre-if":
                         preIfToken = run.Value.AssertString("pre-if");
                         break;
+                    case "steps":
+                        if (!String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                        {
+                            var steps = run.Value.AssertSequence("steps");
+                            var evaluator = executionContext.ToPipelineTemplateEvaluator();
+                            stepsLoaded = evaluator.LoadCompositeSteps(steps);
+                            break;
+                        }
+                        throw new Exception("You aren't supposed to be using Composite Actions yet!");
                     default:
                         Trace.Info($"Ignore run property {runsKey}.");
                         break;
@@ -402,6 +458,21 @@ private ActionExecutionData ConvertRuns(
                         };
                     }
                 }
+                else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !String.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
+                {
+                    if (stepsLoaded == null)
+                    {
+                        throw new ArgumentNullException($"No steps provided.");
+                    }
+                    else
+                    {
+                        return new CompositeActionExecutionData()
+                        {
+                            Steps = stepsLoaded,
+                            Environment = envToken
+                        };
+                    }
+                }
                 else
                 {
                     throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead.");

src/Runner.Worker/ExecutionContext.cs

@@ -105,6 +105,9 @@ public interface IExecutionContext : IRunnerService
         // others
         void ForceTaskComplete();
         void RegisterPostJobStep(IStep step);
+        IStep RegisterCompositeStep(IStep step, DictionaryContextData inputsData, Dictionary<string, string> envData);
+        void EnqueueAllCompositeSteps(Queue<IStep> steps);
+        public void SetEnvironmentVariables(Dictionary<string, string> dict);
     }
 
     public sealed class ExecutionContext : RunnerService, IExecutionContext
@@ -169,7 +172,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext
 
         public bool EchoOnActionCommand { get; set; }
 
-
         public TaskResult? Result
         {
             get
@@ -266,6 +268,74 @@ public void RegisterPostJobStep(IStep step)
             Root.PostJobSteps.Push(step);
         }
 
+        /*
+            RegisterCompositeStep is a helper function used in CompositeActionHandler::RunAsync to 
+            add a child node, aka a step, to the current job to the front of the queue for processing. 
+        */
+        public IStep RegisterCompositeStep(IStep step, DictionaryContextData inputsData, Dictionary<string, string> envData)
+        {
+            // ~Brute Force Method~
+            // There is no way to put this current job in front of the queue in < O(n) time where n = # of elements in JobSteps
+            // Everytime we add a new step, you could requeue every item to put those steps from that stack in JobSteps which 
+            // would result in O(n) for each time we add a composite action step where n = number of jobSteps which would compound 
+            // to O(n*m) where m = number of composite steps
+            // var temp = Root.JobSteps.ToArray();
+            // Root.JobSteps.Clear();
+            // Root.JobSteps.Enqueue(step);
+            // foreach(var s in temp)
+            //     Root.JobSteps.Enqueue(s);
+
+            // ~Optimized Method~
+            // Alterative solution: We add to another temp Queue
+            // After we add all the transformed composite steps to this temp queue, we requeue the whole JobSteps accordingly in EnqueueAllCompositeSteps()
+            // where the queued composite steps are at the front of the JobSteps Queue and the rest of the jobs maintain its order and are 
+            // placed after the queued composite steps
+            // This will take only O(n+m) time which would be just as efficient if we refactored the code of JobSteps to a PriorityQueue
+            // This temp Queue is created in the CompositeActionHandler. 
+            var newGuid = Guid.NewGuid();
+            step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null);
+            step.ExecutionContext.ExpressionValues["inputs"] = inputsData;
+
+            // Add the composite action environment variables to each step.
+            // If the key already exists, we override it since the composite action env variables will have higher precedence
+            // Note that for each composite action step, it's environment variables will be set in the StepRunner automatically
+            step.ExecutionContext.SetEnvironmentVariables(envData);
+
+            return step;
+        }
+
+        public void SetEnvironmentVariables(Dictionary<string, string> dict)
+        {
+            this.EnvironmentVariables = dict;
+        }
+
+        // Add Composite Steps first and then requeue the rest of the job steps. 
+        public void EnqueueAllCompositeSteps(Queue<IStep> steps)
+        {
+            // TODO: For UI purposes, look at figuring out how to condense steps in one node
+            //  maybe use "this" instead of "Root"?
+            if (Root.JobSteps != null)
+            {
+                var temp = Root.JobSteps.ToArray();
+                Root.JobSteps.Clear();
+                foreach (var cs in steps)
+                {
+                    Root.JobSteps.Enqueue(cs);
+                }
+                foreach (var s in temp)
+                {
+                    Root.JobSteps.Enqueue(s);
+                }
+            }
+            else
+            {
+                Root.JobSteps = new Queue<IStep>();
+                foreach (var cs in steps)
+                {
+                    Root.JobSteps.Enqueue(cs);
+                }
+            }
+        }
         public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null)
         {
             Trace.Entering();