Even Richer Workflow Inputs

client/galaxy/scripts/mvc/tool/tool-form-composite.js

@@ -598,6 +598,9 @@ var View = Backbone.View.extend({
                 if (!input_def.step_linked) {
                     if (this._isDataStep(step)) {
                         validated = input_value && input_value.values && input_value.values.length > 0;
+                        if (!validated && input_def.optional) {
+                            validated = true;
+                        }
                     } else {
                         validated =
                             input_def.optional ||

client/galaxy/scripts/mvc/workflow/workflow-node.js

@@ -297,6 +297,7 @@ var Node = Backbone.Model.extend({
                 if (node.type == "parameter_input") {
                     node.output_terminals[output.name].attributes.type = output.type;
                 }
+                node.output_terminals[output.name].optional = output.optional;
                 node.output_terminals[output.name].destroyInvalidConnections();
             }
         });

client/galaxy/scripts/mvc/workflow/workflow-terminals.js

@@ -212,9 +212,16 @@ var OutputTerminal = Terminal.extend({
     initialize: function(attr) {
         Terminal.prototype.initialize.call(this, attr);
         this.datatypes = attr.datatypes;
+        this.optional = attr.optional;
         this.force_datatype = attr.force_datatype;
     },
-
+    /*
+    update: function(output) {
+        // This isn't needed for datatypes or optional because WorkflowNode.update_field_data just updates the
+        // properties directly, but an upshot of that is that connected non-optional inputs that become optional
+        // don't disconnect from non-optional inputs.
+    },
+    */
     resetMappingIfNeeded: function() {
         // If inputs were only mapped over to preserve
         // an output just disconnected reset these...
@@ -312,7 +319,7 @@ var BaseInputTerminal = Terminal.extend({
                 if (
                     firstOutput.isCollection ||
                     firstOutput.isMappedOver() ||
-                    firstOutput.datatypes.indexOf("input_collection") > 0
+                    firstOutput.datatypes.indexOf("input") > 0
                 ) {
                     return true;
                 } else {
@@ -359,7 +366,6 @@ var BaseInputTerminal = Terminal.extend({
                 if (
                     other_datatype == "input" ||
                     other_datatype == "_sniff_" ||
-                    other_datatype == "input_collection" ||
                     window.workflow_globals.app.isSubType(cat_outputs[other_datatype_i], thisDatatype)
                 ) {
                     return new ConnectionAcceptable(true, null);
@@ -371,6 +377,12 @@ var BaseInputTerminal = Terminal.extend({
             `Effective output data type(s) [${cat_outputs}] do not appear to match input type(s) [${this.datatypes}].`
         );
     },
+    _producesAcceptableDatatypeAndOptionalness(other) {
+        if (!this.optional && !this.multiple && other.optional) {
+            return new ConnectionAcceptable(false, "Cannot connect an optional output to a non-optional input");
+        }
+        return this._producesAcceptableDatatype(other);
+    },
     _otherCollectionType: function(other) {
         var otherCollectionType = NULL_COLLECTION_TYPE_DESCRIPTION;
         if (other.isCollection) {
@@ -388,6 +400,7 @@ var InputTerminal = BaseInputTerminal.extend({
     update: function(input) {
         this.datatypes = input.extensions;
         this.multiple = input.multiple;
+        this.optional = input.optional;
         this.collection = false;
     },
     connect: function(connector) {
@@ -420,16 +433,16 @@ var InputTerminal = BaseInputTerminal.extend({
                 }
             }
             if (thisMapOver.isCollection && thisMapOver.canMatch(otherCollectionType)) {
-                return this._producesAcceptableDatatype(other);
+                return this._producesAcceptableDatatypeAndOptionalness(other);
             } else if (this.multiple && new CollectionTypeDescription("list").canMatch(otherCollectionType)) {
                 // This handles the special case of a list input being connected to a multiple="true" data input.
                 // Nested lists would be correctly mapped over by the above condition.
-                return this._producesAcceptableDatatype(other);
+                return this._producesAcceptableDatatypeAndOptionalness(other);
             } else {
                 //  Need to check if this would break constraints...
                 var mappingConstraints = this._mappingConstraints();
                 if (mappingConstraints.every(_.bind(otherCollectionType.canMatch, otherCollectionType))) {
-                    return this._producesAcceptableDatatype(other);
+                    return this._producesAcceptableDatatypeAndOptionalness(other);
                 } else {
                     if (thisMapOver.isCollection) {
                         // incompatible collection type attached
@@ -458,19 +471,25 @@ var InputTerminal = BaseInputTerminal.extend({
                 "Cannot attach non-collection outputs to mapped over inputs, consider disconnecting inputs and outputs to reset this input's mapping."
             );
         }
-        return this._producesAcceptableDatatype(other);
+        return this._producesAcceptableDatatypeAndOptionalness(other);
     }
 });
 
 var InputParameterTerminal = BaseInputTerminal.extend({
     update: function(input) {
         this.type = input.type;
+        this.optional = input.optional;
     },
     connect: function(connector) {
         BaseInputTerminal.prototype.connect.call(this, connector);
     },
+    effectiveType: function(parameterType) {
+        return parameterType == "select" ? "text" : parameterType;
+    },
     attachable: function(other) {
-        return new ConnectionAcceptable(this.type == other.attributes.type, "");
+        const effectiveThisType = this.effectiveType(this.type);
+        const effectiveOtherType = this.effectiveType(other.attributes.type);
+        return new ConnectionAcceptable(effectiveThisType == effectiveOtherType, "");
     }
 });
 
@@ -480,6 +499,7 @@ var InputCollectionTerminal = BaseInputTerminal.extend({
         this.collection = true;
         this.collection_type = input.collection_type;
         this.datatypes = input.extensions;
+        this.optional = input.optional;
         var collectionTypes = [];
         if (input.collection_types) {
             _.each(input.collection_types, collectionType => {
@@ -550,7 +570,7 @@ var InputCollectionTerminal = BaseInputTerminal.extend({
             );
             if (canMatch) {
                 // Only way a direct match...
-                return this._producesAcceptableDatatype(other);
+                return this._producesAcceptableDatatypeAndOptionalness(other);
                 // Otherwise we need to mapOver
             } else if (thisMapOver.isCollection) {
                 // In this case, mapOver already set and we didn't match skipping...
@@ -574,7 +594,7 @@ var InputCollectionTerminal = BaseInputTerminal.extend({
                 //  Need to check if this would break constraints...
                 var mappingConstraints = this._mappingConstraints();
                 if (mappingConstraints.every(d => effectiveMapOver.canMatch(d))) {
-                    return this._producesAcceptableDatatype(other);
+                    return this._producesAcceptableDatatypeAndOptionalness(other);
                 } else {
                     if (this.node.hasConnectedMappedInputTerminals()) {
                         return new ConnectionAcceptable(
@@ -601,6 +621,7 @@ var OutputCollectionTerminal = Terminal.extend({
     initialize: function(attr) {
         Terminal.prototype.initialize.call(this, attr);
         this.datatypes = attr.datatypes;
+        this.optional = attr.optional;
         this.force_datatype = attr.force_datatype;
         if (attr.collection_type) {
             this.collectionType = new CollectionTypeDescription(attr.collection_type);
@@ -614,6 +635,7 @@ var OutputCollectionTerminal = Terminal.extend({
         this.isCollection = true;
     },
     update: function(output) {
+        this.optional = output.optional;
         var newCollectionType;
         if (output.collection_type) {
             newCollectionType = new CollectionTypeDescription(output.collection_type);

client/galaxy/scripts/mvc/workflow/workflow-view-data.js

@@ -47,7 +47,7 @@ const DataOutputView = Backbone.View.extend({
         let label = output.label || output.name;
         const node = this.nodeView.node;
 
-        const isInput = output.extensions.indexOf("input") >= 0 || output.extensions.indexOf("input_collection") >= 0;
+        const isInput = output.extensions.indexOf("input") >= 0;
         if (!isInput) {
             label = `${label} (${output.force_datatype || output.extensions.join(", ")})`;
         }

client/galaxy/scripts/mvc/workflow/workflow-view-terminals.js

@@ -411,7 +411,8 @@ var OutputTerminalView = BaseOutputTerminalView.extend({
         return new Terminals.OutputTerminal({
             element: this.el,
             datatypes: type,
-            force_datatype: output.force_datatype
+            force_datatype: output.force_datatype,
+            optional: output.optional
         });
     }
 });
@@ -427,7 +428,8 @@ var OutputCollectionTerminalView = BaseOutputTerminalView.extend({
             collection_type: collection_type,
             collection_type_source: collection_type_source,
             datatypes: output.extensions,
-            force_datatype: output.force_datatype
+            force_datatype: output.force_datatype,
+            optional: output.optional
         });
     }
 });
@@ -436,9 +438,10 @@ var OutputParameterTerminalView = BaseOutputTerminalView.extend({
     terminalMappingClass: Terminals.TerminalMapping,
     terminalMappingViewClass: TerminalMappingView,
     terminalForOutput: function(output) {
-        return new Terminals.OutputCollectionTerminal({
+        return new Terminals.OutputParameterTerminal({
             element: this.el,
-            type: output.type
+            type: output.type,
+            optional: output.optional
         });
     }
 });

client/galaxy/scripts/mvc/workflow/workflow-view.js

@@ -689,7 +689,7 @@ export default Backbone.View.extend({
         Utils.request({
             url: `${getAppRoot()}api/workflows/${self.options.id}`,
             type: "PUT",
-            data: { workflow: self.workflow.to_simple() },
+            data: { workflow: self.workflow.to_simple(), from_tool_form: true },
             success: function(data) {
                 var body = $("<div/>").text(data.message);
                 if (data.errors) {
@@ -743,6 +743,7 @@ export default Backbone.View.extend({
                     data: {
                         workflow_name: rename_name,
                         workflow_annotation: rename_annotation,
+                        from_tool_form: true,
                         workflow_data: function() {
                             return JSON.stringify(self.workflow.to_simple());
                         }

client/galaxy/scripts/qunit/tests/workflow_editor_tests.js

@@ -39,7 +39,7 @@ QUnit.module("Input terminal model test", {
     beforeEach: function() {
         testApp.create();
         this.node = new Node(create_app(), {});
-        this.input = { extensions: ["txt"], multiple: false };
+        this.input = { extensions: ["txt"], multiple: false, optional: false };
         this.input_terminal = new Terminals.InputTerminal({ input: this.input });
         this.input_terminal.node = this.node;
     },
@@ -65,7 +65,7 @@ QUnit.module("Input terminal model test", {
         this.input_terminal.connectors = [];
     },
     test_accept: function(other) {
-        other = other || { node: {}, datatypes: ["txt"] };
+        other = other || { node: {}, datatypes: ["txt"], optional: false };
         if (!other.mapOver) {
             other.mapOver = function() {
                 return Terminals.NULL_COLLECTION_TYPE_DESCRIPTION;
@@ -151,6 +151,20 @@ QUnit.test("can accept inputs", function(assert) {
     assert.ok(this.test_accept(other));
 });
 
+QUnit.test("can't connect non-optional", function(assert) {
+    var other = { node: {}, datatypes: ["input"], optional: true };
+    assert.ok(!this.test_accept(other));
+});
+
+QUnit.test("multiple inputs can accept optional outputs regardless", function(assert) {
+    // Galaxy multiple inputs have an optional field but it is hard to resolve that
+    // completely until runtime.
+    var other = { node: {}, datatypes: ["input"], optional: true };
+    var self = this;
+    this.multiple();
+    assert.ok(this.test_accept(other));
+});
+
 QUnit.test("input type can accept any datatype", function(assert) {
     this.input.extensions = ["input"];
     this.input_terminal.update(this.input);
@@ -304,6 +318,17 @@ QUnit.test("Collection output can connect to same collection input type", functi
     );
 });
 
+QUnit.test("Optiona collection output can not connect to required collection input", function(assert) {
+    var inputTerminal = this.input_terminal;
+    var outputTerminal = new Terminals.OutputCollectionTerminal({
+        datatypes: "txt",
+        collection_type: "list",
+        optional: true
+    });
+    outputTerminal.node = {};
+    assert.ok(!inputTerminal.canAccept(outputTerminal).canAccept);
+});
+
 QUnit.test("Collection output cannot connect to different collection input type", function(assert) {
     var self = this;
     var inputTerminal = self.input_terminal;

lib/galaxy/managers/workflows.py

@@ -300,6 +300,7 @@ def build_workflow_from_raw_description(
         create_stored_workflow=True,
         exact_tools=True,
         fill_defaults=False,
+        from_tool_form=False,
     ):
         data = raw_workflow_description.as_dict
         # Put parameters in workflow mode
@@ -315,6 +316,7 @@ def build_workflow_from_raw_description(
             name=name,
             exact_tools=exact_tools,
             fill_defaults=fill_defaults,
+            from_tool_form=from_tool_form,
         )
         if 'uuid' in data:
             workflow.uuid = data['uuid']
@@ -624,7 +626,7 @@ def _workflow_to_dict_editor(self, trans, stored, workflow, tooltip=True, is_sub
                 'label': module.label,
                 'content_id': module.get_content_id(),
                 'name': module.get_name(),
-                'tool_state': module.get_state(),
+                'tool_state': module.get_tool_state(),
                 'errors': module.get_errors(),
                 'inputs': module.get_all_inputs(connectable_only=True),
                 'outputs': module.get_all_outputs(),
@@ -823,10 +825,7 @@ def _workflow_to_dict_export(self, trans, stored=None, workflow=None):
             annotation_str = self.get_item_annotation_str(trans.sa_session, trans.user, step) or ''
             content_id = module.get_content_id()
             # Export differences for backward compatibility
-            if module.type == 'tool':
-                tool_state = module.get_state(nested=False)
-            else:
-                tool_state = module.state.inputs
+            tool_state = module.get_export_state()
             # Step info
             step_dict = {
                 'id': step.order_index,
@@ -885,9 +884,10 @@ def _workflow_to_dict_export(self, trans, stored=None, workflow=None):
             # Data inputs, legacy section not used anywhere within core
             input_dicts = []
             step_state = module.state.inputs or {}
-            if "name" in step_state and module.type != 'tool':
-                name = step_state.get("name")
-                input_dicts.append({"name": name, "description": annotation_str})
+            if module.type != 'tool':
+                name = step_state.get("name") or module.label
+                if name:
+                    input_dicts.append({"name": name, "description": annotation_str})
             for name, val in step_state.items():
                 input_type = type(val)
                 if input_type == RuntimeValue:

lib/galaxy/model/__init__.py

@@ -4658,6 +4658,15 @@ def __init__(self):
     def tool_uuid(self):
         return self.dynamic_tool and self.dynamic_tool.uuid
 
+    @property
+    def input_default_value(self):
+        tool_inputs = self.tool_inputs
+        tool_state = tool_inputs
+        default_value = tool_state.get("default")
+        if default_value:
+            default_value = json.loads(default_value)["value"]
+        return default_value
+
     def get_input(self, input_name):
         for step_input in self.inputs:
             if step_input.name == input_name:
@@ -5115,6 +5124,8 @@ def to_dict(self, view='collection', value_mapper=None, step_details=False, lega
                             if output_step_type in ['data_input', 'data_collection_input']:
                                 src = "hda" if output_step_type == 'data_input' else 'hdca'
                                 for job_input in job.input_datasets:
+                                    if not job_input.dataset:
+                                        continue
                                     if job_input.name == step_input.input_name:
                                         inputs[str(step_input.output_step.order_index)] = {
                                             "id": job_input.dataset_id, "src": src,
@@ -5150,6 +5161,8 @@ def to_dict(self, view='collection', value_mapper=None, step_details=False, lega
             output_values = {}
             for output_param in self.output_values:
                 label = output_param.workflow_output.label
+                if not label:
+                    continue
                 output_values[label] = output_param.value
             rval['output_values'] = output_values
 

lib/galaxy/tool_util/parser/factory.py

@@ -10,7 +10,7 @@
 from .cwl import CwlToolSource
 from .interface import InputSource
 from .xml import XmlInputSource, XmlToolSource
-from .yaml import YamlToolSource
+from .yaml import YamlInputSource, YamlToolSource
 from ..fetcher import ToolLocationFetcher
 
 log = logging.getLogger(__name__)
@@ -75,14 +75,17 @@ def get_tool_source_from_representation(tool_format, tool_representation):
 
 
 def get_input_source(content):
-    """Wrap an XML element in a XmlInputSource if needed.
+    """Wrap dicts or XML elements as InputSource if needed.
 
     If the supplied content is already an InputSource object,
     it is simply returned. This allow Galaxy to uniformly
     consume using the tool input source interface.
     """
     if not isinstance(content, InputSource):
-        content = XmlInputSource(content)
+        if isinstance(content, dict):
+            content = YamlInputSource(content)
+        else:
+            content = XmlInputSource(content)
     return content