PUBDEV-7778: minimal integration of TargetEncoding into AutoML

h2o-automl/build.gradle

@@ -5,10 +5,12 @@ dependencies {
   compile project(":h2o-core")
   compile project(":h2o-algos")
   compileOnly project(":h2o-ext-xgboost")
+  compileOnly project(":h2o-ext-target-encoder")
 
   // Test dependencies only
   testCompile project(":h2o-test-support")
   testCompile project(":h2o-ext-xgboost")
+  testCompile project(":h2o-ext-target-encoder")
   testRuntimeOnly project(":${defaultWebserverModule}")
 }
 

h2o-automl/src/main/java/ai/h2o/automl/AutoML.java

@@ -9,6 +9,7 @@
 import ai.h2o.automl.events.EventLogEntry.Stage;
 import ai.h2o.automl.StepDefinition.Alias;
 import ai.h2o.automl.leaderboard.*;
+import ai.h2o.automl.preprocessing.PreprocessingStep;
 import hex.Model;
 import hex.ScoreKeeper.StoppingMetric;
 import hex.splitframe.ShuffleSplitFrame;
@@ -183,6 +184,7 @@ public Class<AutoMLV99.AutoMLKeyV3> makeSchema() {
   private long[] _originalTrainingFrameChecksums;
   private transient NonBlockingHashMap<Key, String> _trackedKeys = new NonBlockingHashMap<>();
   private transient ModelingStep[] _executionPlan;
+  private transient PreprocessingStep[] _preprocessing;
 
   public AutoML() {
     super(null);
@@ -206,6 +208,7 @@ public AutoML(Key<AutoML> key, Date startTime, AutoMLBuildSpec buildSpec) {
 
       prepareData();
       initLeaderboard();
+      initPreprocessing();
       planWork();
       _modelingStepsExecutor = new ModelingStepsExecutor(_leaderboard, _eventLog, _runCountdown);
     } catch (Exception e) {
@@ -343,10 +346,22 @@ private void initLeaderboard() {
     _leaderboard.setExtensionsProvider(createLeaderboardExtensionProvider(this));
   }
 
+  private void initPreprocessing() {
+    _preprocessing = _buildSpec.build_models.preprocessing == null 
+            ? null 
+            : Arrays.stream(_buildSpec.build_models.preprocessing)
+                .map(def -> def.newPreprocessingStep(this))
+                .toArray(PreprocessingStep[]::new);
+  }
+
+  PreprocessingStep[] getPreprocessing() {
+    return _preprocessing;
+  }
+
   ModelingStep[] getExecutionPlan() {
     return _executionPlan == null ? (_executionPlan = _modelingStepsRegistry.getOrderedSteps(_buildSpec.build_models.modeling_plan, this)) : _executionPlan;
   }
-
+  
   void planWork() {
     Set<IAlgo> skippedAlgos = new HashSet<>();
     if (_buildSpec.build_models.exclude_algos != null) {
@@ -480,7 +495,7 @@ public boolean keepRunning() {
     return !_runCountdown.timedOut() && remainingModels() > 0;
   }
 
-  boolean isCVEnabled() {
+  public boolean isCVEnabled() {
     return _buildSpec.build_control.nfolds > 0 || _buildSpec.input_spec.fold_column != null;
   }
 
@@ -599,13 +614,19 @@ private void prepareData() {
 
   private void learn() {
     List<ModelingStep> executed = new ArrayList<>();
+    if (_preprocessing != null) {
+      for (PreprocessingStep preprocessingStep : _preprocessing) preprocessingStep.prepare();
+    }
     for (ModelingStep step : getExecutionPlan()) {
         if (!exceededSearchLimits(step)) {
           if (_modelingStepsExecutor.submit(step, job())) {
             executed.add(step);
           }
         }
     }
+    if (_preprocessing != null) {
+      for (PreprocessingStep preprocessingStep : _preprocessing) preprocessingStep.dispose();
+    }
     _actualModelingSteps = _modelingStepsRegistry.createDefinitionPlanFromSteps(executed.toArray(new ModelingStep[0]));
     eventLog().info(Stage.Workflow, "Actual modeling steps: "+Arrays.toString(_actualModelingSteps));
   }
@@ -622,13 +643,13 @@ private int nextInstanceCounter(String algoName, String type) {
     return _instanceCounters.get(key).incrementAndGet();
   }
 
-  Key makeKey(String algoName, String type, boolean with_counter) {
+  public Key makeKey(String algoName, String type, boolean with_counter) {
     String counterStr = with_counter ? "_" + nextInstanceCounter(algoName, type) : "";
     String prefix = StringUtils.isNullOrEmpty(type) ? algoName : algoName+"_"+type+"_";
     return Key.make(prefix + counterStr + "_AutoML_" + timestampFormatForKeys.get().format(_startTime));
   }
 
-  void trackKey(Key key) {
+  public void trackKey(Key key) {
     _trackedKeys.put(key, Arrays.toString(Thread.currentThread().getStackTrace()));
   }
 
@@ -653,7 +674,7 @@ private boolean exceededSearchLimits(ModelingStep step) {
   //*****************  Clean Up + other utility functions *****************//
 
   /**
-   * Delete the AutoML-related objects, but leave the grids and models that it built.
+   * Delete the AutoML-related objects, including the grids and models that it built if cascade=true
    */
   @Override
   protected Futures remove_impl(Futures fs, boolean cascade) {
@@ -675,7 +696,11 @@ protected Futures remove_impl(Futures fs, boolean cascade) {
       Frame.deleteTempFrameAndItsNonSharedVecs(_trainingFrame, _origTrainingFrame);
     if (leaderboard() != null) leaderboard().remove(fs, cascade);
     if (eventLog() != null) eventLog().remove(fs, cascade);
-
+    if (cascade && _preprocessing != null) {
+      for (PreprocessingStep preprocessingStep : _preprocessing) {
+        preprocessingStep.remove();
+      }
+    }
     for (Key key : _trackedKeys.keySet()) Keyed.remove(key, fs, true);
 
     return super.remove_impl(fs, cascade);

h2o-automl/src/main/java/ai/h2o/automl/AutoMLBuildSpec.java

@@ -1,5 +1,6 @@
 package ai.h2o.automl;
 
+import ai.h2o.automl.preprocessing.PreprocessingStepDefinition;
 import hex.Model;
 import hex.ScoreKeeper.StoppingMetric;
 import hex.grid.HyperSpaceSearchCriteria;
@@ -170,6 +171,7 @@ public static final class AutoMLBuildModels extends Iced {
     public StepDefinition[] modeling_plan;
     public double exploitation_ratio = 0;
     public AutoMLCustomParameters algo_parameters = new AutoMLCustomParameters();
+    public PreprocessingStepDefinition[] preprocessing;
   }
 
   public static final class AutoMLCustomParameters extends Iced {

h2o-automl/src/main/java/ai/h2o/automl/ModelingStep.java

@@ -9,6 +9,9 @@
 import ai.h2o.automl.WorkAllocations.JobType;
 import ai.h2o.automl.WorkAllocations.Work;
 import ai.h2o.automl.leaderboard.Leaderboard;
+import ai.h2o.automl.preprocessing.PreprocessingConfig;
+import ai.h2o.automl.preprocessing.PreprocessingStep;
+import ai.h2o.automl.preprocessing.PreprocessingStepDefinition;
 import hex.Model;
 import hex.Model.Parameters.FoldAssignmentScheme;
 import hex.ModelBuilder;
@@ -27,9 +30,8 @@
 import water.util.EnumUtils;
 import water.util.Log;
 
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Map;
+import java.util.*;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 
 /**
@@ -46,6 +48,7 @@ protected enum SeedPolicy {
         Incremental
     }
 
+    static Predicate<Work> isDefaultModel = w -> w._type == JobType.ModelBuild;
     static Predicate<Work> isExplorationWork = w -> w._type == JobType.ModelBuild || w._type == JobType.HyperparamSearch;
     static Predicate<Work> isExploitationWork = w -> w._type == JobType.Selection;
 
@@ -55,6 +58,7 @@ protected <MP extends Model.Parameters> Job<Grid> startSearch(
             final Map<String, Object[]> hyperParams,
             final HyperSpaceSearchCriteria searchCriteria)
     {
+        applyPreprocessing(baseParams);
         aml().eventLog().info(Stage.ModelTraining, "AutoML: starting "+resultKey+" hyperparameter search")
                 .setNamedValue("start_"+_algo+"_"+_id, new Date(), EventLogEntry.epochFormat.get());
         return GridSearch.startGridSearch(
@@ -72,6 +76,7 @@ protected <M extends Model, MP extends Model.Parameters> Job<M> startModel(
             final MP params
     ) {
         Job<M> job = new Job<>(resultKey, ModelBuilder.javaName(_algo.urlName()), _description);
+        applyPreprocessing(params);
         ModelBuilder builder = ModelBuilder.make(_algo.urlName(), job, (Key<Model>) resultKey);
         builder._parms = params;
         aml().eventLog().info(Stage.ModelTraining, "AutoML: starting "+resultKey+" model training")
@@ -81,9 +86,9 @@ protected <M extends Model, MP extends Model.Parameters> Job<M> startModel(
             return builder.trainModelOnH2ONode();
         } catch (H2OIllegalArgumentException exception) {
             aml().eventLog().warn(Stage.ModelTraining, "Skipping training of model "+resultKey+" due to exception: "+exception);
+            onDone(null);
             return null;
         }
-
     }
 
     private transient AutoML _aml;
@@ -93,6 +98,7 @@ protected <M extends Model, MP extends Model.Parameters> Job<M> startModel(
     protected int _weight;
     protected AutoML.Constraint[] _ignoredConstraints = new AutoML.Constraint[0];  // whether or not to ignore the max_models/max_runtime constraints
     protected String _description;
+    private final transient List<Consumer<Job>> _onDone = new ArrayList<>();
 
     StepDefinition _fromDef;
 
@@ -112,6 +118,13 @@ protected ModelingStep(IAlgo algo, String id, int weight, AutoML autoML) {
 
     protected abstract Job startJob();
 
+    protected void onDone(Job job) {
+        for (Consumer<Job> exec : _onDone) {
+            exec.accept(job);
+        }
+        _onDone.clear();
+    };
+
     protected AutoML aml() {
         return _aml;
     }
@@ -160,7 +173,7 @@ protected void setCommonModelBuilderParams(Model.Parameters params) {
         setCrossValidationParams(params);
         setWeightingParams(params);
         setClassBalancingParams(params);
-        
+
         params._keep_cross_validation_models = buildSpec.build_control.keep_cross_validation_models;
         params._keep_cross_validation_fold_assignment = buildSpec.build_control.nfolds != 0 && buildSpec.build_control.keep_cross_validation_fold_assignment;
         params._export_checkpoints_dir = buildSpec.build_control.export_checkpoints_dir;
@@ -199,7 +212,18 @@ protected void setCustomParams(Model.Parameters params) {
         if (customParams == null) return;
         customParams.applyCustomParameters(_algo, params);
     }
-
+
+    protected void applyPreprocessing(Model.Parameters params) {
+        if (aml().getPreprocessing() == null) return;
+        for (PreprocessingStep preprocessingStep : aml().getPreprocessing()) {
+            PreprocessingStep.Completer complete = preprocessingStep.apply(params, getPreprocessingConfig());
+            _onDone.add(j -> complete.run());
+        }
+    }
+
+    protected PreprocessingConfig getPreprocessingConfig() {
+        return new PreprocessingConfig();
+    }
 
     /**
      * Configures early-stopping for the model or set of models to be built.
@@ -340,6 +364,7 @@ protected Job<M> trainModel(Key<M> key, Model.Parameters parms) {
                 Work work = getAllocatedWork();
 //                double maxAssignedTimeSecs = aml().timeRemainingMs() / 1e3; // legacy
                 double maxAssignedTimeSecs = aml().timeRemainingMs() * getWorkAllocations().remainingWorkRatio(work) / 1e3; //including default models in the distribution of the time budget.
+//                double maxAssignedTimeSecs = aml().timeRemainingMs() * getWorkAllocations().remainingWorkRatio(work, isDefaultModel) / 1e3; //PUBDEV-7595
                 parms._max_runtime_secs = parms._max_runtime_secs == 0
                         ? maxAssignedTimeSecs
                         : Math.min(parms._max_runtime_secs, maxAssignedTimeSecs);
@@ -350,7 +375,6 @@ protected Job<M> trainModel(Key<M> key, Model.Parameters parms) {
                     : "Time assigned for "+key+": "+parms._max_runtime_secs+"s");
             return startModel(key, parms);
         }
-
     }
 
     /**

h2o-automl/src/main/java/ai/h2o/automl/ModelingStepsExecutor.java

@@ -16,7 +16,6 @@
 import water.util.Log;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
@@ -76,11 +75,18 @@ void stop() {
     boolean submit(ModelingStep step, Job parentJob) {
         if (step.canRun()) {
             Job job = step.startJob();
-            if (job == null) {
-                skip(step._description, step.getAllocatedWork(), parentJob);
-            } else {
-                monitor(job, step.getAllocatedWork(), parentJob, ArrayUtils.contains(step._ignoredConstraints, Constraint.TIMEOUT));
-                return true;
+            try {
+                if (job==null) {
+                    skip(step._description, step.getAllocatedWork(), parentJob);
+                } else {
+                    monitor(job,
+                            step.getAllocatedWork(),
+                            parentJob,
+                            ArrayUtils.contains(step._ignoredConstraints, Constraint.TIMEOUT));
+                    return true;
+                }
+            } finally {
+                step.onDone(job);
             }
         }
         return false;
@@ -145,13 +151,13 @@ void monitor(Job job, Work work, Job parentJob, boolean ignoreTimeout) {
             } else if (job.get() == null) {
                 eventLog.info(Stage.ModelTraining, jobDescription + " cancelled");
             } else {
+                eventLog.debug(Stage.ModelTraining, jobDescription + " complete");
                 ModelContainer<?> container = (ModelContainer) job.get();
                 int totalModelsBuilt = container.getModelCount();
                 if (totalModelsBuilt > lastTotalModelsBuilt) {
                     eventLog.debug(Stage.ModelTraining, "Built: "+totalModelsBuilt+" models for "+work._type+" : "+jobDescription);
                     this.addModels(container);
                 }
-                eventLog.debug(Stage.ModelTraining, jobDescription + " complete");
             }
         } else if (JobType.ModelBuild == work._type) {
             if (job.isCrashed()) {
@@ -160,7 +166,7 @@ void monitor(Job job, Work work, Job parentJob, boolean ignoreTimeout) {
                 eventLog.info(Stage.ModelTraining, jobDescription + " cancelled");
             } else {
                 eventLog.debug(Stage.ModelTraining, jobDescription + " complete");
-                this.addModel((Model) job.get());
+                this.addModel((Model)job.get());
             }
         }
 

h2o-automl/src/main/java/ai/h2o/automl/modeling/DeepLearningStepsProvider.java

@@ -1,6 +1,8 @@
 package ai.h2o.automl.modeling;
 
 import ai.h2o.automl.*;
+import ai.h2o.automl.preprocessing.PreprocessingConfig;
+import ai.h2o.automl.preprocessing.TargetEncoding;
 import hex.deeplearning.DeepLearningModel;
 import hex.deeplearning.DeepLearningModel.DeepLearningParameters;
 import hex.grid.Grid;
@@ -22,6 +24,14 @@ static abstract class DeepLearningModelStep extends ModelingStep.ModelStep<DeepL
             public DeepLearningModelStep(String id, int weight, AutoML autoML) {
                 super(Algo.DeepLearning, id, weight, autoML);
             }
+
+            @Override
+            protected PreprocessingConfig getPreprocessingConfig() {
+                //TE useless for DNN
+                PreprocessingConfig config = super.getPreprocessingConfig();
+                config.put(TargetEncoding.CONFIG_PREPARE_CV_ONLY, aml().isCVEnabled());
+                return config;
+            }
         }
 
         static abstract class DeepLearningGridStep extends ModelingStep.GridStep<DeepLearningModel> {
@@ -39,7 +49,14 @@ DeepLearningParameters prepareModelParameters() {
 
                 return dlParameters;
             }
-
+
+            @Override
+            protected PreprocessingConfig getPreprocessingConfig() {
+                //TE useless for DNN
+                PreprocessingConfig config = super.getPreprocessingConfig();
+                config.put(TargetEncoding.CONFIG_PREPARE_CV_ONLY, aml().isCVEnabled());
+                return config;
+            }
 
             Map<String, Object[]> prepareSearchParams() {
                 Map<String, Object[]> searchParams = new HashMap<>();

h2o-automl/src/main/java/ai/h2o/automl/modeling/GLMStepsProvider.java

@@ -1,6 +1,9 @@
 package ai.h2o.automl.modeling;
 
 import ai.h2o.automl.*;
+import ai.h2o.automl.preprocessing.PreprocessingConfig;
+import ai.h2o.automl.preprocessing.PreprocessingStepDefinition;
+import ai.h2o.automl.preprocessing.TargetEncoding;
 import hex.Model;
 import hex.glm.GLMModel;
 import hex.glm.GLMModel.GLMParameters;
@@ -35,6 +38,15 @@ GLMParameters prepareModelParameters() {
                                 : GLMParameters.Family.gaussian;  // TODO: other continuous distributions!
                 return glmParameters;
             }
+
+            @Override
+            protected PreprocessingConfig getPreprocessingConfig() {
+                //GLM (the exception as usual) doesn't support targetencoding if CV is enabled
+                // because it is initializing its lambdas + other params before CV (preventing changes in train frame during CV).
+                PreprocessingConfig config = super.getPreprocessingConfig();
+                config.put(TargetEncoding.CONFIG_PREPARE_CV_ONLY, aml().isCVEnabled()); 
+                return config;
+            }
         }