CHE-11042 Move PluginBroking to KubernetesRuntime#internalStart (#11061)

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java

@@ -38,79 +38,85 @@
  * @author Anton Korneta
  * @author Alexander Garagatyi
  */
-@Singleton
-public class KubernetesEnvironmentProvisioner {
+public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironment> {
 
-  private final boolean pvcEnabled;
-  private final WorkspaceVolumesStrategy volumesStrategy;
-  private final UniqueNamesProvisioner<KubernetesEnvironment> uniqueNamesProvisioner;
-  private final ServersConverter<KubernetesEnvironment> serversConverter;
-  private final EnvVarsConverter envVarsConverter;
-  private final RestartPolicyRewriter restartPolicyRewriter;
-  private final RamLimitProvisioner ramLimitProvisioner;
-  private final InstallerServersPortProvisioner installerServersPortProvisioner;
-  private final LogsVolumeMachineProvisioner logsVolumeMachineProvisioner;
-  private final SecurityContextProvisioner securityContextProvisioner;
-  private final PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner;
-  private final IngressTlsProvisioner externalServerIngressTlsProvisioner;
-  private final ImagePullSecretProvisioner imagePullSecretProvisioner;
-  private final ProxySettingsProvisioner proxySettingsProvisioner;
+  void provision(T k8sEnv, RuntimeIdentity identity) throws InfrastructureException;
 
-  @Inject
-  public KubernetesEnvironmentProvisioner(
-      @Named("che.infra.kubernetes.pvc.enabled") boolean pvcEnabled,
-      UniqueNamesProvisioner<KubernetesEnvironment> uniqueNamesProvisioner,
-      ServersConverter<KubernetesEnvironment> serversConverter,
-      EnvVarsConverter envVarsConverter,
-      RestartPolicyRewriter restartPolicyRewriter,
-      WorkspaceVolumesStrategy volumesStrategy,
-      RamLimitProvisioner ramLimitProvisioner,
-      InstallerServersPortProvisioner installerServersPortProvisioner,
-      LogsVolumeMachineProvisioner logsVolumeMachineProvisioner,
-      SecurityContextProvisioner securityContextProvisioner,
-      PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner,
-      IngressTlsProvisioner externalServerIngressTlsProvisioner,
-      ImagePullSecretProvisioner imagePullSecretProvisioner,
-      ProxySettingsProvisioner proxySettingsProvisioner) {
-    this.pvcEnabled = pvcEnabled;
-    this.volumesStrategy = volumesStrategy;
-    this.uniqueNamesProvisioner = uniqueNamesProvisioner;
-    this.serversConverter = serversConverter;
-    this.envVarsConverter = envVarsConverter;
-    this.restartPolicyRewriter = restartPolicyRewriter;
-    this.ramLimitProvisioner = ramLimitProvisioner;
-    this.installerServersPortProvisioner = installerServersPortProvisioner;
-    this.logsVolumeMachineProvisioner = logsVolumeMachineProvisioner;
-    this.securityContextProvisioner = securityContextProvisioner;
-    this.podTerminationGracePeriodProvisioner = podTerminationGracePeriodProvisioner;
-    this.externalServerIngressTlsProvisioner = externalServerIngressTlsProvisioner;
-    this.imagePullSecretProvisioner = imagePullSecretProvisioner;
-    this.proxySettingsProvisioner = proxySettingsProvisioner;
-  }
+  @Singleton
+  class KubernetesEnvironmentProvisionerImpl
+      implements KubernetesEnvironmentProvisioner<KubernetesEnvironment> {
 
-  public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
-      throws InfrastructureException {
-    // 1 stage - update environment according Infrastructure specific
-    installerServersPortProvisioner.provision(k8sEnv, identity);
-    if (pvcEnabled) {
-      logsVolumeMachineProvisioner.provision(k8sEnv, identity);
-    }
+    private boolean pvcEnabled;
+    private WorkspaceVolumesStrategy volumesStrategy;
+    private UniqueNamesProvisioner<KubernetesEnvironment> uniqueNamesProvisioner;
+    private ServersConverter<KubernetesEnvironment> serversConverter;
+    private EnvVarsConverter envVarsConverter;
+    private RestartPolicyRewriter restartPolicyRewriter;
+    private RamLimitProvisioner ramLimitProvisioner;
+    private InstallerServersPortProvisioner installerServersPortProvisioner;
+    private LogsVolumeMachineProvisioner logsVolumeMachineProvisioner;
+    private SecurityContextProvisioner securityContextProvisioner;
+    private PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner;
+    private IngressTlsProvisioner externalServerIngressTlsProvisioner;
+    private ImagePullSecretProvisioner imagePullSecretProvisioner;
+    private ProxySettingsProvisioner proxySettingsProvisioner;
 
-    // 2 stage - converting Che model env to Kubernetes env
-    serversConverter.provision(k8sEnv, identity);
-    envVarsConverter.provision(k8sEnv, identity);
-    if (pvcEnabled) {
-      volumesStrategy.provision(k8sEnv, identity);
+    @Inject
+    public KubernetesEnvironmentProvisionerImpl(
+        @Named("che.infra.kubernetes.pvc.enabled") boolean pvcEnabled,
+        UniqueNamesProvisioner<KubernetesEnvironment> uniqueNamesProvisioner,
+        ServersConverter<KubernetesEnvironment> serversConverter,
+        EnvVarsConverter envVarsConverter,
+        RestartPolicyRewriter restartPolicyRewriter,
+        WorkspaceVolumesStrategy volumesStrategy,
+        RamLimitProvisioner ramLimitProvisioner,
+        InstallerServersPortProvisioner installerServersPortProvisioner,
+        LogsVolumeMachineProvisioner logsVolumeMachineProvisioner,
+        SecurityContextProvisioner securityContextProvisioner,
+        PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner,
+        IngressTlsProvisioner externalServerIngressTlsProvisioner,
+        ImagePullSecretProvisioner imagePullSecretProvisioner,
+        ProxySettingsProvisioner proxySettingsProvisioner) {
+      this.pvcEnabled = pvcEnabled;
+      this.volumesStrategy = volumesStrategy;
+      this.uniqueNamesProvisioner = uniqueNamesProvisioner;
+      this.serversConverter = serversConverter;
+      this.envVarsConverter = envVarsConverter;
+      this.restartPolicyRewriter = restartPolicyRewriter;
+      this.ramLimitProvisioner = ramLimitProvisioner;
+      this.installerServersPortProvisioner = installerServersPortProvisioner;
+      this.logsVolumeMachineProvisioner = logsVolumeMachineProvisioner;
+      this.securityContextProvisioner = securityContextProvisioner;
+      this.podTerminationGracePeriodProvisioner = podTerminationGracePeriodProvisioner;
+      this.externalServerIngressTlsProvisioner = externalServerIngressTlsProvisioner;
+      this.imagePullSecretProvisioner = imagePullSecretProvisioner;
+      this.proxySettingsProvisioner = proxySettingsProvisioner;
     }
 
-    // 3 stage - add Kubernetes env items
-    restartPolicyRewriter.provision(k8sEnv, identity);
-    uniqueNamesProvisioner.provision(k8sEnv, identity);
-    ramLimitProvisioner.provision(k8sEnv, identity);
-    externalServerIngressTlsProvisioner.provision(k8sEnv, identity);
-    securityContextProvisioner.provision(k8sEnv, identity);
-    podTerminationGracePeriodProvisioner.provision(k8sEnv, identity);
-    imagePullSecretProvisioner.provision(k8sEnv, identity);
-    proxySettingsProvisioner.provision(k8sEnv, identity);
+    public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
+        throws InfrastructureException {
+      // 1 stage - update environment according Infrastructure specific
+      installerServersPortProvisioner.provision(k8sEnv, identity);
+      if (pvcEnabled) {
+        logsVolumeMachineProvisioner.provision(k8sEnv, identity);
+      }
+
+      // 2 stage - converting Che model env to Kubernetes env
+      serversConverter.provision(k8sEnv, identity);
+      envVarsConverter.provision(k8sEnv, identity);
+      if (pvcEnabled) {
+        volumesStrategy.provision(k8sEnv, identity);
+      }
+
+      // 3 stage - add Kubernetes env items
+      restartPolicyRewriter.provision(k8sEnv, identity);
+      uniqueNamesProvisioner.provision(k8sEnv, identity);
+      ramLimitProvisioner.provision(k8sEnv, identity);
+      externalServerIngressTlsProvisioner.provision(k8sEnv, identity);
+      securityContextProvisioner.provision(k8sEnv, identity);
+      podTerminationGracePeriodProvisioner.provision(k8sEnv, identity);
+      imagePullSecretProvisioner.provision(k8sEnv, identity);
+      proxySettingsProvisioner.provision(k8sEnv, identity);
+    }
   }
 }

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java

@@ -70,9 +70,14 @@ protected void configure() {
 
     bind(RuntimeInfrastructure.class).to(KubernetesInfrastructure.class);
 
+    bind(new TypeLiteral<KubernetesEnvironmentProvisioner<KubernetesEnvironment>>() {})
+        .to(KubernetesEnvironmentProvisioner.KubernetesEnvironmentProvisionerImpl.class);
+
     install(new FactoryModuleBuilder().build(KubernetesRuntimeContextFactory.class));
 
-    install(new FactoryModuleBuilder().build(KubernetesRuntimeFactory.class));
+    install(
+        new FactoryModuleBuilder()
+            .build(new TypeLiteral<KubernetesRuntimeFactory<KubernetesEnvironment>>() {}));
     install(new FactoryModuleBuilder().build(KubernetesBootstrapperFactory.class));
     install(new FactoryModuleBuilder().build(StartSynchronizerFactory.class));
 

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfrastructure.java

@@ -17,20 +17,17 @@
 import java.util.Set;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import org.eclipse.che.api.core.ValidationException;
 import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
 import org.eclipse.che.api.core.notification.EventService;
 import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
 import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
-import org.eclipse.che.api.workspace.server.spi.RuntimeContext;
 import org.eclipse.che.api.workspace.server.spi.RuntimeInfrastructure;
 import org.eclipse.che.api.workspace.server.spi.environment.InternalEnvironment;
 import org.eclipse.che.api.workspace.server.spi.provision.InternalEnvironmentProvisioner;
 import org.eclipse.che.workspace.infrastructure.docker.environment.dockerimage.DockerImageEnvironment;
 import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.convert.DockerImageEnvironmentConverter;
-import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner;
 
 /** @author Sergii Leshchenko */
 @Singleton
@@ -40,59 +37,34 @@ public class KubernetesInfrastructure extends RuntimeInfrastructure {
 
   private final DockerImageEnvironmentConverter dockerImageEnvConverter;
   private final KubernetesRuntimeContextFactory runtimeContextFactory;
-  private final KubernetesEnvironmentProvisioner k8sEnvProvisioner;
   private final KubernetesRuntimeStateCache runtimeStatusesCache;
-  private final SidecarToolingProvisioner toolingProvisioner;
 
   @Inject
   public KubernetesInfrastructure(
       EventService eventService,
       KubernetesRuntimeContextFactory runtimeContextFactory,
-      KubernetesEnvironmentProvisioner k8sEnvProvisioner,
       Set<InternalEnvironmentProvisioner> internalEnvProvisioners,
       DockerImageEnvironmentConverter dockerImageEnvConverter,
-      KubernetesRuntimeStateCache runtimeStatusesCache,
-      SidecarToolingProvisioner toolingProvisioner) {
+      KubernetesRuntimeStateCache runtimeStatusesCache) {
     super(
         NAME,
         ImmutableSet.of(KubernetesEnvironment.TYPE, DockerImageEnvironment.TYPE),
         eventService,
         internalEnvProvisioners);
     this.runtimeContextFactory = runtimeContextFactory;
-    this.k8sEnvProvisioner = k8sEnvProvisioner;
     this.dockerImageEnvConverter = dockerImageEnvConverter;
     this.runtimeStatusesCache = runtimeStatusesCache;
-    this.toolingProvisioner = toolingProvisioner;
   }
 
   @Override
   public Set<RuntimeIdentity> getIdentities() throws InfrastructureException {
     return runtimeStatusesCache.getIdentities();
   }
 
-  @Override
-  public RuntimeContext prepare(RuntimeIdentity id, InternalEnvironment environment)
-      throws ValidationException, InfrastructureException {
-
-    // Sidecar-based tooling for now supports k8s/OS env only
-    // Convert other env types to a k8s type to use this tooling with other env types
-    final KubernetesEnvironment kubernetesEnvironment = asKubernetesEnv(environment);
-
-    // We need to provision development tooling here because there is environment variables
-    // provisioning in the superclass which might be important
-    toolingProvisioner.provision(id, kubernetesEnvironment);
-
-    return super.prepare(id, kubernetesEnvironment);
-  }
-
   @Override
   protected KubernetesRuntimeContext internalPrepare(
       RuntimeIdentity id, InternalEnvironment environment) throws InfrastructureException {
-    final KubernetesEnvironment kubernetesEnvironment = asKubernetesEnv(environment);
-
-    k8sEnvProvisioner.provision(kubernetesEnvironment, id);
-
-    return runtimeContextFactory.create(kubernetesEnvironment, id, this);
+    return runtimeContextFactory.create(asKubernetesEnv(environment), id, this);
   }
 
   private KubernetesEnvironment asKubernetesEnv(InternalEnvironment source)

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInternalRuntime.java

@@ -64,6 +64,7 @@
 import org.eclipse.che.api.workspace.server.spi.RuntimeStartInterruptedException;
 import org.eclipse.che.api.workspace.server.spi.StateException;
 import org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig;
+import org.eclipse.che.api.workspace.server.spi.provision.InternalEnvironmentProvisioner;
 import org.eclipse.che.commons.env.EnvironmentContext;
 import org.eclipse.che.workspace.infrastructure.kubernetes.bootstrapper.KubernetesBootstrapperFactory;
 import org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache;
@@ -79,16 +80,16 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.server.KubernetesServerResolver;
 import org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool;
 import org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher;
+import org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * @author Sergii Leshchenko
  * @author Anton Korneta
  */
-public class KubernetesInternalRuntime<
-        T extends KubernetesRuntimeContext<? extends KubernetesEnvironment>>
-    extends InternalRuntime<T> {
+public class KubernetesInternalRuntime<E extends KubernetesEnvironment>
+    extends InternalRuntime<KubernetesRuntimeContext<E>> {
 
   private static final Logger LOG = LoggerFactory.getLogger(KubernetesInternalRuntime.class);
 
@@ -106,6 +107,9 @@ public class KubernetesInternalRuntime<
   private final KubernetesRuntimeStateCache runtimeStates;
   private final KubernetesMachineCache machines;
   private final StartSynchronizer startSynchronizer;
+  private final Set<InternalEnvironmentProvisioner> internalEnvironmentProvisioners;
+  private final KubernetesEnvironmentProvisioner<E> kubernetesEnvironmentProvisioner;
+  private final SidecarToolingProvisioner toolingProvisioner;
 
   @Inject
   public KubernetesInternalRuntime(
@@ -123,7 +127,10 @@ public KubernetesInternalRuntime(
       KubernetesRuntimeStateCache runtimeStates,
       KubernetesMachineCache machines,
       StartSynchronizerFactory startSynchronizerFactory,
-      @Assisted T context,
+      Set<InternalEnvironmentProvisioner> internalEnvironmentProvisioners,
+      KubernetesEnvironmentProvisioner<E> kubernetesEnvironmentProvisioner,
+      SidecarToolingProvisioner toolingProvisioner,
+      @Assisted KubernetesRuntimeContext<E> context,
       @Assisted KubernetesNamespace namespace,
       @Assisted List<Warning> warnings) {
     super(context, urlRewriter, warnings);
@@ -140,17 +147,37 @@ public KubernetesInternalRuntime(
     this.executor = sharedPool.getExecutor();
     this.runtimeStates = runtimeStates;
     this.machines = machines;
+    this.toolingProvisioner = toolingProvisioner;
+    this.kubernetesEnvironmentProvisioner = kubernetesEnvironmentProvisioner;
+    this.internalEnvironmentProvisioners = internalEnvironmentProvisioners;
     this.startSynchronizer = startSynchronizerFactory.create(context.getIdentity());
   }
 
   @Override
   protected void internalStart(Map<String, String> startOptions) throws InfrastructureException {
-    KubernetesRuntimeContext<? extends KubernetesEnvironment> context = getContext();
+    KubernetesRuntimeContext<E> context = getContext();
     String workspaceId = context.getIdentity().getWorkspaceId();
     try {
       startSynchronizer.setStartThread();
       startSynchronizer.start();
 
+      // Tooling side car provisioner should be applied before other provisioners
+      // because new machines may be provisioned there
+      toolingProvisioner.provision(context.getIdentity(), context.getEnvironment());
+
+      startSynchronizer.checkFailure();
+
+      // Workspace API provisioners should be reapplied here to bring needed
+      // changed into new machines that came during tooling provisioning
+      for (InternalEnvironmentProvisioner envProvisioner : internalEnvironmentProvisioners) {
+        envProvisioner.provision(context.getIdentity(), context.getEnvironment());
+      }
+
+      // Infrastructure specific provisioner should be applied last
+      // because it converts all Workspace API model objects that comes
+      // from previous provisioners into infrastructure specific objects
+      kubernetesEnvironmentProvisioner.provision(context.getEnvironment(), context.getIdentity());
+
       volumesStrategy.prepare(context.getEnvironment(), workspaceId);
 
       startSynchronizer.checkFailure();

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeContext.java

@@ -30,7 +30,7 @@
 /** @author Sergii Leshchenko */
 public class KubernetesRuntimeContext<T extends KubernetesEnvironment> extends RuntimeContext<T> {
 
-  private final KubernetesRuntimeFactory runtimeFactory;
+  private final KubernetesRuntimeFactory<T> runtimeFactory;
   private final KubernetesNamespaceFactory namespaceFactory;
   private final String websocketOutputEndpoint;
   private final KubernetesRuntimeStateCache runtimeStatuses;
@@ -39,7 +39,7 @@ public class KubernetesRuntimeContext<T extends KubernetesEnvironment> extends R
   public KubernetesRuntimeContext(
       @Named("che.websocket.endpoint") String cheWebsocketEndpoint,
       KubernetesNamespaceFactory namespaceFactory,
-      KubernetesRuntimeFactory runtimeFactory,
+      KubernetesRuntimeFactory<T> runtimeFactory,
       KubernetesRuntimeStateCache runtimeStatuses,
       @Assisted T kubernetesEnvironment,
       @Assisted RuntimeIdentity identity,

infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesRuntimeFactory.java

@@ -17,9 +17,7 @@
 import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace;
 
 /** @author Sergii Leshchenko */
-public interface KubernetesRuntimeFactory {
-  KubernetesInternalRuntime<KubernetesRuntimeContext<? extends KubernetesEnvironment>> create(
-      KubernetesRuntimeContext<? extends KubernetesEnvironment> context,
-      KubernetesNamespace namespace,
-      List<Warning> warnings);
+public interface KubernetesRuntimeFactory<E extends KubernetesEnvironment> {
+  KubernetesInternalRuntime<E> create(
+      KubernetesRuntimeContext<E> context, KubernetesNamespace namespace, List<Warning> warnings);
 }

infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java

@@ -15,6 +15,7 @@
 import static org.mockito.Mockito.inOrder;
 
 import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
+import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesEnvironmentProvisioner.KubernetesEnvironmentProvisionerImpl;
 import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
 import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy;
 import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ImagePullSecretProvisioner;
@@ -67,7 +68,7 @@ public class KubernetesEnvironmentProvisionerTest {
   @BeforeMethod
   public void setUp() {
     osInfraProvisioner =
-        new KubernetesEnvironmentProvisioner(
+        new KubernetesEnvironmentProvisionerImpl(
             true,
             uniqueNamesProvisioner,
             serversProvisioner,