Skip to content

Commit

Permalink
Epankou/bug mic removal sync ewc 374 (#174)
Browse files Browse the repository at this point in the history 
* Bug(fix) microservice removal: networking microsrevice isn't removed (EWC-424)

* Bug(fix) microservice removal: removed extra host mapping (EWC-424)

* Bug(fix) microservice removal: simplified getRunningContainers method (EWC-424)

* Bug(fix) route update: added missing method for route update (EWC-374)

* Bug(fix) mic removal update: added synchronization (EWC-374)

* Bug(fix) mic removal update: added synchronization (EWC-374)

* Bug(fix) mic removal update: added synchronization (EWC-374)
  • Loading branch information
@epankou @mchepelev
epankou authored and mchepelev committed Dec 29, 2018
1 parent 1b1f7b6 commit fb1c826
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 58 deletions.
6 changes: 4 additions & 2 deletions daemon/src/org/eclipse/iofog/microservice/Microservice.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
*/
public class Microservice {

public static final Object deleteLock = new Object();
private final String microserviceUuid; //container name
private final String imageName;
private List<PortMapping> portMappings;
Expand All @@ -34,6 +35,7 @@ public class Microservice {
private long logSize;
private List<VolumeMapping> volumeMappings;
private boolean isUpdating;

private boolean delete;
private boolean deleteWithCleanup;

Expand Down Expand Up @@ -123,11 +125,11 @@ public void setVolumeMappings(List<VolumeMapping> volumeMappings) {
this.volumeMappings = volumeMappings;
}

public boolean isUpdating() {
public synchronized boolean isUpdating() {
return isUpdating;
}

public void setUpdating(boolean updating) {
public synchronized void setUpdating(boolean updating) {
isUpdating = updating;
}

Expand Down
22 changes: 15 additions & 7 deletions daemon/src/org/eclipse/iofog/process_manager/ContainerManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
*******************************************************************************/
package org.eclipse.iofog.process_manager;

import com.github.dockerjava.api.exception.ConflictException;
import com.github.dockerjava.api.model.Container;
import com.github.dockerjava.api.model.Image;
import org.eclipse.iofog.microservice.Microservice;
Expand All @@ -22,6 +23,8 @@

import java.util.Optional;

import static org.eclipse.iofog.microservice.Microservice.deleteLock;

/**
* provides methods to manage Docker containers
*
Expand Down Expand Up @@ -132,12 +135,12 @@ private void stopContainer(String microserviceUuid) {
* removes a {@link Container} by Microservice uuid
*/
private void removeContainerByMicroserviceUuid(String microserviceUuid, boolean withCleanUp) {

Optional<Container> containerOptional = docker.getContainer(microserviceUuid);

if (containerOptional.isPresent()) {
Container container = containerOptional.get();
removeContainer(container.getId(), container.getImageId(), withCleanUp);
synchronized (deleteLock) {
Optional<Container> containerOptional = docker.getContainer(microserviceUuid);
if (containerOptional.isPresent()) {
Container container = containerOptional.get();
removeContainer(container.getId(), container.getImageId(), withCleanUp);
}
}
}

Expand All @@ -147,7 +150,12 @@ private void removeContainer(String containerId, String imageId, boolean withCle
docker.stopContainer(containerId);
docker.removeContainer(containerId, withCleanUp);
if (withCleanUp) {
docker.removeImageById(imageId);
try {
docker.removeImageById(imageId);
} catch (ConflictException ex) {
LoggingService.logInfo(MODULE_NAME, String.format("Image for container \"%s\" hasn't been removed", containerId));
LoggingService.logInfo(MODULE_NAME, ex.getMessage().replace("\n", ""));
}
}

LoggingService.logInfo(MODULE_NAME, String.format("container \"%s\" removed", containerId));
Expand Down
111 changes: 62 additions & 49 deletions daemon/src/org/eclipse/iofog/process_manager/ProcessManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.eclipse.iofog.microservice.MicroserviceState;
import org.eclipse.iofog.microservice.MicroserviceStatus;
import org.eclipse.iofog.status_reporter.StatusReporter;
import org.eclipse.iofog.utils.Constants;
import org.eclipse.iofog.utils.Constants.ModulesStatus;
import org.eclipse.iofog.utils.configuration.Configuration;

Expand All @@ -28,9 +29,9 @@
import java.util.stream.Stream;

import static java.lang.String.format;
import static org.eclipse.iofog.microservice.Microservice.deleteLock;
import static org.eclipse.iofog.process_manager.ContainerTask.Tasks.*;
import static org.eclipse.iofog.utils.Constants.ControllerStatus.OK;
import static org.eclipse.iofog.utils.Constants.IOFOG_DOCKER_CONTAINER_NAME_PREFIX;
import static org.eclipse.iofog.utils.Constants.PROCESS_MANAGER;

/**
Expand Down Expand Up @@ -168,18 +169,69 @@ private void handleLatestMicroservices() {
}

private void deleteRemainingMicroservices() {
Set<Microservice> allAgentMicroservices = Stream.concat(
microserviceManager.getLatestMicroservices().stream(),
microserviceManager.getCurrentMicroservices().stream()
Set<String> latestMicroserviceUuids = microserviceManager.getLatestMicroservices().stream()
.map(Microservice::getMicroserviceUuid).collect(Collectors.toSet());
Set<String> currentMicroserviceUuids = microserviceManager.getCurrentMicroservices().stream()
.map(Microservice::getMicroserviceUuid).collect(Collectors.toSet());
List<Container> runningContainers;
synchronized (deleteLock) {
runningContainers = docker.getRunningContainers();
}
Set<String> runningContainerNames = runningContainers.stream()
.map(docker::getContainerName)
.collect(Collectors.toSet());

Set<String> allMicroserviceUuids = Stream.concat(
Stream.concat(
latestMicroserviceUuids.stream(),
currentMicroserviceUuids.stream()
),
runningContainers.stream()
.map(docker::getContainerMicroserviceUuid)
)
.collect(Collectors.toSet());
deleteOldAgentMicroservices();
deleteObsoleteAgentMicroservices(allAgentMicroservices);
deleteNonAgentMicroservices(allAgentMicroservices);

Set<String> oldAgentMicroserviceUuids = new HashSet<>();
Set<String> unknownMicroserviceUuids = new HashSet<>();

for (String uuid : allMicroserviceUuids) {
boolean isCurrentMicroserviceUuid = currentMicroserviceUuids.contains(uuid);
boolean isLatestMicroserviceUuid = latestMicroserviceUuids.contains(uuid);

if (isCurrentMicroserviceUuid && !isLatestMicroserviceUuid) {
oldAgentMicroserviceUuids.add(uuid);
} else if (!isCurrentMicroserviceUuid
&& !isLatestMicroserviceUuid
&& runningContainerNames.contains(Constants.IOFOG_DOCKER_CONTAINER_NAME_PREFIX + uuid)) {
unknownMicroserviceUuids.add(uuid);
} else if (!isCurrentMicroserviceUuid
&& !isLatestMicroserviceUuid
&& Configuration.isWatchdogEnabled()) {
unknownMicroserviceUuids.add(uuid);
}
}

deleteOldAgentContainers(oldAgentMicroserviceUuids);
deleteUnknownContainers(unknownMicroserviceUuids);
}

private void deleteOldAgentContainers(Set<String> oldAgentContainerNames) {
oldAgentContainerNames.forEach(name -> {
MicroserviceStatus status = new MicroserviceStatus(MicroserviceState.NOT_RUNNING);
StatusReporter.setProcessManagerStatus().setMicroservicesStatus(name, status);
disableMicroserviceFeaturesBeforeRemoval(name);
addTask(new ContainerTask(REMOVE, name));
});
}

private void deleteUnknownContainers(Set<String> unknownContainerNames) {
unknownContainerNames.forEach(name -> addTask(new ContainerTask(REMOVE, name)));
}

private void updateRunningMicroservicesCount() {
StatusReporter.setProcessManagerStatus().setRunningMicroservicesCount(docker.getRunningIofogContainers().size());
synchronized (deleteLock) {
StatusReporter.setProcessManagerStatus().setRunningMicroservicesCount(docker.getRunningIofogContainers().size());
}
}

private void updateCurrentMicroservices() {
Expand All @@ -189,50 +241,11 @@ private void updateCurrentMicroservices() {
microserviceManager.setCurrentMicroservices(currentMicroservices);
}

/**
* Deletes microservices which don't present in latest microservices list but do present in current microservices list
*/
private void deleteOldAgentMicroservices() {
microserviceManager.getCurrentMicroservices().stream()
.filter(microservice -> !microserviceManager.getLatestMicroservices().contains(microservice))
.forEach(microservice -> {
MicroserviceStatus status = new MicroserviceStatus(MicroserviceState.NOT_RUNNING);
StatusReporter.setProcessManagerStatus().setMicroservicesStatus(microservice.getMicroserviceUuid(), status);
disableMicroserviceFeaturesBeforeRemoval(microservice.getMicroserviceUuid());
addTask(new ContainerTask(REMOVE, microservice.getMicroserviceUuid()));
});
}

/**
* Deletes obsolete agent microservices that aren't present in current or latest microservices list
* @param allAgentMicroservices all microservices run by iofog agent
*/
private void deleteObsoleteAgentMicroservices(Set<Microservice> allAgentMicroservices) {
docker.getRunningIofogContainers().stream()
.map(container -> docker.getContainerMicroserviceUuid(container))
.filter(microserviceUuid -> allAgentMicroservices.stream()
.noneMatch(microservice -> microservice.getMicroserviceUuid().equals(microserviceUuid)))
.forEach(microserviceUuid -> addTask(new ContainerTask(REMOVE, microserviceUuid)));
}

/**
* Deletes any microservices which don't belong to iofog agent
* @param allAgentMicroservices all microservices run by iofog agent
*/
private void deleteNonAgentMicroservices(Set<Microservice> allAgentMicroservices) {
if (Configuration.isWatchdogEnabled()) {
docker.getRunningNonIofogContainers().stream()
.map(container -> docker.getContainerMicroserviceUuid(container))
.filter(microserviceUuid -> allAgentMicroservices.stream()
.noneMatch(microservice -> microservice.getMicroserviceUuid().equals(microserviceUuid)))
.forEach(microserviceUuid -> addTask(new ContainerTask(REMOVE, microserviceUuid)));
}
}

private boolean shouldContainerBeUpdated(Microservice microservice, Container container, MicroserviceStatus status) {
boolean isNotRunning = !MicroserviceState.RUNNING.equals(status.getStatus());
boolean areNotEqual = !docker.areMicroserviceAndContainerEqual(container.getId(), microservice);
return isNotRunning || areNotEqual;
boolean isRebuild = microservice.isRebuild();
return isNotRunning || areNotEqual || isRebuild;
}

/**
Expand Down

0 comments on commit fb1c826

Please sign in to comment.