Skip to content

Commit

Permalink
feat(ComponentPortlet & ImportCDX): Validate VCS URL while creating a…
Browse files Browse the repository at this point in the history 
…nd updating component and sanitize GitHub domain Repo URLs. Packages with invalid VCS URLs in the SBOM will be created as orphan packages

Signed-off-by: afsahsyeda <afsah.syeda@siemens-healthineers.com>
  • Loading branch information
@afsahsyeda
afsahsyeda committed Apr 13, 2024
1 parent 5e49b79 commit 7810ffa
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 25 deletions.
42 changes: 33 additions & 9 deletions backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,21 @@ private Map<String, List<org.cyclonedx.model.Component>> getVcsToComponentMap(Li
.filter(Objects::nonNull)
.filter(ref -> ExternalReference.Type.VCS.equals(ref.getType()))
.map(ExternalReference::getUrl)
.map(String::toLowerCase)
.map(url -> StringUtils.removeEnd(url, DOT_GIT))
.map(url -> sanitizeVcsUrl(url))
.map(url -> new AbstractMap.SimpleEntry<>(url, comp)))
.collect(Collectors.groupingBy(e -> e.getKey(),
Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
}

private String sanitizeVcsUrl(String url) {
try {
return componentDatabaseHandler.validateVCS(url);
} catch (SW360Exception e) {
log.error(e.getMessage());
return "";
}
}

@SuppressWarnings("unchecked")
public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent attachmentContent, String projectId, User user) {
RequestSummary requestSummary = new RequestSummary();
Expand Down Expand Up @@ -186,8 +194,13 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
// Getting List of org.cyclonedx.model.Component from the Bom
List<org.cyclonedx.model.Component> components = CommonUtils.nullToEmptyList(bom.getComponents());

long vcsCount = components.stream().map(org.cyclonedx.model.Component::getExternalReferences)
.filter(Objects::nonNull).flatMap(List::stream).map(ExternalReference::getType).filter(typeFilter).count();
/*long vcsCount = components.stream().map(org.cyclonedx.model.Component::getExternalReferences)
.filter(Objects::nonNull).flatMap(List::stream).map(ExternalReference::getType).filter(typeFilter).count();*/

//get the count of keys in vcsToComponentMa. This count should not include the key that is empty or null

long vcsCount = getVcsToComponentMap(components).keySet().stream().filter(str -> str != "").count();

long componentsCount = components.size();
org.cyclonedx.model.Component compMetadata = bomMetadata.getComponent();
Map<String, List<org.cyclonedx.model.Component>> vcsToComponentMap = new HashMap<>();
Expand All @@ -198,12 +211,15 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
} else {

vcsToComponentMap = getVcsToComponentMap(components);
final Map<String, List<org.cyclonedx.model.Component>> validVCSToComponentMap = vcsToComponentMap
.entrySet().stream().filter(entry -> entry.getKey() != "")
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
if (componentsCount == vcsCount) {

requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);
requestSummary = importSbomAsProject(compMetadata, validVCSToComponentMap, projectId, attachmentContent);
} else if (componentsCount > vcsCount) {

requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);
requestSummary = importSbomAsProject(compMetadata, validVCSToComponentMap, projectId, attachmentContent);

if (requestSummary.requestStatus.equals(RequestStatus.SUCCESS)) {

Expand Down Expand Up @@ -236,7 +252,8 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a

for (org.cyclonedx.model.Component comp : components) {
if (CommonUtils.isNullOrEmptyCollection(comp.getExternalReferences())
|| comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0) {
|| comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0
|| validVCSToComponentMap.values().stream().flatMap(List::stream).noneMatch(c -> c.equals(comp))) {

final var fullName = SW360Utils.getVersionedName(comp.getName(), comp.getVersion());
final var licenses = getLicenseFromBomComponent(comp);
Expand Down Expand Up @@ -574,8 +591,14 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c
comp.setId(compAddSummary.getId());
} else {
// in case of more than 1 duplicate found, then continue and show error message in UI.
log.warn("found multiple components: " + comp.getName());
duplicateComponents.add(comp.getName());
if (compAddSummary.getRequestStatus() == AddDocumentRequestStatus.DUPLICATE) {
log.warn("found multiple components: " + comp.getName());
duplicateComponents.add(comp.getName());
}
else if (compAddSummary.getRequestStatus() == AddDocumentRequestStatus.INVALID_INPUT) {
log.error("VCS is not a valid URL " + comp.getVcs());
invalidReleases.add(comp.getName());
}
continue;
}

Expand Down Expand Up @@ -964,4 +987,5 @@ private Package createPackage(org.cyclonedx.model.Component componentFromBom, Re
}
return pckg;
}

}
75 changes: 62 additions & 13 deletions ...d/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@

import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.file.*;
Expand All @@ -84,6 +85,8 @@
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.*;

import static com.google.common.base.Strings.isNullOrEmpty;
Expand Down Expand Up @@ -455,16 +458,28 @@ private void setMainLicenses(Component component) {
* Add new release to the database
*/
public AddDocumentRequestSummary addComponent(Component component, String user) throws SW360Exception {
if(isDuplicateUsingVcs(component.getVcs())){
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByVCS(component.getVcs().toLowerCase());
if (duplicates.size() == 1) {
duplicates.forEach(addDocumentRequestSummary::setId);
String vcsUrl = component.getVcs();
if (isNotNullEmptyOrWhitespace(vcsUrl)) {
try {
vcsUrl = validateVCS(vcsUrl);
component.setVcs(vcsUrl);
} catch (SW360Exception e) {
return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.INVALID_INPUT);
}
return addDocumentRequestSummary;

}else if(isDuplicate(component, true)) {
if (isDuplicateUsingVcs(vcsUrl)){
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByVCS(component.getVcs().toLowerCase());
if (duplicates.size() == 1) {
duplicates.forEach(addDocumentRequestSummary::setId);
}
return addDocumentRequestSummary;

}
}

if(isDuplicate(component, true)) {
final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary()
.setRequestStatus(AddDocumentRequestStatus.DUPLICATE);
Set<String> duplicates = componentRepository.getComponentIdsByName(component.getName(), true);
Expand Down Expand Up @@ -590,7 +605,7 @@ public AddDocumentRequestSummary addRelease(Release release, User user) throws S
}

private boolean isDuplicate(Component component, boolean caseInsenstive){
return isDuplicate(component.getName(), caseInsenstive);
return isDuplicate(component.getName(), caseInsenstive) && isDuplicateUsingVcs(component.getVcs());
}

private boolean isDuplicate(Release release){
Expand All @@ -606,9 +621,6 @@ private boolean isDuplicate(String componentName, boolean caseInsenstive) {
}

private boolean isDuplicateUsingVcs(String vcsUrl){
if (isNullEmptyOrWhitespace(vcsUrl)) {
return false;
}
Set<String> duplicates = componentRepository.getComponentIdsByVCS(vcsUrl.toLowerCase());
return duplicates.size()>0;
}
Expand Down Expand Up @@ -697,6 +709,17 @@ public RequestStatus updateComponent(Component component, User user, boolean for
if (categories == null || categories.isEmpty()) {
component.setCategories(ImmutableSet.of(DEFAULT_CATEGORY));
}

String vcsUrl = component.getVcs();
if (isNotNullEmptyOrWhitespace(vcsUrl)) {
try {
vcsUrl = validateVCS(vcsUrl);
component.setVcs(vcsUrl);
} catch (SW360Exception e) {
return RequestStatus.INVALID_INPUT;
}
}

// Prepare component for database
prepareComponent(component);

Expand Down Expand Up @@ -818,7 +841,7 @@ private void updateEccStatusForRelease(Component component) {
}

private boolean changeWouldResultInDuplicate(Component before, Component after) {
if (before.getName().equals(after.getName())) {
if (before.getName().equals(after.getName()) && before.getVcs().equals(after.getVcs())) {
// sth else was changed, not one of the duplication relevant properties
return false;
}
Expand Down Expand Up @@ -3186,4 +3209,30 @@ private AttachmentContent makeAttachmentContent(AttachmentContent content) {
throw new RuntimeException(e);
}
}

public String validateVCS(String vcs) throws SW360Exception {
// GitHub repository URL Format: https://github.com/supplier/name
if (vcs.toLowerCase().contains("github.com")) {
//split the URL which is the VCS in path segments using java.net.URI
URI uri = URI.create(vcs);
String[] urlParts = uri.getPath().split("/");
if (urlParts.length >= 3) {
String firstSegment = urlParts[1];
String secondSegment = urlParts[2].replaceAll("\\.git.*", "").replaceAll("#.*", ""); // Remove .git and anything after, and also remove anything after #
String sanitizedVCS = "https://github.com/" + firstSegment + "/" + secondSegment;
return sanitizedVCS;
} else {
log.error("Invalid GitHub repository URL: " + vcs);
throw new SW360Exception("Invalid GitHub repository URL: " + vcs).setErrorCode(HttpStatus.SC_BAD_REQUEST);
}
} else {
//repository URL formats for other domains yet to be defined
if (!CommonUtils.isValidUrl(vcs)) {
log.error("Invalid VCS URL: " + vcs);
throw new SW360Exception("Invalid VCS URL: " + vcs).setErrorCode(HttpStatus.SC_BAD_REQUEST);
}
return vcs;
}
}

}
4 changes: 3 additions & 1 deletion frontend/sw360-portlet/src/main/java/org/eclipse/sw360/portal/common/ErrorMessages.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class ErrorMessages {
public static final String COMPONENT_NOT_ADDED = "Component could not be added.";
public static final String COMPONENT_DUPLICATE = "A component with the same name or VCS already exists.";
public static final String COMPONENT_NAMING_ERROR = "Name and Categories of component cannot contain only space characters.";
public static final String INVALID_VCS_OR_LINKED_DOCUMENT = "Invalid VCS or linked document";
public static final String RELEASE_NOT_ADDED = "Release could not be added.";
public static final String RELEASE_DUPLICATE = "A release with the same name and version already exists.";
public static final String RELEASE_NAME_VERSION_ERROR = "Name and version of release cannot contain only space characters.";
Expand Down Expand Up @@ -71,7 +72,7 @@ public class ErrorMessages {
public static final String ERROR_VENDOR = "Error: Invalid vendor Name or Url.";
public static final String ERROR_BULK_DELETING = "Error while bulk deleting";
public static final String ERROR_BULK_DELETING_IN_BACKEND = "Error while bulk deleting in backend.";

public static final String PACKAGE_NOT_ADDED = "Package could not be added.";
public static final String PACKAGE_DUPLICATE = "A package with the same name and version already exists.";
public static final String PACKAGE_NAME_VERSION_ERROR = "Name and version of the package cannot contain only space characters.";
Expand All @@ -93,6 +94,7 @@ public class ErrorMessages {
.add(COMPONENT_NOT_ADDED)
.add(COMPONENT_DUPLICATE)
.add(COMPONENT_NAMING_ERROR)
.add(INVALID_VCS_OR_LINKED_DOCUMENT)
.add(RELEASE_NOT_ADDED)
.add(RELEASE_DUPLICATE)
.add(RELEASE_NAME_VERSION_ERROR)
Expand Down
11 changes: 10 additions & 1 deletion ...-portlet/src/main/java/org/eclipse/sw360/portal/portlets/components/ComponentPortlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2318,11 +2318,13 @@ public void updateComponent(ActionRequest request, ActionResponse response) thro
RequestStatus requestStatus = client.updateComponent(component, user);
setSessionMessage(request, requestStatus, "Component", "update", component.getName());
if (RequestStatus.DUPLICATE.equals(requestStatus) || RequestStatus.DUPLICATE_ATTACHMENT.equals(requestStatus) ||
RequestStatus.NAMINGERROR.equals(requestStatus)) {
RequestStatus.NAMINGERROR.equals(requestStatus) || RequestStatus.INVALID_INPUT.equals(requestStatus) ) {
if(RequestStatus.DUPLICATE.equals(requestStatus))
setSW360SessionError(request, ErrorMessages.COMPONENT_DUPLICATE);
else if (RequestStatus.NAMINGERROR.equals(requestStatus))
setSW360SessionError(request, ErrorMessages.COMPONENT_NAMING_ERROR);
else if (RequestStatus.INVALID_INPUT.equals(requestStatus))
setSW360SessionError(request, ErrorMessages.INVALID_VCS_OR_LINKED_DOCUMENT);
else
setSW360SessionError(request, ErrorMessages.DUPLICATE_ATTACHMENT);
response.setRenderParameter(PAGENAME, PAGENAME_EDIT);
Expand Down Expand Up @@ -2360,6 +2362,11 @@ else if (RequestStatus.NAMINGERROR.equals(requestStatus))
response.setRenderParameter(PAGENAME, PAGENAME_EDIT);
prepareRequestForEditAfterDuplicateOrNamingError(request, component);
break;
case INVALID_INPUT:
setSW360SessionError(request, ErrorMessages.INVALID_VCS_OR_LINKED_DOCUMENT);
response.setRenderParameter(PAGENAME, PAGENAME_EDIT);
prepareRequestForEditAfterDuplicateOrNamingError(request, component);
break;
default:
setSW360SessionError(request, ErrorMessages.COMPONENT_NOT_ADDED);
response.setRenderParameter(PAGENAME, PAGENAME_VIEW);
Expand Down Expand Up @@ -2417,6 +2424,8 @@ public void updateRelease(ActionRequest request, ActionResponse response) throws
setSW360SessionError(request, ErrorMessages.RELEASE_DUPLICATE);
else if (RequestStatus.NAMINGERROR.equals(requestStatus))
setSW360SessionError(request, ErrorMessages.RELEASE_NAME_VERSION_ERROR);
else if (RequestStatus.INVALID_INPUT.equals(requestStatus))
setSW360SessionError(request, ErrorMessages.INVALID_VCS_OR_LINKED_DOCUMENT);
else
setSW360SessionError(request, ErrorMessages.DUPLICATE_ATTACHMENT);
response.setRenderParameter(PAGENAME, PAGENAME_EDIT_RELEASE);
Expand Down
2 changes: 1 addition & 1 deletion frontend/sw360-portlet/src/main/resources/content/Language.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -858,7 +858,7 @@ link.to.project=Link to Project
link.to.projects=Link to Projects
list.of.components.without.version.information=List of Components without version information
list.of.invalid.packages.without.purl.or.name.or.version=List of invalid Packages without purl or name or version
list.of.packages.without.vcs.information=List of Packages without VCS information
list.of.packages.without.vcs.information=List of Packages with missing or invalid VCS information
list.view=List View
loading=Loading...
loading.data.for.step.1.please.wait=Loading data for step 1, please wait...
Expand Down

0 comments on commit 7810ffa

Please sign in to comment.