Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(ComponentPortlet & ImportCDX): Validate VCS URL #2408

Merged
merged 1 commit into from
Sep 20, 2024
Merged

feat(ComponentPortlet & ImportCDX): Validate VCS URL #2408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 55 additions & 20 deletions backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,9 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.AbstractMap;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.*;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -93,7 +87,6 @@ public class CycloneDxBOMImporter {
private static final String DOT_GIT = ".git";
private static final String SLASH = "/";
private static final String DOT = ".";
private static final String HASH = "#";
private static final String HYPHEN = "-";
private static final String JOINER = "||";
private static final Pattern THIRD_SLASH_PATTERN = Pattern.compile("[^/]*(/[^/]*){2}");
Expand Down Expand Up @@ -142,13 +135,13 @@ public CycloneDxBOMImporter(ProjectDatabaseHandler projectDatabaseHandler, Compo
* @return Map<String, List<org.cyclonedx.model.Component>>
*/
private Map<String, List<org.cyclonedx.model.Component>> getVcsToComponentMap(List<org.cyclonedx.model.Component> components) {
return components.stream().filter(Objects::nonNull)
return components.parallelStream().filter(Objects::nonNull)
.flatMap(comp -> CommonUtils.nullToEmptyList(comp.getExternalReferences()).stream()
.filter(Objects::nonNull)
.filter(ref -> ExternalReference.Type.VCS.equals(ref.getType()))
.map(ExternalReference::getUrl)
.map(String::toLowerCase)
.map(url -> StringUtils.removeEnd(url, DOT_GIT))
.map(url -> sanitizeVCS(url))
.filter(url -> CommonUtils.isValidUrl(url))
.map(url -> new AbstractMap.SimpleEntry<>(url, comp)))
.collect(Collectors.groupingBy(e -> e.getKey(),
Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
Expand Down Expand Up @@ -186,8 +179,7 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
// Getting List of org.cyclonedx.model.Component from the Bom
List<org.cyclonedx.model.Component> components = CommonUtils.nullToEmptyList(bom.getComponents());

long vcsCount = components.stream().map(org.cyclonedx.model.Component::getExternalReferences)
.filter(Objects::nonNull).flatMap(List::stream).map(ExternalReference::getType).filter(typeFilter).count();
long vcsCount = getVcsToComponentMap(components).size();
long componentsCount = components.size();
org.cyclonedx.model.Component compMetadata = bomMetadata.getComponent();
Map<String, List<org.cyclonedx.model.Component>> vcsToComponentMap = new HashMap<>();
Expand All @@ -196,7 +188,6 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
vcsToComponentMap.put("", components);
requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent);
} else {

vcsToComponentMap = getVcsToComponentMap(components);
if (componentsCount == vcsCount) {

Expand Down Expand Up @@ -236,7 +227,8 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a

for (org.cyclonedx.model.Component comp : components) {
if (CommonUtils.isNullOrEmptyCollection(comp.getExternalReferences())
|| comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0) {
|| comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0
|| !containsComp(vcsToComponentMap, comp)) {

final var fullName = SW360Utils.getVersionedName(comp.getName(), comp.getVersion());
final var licenses = getLicenseFromBomComponent(comp);
Expand Down Expand Up @@ -282,6 +274,7 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a
}
}
}

RequestStatus updateStatus = projectDatabaseHandler.updateProject(project, user);
if (RequestStatus.SUCCESS.equals(updateStatus)) {
log.info("linking packages to project successfull: " + projId);
Expand Down Expand Up @@ -411,7 +404,6 @@ public RequestSummary importSbomAsProject(org.cyclonedx.model.Component compMeta
summary.setMessage("Invalid Projct metadata present in SBOM or Multiple project with same name and version is already present in SW360!");
return summary;
}

}
} catch (SW360Exception e) {
log.error("An error occured while importing project from SBOM: " + e.getMessage());
Expand Down Expand Up @@ -583,7 +575,6 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c
}

for (org.cyclonedx.model.Component bomComp : entry.getValue()) {

Set<String> licenses = getLicenseFromBomComponent(bomComp);
release = createRelease(bomComp.getVersion(), comp, licenses);
if (CommonUtils.isNullEmptyOrWhitespace(release.getVersion()) ) {
Expand All @@ -595,6 +586,7 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c

try {
AddDocumentRequestSummary relAddSummary = componentDatabaseHandler.addRelease(release, user);

if (CommonUtils.isNotNullEmptyOrWhitespace(relAddSummary.getId())) {
release.setId(relAddSummary.getId());
if (AddDocumentRequestStatus.SUCCESS.equals(relAddSummary.getRequestStatus())) {
Expand Down Expand Up @@ -626,6 +618,17 @@ private Map<String, String> importAllComponentsAsPackages(Map<String, List<org.c
} else {
comp.setMainLicenseIds(licenses);
}
if (CommonUtils.isNullEmptyOrWhitespace(comp.getVcs())) {
for (ExternalReference extRef : CommonUtils.nullToEmptyList(bomComp.getExternalReferences())) {
if (Type.VCS.equals(extRef.getType())) {
comp.setVcs(entry.getKey());
} else if (Type.MAILING_LIST.equals(extRef.getType())) {
comp.setMailinglist(CommonUtils.nullToEmptyString(extRef.getUrl()));
} else if (Type.SUPPORT.equals(extRef.getType())) {
comp.setWiki(CommonUtils.nullToEmptyString(extRef.getUrl()));
}
}
}

RequestStatus updateStatus = componentDatabaseHandler.updateComponent(comp, user, true);
if (RequestStatus.SUCCESS.equals(updateStatus)) {
Expand Down Expand Up @@ -874,7 +877,7 @@ private Release createRelease(org.cyclonedx.model.Component componentFromBom, Co
}
for (ExternalReference extRef : CommonUtils.nullToEmptyList(componentFromBom.getExternalReferences())) {
if (Type.VCS.equals(extRef.getType())) {
String repoUrl = CommonUtils.nullToEmptyString(extRef.getUrl());
String repoUrl = CommonUtils.nullToEmptyString(StringUtils.removeEnd(extRef.getUrl(), DOT_GIT));
Repository repo = new Repository(repoUrl);
if (repoUrl.toLowerCase().contains("github")) {
repo.setRepositorytype(RepositoryType.GIT);
Expand Down Expand Up @@ -913,8 +916,10 @@ private String getComponentNameFromVCS(String vcsUrl){
Matcher firstSlashMatcher = FIRST_SLASH_PATTERN.matcher(compName);
if (firstSlashMatcher.find()) {
compName = firstSlashMatcher.group(1);
compName = StringUtils.substringBefore(compName, HASH);
compName = compName.replaceAll(SLASH, ".");
if (vcsUrl.toLowerCase().contains("github.com")) {
compName = compName.replaceAll("\\.git.*", "").replaceAll("#.*", "");
}
}
}

Expand Down Expand Up @@ -976,4 +981,34 @@ public String getComponetNameById(String id, User user) throws SW360Exception {
Component comp = componentDatabaseHandler.getComponent(id, user);
return comp.getName();
}

/*
* Sanitize different repository URLS based on their defined schema
*/
public String sanitizeVCS(String vcs) {
// GitHub repository URL Format: https://github.com/supplier/name
if (vcs.toLowerCase().contains("github.com")) {
URI uri = URI.create(vcs);
String[] urlParts = uri.getPath().split("/");
if (urlParts.length >= 3) {
String firstSegment = urlParts[1];
String secondSegment = urlParts[2].replaceAll("\\.git.*", "").replaceAll("#.*", "");
vcs = "https://github.com/" + firstSegment + "/" + secondSegment;
return vcs;
} else {
log.error("Invalid GitHub repository URL: " + vcs);
}
}
// Other formats yet to be defined
return vcs;
}

public static boolean containsComp(Map<String, List<org.cyclonedx.model.Component>> map, org.cyclonedx.model.Component element) {
for (List<org.cyclonedx.model.Component> list : map.values()) {
if (list.contains(element)) {
return true;
}
}
return false;
}
}
Loading