diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java b/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java index ef2a50fbca..cfe7d08f26 100644 --- a/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java +++ b/backend/src-common/src/main/java/org/eclipse/sw360/cyclonedx/CycloneDxBOMImporter.java @@ -12,15 +12,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; +import java.net.URI; import java.nio.charset.Charset; -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -93,7 +87,6 @@ public class CycloneDxBOMImporter { private static final String DOT_GIT = ".git"; private static final String SLASH = "/"; private static final String DOT = "."; - private static final String HASH = "#"; private static final String HYPHEN = "-"; private static final String JOINER = "||"; private static final Pattern THIRD_SLASH_PATTERN = Pattern.compile("[^/]*(/[^/]*){2}"); @@ -142,13 +135,13 @@ public CycloneDxBOMImporter(ProjectDatabaseHandler projectDatabaseHandler, Compo * @return Map> */ private Map> getVcsToComponentMap(List components) { - return components.stream().filter(Objects::nonNull) + return components.parallelStream().filter(Objects::nonNull) .flatMap(comp -> CommonUtils.nullToEmptyList(comp.getExternalReferences()).stream() .filter(Objects::nonNull) .filter(ref -> ExternalReference.Type.VCS.equals(ref.getType())) .map(ExternalReference::getUrl) - .map(String::toLowerCase) - .map(url -> StringUtils.removeEnd(url, DOT_GIT)) + .map(url -> sanitizeVCS(url)) + .filter(url -> CommonUtils.isValidUrl(url)) .map(url -> new AbstractMap.SimpleEntry<>(url, comp))) .collect(Collectors.groupingBy(e -> e.getKey(), Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); @@ -186,8 +179,7 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a // Getting List of org.cyclonedx.model.Component from the Bom List components = CommonUtils.nullToEmptyList(bom.getComponents()); - long vcsCount = components.stream().map(org.cyclonedx.model.Component::getExternalReferences) - .filter(Objects::nonNull).flatMap(List::stream).map(ExternalReference::getType).filter(typeFilter).count(); + long vcsCount = getVcsToComponentMap(components).size(); long componentsCount = components.size(); org.cyclonedx.model.Component compMetadata = bomMetadata.getComponent(); Map> vcsToComponentMap = new HashMap<>(); @@ -196,7 +188,6 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a vcsToComponentMap.put("", components); requestSummary = importSbomAsProject(compMetadata, vcsToComponentMap, projectId, attachmentContent); } else { - vcsToComponentMap = getVcsToComponentMap(components); if (componentsCount == vcsCount) { @@ -236,7 +227,8 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a for (org.cyclonedx.model.Component comp : components) { if (CommonUtils.isNullOrEmptyCollection(comp.getExternalReferences()) - || comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0) { + || comp.getExternalReferences().stream().map(ExternalReference::getType).filter(typeFilter).count() == 0 + || !containsComp(vcsToComponentMap, comp)) { final var fullName = SW360Utils.getVersionedName(comp.getName(), comp.getVersion()); final var licenses = getLicenseFromBomComponent(comp); @@ -282,6 +274,7 @@ public RequestSummary importFromBOM(InputStream inputStream, AttachmentContent a } } } + RequestStatus updateStatus = projectDatabaseHandler.updateProject(project, user); if (RequestStatus.SUCCESS.equals(updateStatus)) { log.info("linking packages to project successfull: " + projId); @@ -411,7 +404,6 @@ public RequestSummary importSbomAsProject(org.cyclonedx.model.Component compMeta summary.setMessage("Invalid Projct metadata present in SBOM or Multiple project with same name and version is already present in SW360!"); return summary; } - } } catch (SW360Exception e) { log.error("An error occured while importing project from SBOM: " + e.getMessage()); @@ -583,7 +575,6 @@ private Map importAllComponentsAsPackages(Map licenses = getLicenseFromBomComponent(bomComp); release = createRelease(bomComp.getVersion(), comp, licenses); if (CommonUtils.isNullEmptyOrWhitespace(release.getVersion()) ) { @@ -595,6 +586,7 @@ private Map importAllComponentsAsPackages(Map importAllComponentsAsPackages(Map= 3) { + String firstSegment = urlParts[1]; + String secondSegment = urlParts[2].replaceAll("\\.git.*", "").replaceAll("#.*", ""); + vcs = "https://github.com/" + firstSegment + "/" + secondSegment; + return vcs; + } else { + log.error("Invalid GitHub repository URL: " + vcs); + } + } + // Other formats yet to be defined + return vcs; + } + + public static boolean containsComp(Map> map, org.cyclonedx.model.Component element) { + for (List list : map.values()) { + if (list.contains(element)) { + return true; + } + } + return false; + } } diff --git a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java index 9b1f2af83d..d44d0b11fb 100644 --- a/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java +++ b/backend/src-common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentDatabaseHandler.java @@ -76,6 +76,7 @@ import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; import java.nio.file.*; @@ -455,27 +456,35 @@ private void setMainLicenses(Component component) { * Add new release to the database */ public AddDocumentRequestSummary addComponent(Component component, String user) throws SW360Exception { - if(isDuplicateUsingVcs(component, true)){ - final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() - .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); - Set duplicates = componentRepository.getComponentIdsByVCS(component.getVcs(), true); - if (duplicates.size() == 1) { - duplicates.forEach(addDocumentRequestSummary::setId); + if (isNotNullEmptyOrWhitespace(component.getVcs())) { + String vcsUrl = component.getVcs(); + if (isDuplicateUsingVcs(vcsUrl, true)){ + final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() + .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); + Set duplicates = componentRepository.getComponentIdsByVCS(component.getVcs(), true); + if (duplicates.size() == 1) { + duplicates.forEach(addDocumentRequestSummary::setId); + } + return addDocumentRequestSummary; } - return addDocumentRequestSummary; + if (!CommonUtils.isValidUrl(vcsUrl)) { + log.error("Invalid VCS URL: " + vcsUrl); + return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.INVALID_INPUT); + } + } - }else if(isDuplicate(component, true)) { - final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() - .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); - Set duplicates = componentRepository.getComponentIdsByName(component.getName(), true); - if (duplicates.size() == 1) { - duplicates.forEach(addDocumentRequestSummary::setId); + if (component.getName().trim().length() == 0) { + return new AddDocumentRequestSummary().setRequestStatus(AddDocumentRequestStatus.NAMINGERROR); + } else { + if (isDuplicate(component.getName(), true)) { + final AddDocumentRequestSummary addDocumentRequestSummary = new AddDocumentRequestSummary() + .setRequestStatus(AddDocumentRequestStatus.DUPLICATE); + Set duplicates = componentRepository.getComponentIdsByName(component.getName(), true); + if (duplicates.size() == 1) { + duplicates.forEach(addDocumentRequestSummary::setId); + } + return addDocumentRequestSummary; } - return addDocumentRequestSummary; - } - if(component.getName().trim().length() == 0) { - return new AddDocumentRequestSummary() - .setRequestStatus(AddDocumentRequestStatus.NAMINGERROR); } if (!isDependenciesExistInComponent(component)) { @@ -589,31 +598,27 @@ public AddDocumentRequestSummary addRelease(Release release, User user) throws S .setId(id); } - private boolean isDuplicate(Component component, boolean caseInsenstive){ - return isDuplicate(component.getName(), caseInsenstive); + private boolean isDuplicate(Component component, boolean caseInsensitive){ + return isDuplicate(component.getName(), caseInsensitive); } private boolean isDuplicate(Release release){ return isDuplicate(release.getName(), release.getVersion()); } - private boolean isDuplicate(String componentName, boolean caseInsenstive) { + public boolean isDuplicate(String componentName, boolean caseInsensitive) { if (isNullEmptyOrWhitespace(componentName)) { return false; } - Set duplicates = componentRepository.getComponentIdsByName(componentName, caseInsenstive); + Set duplicates = componentRepository.getComponentIdsByName(componentName.trim(), caseInsensitive); return duplicates.size()>0; } - private boolean isDuplicateUsingVcs(Component component, boolean caseInsenstive){ - return isDuplicateUsingVcs(component.getVcs(), caseInsenstive); - } - - private boolean isDuplicateUsingVcs(String vcsUrl, boolean caseInsenstive){ + private boolean isDuplicateUsingVcs(String vcsUrl, boolean caseInsensitive){ if (isNullEmptyOrWhitespace(vcsUrl)) { return false; } - Set duplicates = componentRepository.getComponentIdsByVCS(vcsUrl, caseInsenstive); + Set duplicates = componentRepository.getComponentIdsByVCS(vcsUrl, caseInsensitive); return duplicates.size()>0; } @@ -625,9 +630,9 @@ private boolean isDuplicate(String releaseName, String releaseVersion) { return duplicates.size()>0; } - private void isDuplicateComponent(List componentNames, boolean caseInsenstive) { + private void isDuplicateComponent(List componentNames, boolean caseInsensitive) { for (String name : componentNames) { - if(!isDuplicate(name, caseInsenstive)) + if(!isDuplicate(name, caseInsensitive)) listComponentName.add(name); } } @@ -694,13 +699,24 @@ public RequestStatus updateComponent(Component component, User user) throws SW36 public RequestStatus updateComponent(Component component, User user, boolean forceUpdate) throws SW360Exception { removeLeadingTrailingWhitespace(component); String name = component.getName(); + String vcs = component.getVcs(); + if (name == null || name.isEmpty()) { return RequestStatus.NAMINGERROR; } + + if (isNotNullEmptyOrWhitespace(vcs)) { + if (!CommonUtils.isValidUrl(vcs)) { + log.error("Invalid VCS URL: " + vcs); + return RequestStatus.INVALID_INPUT; + } + } + Set categories = component.getCategories(); if (categories == null || categories.isEmpty()) { component.setCategories(ImmutableSet.of(DEFAULT_CATEGORY)); } + // Prepare component for database prepareComponent(component); @@ -822,12 +838,17 @@ private void updateEccStatusForRelease(Component component) { } private boolean changeWouldResultInDuplicate(Component before, Component after) { - if (before.getName().equals(after.getName())) { - // sth else was changed, not one of the duplication relevant properties - return false; + String beforeVCS = isNullEmptyOrWhitespace(before.getVcs()) ? "" : before.getVcs().trim(); + String afterVCS = isNullEmptyOrWhitespace(after.getVcs()) ? "" : after.getVcs().trim(); + + if (before.getName().trim().equalsIgnoreCase(after.getName().trim())) { + if (beforeVCS.equalsIgnoreCase(afterVCS)) { + return false; + } + return isDuplicateUsingVcs(afterVCS, true); } - return isDuplicate(after, false); + return isDuplicate(after, true); } private boolean duplicateAttachmentExist(Component component) { @@ -837,7 +858,6 @@ private boolean duplicateAttachmentExist(Component component) { return false; } - private void updateComponentInternal(Component updated, Component current, User user) { updateModifiedFields(updated, user.getEmail()); // Update the database with the component @@ -2017,7 +2037,13 @@ public Map getAllComponentsIdMap() { } public List getAllComponentsWithVCS() { - final List components = componentRepository.getComponentsByVCS(); + final List componentsWithVCS = componentRepository.getComponentsByVCS(); + final List components = new ArrayList<>(); + for (Component component : componentsWithVCS) { + if (!CommonUtils.isNullOrEmptyCollection(component.getReleaseIds())) { + components.add(component); + } + } return components; } @@ -3069,7 +3095,6 @@ public RequestStatus uploadSourceCodeAttachmentToReleases() { components.forEach(c -> { String VCS = c.getVcs(); - log.info(String.format("SRC Upload: %s %s", c.getId(), VCS)); // Add more domains in the future and include the download logic accordingly if (VCS.toLowerCase().contains("github.com")) { for (String r_id : c.getReleaseIds()) { @@ -3085,6 +3110,7 @@ public RequestStatus uploadSourceCodeAttachmentToReleases() { releasesWithoutSRC.add(r.getId()); String version = r.getVersion(); Release originalReleaseData = r.deepCopy(); + log.info(String.format("SRC Upload: %s %s", c.getVcs(), version)); for (String format : formats) { String downloadURL = String.format(format, c.getVcs(), version); @@ -3108,6 +3134,8 @@ public RequestStatus uploadSourceCodeAttachmentToReleases() { log.error( "SRC Upload: Error while downloading the source code zip file for release:" + r.getId() + " " + e); + } catch (Exception e) { + log.error("An exception occured while uploading source:" + r.getId() + " " + e); } } } @@ -3143,7 +3171,7 @@ private boolean isValidURL(String url) { } else { return false; } - } catch (IOException e) { + } catch (Exception e) { log.error("Error while checking the validity of the URL " + url, e); return false; }