Skip to content

Commit

Permalink
Implement file storage plugin
Browse files Browse the repository at this point in the history
Signed-off-by: nscuro <[email protected]>
  • Loading branch information
nscuro committed Jan 31, 2025
1 parent 425cabc commit c0b0efc
Show file tree
Hide file tree
Showing 23 changed files with 1,948 additions and 128 deletions.
6 changes: 6 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,12 @@
<version>${lib.testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>minio</artifactId>
<version>${lib.testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
Expand Down
13 changes: 6 additions & 7 deletions src/main/java/org/dependencytrack/event/BomUploadEvent.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@

import alpine.event.framework.AbstractChainableEvent;
import org.dependencytrack.model.Project;

import java.io.File;
import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;

/**
* Defines an event triggered when a bill-of-material (bom) document is submitted.
Expand All @@ -32,18 +31,18 @@
public class BomUploadEvent extends AbstractChainableEvent {

private final Project project;
private final File file;
private final FileMetadata fileMetadata;

public BomUploadEvent(final Project project, final File file) {
public BomUploadEvent(final Project project, final FileMetadata fileMetadata) {
this.project = project;
this.file = file;
this.fileMetadata = fileMetadata;
}

public Project getProject() {
return project;
}

public File getFile() {
return file;
public FileMetadata getFileMetadata() {
return fileMetadata;
}
}
74 changes: 33 additions & 41 deletions src/main/java/org/dependencytrack/resources/v1/BomResource.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,6 @@
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.security.SecurityRequirements;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.json.Json;
import jakarta.json.JsonArray;
import jakarta.json.JsonReader;
import jakarta.json.JsonString;
import jakarta.validation.Validator;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BOMInputStream;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -72,26 +55,44 @@
import org.dependencytrack.parser.cyclonedx.CycloneDxValidator;
import org.dependencytrack.parser.cyclonedx.InvalidBomException;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.plugin.PluginManager;
import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;
import org.dependencytrack.resources.v1.problems.InvalidBomProblemDetails;
import org.dependencytrack.resources.v1.problems.ProblemDetails;
import org.dependencytrack.resources.v1.vo.BomSubmitRequest;
import org.dependencytrack.resources.v1.vo.BomUploadResponse;
import org.dependencytrack.storage.FileStorage;
import org.glassfish.jersey.media.multipart.BodyPartEntity;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataParam;

import jakarta.json.Json;
import jakarta.json.JsonArray;
import jakarta.json.JsonReader;
import jakarta.json.JsonString;
import jakarta.validation.Validator;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.DefaultValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.security.Principal;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Set;
import java.util.UUID;

import static java.util.function.Predicate.not;
import static org.dependencytrack.model.ConfigPropertyConstants.BOM_VALIDATION_MODE;
Expand Down Expand Up @@ -330,7 +331,7 @@ public Response uploadBom(@Parameter(required = true) BomSubmitRequest request)
final String trimmedProjectName = StringUtils.trimToNull(request.getProjectName());
if (request.isLatestProjectVersion()) {
final Project oldLatest = qm.getLatestProjectVersion(trimmedProjectName);
if(oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
if (oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
return Response.status(Response.Status.FORBIDDEN)
.entity("Cannot create latest version for project with this name. Access to current latest " +
"version is forbidden!")
Expand Down Expand Up @@ -436,7 +437,7 @@ public Response uploadBom(
}
if (isLatest) {
final Project oldLatest = qm.getLatestProjectVersion(trimmedProjectName);
if(oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
if (oldLatest != null && !qm.hasAccess(super.getPrincipal(), oldLatest)) {
return Response.status(Response.Status.FORBIDDEN)
.entity("Cannot create latest version for project with this name. Access to current latest " +
"version is forbidden!")
Expand Down Expand Up @@ -467,17 +468,17 @@ private Response process(QueryManager qm, Project project, String encodedBomData
return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build();
}

final File bomFile;
final FileMetadata bomFileMetadata;
try (final var encodedInputStream = new ByteArrayInputStream(encodedBomData.getBytes(StandardCharsets.UTF_8));
final var decodedInputStream = Base64.getDecoder().wrap(encodedInputStream);
final var byteOrderMarkInputStream = new BOMInputStream(decodedInputStream)) {
bomFile = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
bomFileMetadata = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
} catch (IOException e) {
LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}

final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFile);
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFileMetadata);
qm.createWorkflowSteps(bomUploadEvent.getChainIdentifier());
Event.dispatch(bomUploadEvent);

Expand All @@ -500,18 +501,18 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build();
}

final File bomFile;
final FileMetadata bomFileMetadata;
try (final var inputStream = bodyPartEntity.getInputStream();
final var byteOrderMarkInputStream = new BOMInputStream(inputStream)) {
bomFile = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
bomFileMetadata = validateAndStoreBom(IOUtils.toByteArray(byteOrderMarkInputStream), project);
} catch (IOException e) {
LOGGER.error("An unexpected error occurred while validating or storing a BOM uploaded to project: " + project.getUuid(), e);
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}

// todo: make option to combine all the bom data so components are reconciled in a single pass.
// todo: https://github.com/DependencyTrack/dependency-track/issues/130
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFile);
final BomUploadEvent bomUploadEvent = new BomUploadEvent(qm.detach(Project.class, project.getId()), bomFileMetadata);

qm.createWorkflowSteps(bomUploadEvent.getChainIdentifier());
Event.dispatch(bomUploadEvent);
Expand All @@ -526,21 +527,12 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
return Response.ok().build();
}

private File validateAndStoreBom(final byte[] bomBytes, final Project project) throws IOException {
private FileMetadata validateAndStoreBom(final byte[] bomBytes, final Project project) throws IOException {
validate(bomBytes, project);

// TODO: Store externally so other instances of the API server can pick it up.
// https://github.com/CycloneDX/cyclonedx-bom-repo-server
final java.nio.file.Path tmpPath = Files.createTempFile("dtrack-bom-%s".formatted(project.getUuid()), null);
final File tmpFile = tmpPath.toFile();
tmpFile.deleteOnExit();

LOGGER.debug("Writing BOM for project %s to %s".formatted(project.getUuid(), tmpPath));
try (final var tmpOutputStream = Files.newOutputStream(tmpPath, StandardOpenOption.WRITE)) {
tmpOutputStream.write(bomBytes);
try (final var fileStorage = PluginManager.getInstance().getExtension(FileStorage.class)) {
return fileStorage.store("bom-upload/%s/%s".formatted(project.getUuid(), UUID.randomUUID()), bomBytes);
}

return tmpFile;
}

static void validate(final byte[] bomBytes, final Project project) {
Expand Down Expand Up @@ -634,7 +626,7 @@ private static boolean shouldValidate(final Project project) {
.map(org.dependencytrack.model.Tag::getName)
.anyMatch(validationModeTags::contains);
return (validationMode == BomValidationMode.ENABLED_FOR_TAGS && doTagsMatch)
|| (validationMode == BomValidationMode.DISABLED_FOR_TAGS && !doTagsMatch);
|| (validationMode == BomValidationMode.DISABLED_FOR_TAGS && !doTagsMatch);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) OWASP Foundation. All Rights Reserved.
*/
package org.dependencytrack.storage;

import org.dependencytrack.plugin.api.ExtensionFactory;
import org.dependencytrack.plugin.api.ExtensionPoint;
import org.dependencytrack.plugin.api.Plugin;

import java.util.Collection;
import java.util.List;

/**
* @since 5.6.0
*/
public final class DefaultFileStoragePlugin implements Plugin {

@Override
public Collection<? extends ExtensionFactory<? extends ExtensionPoint>> extensionFactories() {
return List.of(
new LocalFileStorageFactory(),
new MemoryFileStorageFactory(),
new S3FileStorageFactory());
}

}
110 changes: 110 additions & 0 deletions src/main/java/org/dependencytrack/storage/FileStorage.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
/*
* This file is part of Dependency-Track.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* Copyright (c) OWASP Foundation. All Rights Reserved.
*/
package org.dependencytrack.storage;

import org.dependencytrack.plugin.api.ExtensionPoint;
import org.dependencytrack.proto.storage.v1alpha1.FileMetadata;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.regex.Pattern;

import static java.util.Objects.requireNonNull;

/**
* @since 5.6.0
*/
public interface FileStorage extends ExtensionPoint {

String METADATA_KEY_SHA256_DIGEST = "sha256_digest";
Pattern VALID_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_/\\-.]+");

/**
* Persist data to a file in storage.
* <p>
* Storage providers may transparently perform additional steps,
* such as encryption and compression.
*
* @param fileName Name of the file. This fileName is not guaranteed to be reflected
* in storage as-is. It may be modified or changed entirely.
* @param content Data to store.
* @return Metadata of the stored file.
* @throws IOException When storing the file failed.
*/
FileMetadata store(final String fileName, final byte[] content) throws IOException;

/**
* Retrieves a file from storage.
* <p>
* Storage providers may transparently perform additional steps,
* such as integrity verification, decryption and decompression.
* <p>
* Trying to retrieve a file from a different storage provider
* is an illegal operation and yields an exception.
*
* @param fileMetadata Metadata of the file to retrieve.
* @return The file's content.
* @throws IOException When retrieving the file failed.
* @throws FileNotFoundException When the requested file was not found.
*/
byte[] get(final FileMetadata fileMetadata) throws IOException;

/**
* Deletes a file from storage.
* <p>
* Trying to delete a file from a different storage provider
* is an illegal operation and yields an exception.
*
* @param fileMetadata Metadata of the file to delete.
* @return {@code true} when the file was deleted, otherwise {@code false}.
* @throws IOException When deleting the file failed.
*/
boolean delete(final FileMetadata fileMetadata) throws IOException;

// TODO: deleteMany. Some remote storage backends support batch deletes.
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html

static void requireValidFileName(final String fileName) {
requireNonNull(fileName, "fileName must not be null");

if (!VALID_NAME_PATTERN.matcher(fileName).matches()) {
throw new IllegalArgumentException("fileName must match pattern: " + VALID_NAME_PATTERN.pattern());
}
}

class ExtensionPointMetadata implements org.dependencytrack.plugin.api.ExtensionPointMetadata<FileStorage> {

@Override
public String name() {
return "file.storage";
}

@Override
public boolean required() {
return true;
}

@Override
public Class<FileStorage> extensionPointClass() {
return FileStorage.class;
}

}

}
Loading

0 comments on commit c0b0efc

Please sign in to comment.