From a05dfa778f48791d1d8f957e6a6870dd4abfd40e Mon Sep 17 00:00:00 2001 From: Chris K Wensel Date: Tue, 5 Dec 2023 12:17:47 -0800 Subject: [PATCH] cloudwatch activity support this begins to introduce *Ref dependencies for buckets allowing a s3:///prefix url to ref a bucket component and include the bucket name this gets around breaking json syntax for substitutions, and allows the cloudwatch activity to introduce new resource permissions directly on the Bucket object --- ...terless.java-common-conventions.gradle.kts | 3 +- .../main/java/clusterless/cls/util/URIs.java | 50 +++++++++-- .../java/clusterless/cls/util/URIsTest.java | 11 ++- .../cls/managed/component/Component.java | 1 + .../cls/model/deploy/Extensible.java | 7 ++ .../cls/substrate/aws/sdk/CloudWatchLogs.java | 2 +- .../aws/construct/ActivityConstruct.java | 6 +- .../substrate/aws/construct/ArcConstruct.java | 4 +- .../construct/EgressBoundaryConstruct.java | 4 +- ...onstruct.java => ExtensibleConstruct.java} | 31 ++++--- .../construct/IngressBoundaryConstruct.java | 8 +- .../aws/construct/ResourceConstruct.java | 7 +- .../cls/substrate/aws/managed/ManagedApp.java | 8 +- .../aws/managed/ManagedComponentContext.java | 2 +- .../aws/managed/ManagedConstruct.java | 14 ++-- .../cloudwatch/CloudWatchExportActivity.java | 13 +-- .../CloudWatchExportActivityConstruct.java | 82 ++++++++++++------- .../CloudWatchExportActivityProvider.java | 23 +++++- .../aws/arc/batch/BatchExecArcConstruct.java | 2 +- ...requentS3PutStrategyBoundaryConstruct.java | 6 +- ...requentS3PutStrategyBoundaryConstruct.java | 6 +- .../batch/ComputeResourceConstruct.java | 5 +- .../EventBridgeResourceConstruct.java | 10 +-- .../GlueDatabaseResourceConstruct.java | 5 +- .../table/GlueTableResourceConstruct.java | 7 +- .../s3/S3BucketResourceConstruct.java | 5 +- .../aws/cdk/bootstrap/BootstrapStack.java | 20 ++--- .../aws/cdk/lifecycle/Lifecycle.java | 26 +++--- .../CloudWatchExportActivityProps.java | 16 ++-- .../CloudWatchExportActivityHandlerTest.java | 4 +- .../CloudWatchExportActivityHandler.java | 6 +- 31 files changed, 255 insertions(+), 139 deletions(-) rename clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/{ModelConstruct.java => ExtensibleConstruct.java} (71%) diff --git a/build-logic/src/main/kotlin/clusterless.java-common-conventions.gradle.kts b/build-logic/src/main/kotlin/clusterless.java-common-conventions.gradle.kts index 631ad1e4..573e5485 100644 --- a/build-logic/src/main/kotlin/clusterless.java-common-conventions.gradle.kts +++ b/build-logic/src/main/kotlin/clusterless.java-common-conventions.gradle.kts @@ -45,6 +45,7 @@ else repositories { mavenCentral() +// mavenLocal() // maven { // url = uri("https://maven.pkg.github.com/heretical/*") // @@ -98,7 +99,7 @@ dependencies { constraints { // manage dependency versions here - val commons = "0.9" + val commons = "0.10" implementationAndTestFixture("io.clusterless:clusterless-commons-core:$commons") implementationAndTestFixture("io.clusterless:clusterless-commons-aws:$commons") diff --git a/clusterless-common/src/main/java/clusterless/cls/util/URIs.java b/clusterless-common/src/main/java/clusterless/cls/util/URIs.java index 8403b695..1c73f5b4 100644 --- a/clusterless-common/src/main/java/clusterless/cls/util/URIs.java +++ b/clusterless-common/src/main/java/clusterless/cls/util/URIs.java @@ -136,7 +136,7 @@ public static String asKeyPath(URI uri) { * Returns the path part of the uri without the leading slash, for use in S3 request. * * @param uri - * @return + * @return null if the path is empty */ public static String asKey(URI uri) { if (uri == null) { @@ -145,19 +145,57 @@ public static String asKey(URI uri) { String normalize = uri.normalize().getPath(); - if (normalize.isEmpty()) { + return asKey(normalize); + } + + /** + * Returns the path part of the uri without the leading slash, for use in S3 request. + * + * @param path + * @return null if the path is empty + */ + public static String asKey(String path) { + if (path == null || path.isEmpty()) { return null; } - if (normalize.charAt(0) == '/') { - if (normalize.length() == 1) { + if (path.charAt(0) == '/') { + if (path.length() == 1) { return null; } - return normalize.substring(1); + return path.substring(1); + } + + return path; + } + + public static String asKeyPrefix(URI path) { + if (path == null) { + return null; } - return normalize; + return asKeyPrefix(path.getPath()); + } + + /** + * Removes the first slash and trailing slash, if any. + * + * @param path the path to remove the first and trailing slash from + * @return the path without the first and trailing slash + */ + public static String asKeyPrefix(String path) { + path = asKey(path); + + if (path == null || path.isEmpty()) { + return null; + } + + if (path.charAt(path.length() - 1) == '/') { + return path.substring(0, path.length() - 1); + } + + return path; } public static URI fromTo(URI fromBase, URI from, URI toBase) { diff --git a/clusterless-common/src/test/java/clusterless/cls/util/URIsTest.java b/clusterless-common/src/test/java/clusterless/cls/util/URIsTest.java index aeed5fa8..1cd041a8 100644 --- a/clusterless-common/src/test/java/clusterless/cls/util/URIsTest.java +++ b/clusterless-common/src/test/java/clusterless/cls/util/URIsTest.java @@ -79,7 +79,6 @@ void asKey() { Assertions.assertEquals("foo/", URIs.asKey(URI.create("s3://bucket/foo//"))); Assertions.assertEquals("foo/", URIs.asKey(URI.create("s3://bucket//foo//"))); Assertions.assertNull(URIs.asKey(URI.create("/"))); - Assertions.assertNull(URIs.asKey(URI.create("/"))); Assertions.assertNull(URIs.asKey(URI.create("s3://bucket"))); Assertions.assertNull(URIs.asKey(URI.create("s3://bucket/"))); } @@ -93,11 +92,19 @@ void asPath() { Assertions.assertEquals("foo/", URIs.asKeyPath(URI.create("s3://bucket/foo//"))); Assertions.assertEquals("foo/", URIs.asKeyPath(URI.create("s3://bucket//foo//"))); Assertions.assertNull(URIs.asKeyPath(URI.create("/"))); - Assertions.assertNull(URIs.asKeyPath(URI.create("/"))); Assertions.assertNull(URIs.asKeyPath(URI.create("s3://bucket"))); Assertions.assertNull(URIs.asKeyPath(URI.create("s3://bucket/"))); } + @Test + void asKeyPrefix() { + Assertions.assertEquals("foo", URIs.asKeyPrefix(URI.create("s3://bucket/foo"))); + Assertions.assertEquals("foo", URIs.asKeyPrefix(URI.create("s3://bucket/foo/"))); + Assertions.assertEquals("foo", URIs.asKeyPrefix("/foo")); + Assertions.assertEquals("foo", URIs.asKeyPrefix("/foo/")); + Assertions.assertNull(URIs.asKeyPrefix("/")); + } + public static Stream copyAppend() { return Stream.of( Arguments.arguments("s3://from/1", "s3://from/", new String[]{"1"}), diff --git a/clusterless-model/src/main/java/clusterless/cls/managed/component/Component.java b/clusterless-model/src/main/java/clusterless/cls/managed/component/Component.java index babdf336..2adb274a 100644 --- a/clusterless-model/src/main/java/clusterless/cls/managed/component/Component.java +++ b/clusterless-model/src/main/java/clusterless/cls/managed/component/Component.java @@ -12,4 +12,5 @@ * */ public interface Component { + String name(); } diff --git a/clusterless-model/src/main/java/clusterless/cls/model/deploy/Extensible.java b/clusterless-model/src/main/java/clusterless/cls/model/deploy/Extensible.java index f58b7ee6..3b9f21b8 100644 --- a/clusterless-model/src/main/java/clusterless/cls/model/deploy/Extensible.java +++ b/clusterless-model/src/main/java/clusterless/cls/model/deploy/Extensible.java @@ -9,6 +9,7 @@ package clusterless.cls.model.deploy; import clusterless.cls.json.ExtensibleResolver; +import clusterless.cls.json.JsonRequiredProperty; import clusterless.cls.json.Views; import clusterless.cls.model.Model; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -30,6 +31,8 @@ public abstract class Extensible extends Model { // can't use @JsonRequiredProperty here @JsonView(Views.Required.class) String type; + @JsonRequiredProperty + String name; boolean exclude = false; public Extensible() { @@ -60,6 +63,10 @@ public String type() { return type; } + public String name() { + return name; + } + /** * @return true if this object should be excluded when creating constructs */ diff --git a/clusterless-substrate-aws-common/src/main/java/clusterless/cls/substrate/aws/sdk/CloudWatchLogs.java b/clusterless-substrate-aws-common/src/main/java/clusterless/cls/substrate/aws/sdk/CloudWatchLogs.java index be58d0f0..0853d7c4 100644 --- a/clusterless-substrate-aws-common/src/main/java/clusterless/cls/substrate/aws/sdk/CloudWatchLogs.java +++ b/clusterless-substrate-aws-common/src/main/java/clusterless/cls/substrate/aws/sdk/CloudWatchLogs.java @@ -54,7 +54,7 @@ public Response createExportLogGroupTask(String taskName, String logGroupName, S .from(from.toEpochMilli()) .to(to.toEpochMilli()) .destination(destination.getHost()) - .destinationPrefix(URIs.asKey(destination)) + .destinationPrefix(URIs.asKeyPrefix(destination)) .logStreamNamePrefix(logStreamPrefix) .build(); diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ActivityConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ActivityConstruct.java index 4f849da2..feadbbb9 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ActivityConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ActivityConstruct.java @@ -16,8 +16,8 @@ /** * */ -public class ActivityConstruct extends ModelConstruct implements ActivityComponent { - public ActivityConstruct(@NotNull ManagedComponentContext context, @NotNull M model, @NotNull String id) { - super(context, model, id); +public class ActivityConstruct extends ExtensibleConstruct implements ActivityComponent { + public ActivityConstruct(@NotNull ManagedComponentContext context, @NotNull M model) { + super(context, model); } } diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ArcConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ArcConstruct.java index a63b91f4..4f57294d 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ArcConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ArcConstruct.java @@ -30,12 +30,12 @@ /** * */ -public abstract class ArcConstruct> extends ModelConstruct implements ArcComponent { +public abstract class ArcConstruct> extends ExtensibleConstruct implements ArcComponent { private final Lazy manifestBucket = Lazy.of(() -> BootstrapStores.manifestBucket(this)); public ArcConstruct(@NotNull ManagedComponentContext context, @NotNull M model) { - super(context, model, model.name()); + super(context, model); } protected void grantManifestAndDatasetPermissionsTo(IGrantable grantable) { diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/EgressBoundaryConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/EgressBoundaryConstruct.java index 25db9bb3..82c6c285 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/EgressBoundaryConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/EgressBoundaryConstruct.java @@ -16,8 +16,8 @@ /** * */ -public class EgressBoundaryConstruct extends ModelConstruct implements BoundaryComponent { +public abstract class EgressBoundaryConstruct extends ExtensibleConstruct implements BoundaryComponent { public EgressBoundaryConstruct(@NotNull ManagedComponentContext context, @NotNull M model) { - super(context, model, model.name()); + super(context, model); } } diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ModelConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ExtensibleConstruct.java similarity index 71% rename from clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ModelConstruct.java rename to clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ExtensibleConstruct.java index 577f9ef4..496eee86 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ModelConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ExtensibleConstruct.java @@ -8,7 +8,6 @@ package clusterless.cls.substrate.aws.construct; -import clusterless.cls.model.Model; import clusterless.cls.model.deploy.Extensible; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.managed.ManagedConstruct; @@ -31,28 +30,38 @@ /** * */ -public class ModelConstruct extends ManagedConstruct { - private static final Logger LOG = LogManager.getLogger(ModelConstruct.class); +public class ExtensibleConstruct extends ManagedConstruct { + private static final Logger LOG = LogManager.getLogger(ExtensibleConstruct.class); private final Map buckets = new HashMap<>(); // cache the construct to prevent collisions private final Map tables = new HashMap<>(); // cache the construct to prevent collisions - private final M model; + private final E model; - public ModelConstruct(@NotNull ManagedComponentContext context, @NotNull M model, @NotNull String id) { - super(context, uniqueId(model, id)); + public ExtensibleConstruct(@NotNull ManagedComponentContext context, @NotNull E model) { + super(context, uniqueId(model, Label.of(model.name()))); this.model = model; } - private static Label uniqueId(@NotNull Model model, @NotNull String id) { + public ExtensibleConstruct(@NotNull ManagedComponentContext context, @NotNull E model, @NotNull Label discriminator) { + super(context, uniqueId(model, discriminator)); + this.model = model; + } + + private static Label uniqueId(@NotNull Extensible model, @NotNull Label discriminator) { return model .label() - .with(id); + .with(discriminator); } - public M model() { + @NotNull + public E model() { return model; } + public String name() { + return model().name(); + } + protected String id(String value) { return model() .label() @@ -61,8 +70,8 @@ protected String id(String value) { } protected R constructWithinHandler(Supplier supplier) { - if (model() instanceof Extensible) { - return ErrorsUtil.construct(((Extensible) model()).type(), supplier, LOG); + if (model() != null) { + return ErrorsUtil.construct(model().type(), supplier, LOG); } return ErrorsUtil.construct(null, supplier, LOG); diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/IngressBoundaryConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/IngressBoundaryConstruct.java index 54601d3f..1676e64a 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/IngressBoundaryConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/IngressBoundaryConstruct.java @@ -16,12 +16,8 @@ /** * */ -public class IngressBoundaryConstruct extends ModelConstruct implements BoundaryComponent { +public abstract class IngressBoundaryConstruct extends ExtensibleConstruct implements BoundaryComponent { public IngressBoundaryConstruct(@NotNull ManagedComponentContext context, @NotNull M model) { - super(context, model, model.name()); - } - - public IngressBoundaryConstruct(@NotNull ManagedComponentContext context, @NotNull M model, @NotNull String id) { - super(context, model, id); + super(context, model); } } diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ResourceConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ResourceConstruct.java index 7e705199..a7776370 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ResourceConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/construct/ResourceConstruct.java @@ -11,13 +11,14 @@ import clusterless.cls.managed.component.ResourceComponent; import clusterless.cls.model.deploy.Resource; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; +import clusterless.commons.naming.Label; import org.jetbrains.annotations.NotNull; /** * */ -public class ResourceConstruct extends ModelConstruct implements ResourceComponent { - public ResourceConstruct(@NotNull ManagedComponentContext context, @NotNull M model, @NotNull String id) { - super(context, model, id); +public class ResourceConstruct extends ExtensibleConstruct implements ResourceComponent { + public ResourceConstruct(@NotNull ManagedComponentContext context, @NotNull M model, @NotNull Label discriminator) { + super(context, model, discriminator); } } diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedApp.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedApp.java index 04a10581..534300ed 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedApp.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedApp.java @@ -15,6 +15,8 @@ import clusterless.commons.naming.Stage; import clusterless.commons.naming.Version; import clusterless.commons.substrate.aws.cdk.scoped.ScopedApp; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; import software.amazon.awscdk.AppProps; import software.amazon.awscdk.TagProps; import software.constructs.Construct; @@ -26,6 +28,7 @@ * */ public class ManagedApp extends ScopedApp implements Managed { + private final Multimap, Construct> constructs = LinkedListMultimap.create(); private final List deployableModel; private final List stacks = new LinkedList<>(); @@ -51,11 +54,14 @@ public ManagedApp(String name, String version, String stage, List de applyTags(); } + public Multimap, Construct> constructs() { + return constructs; + } + public List projectModels() { return deployableModel; } - public List stacks() { return stacks; } diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedComponentContext.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedComponentContext.java index fa13b00d..8b10f21a 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedComponentContext.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedComponentContext.java @@ -45,7 +45,7 @@ public DatasetResolver resolver() { return resolver; } - public ManagedApp managedProject() { + public ManagedApp managedApp() { return managedApp; } diff --git a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedConstruct.java b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedConstruct.java index 303280ad..2fd4cde0 100644 --- a/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedConstruct.java +++ b/clusterless-substrate-aws-construct-common/src/main/java/clusterless/cls/substrate/aws/managed/ManagedConstruct.java @@ -46,32 +46,30 @@ protected Placement placement() { .build(); } - protected void addIdRefFor(Resource resource, Construct construct, String value, String description) { + protected void exportIdRefFor(Resource resource, Construct construct, String value, String description) { Ref ref = Ref.ref() .withResourceNs(resource.resourceNs()) .withResourceType(resource.resourceType()) .withResourceName(resource.name()); - addIdRefFor(ref, construct, value, description); + exportIdRefFor(ref, construct, value, description); } - protected void addArnRefFor(Resource resource, Construct construct, String value, String description) { + protected void exportArnRefFor(Resource resource, Construct construct, String value, String description) { Ref ref = Ref.ref() .withResourceNs(resource.resourceNs()) .withResourceType(resource.resourceType()) .withResourceName(resource.name()); - addArnRefFor(ref, construct, value, description); + exportArnRefFor(ref, construct, value, description); } - protected void addNameRefFor(Resource resource, Construct construct, String value, String description) { + protected void exportNameRefFor(Resource resource, Construct construct, String value, String description) { Ref ref = Ref.ref() .withResourceNs(resource.resourceNs()) .withResourceType(resource.resourceType()) .withResourceName(resource.name()); - addNameRefFor(ref, construct, value, description); + exportNameRefFor(ref, construct, value, description); } - - } diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivity.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivity.java index e4dc5c0f..e285970b 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivity.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivity.java @@ -21,11 +21,10 @@ public class CloudWatchExportActivity extends Activity { @JsonRequiredProperty private String logGroupName; - private String logStreamPrefix; - + private String bucketRef; @JsonRequiredProperty - private URI destinationURI; + private URI pathURI; LambdaJavaRuntimeProps runtimeProps = new LambdaJavaRuntimeProps( Memory.MEM_1_024MB, @@ -44,8 +43,12 @@ public String logStreamPrefix() { return logStreamPrefix; } - public URI destinationURI() { - return destinationURI; + public String bucketRef() { + return bucketRef; + } + + public URI pathURI() { + return pathURI; } public LambdaJavaRuntimeProps runtimeProps() { diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityConstruct.java index 589c07dc..847f38cb 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityConstruct.java @@ -12,10 +12,12 @@ import clusterless.cls.substrate.aws.construct.ActivityConstruct; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.props.Lookup; +import clusterless.cls.substrate.aws.resource.s3.S3BucketResourceConstruct; import clusterless.cls.substrate.aws.resources.Assets; import clusterless.cls.substrate.aws.resources.Functions; import clusterless.cls.substrate.aws.resources.Rules; import clusterless.cls.util.Env; +import clusterless.cls.util.URIs; import clusterless.commons.collection.OrderedSafeMaps; import clusterless.commons.naming.Label; import clusterless.commons.substrate.aws.cdk.construct.LambdaLogGroupConstruct; @@ -35,6 +37,7 @@ import software.amazon.awscdk.services.s3.Bucket; import software.amazon.awscdk.services.s3.IBucket; +import java.net.URI; import java.time.temporal.TemporalUnit; import java.util.List; import java.util.Map; @@ -47,7 +50,52 @@ public class CloudWatchExportActivityConstruct extends ActivityConstruct environment = Env.toEnv(activityProps); - String functionName = Functions.functionName(this, model().name(), "Int"); + String functionName = Functions.functionName(this, model().name(), "Act"); Label functionLabel = Label.of(model().name()).with("Int"); Function function = Function.Builder.create(this, functionLabel.camelCase()) .functionName(functionName) @@ -87,34 +135,6 @@ public CloudWatchExportActivityConstruct(@NotNull ManagedComponentContext contex declaredLogGroup.grant(function, "logs:CreateExportTask"); - IBucket destinationBucket = Bucket.fromBucketName(this, "DestinationBucket", model.destinationURI().getHost()); - - String region = context.deployable().placement().region(); - ServicePrincipal principal = new ServicePrincipal(ServicePrincipal.servicePrincipalName("logs.amazonaws.com")); - - Grant grant = destinationBucket - .grantWrite(principal); - - grant.assertSuccess(); - - // https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/S3ExportTasks.html - String account = context.deployable().placement().account(); - AddToResourcePolicyResult policyResult = destinationBucket.addToResourcePolicy(PolicyStatement.Builder.create() - .principals(List.of(principal)) - .actions(List.of("s3:GetBucketAcl")) - .resources(List.of(destinationBucket.getBucketArn())) - .conditions(OrderedSafeMaps.of( - "StringEquals", Map.of("aws:SourceAccount", List.of(account)), - "ArnLike", Map.of("aws:SourceArn", List.of("arn:aws:logs:%s:%s:%s:*".formatted(region, account, model.logGroupName()))) - ) - ) - .effect(Effect.ALLOW) - .build()); - - if (policyResult.getStatementAdded()) { - throw new IllegalStateException("failed to add policy statement to bucket: " + destinationBucket.getBucketName()); - } - // https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-cron-expressions.html if (temporalUnit.getDuration().toMinutes() > 60) { throw new UnsupportedOperationException("interval greater than 60 minutes: " + model().interval()); diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityProvider.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityProvider.java index 90b68a41..13f90577 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityProvider.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/activity/cloudwatch/CloudWatchExportActivityProvider.java @@ -17,9 +17,28 @@ */ @ProvidesComponent( type = "aws:core:cloudWatchExport", - synopsis = "Export CloudWatch logs.", + synopsis = "Export CloudWatch logs to S3.", description = """ - Periodically will extract a CloudWatch log group to an S3 bucket. + Periodically will extract a CloudWatch log group to an S3 bucket, under a prefix. + + interval: Fourths|Sixth|Twelfths|etc + (Future versions will support rates and cron expressions.) + + logGroupName: string + The name of the log group to export. e.g '/aws/lambda/my-lambda' + + logStreamPrefix: string (optional) + The prefix of the log streams to export. + + bucketRef: string (optional, see below) + The reference to the bucket created in this project to export to, created by `aws:core:s3Bucket`. + + pathURI: an s3 URI + The destination URI to export to. + Use `s3:///prefix` or `/prefix` so that he bucketRef value is used for the bucket name. + + If a bucket name is given in the URI, the bucket must allow 'logs.amazonaws.com' permission + for 's3:GetBucketAcl'. """ ) public class CloudWatchExportActivityProvider implements ActivityComponentService { diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/arc/batch/BatchExecArcConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/arc/batch/BatchExecArcConstruct.java index f8e77f06..da6f5141 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/arc/batch/BatchExecArcConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/arc/batch/BatchExecArcConstruct.java @@ -138,7 +138,7 @@ protected IManagedComputeEnvironment resolveComputeEnvironment(String computeEnv LOG.info("resolving computeEnvironment ref: {}", computeEnvironmentRef); - return resolveArnRef(computeEnvironmentRef, arn -> { + return importArnRef(computeEnvironmentRef, arn -> { LOG.info("using computeEnvironment arn: {}", arn); return FargateComputeEnvironment.fromFargateComputeEnvironmentArn( this, diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/FrequentS3PutStrategyBoundaryConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/FrequentS3PutStrategyBoundaryConstruct.java index f2a9ba72..955e3172 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/FrequentS3PutStrategyBoundaryConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/FrequentS3PutStrategyBoundaryConstruct.java @@ -11,7 +11,7 @@ import clusterless.aws.lambda.boundary.frequents3put.FrequentS3PutBoundaryProps; import clusterless.cls.model.deploy.SinkDataset; import clusterless.cls.model.manifest.ManifestState; -import clusterless.cls.substrate.aws.construct.ModelConstruct; +import clusterless.cls.substrate.aws.construct.ExtensibleConstruct; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.props.Lookup; import clusterless.cls.substrate.aws.resource.s3.S3BucketResourceConstruct; @@ -47,11 +47,11 @@ /** * */ -public class FrequentS3PutStrategyBoundaryConstruct extends ModelConstruct { +public class FrequentS3PutStrategyBoundaryConstruct extends ExtensibleConstruct { private static final Logger LOG = LogManager.getLogger(FrequentS3PutStrategyBoundaryConstruct.class); public FrequentS3PutStrategyBoundaryConstruct(@NotNull ManagedComponentContext context, @NotNull S3PutListenerBoundary model) { - super(context, model, Label.of("Frequent").with(model.name()).camelCase()); + super(context, model, Label.of("Frequent").with(model.name())); // confirm unit exits TemporalUnit temporalUnit = IntervalUnits.find(model().lotUnit()); diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/InfrequentS3PutStrategyBoundaryConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/InfrequentS3PutStrategyBoundaryConstruct.java index f3ad4284..d7a7f018 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/InfrequentS3PutStrategyBoundaryConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/boundary/s3put/InfrequentS3PutStrategyBoundaryConstruct.java @@ -11,7 +11,7 @@ import clusterless.aws.lambda.boundary.s3put.S3PutBoundaryProps; import clusterless.cls.model.deploy.SinkDataset; import clusterless.cls.model.manifest.ManifestState; -import clusterless.cls.substrate.aws.construct.ModelConstruct; +import clusterless.cls.substrate.aws.construct.ExtensibleConstruct; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.props.Lookup; import clusterless.cls.substrate.aws.resource.s3.S3BucketResourceConstruct; @@ -47,11 +47,11 @@ /** * */ -public class InfrequentS3PutStrategyBoundaryConstruct extends ModelConstruct { +public class InfrequentS3PutStrategyBoundaryConstruct extends ExtensibleConstruct { private static final Logger LOG = LogManager.getLogger(InfrequentS3PutStrategyBoundaryConstruct.class); public InfrequentS3PutStrategyBoundaryConstruct(@NotNull ManagedComponentContext context, @NotNull S3PutListenerBoundary model) { - super(context, model, Label.of("Infrequent").with(model.name()).camelCase()); + super(context, model, Label.of("Infrequent").with(model.name())); // confirm unit exits TemporalUnit temporalUnit = IntervalUnits.find(model().lotUnit()); diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/batch/ComputeResourceConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/batch/ComputeResourceConstruct.java index aa4d7d25..6ac0877e 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/batch/ComputeResourceConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/batch/ComputeResourceConstruct.java @@ -12,6 +12,7 @@ import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.resources.Vpcs; import clusterless.cls.substrate.aws.util.TagsUtil; +import clusterless.commons.naming.Label; import clusterless.commons.substrate.aws.cdk.naming.ResourceNames; import org.jetbrains.annotations.NotNull; import software.amazon.awscdk.services.batch.FargateComputeEnvironment; @@ -25,7 +26,7 @@ public class ComputeResourceConstruct extends ResourceConstruct private final IManagedComputeEnvironment computeEnvironment; public ComputeResourceConstruct(@NotNull ManagedComponentContext context, @NotNull ComputeResource model) { - super(context, model, model.computeEnvironmentName()); + super(context, model, Label.of(model.computeEnvironmentName())); String name = ResourceNames.regionUniqueScopedName(this, model().computeEnvironmentName()); @@ -49,7 +50,7 @@ public ComputeResourceConstruct(@NotNull ManagedComponentContext context, @NotNu String computeEnvironmentArn = computeEnvironment.getComputeEnvironmentArn(); - addArnRefFor(model(), (Construct) computeEnvironment, computeEnvironmentArn, "compute environment arn"); + exportArnRefFor(model(), (Construct) computeEnvironment, computeEnvironmentArn, "compute environment arn"); } public IManagedComputeEnvironment computeEnvironment() { diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/eventbridge/EventBridgeResourceConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/eventbridge/EventBridgeResourceConstruct.java index a1f3bf57..b3d3d59c 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/eventbridge/EventBridgeResourceConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/eventbridge/EventBridgeResourceConstruct.java @@ -9,7 +9,7 @@ package clusterless.cls.substrate.aws.resource.eventbridge; import clusterless.cls.managed.component.ResourceComponent; -import clusterless.cls.substrate.aws.construct.ModelConstruct; +import clusterless.cls.substrate.aws.construct.ExtensibleConstruct; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.commons.naming.Label; import org.jetbrains.annotations.NotNull; @@ -18,19 +18,19 @@ /** * */ -public class EventBridgeResourceConstruct extends ModelConstruct implements ResourceComponent { +public class EventBridgeResourceConstruct extends ExtensibleConstruct implements ResourceComponent { private final EventBus eventBus; public EventBridgeResourceConstruct(@NotNull ManagedComponentContext context, @NotNull EventBridgeResource model) { - super(context, model, model.eventBusName()); + super(context, model, Label.of(model.eventBusName())); eventBus = EventBus.Builder.create(this, Label.of(model.eventBusName()).camelCase()) .eventBusName(model.eventBusName()) .build(); - addArnRefFor(model(), eventBus(), eventBus().getEventBusArn(), "event bus arn"); - addNameRefFor(model(), eventBus(), model().eventBusName(), "event bus name"); + exportArnRefFor(model(), eventBus(), eventBus().getEventBusArn(), "event bus arn"); + exportNameRefFor(model(), eventBus(), model().eventBusName(), "event bus name"); } public EventBus eventBus() { diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/glue/database/GlueDatabaseResourceConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/glue/database/GlueDatabaseResourceConstruct.java index b59731d0..5953e89b 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/glue/database/GlueDatabaseResourceConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/glue/database/GlueDatabaseResourceConstruct.java @@ -12,6 +12,7 @@ import clusterless.cls.substrate.aws.construct.ResourceConstruct; import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.util.TagsUtil; +import clusterless.commons.naming.Label; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -25,7 +26,7 @@ public class GlueDatabaseResourceConstruct extends ResourceConstruct { + IDatabase database = importArnRef(model().databaseRef(), arn -> { LOG.info("using database arn: {}", arn); return Database.fromDatabaseArn(this, "Database", arn); }); @@ -72,7 +73,7 @@ public GlueTableResourceConstruct(@NotNull ManagedComponentContext context, @Not TagsUtil.applyTags(table, model().tags()); - addArnRefFor(model(), table, table.getTableArn(), "glue table arn"); + exportArnRefFor(model(), table, table.getTableArn(), "glue table arn"); } private DataFormat formatFrom(String format) { diff --git a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/s3/S3BucketResourceConstruct.java b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/s3/S3BucketResourceConstruct.java index 11061da7..c5cf936d 100644 --- a/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/s3/S3BucketResourceConstruct.java +++ b/clusterless-substrate-aws-construct-core/src/main/java/clusterless/cls/substrate/aws/resource/s3/S3BucketResourceConstruct.java @@ -13,6 +13,7 @@ import clusterless.cls.substrate.aws.managed.ManagedComponentContext; import clusterless.cls.substrate.aws.resources.Buckets; import clusterless.cls.substrate.aws.util.TagsUtil; +import clusterless.commons.naming.Label; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -31,7 +32,7 @@ public class S3BucketResourceConstruct extends ResourceConstruct> containers) { + private static void construct(ManagedComponentContext context, Map> containers) { Multimap, Construct> map = LinkedListMultimap.create(); containers.entrySet().stream().filter(e -> e.getValue() != null).forEach(e -> { @@ -231,11 +228,18 @@ private static void construct(ComponentContext context, Map map.put(type, (Construct) component)); }); + // add dependencies across constructs within this managed stack for (Construct resource : map.get(ResourceConstruct.class)) { map.get(ArcConstruct.class).forEach(c -> c.getNode().addDependency(resource)); + map.get(ActivityConstruct.class).forEach(c -> c.getNode().addDependency(resource)); map.get(EgressBoundaryConstruct.class).forEach(c -> c.getNode().addDependency(resource)); map.get(IngressBoundaryConstruct.class).forEach(c -> c.getNode().addDependency(resource)); } + + // enable cross stack references + context.managedApp() + .constructs() + .putAll(map); } private static Optional> lookupModel(Component component) { @@ -245,6 +249,9 @@ private static Optional> lookupModel(Component compon if (component instanceof ResourceConstruct) { return Optional.of(ResourceConstruct.class); } + if (component instanceof ActivityConstruct) { + return Optional.of(ActivityConstruct.class); + } if (component instanceof EgressBoundaryConstruct) { return Optional.of(EgressBoundaryConstruct.class); } @@ -291,5 +298,4 @@ private static Set verify(String propertyName, Set values) { return values; } - } diff --git a/clusterless-substrate-aws-lambda-transform-model/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityProps.java b/clusterless-substrate-aws-lambda-transform-model/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityProps.java index f01efd8f..b98f0e53 100644 --- a/clusterless-substrate-aws-lambda-transform-model/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityProps.java +++ b/clusterless-substrate-aws-lambda-transform-model/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityProps.java @@ -20,7 +20,7 @@ public class CloudWatchExportActivityProps implements Struct { String logGroupName; String logStreamPrefix; @JsonRequiredProperty - URI destinationURI; + URI pathURI; public static Builder builder() { return Builder.builder(); @@ -38,15 +38,15 @@ public String logStreamPrefix() { return logStreamPrefix; } - public URI destinationURI() { - return destinationURI; + public URI pathURI() { + return pathURI; } public static final class Builder { String interval; String logGroupName; String logStreamPrefix; - URI destinationURI; + URI pathURI; private Builder() { } @@ -70,17 +70,17 @@ public Builder withLogStreamPrefix(String logStreamPrefix) { return this; } - public Builder withDestinationURI(URI destinationURI) { - this.destinationURI = destinationURI; + public Builder withPathURI(URI pathURI) { + this.pathURI = pathURI; return this; } public CloudWatchExportActivityProps build() { CloudWatchExportActivityProps cloudWatchExportActivityProps = new CloudWatchExportActivityProps(); - cloudWatchExportActivityProps.destinationURI = this.destinationURI; - cloudWatchExportActivityProps.interval = this.interval; + cloudWatchExportActivityProps.pathURI = this.pathURI; cloudWatchExportActivityProps.logGroupName = this.logGroupName; cloudWatchExportActivityProps.logStreamPrefix = this.logStreamPrefix; + cloudWatchExportActivityProps.interval = this.interval; return cloudWatchExportActivityProps; } } diff --git a/clusterless-substrate-aws-lambda-transform/src/integrationTest/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandlerTest.java b/clusterless-substrate-aws-lambda-transform/src/integrationTest/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandlerTest.java index c6a5f299..78a3ca93 100644 --- a/clusterless-substrate-aws-lambda-transform/src/integrationTest/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandlerTest.java +++ b/clusterless-substrate-aws-lambda-transform/src/integrationTest/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandlerTest.java @@ -12,6 +12,7 @@ import clusterless.aws.lambda.TestLots; import clusterless.aws.lambda.transform.json.event.AWSEvent; import clusterless.cls.json.JSONUtil; +import clusterless.cls.util.URIs; import clusterless.commons.temporal.IntervalUnit; import com.adelean.inject.resources.junit.jupiter.GivenJsonResource; import com.adelean.inject.resources.junit.jupiter.TestWithResources; @@ -22,7 +23,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; @@ -40,7 +40,7 @@ public class CloudWatchExportActivityHandlerTest extends LocalStackBase { protected CloudWatchExportActivityProps getProps() { return CloudWatchExportActivityProps.builder() .withLogGroupName("test-log-group") - .withDestinationURI(URI.create("s3://%s/test-prefix/".formatted(bucketName()))) + .withPathURI(URIs.create("s3", bucketName(), "/test-prefix/")) .withInterval(IntervalUnit.TWELFTHS.name()) .build(); } diff --git a/clusterless-substrate-aws-lambda-transform/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandler.java b/clusterless-substrate-aws-lambda-transform/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandler.java index 43f1afab..bd07df72 100644 --- a/clusterless-substrate-aws-lambda-transform/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandler.java +++ b/clusterless-substrate-aws-lambda-transform/src/main/java/clusterless/aws/lambda/activity/cloudwatch/CloudWatchExportActivityHandler.java @@ -83,7 +83,7 @@ public void handleEvent(AWSEvent event, Context context, CloudWatchExportActivit String taskName = "export-" + interval; String logGroupName = activityProps.logGroupName(); String logStreamPrefix = activityProps.logStreamPrefix(); - URI path = activityProps.destinationURI(); + URI destination = activityProps.pathURI(); if (!eventObserver.enableExport()) { LOG.info("export disabled"); @@ -91,11 +91,11 @@ public void handleEvent(AWSEvent event, Context context, CloudWatchExportActivit } getStopwatch.start(); - CloudWatchLogs.Response response = cloudWatchLogs.createExportLogGroupTask(taskName, logGroupName, logStreamPrefix, path, startTimeInclusive, endTimeInclusive); + CloudWatchLogs.Response response = cloudWatchLogs.createExportLogGroupTask(taskName, logGroupName, logStreamPrefix, destination, startTimeInclusive, endTimeInclusive); getStopwatch.stop(); response.isSuccessOrThrowRuntime( - r -> String.format("unable to create export task: %s, %s, %s", logGroupName, path, r.errorMessage()) + r -> String.format("unable to create export task: %s, %s, %s", logGroupName, destination, r.errorMessage()) ); Duration getElapsed = getStopwatch.elapsed();