Implement layout="zip" for Lambda/GCF, skipping lambdex

src/python/pants/backend/awslambda/python/rules.py

@@ -12,8 +12,14 @@
     PythonAwsLambdaIncludeRequirements,
     PythonAwsLambdaRuntime,
 )
-from pants.backend.python.util_rules import pex_from_targets
-from pants.backend.python.util_rules.faas import BuildLambdexRequest, PythonFaaSCompletePlatforms
+from pants.backend.python.util_rules.faas import (
+    BuildLambdexRequest,
+    BuildPythonFaaSRequest,
+    PythonFaaSCompletePlatforms,
+    PythonFaaSLayout,
+    PythonFaaSLayoutField,
+)
+from pants.backend.python.util_rules.faas import rules as faas_rules
 from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet
 from pants.core.util_rules.environments import EnvironmentField
 from pants.engine.rules import Get, collect_rules, rule
@@ -33,27 +39,49 @@ class PythonAwsLambdaFieldSet(PackageFieldSet):
     complete_platforms: PythonFaaSCompletePlatforms
     output_path: OutputPathField
     environment: EnvironmentField
+    layout: PythonFaaSLayoutField
 
 
 @rule(desc="Create Python AWS Lambda", level=LogLevel.DEBUG)
 async def package_python_awslambda(
     field_set: PythonAwsLambdaFieldSet,
 ) -> BuiltPackage:
+    layout = PythonFaaSLayout(field_set.layout.value)
+
+    if layout is PythonFaaSLayout.LAMBDEX:
+        return await Get(
+            BuiltPackage,
+            BuildLambdexRequest(
+                address=field_set.address,
+                target_name=PythonAWSLambda.alias,
+                complete_platforms=field_set.complete_platforms,
+                runtime=field_set.runtime,
+                handler=field_set.handler,
+                output_path=field_set.output_path,
+                include_requirements=field_set.include_requirements.value,
+                script_handler=None,
+                script_module=None,
+                # The AWS-facing handler function is always lambdex_handler.handler, which is the
+                # wrapper injected by lambdex that manages invocation of the actual handler.
+                handler_log_message="lambdex_handler.handler",
+            ),
+        )
+
     return await Get(
         BuiltPackage,
-        BuildLambdexRequest(
+        BuildPythonFaaSRequest(
             address=field_set.address,
             target_name=PythonAWSLambda.alias,
             complete_platforms=field_set.complete_platforms,
             runtime=field_set.runtime,
             handler=field_set.handler,
+            layout=layout,
             output_path=field_set.output_path,
             include_requirements=field_set.include_requirements.value,
-            script_handler=None,
-            script_module=None,
-            # The AWS-facing handler function is always lambdex_handler.handler, which is the
-            # wrapper injected by lambdex that manages invocation of the actual handler.
-            handler_log_message="lambdex_handler.handler",
+            # This doesn't matter (just needs to be fixed), but is the default name used by the AWS
+            # console when creating a Python lambda, so is as good as any
+            # https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
+            reexported_handler_module="lambda_function",
         ),
     )
 
@@ -62,5 +90,5 @@ def rules():
     return [
         *collect_rules(),
         UnionRule(PackageFieldSet, PythonAwsLambdaFieldSet),
-        *pex_from_targets.rules(),
+        *faas_rules(),
     ]

src/python/pants/backend/awslambda/python/rules_test.py

@@ -122,7 +122,7 @@ def complete_platform(rule_runner: PythonRuleRunner) -> bytes:
     "major_minor_interpreter",
     all_major_minor_python_versions(Lambdex.default_interpreter_constraints),
 )
-def test_create_hello_world_lambda(
+def test_create_hello_world_lambda_with_lambdex(
     rule_runner: PythonRuleRunner, major_minor_interpreter: str, complete_platform: str, caplog
 ) -> None:
     rule_runner.write_files(
@@ -197,7 +197,7 @@ def handler(event, context):
     ), "Using include_requirements=False should exclude third-party deps"
 
 
-def test_warn_files_targets(rule_runner: PythonRuleRunner, caplog) -> None:
+def test_warn_files_targets_with_lambdex(rule_runner: PythonRuleRunner, caplog) -> None:
     rule_runner.write_files(
         {
             "assets/f.txt": "",
@@ -257,3 +257,68 @@ def handler(event, context):
     assert "assets/f.txt:files" in caplog.text
     assert "assets:relocated" in caplog.text
     assert "assets:resources" not in caplog.text
+
+
+def test_create_hello_world_lambda(rule_runner: PythonRuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "src/python/foo/bar/hello_world.py": dedent(
+                """
+                import mureq
+
+                def handler(event, context):
+                    print('Hello, World!')
+                """
+            ),
+            "src/python/foo/bar/BUILD": dedent(
+                """
+                python_requirement(name="mureq", requirements=["mureq==0.2"])
+                python_sources()
+
+                python_awslambda(
+                    name='lambda',
+                    handler='foo.bar.hello_world:handler',
+                    runtime="python3.7",
+                    layout='zip',
+                )
+                python_awslambda(
+                    name='slimlambda',
+                    include_requirements=False,
+                    handler='foo.bar.hello_world:handler',
+                    runtime="python3.7",
+                    layout='zip',
+                )
+                """
+            ),
+        }
+    )
+
+    zip_file_relpath, content = create_python_awslambda(
+        rule_runner,
+        Address("src/python/foo/bar", target_name="lambda"),
+        expected_extra_log_lines=("    Handler: lambda_function.handler",),
+    )
+    assert "src.python.foo.bar/lambda.zip" == zip_file_relpath
+
+    zipfile = ZipFile(BytesIO(content))
+    names = set(zipfile.namelist())
+    assert "mureq/__init__.py" in names
+    assert "foo/bar/hello_world.py" in names
+    assert (
+        zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler"
+    )
+
+    zip_file_relpath, content = create_python_awslambda(
+        rule_runner,
+        Address("src/python/foo/bar", target_name="slimlambda"),
+        expected_extra_log_lines=("    Handler: lambda_function.handler",),
+    )
+    assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath
+
+    zipfile = ZipFile(BytesIO(content))
+    names = set(zipfile.namelist())
+    assert "mureq/__init__.py" not in names
+    assert "foo/bar/hello_world.py" in names
+    assert (
+        zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler"
+    )

src/python/pants/backend/awslambda/python/target_types.py

@@ -10,6 +10,7 @@
     PythonFaaSCompletePlatforms,
     PythonFaaSDependencies,
     PythonFaaSHandlerField,
+    PythonFaaSLayoutField,
     PythonFaaSRuntimeField,
 )
 from pants.backend.python.util_rules.faas import rules as faas_rules
@@ -29,11 +30,20 @@
 
 
 class PythonAwsLambdaHandlerField(PythonFaaSHandlerField):
+    # This doesn't matter (just needs to be fixed), but is the default name used by the AWS
+    # console when creating a Python lambda, so is as good as any
+    # https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
+    reexported_handler_module = "lambda_function"
+
     help = help_text(
         f"""
         Entry point to the AWS Lambda handler.
 
         {PythonFaaSHandlerField.help}
+
+        This is re-exported at `{reexported_handler_module}.handler` in the resulting package to be
+        used as the configured handler of the Lambda in AWS. It can also be accessed under its
+        source-root-relative module path, for example: `path.to.module.handler_func`.
         """
     )
 
@@ -111,6 +121,7 @@ class PythonAWSLambda(Target):
         PythonAwsLambdaRuntime,
         PythonFaaSCompletePlatforms,
         PythonResolveField,
+        PythonFaaSLayoutField,
         EnvironmentField,
     )
     help = help_text(

src/python/pants/backend/google_cloud_function/python/rules.py

@@ -12,8 +12,14 @@
     PythonGoogleCloudFunctionRuntime,
     PythonGoogleCloudFunctionType,
 )
-from pants.backend.python.util_rules import pex_from_targets
-from pants.backend.python.util_rules.faas import BuildLambdexRequest, PythonFaaSCompletePlatforms
+from pants.backend.python.util_rules.faas import (
+    BuildLambdexRequest,
+    BuildPythonFaaSRequest,
+    PythonFaaSCompletePlatforms,
+    PythonFaaSLayout,
+    PythonFaaSLayoutField,
+)
+from pants.backend.python.util_rules.faas import rules as faas_rules
 from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet
 from pants.core.util_rules.environments import EnvironmentField
 from pants.engine.rules import Get, collect_rules, rule
@@ -33,33 +39,52 @@ class PythonGoogleCloudFunctionFieldSet(PackageFieldSet):
     type: PythonGoogleCloudFunctionType
     output_path: OutputPathField
     environment: EnvironmentField
+    layout: PythonFaaSLayoutField
 
 
 @rule(desc="Create Python Google Cloud Function", level=LogLevel.DEBUG)
 async def package_python_google_cloud_function(
     field_set: PythonGoogleCloudFunctionFieldSet,
 ) -> BuiltPackage:
+    layout = PythonFaaSLayout(field_set.layout.value)
+    if layout is PythonFaaSLayout.LAMBDEX:
+        return await Get(
+            BuiltPackage,
+            BuildLambdexRequest(
+                address=field_set.address,
+                target_name=PythonGoogleCloudFunction.alias,
+                complete_platforms=field_set.complete_platforms,
+                runtime=field_set.runtime,
+                handler=field_set.handler,
+                output_path=field_set.output_path,
+                include_requirements=True,
+                # The GCP-facing handler function is always `main.handler` (We pass `-M main.py -H handler` to
+                # Lambdex to ensure this), which is the wrapper injected by Lambdex that manages invocation of
+                # the actual user-supplied handler function. This arrangement works well since GCF assumes the
+                # handler function is housed in `main.py` in the root of the zip (you can re-direct this by
+                # setting a `GOOGLE_FUNCTION_SOURCE` Google Cloud build environment variable; e.g.:
+                # `gcloud functions deploy {--build-env-vars-file,--set-build-env-vars}`, but it's non-trivial
+                # to do this right or with intended effect) and the handler name you configure GCF with is just
+                # the unqualified function name, which we log here.
+                script_handler="handler",
+                script_module="main.py",
+                handler_log_message="handler",
+            ),
+        )
+
     return await Get(
         BuiltPackage,
-        BuildLambdexRequest(
+        BuildPythonFaaSRequest(
             address=field_set.address,
             target_name=PythonGoogleCloudFunction.alias,
             complete_platforms=field_set.complete_platforms,
             runtime=field_set.runtime,
             handler=field_set.handler,
+            layout=layout,
             output_path=field_set.output_path,
             include_requirements=True,
-            # The GCP-facing handler function is always `main.handler` (We pass `-M main.py -H handler` to
-            # Lambdex to ensure this), which is the wrapper injected by Lambdex that manages invocation of
-            # the actual user-supplied handler function. This arrangement works well since GCF assumes the
-            # handler function is housed in `main.py` in the root of the zip (you can re-direct this by
-            # setting a `GOOGLE_FUNCTION_SOURCE` Google Cloud build environment variable; e.g.:
-            # `gcloud functions deploy {--build-env-vars-file,--set-build-env-vars}`, but it's non-trivial
-            # to do this right or with intended effect) and the handler name you configure GCF with is just
-            # the unqualified function name, which we log here.
-            script_handler="handler",
-            script_module="main.py",
-            handler_log_message="handler",
+            reexported_handler_module=PythonGoogleCloudFunctionHandlerField.reexported_handler_module,
+            log_only_reexported_handler_func=True,
         ),
     )
 
@@ -68,5 +93,5 @@ def rules():
     return [
         *collect_rules(),
         UnionRule(PackageFieldSet, PythonGoogleCloudFunctionFieldSet),
-        *pex_from_targets.rules(),
+        *faas_rules(),
     ]

src/python/pants/backend/google_cloud_function/python/rules_test.py

@@ -130,7 +130,7 @@ def complete_platform(rule_runner: PythonRuleRunner) -> bytes:
     "major_minor_interpreter",
     all_major_minor_python_versions(Lambdex.default_interpreter_constraints),
 )
-def test_create_hello_world_lambda(
+def test_create_hello_world_lambda_with_lambdex(
     rule_runner: PythonRuleRunner, major_minor_interpreter: str, complete_platform: str, caplog
 ) -> None:
     rule_runner.write_files(
@@ -243,3 +243,45 @@ def handler(event, context):
     assert "assets/f.txt:files" in caplog.text
     assert "assets:relocated" in caplog.text
     assert "assets:resources" not in caplog.text
+
+
+def test_create_hello_world_gcf(rule_runner: PythonRuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "src/python/foo/bar/hello_world.py": dedent(
+                """
+                import mureq
+
+                def handler(event, context):
+                    print('Hello, World!')
+                """
+            ),
+            "src/python/foo/bar/BUILD": dedent(
+                """
+                python_requirement(name="mureq", requirements=["mureq==0.2"])
+                python_sources()
+
+                python_google_cloud_function(
+                    name='gcf',
+                    handler='foo.bar.hello_world:handler',
+                    runtime="python37",
+                    layout='zip',
+                    type='event',
+                )
+                """
+            ),
+        }
+    )
+
+    zip_file_relpath, content = create_python_google_cloud_function(
+        rule_runner,
+        Address("src/python/foo/bar", target_name="gcf"),
+        expected_extra_log_lines=("    Handler: handler",),
+    )
+    assert "src.python.foo.bar/gcf.zip" == zip_file_relpath
+
+    zipfile = ZipFile(BytesIO(content))
+    names = set(zipfile.namelist())
+    assert "mureq/__init__.py" in names
+    assert "foo/bar/hello_world.py" in names
+    assert zipfile.read("main.py") == b"from foo.bar.hello_world import handler as handler"

src/python/pants/backend/google_cloud_function/python/target_types.py

@@ -10,6 +10,7 @@
     PythonFaaSCompletePlatforms,
     PythonFaaSDependencies,
     PythonFaaSHandlerField,
+    PythonFaaSLayoutField,
     PythonFaaSRuntimeField,
 )
 from pants.backend.python.util_rules.faas import rules as faas_rules
@@ -29,11 +30,19 @@
 
 
 class PythonGoogleCloudFunctionHandlerField(PythonFaaSHandlerField):
+    # GCP requires "Your main file must be named main.py"
+    # https://cloud.google.com/functions/docs/writing#directory-structure-python
+    reexported_handler_module = "main"
+
     help = help_text(
         f"""
         Entry point to the Google Cloud Function handler.
 
         {PythonFaaSHandlerField.help}
+
+        This is re-exported at `{reexported_handler_module}.handler` in the resulting package to
+        used as the configured handler of the Google Cloud Function in GCP.  It can also be accessed
+        under its source-root-relative module path, for example: `path.to.module.handler_func`.
         """
     )
 
@@ -111,6 +120,7 @@ class PythonGoogleCloudFunction(Target):
         PythonFaaSCompletePlatforms,
         PythonGoogleCloudFunctionType,
         PythonResolveField,
+        PythonFaaSLayoutField,
         EnvironmentField,
     )
     help = help_text(