Skip to content

Commit

Permalink
Revert "Implement layout="zip" for Lambda/GCF, skipping lambdex" (#19071
Browse files Browse the repository at this point in the history 
)

Reverts #19022.

Per
#19032 (comment),
we're going to change the approach for moving away from Lambdex to a
global setting, rather than the per-target `layout` field added in
#19022. That change will simplify things, and involve an extra moving
part that will be cherry-picked to 2.17, so I think it's better to start
from scratch, and then more carefully cherry-pick the parts of #19022
that are relevant.
  • Loading branch information
@huonw
huonw authored May 21, 2023
1 parent 10d7de0 commit a7738f9
Show file tree
Hide file tree
Showing 10 changed files with 54 additions and 685 deletions.
46 changes: 9 additions & 37 deletions src/python/pants/backend/awslambda/python/rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,8 @@
PythonAwsLambdaIncludeRequirements,
PythonAwsLambdaRuntime,
)
from pants.backend.python.util_rules.faas import (
BuildLambdexRequest,
BuildPythonFaaSRequest,
PythonFaaSCompletePlatforms,
PythonFaaSLayout,
PythonFaaSLayoutField,
)
from pants.backend.python.util_rules.faas import rules as faas_rules
from pants.backend.python.util_rules import pex_from_targets
from pants.backend.python.util_rules.faas import BuildLambdexRequest, PythonFaaSCompletePlatforms
from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet
from pants.core.util_rules.environments import EnvironmentField
from pants.engine.rules import Get, collect_rules, rule
Expand All @@ -39,49 +33,27 @@ class PythonAwsLambdaFieldSet(PackageFieldSet):
complete_platforms: PythonFaaSCompletePlatforms
output_path: OutputPathField
environment: EnvironmentField
layout: PythonFaaSLayoutField


@rule(desc="Create Python AWS Lambda", level=LogLevel.DEBUG)
async def package_python_awslambda(
field_set: PythonAwsLambdaFieldSet,
) -> BuiltPackage:
layout = PythonFaaSLayout(field_set.layout.value)

if layout is PythonFaaSLayout.LAMBDEX:
return await Get(
BuiltPackage,
BuildLambdexRequest(
address=field_set.address,
target_name=PythonAWSLambda.alias,
complete_platforms=field_set.complete_platforms,
runtime=field_set.runtime,
handler=field_set.handler,
output_path=field_set.output_path,
include_requirements=field_set.include_requirements.value,
script_handler=None,
script_module=None,
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
handler_log_message="lambdex_handler.handler",
),
)

return await Get(
BuiltPackage,
BuildPythonFaaSRequest(
BuildLambdexRequest(
address=field_set.address,
target_name=PythonAWSLambda.alias,
complete_platforms=field_set.complete_platforms,
runtime=field_set.runtime,
handler=field_set.handler,
layout=layout,
output_path=field_set.output_path,
include_requirements=field_set.include_requirements.value,
# This doesn't matter (just needs to be fixed), but is the default name used by the AWS
# console when creating a Python lambda, so is as good as any
# https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
reexported_handler_module="lambda_function",
script_handler=None,
script_module=None,
# The AWS-facing handler function is always lambdex_handler.handler, which is the
# wrapper injected by lambdex that manages invocation of the actual handler.
handler_log_message="lambdex_handler.handler",
),
)

Expand All @@ -90,5 +62,5 @@ def rules():
return [
*collect_rules(),
UnionRule(PackageFieldSet, PythonAwsLambdaFieldSet),
*faas_rules(),
*pex_from_targets.rules(),
]
69 changes: 2 additions & 67 deletions src/python/pants/backend/awslambda/python/rules_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ def complete_platform(rule_runner: PythonRuleRunner) -> bytes:
"major_minor_interpreter",
all_major_minor_python_versions(Lambdex.default_interpreter_constraints),
)
def test_create_hello_world_lambda_with_lambdex(
def test_create_hello_world_lambda(
rule_runner: PythonRuleRunner, major_minor_interpreter: str, complete_platform: str, caplog
) -> None:
rule_runner.write_files(
Expand Down Expand Up @@ -197,7 +197,7 @@ def handler(event, context):
), "Using include_requirements=False should exclude third-party deps"


def test_warn_files_targets_with_lambdex(rule_runner: PythonRuleRunner, caplog) -> None:
def test_warn_files_targets(rule_runner: PythonRuleRunner, caplog) -> None:
rule_runner.write_files(
{
"assets/f.txt": "",
Expand Down Expand Up @@ -257,68 +257,3 @@ def handler(event, context):
assert "assets/f.txt:files" in caplog.text
assert "assets:relocated" in caplog.text
assert "assets:resources" not in caplog.text


def test_create_hello_world_lambda(rule_runner: PythonRuleRunner) -> None:
rule_runner.write_files(
{
"src/python/foo/bar/hello_world.py": dedent(
"""
import mureq
def handler(event, context):
print('Hello, World!')
"""
),
"src/python/foo/bar/BUILD": dedent(
"""
python_requirement(name="mureq", requirements=["mureq==0.2"])
python_sources()
python_awslambda(
name='lambda',
handler='foo.bar.hello_world:handler',
runtime="python3.7",
layout='zip',
)
python_awslambda(
name='slimlambda',
include_requirements=False,
handler='foo.bar.hello_world:handler',
runtime="python3.7",
layout='zip',
)
"""
),
}
)

zip_file_relpath, content = create_python_awslambda(
rule_runner,
Address("src/python/foo/bar", target_name="lambda"),
expected_extra_log_lines=(" Handler: lambda_function.handler",),
)
assert "src.python.foo.bar/lambda.zip" == zip_file_relpath

zipfile = ZipFile(BytesIO(content))
names = set(zipfile.namelist())
assert "mureq/__init__.py" in names
assert "foo/bar/hello_world.py" in names
assert (
zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler"
)

zip_file_relpath, content = create_python_awslambda(
rule_runner,
Address("src/python/foo/bar", target_name="slimlambda"),
expected_extra_log_lines=(" Handler: lambda_function.handler",),
)
assert "src.python.foo.bar/slimlambda.zip" == zip_file_relpath

zipfile = ZipFile(BytesIO(content))
names = set(zipfile.namelist())
assert "mureq/__init__.py" not in names
assert "foo/bar/hello_world.py" in names
assert (
zipfile.read("lambda_function.py") == b"from foo.bar.hello_world import handler as handler"
)
11 changes: 0 additions & 11 deletions src/python/pants/backend/awslambda/python/target_types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
PythonFaaSCompletePlatforms,
PythonFaaSDependencies,
PythonFaaSHandlerField,
PythonFaaSLayoutField,
PythonFaaSRuntimeField,
)
from pants.backend.python.util_rules.faas import rules as faas_rules
Expand All @@ -30,20 +29,11 @@


class PythonAwsLambdaHandlerField(PythonFaaSHandlerField):
# This doesn't matter (just needs to be fixed), but is the default name used by the AWS
# console when creating a Python lambda, so is as good as any
# https://docs.aws.amazon.com/lambda/latest/dg/python-handler.html
reexported_handler_module = "lambda_function"

help = help_text(
f"""
Entry point to the AWS Lambda handler.
{PythonFaaSHandlerField.help}
This is re-exported at `{reexported_handler_module}.handler` in the resulting package to be
used as the configured handler of the Lambda in AWS. It can also be accessed under its
source-root-relative module path, for example: `path.to.module.handler_func`.
"""
)

Expand Down Expand Up @@ -121,7 +111,6 @@ class PythonAWSLambda(Target):
PythonAwsLambdaRuntime,
PythonFaaSCompletePlatforms,
PythonResolveField,
PythonFaaSLayoutField,
EnvironmentField,
)
help = help_text(
Expand Down
55 changes: 15 additions & 40 deletions src/python/pants/backend/google_cloud_function/python/rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,8 @@
PythonGoogleCloudFunctionRuntime,
PythonGoogleCloudFunctionType,
)
from pants.backend.python.util_rules.faas import (
BuildLambdexRequest,
BuildPythonFaaSRequest,
PythonFaaSCompletePlatforms,
PythonFaaSLayout,
PythonFaaSLayoutField,
)
from pants.backend.python.util_rules.faas import rules as faas_rules
from pants.backend.python.util_rules import pex_from_targets
from pants.backend.python.util_rules.faas import BuildLambdexRequest, PythonFaaSCompletePlatforms
from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet
from pants.core.util_rules.environments import EnvironmentField
from pants.engine.rules import Get, collect_rules, rule
Expand All @@ -39,52 +33,33 @@ class PythonGoogleCloudFunctionFieldSet(PackageFieldSet):
type: PythonGoogleCloudFunctionType
output_path: OutputPathField
environment: EnvironmentField
layout: PythonFaaSLayoutField


@rule(desc="Create Python Google Cloud Function", level=LogLevel.DEBUG)
async def package_python_google_cloud_function(
field_set: PythonGoogleCloudFunctionFieldSet,
) -> BuiltPackage:
layout = PythonFaaSLayout(field_set.layout.value)
if layout is PythonFaaSLayout.LAMBDEX:
return await Get(
BuiltPackage,
BuildLambdexRequest(
address=field_set.address,
target_name=PythonGoogleCloudFunction.alias,
complete_platforms=field_set.complete_platforms,
runtime=field_set.runtime,
handler=field_set.handler,
output_path=field_set.output_path,
include_requirements=True,
# The GCP-facing handler function is always `main.handler` (We pass `-M main.py -H handler` to
# Lambdex to ensure this), which is the wrapper injected by Lambdex that manages invocation of
# the actual user-supplied handler function. This arrangement works well since GCF assumes the
# handler function is housed in `main.py` in the root of the zip (you can re-direct this by
# setting a `GOOGLE_FUNCTION_SOURCE` Google Cloud build environment variable; e.g.:
# `gcloud functions deploy {--build-env-vars-file,--set-build-env-vars}`, but it's non-trivial
# to do this right or with intended effect) and the handler name you configure GCF with is just
# the unqualified function name, which we log here.
script_handler="handler",
script_module="main.py",
handler_log_message="handler",
),
)

return await Get(
BuiltPackage,
BuildPythonFaaSRequest(
BuildLambdexRequest(
address=field_set.address,
target_name=PythonGoogleCloudFunction.alias,
complete_platforms=field_set.complete_platforms,
runtime=field_set.runtime,
handler=field_set.handler,
layout=layout,
output_path=field_set.output_path,
include_requirements=True,
reexported_handler_module=PythonGoogleCloudFunctionHandlerField.reexported_handler_module,
log_only_reexported_handler_func=True,
# The GCP-facing handler function is always `main.handler` (We pass `-M main.py -H handler` to
# Lambdex to ensure this), which is the wrapper injected by Lambdex that manages invocation of
# the actual user-supplied handler function. This arrangement works well since GCF assumes the
# handler function is housed in `main.py` in the root of the zip (you can re-direct this by
# setting a `GOOGLE_FUNCTION_SOURCE` Google Cloud build environment variable; e.g.:
# `gcloud functions deploy {--build-env-vars-file,--set-build-env-vars}`, but it's non-trivial
# to do this right or with intended effect) and the handler name you configure GCF with is just
# the unqualified function name, which we log here.
script_handler="handler",
script_module="main.py",
handler_log_message="handler",
),
)

Expand All @@ -93,5 +68,5 @@ def rules():
return [
*collect_rules(),
UnionRule(PackageFieldSet, PythonGoogleCloudFunctionFieldSet),
*faas_rules(),
*pex_from_targets.rules(),
]
44 changes: 1 addition & 43 deletions src/python/pants/backend/google_cloud_function/python/rules_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ def complete_platform(rule_runner: PythonRuleRunner) -> bytes:
"major_minor_interpreter",
all_major_minor_python_versions(Lambdex.default_interpreter_constraints),
)
def test_create_hello_world_lambda_with_lambdex(
def test_create_hello_world_lambda(
rule_runner: PythonRuleRunner, major_minor_interpreter: str, complete_platform: str, caplog
) -> None:
rule_runner.write_files(
Expand Down Expand Up @@ -243,45 +243,3 @@ def handler(event, context):
assert "assets/f.txt:files" in caplog.text
assert "assets:relocated" in caplog.text
assert "assets:resources" not in caplog.text


def test_create_hello_world_gcf(rule_runner: PythonRuleRunner) -> None:
rule_runner.write_files(
{
"src/python/foo/bar/hello_world.py": dedent(
"""
import mureq
def handler(event, context):
print('Hello, World!')
"""
),
"src/python/foo/bar/BUILD": dedent(
"""
python_requirement(name="mureq", requirements=["mureq==0.2"])
python_sources()
python_google_cloud_function(
name='gcf',
handler='foo.bar.hello_world:handler',
runtime="python37",
layout='zip',
type='event',
)
"""
),
}
)

zip_file_relpath, content = create_python_google_cloud_function(
rule_runner,
Address("src/python/foo/bar", target_name="gcf"),
expected_extra_log_lines=(" Handler: handler",),
)
assert "src.python.foo.bar/gcf.zip" == zip_file_relpath

zipfile = ZipFile(BytesIO(content))
names = set(zipfile.namelist())
assert "mureq/__init__.py" in names
assert "foo/bar/hello_world.py" in names
assert zipfile.read("main.py") == b"from foo.bar.hello_world import handler as handler"
10 changes: 0 additions & 10 deletions src/python/pants/backend/google_cloud_function/python/target_types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
PythonFaaSCompletePlatforms,
PythonFaaSDependencies,
PythonFaaSHandlerField,
PythonFaaSLayoutField,
PythonFaaSRuntimeField,
)
from pants.backend.python.util_rules.faas import rules as faas_rules
Expand All @@ -30,19 +29,11 @@


class PythonGoogleCloudFunctionHandlerField(PythonFaaSHandlerField):
# GCP requires "Your main file must be named main.py"
# https://cloud.google.com/functions/docs/writing#directory-structure-python
reexported_handler_module = "main"

help = help_text(
f"""
Entry point to the Google Cloud Function handler.
{PythonFaaSHandlerField.help}
This is re-exported at `{reexported_handler_module}.handler` in the resulting package to
used as the configured handler of the Google Cloud Function in GCP. It can also be accessed
under its source-root-relative module path, for example: `path.to.module.handler_func`.
"""
)

Expand Down Expand Up @@ -120,7 +111,6 @@ class PythonGoogleCloudFunction(Target):
PythonFaaSCompletePlatforms,
PythonGoogleCloudFunctionType,
PythonResolveField,
PythonFaaSLayoutField,
EnvironmentField,
)
help = help_text(
Expand Down
Loading

0 comments on commit a7738f9

Please sign in to comment.