diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql b/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql index 091b0275e2353..bb14f13ff377f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/schema.graphql @@ -661,6 +661,7 @@ type AssetNode { partitionStats: PartitionStats metadataEntries: [MetadataEntry!]! tags: [DefinitionTag!]! + kinds: [String!]! op: SolidDefinition opName: String opNames: [String!]! diff --git a/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts b/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts index dcb531d01d46f..87f890b0f169d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/graphql/types.ts @@ -448,6 +448,7 @@ export type AssetNode = { isPartitioned: Scalars['Boolean']['output']; jobNames: Array; jobs: Array; + kinds: Array; latestMaterializationByPartition: Array>; latestRunForPartition: Maybe; metadataEntries: Array< @@ -6519,6 +6520,7 @@ export const buildAssetNode = ( overrides && overrides.hasOwnProperty('isPartitioned') ? overrides.isPartitioned! : true, jobNames: overrides && overrides.hasOwnProperty('jobNames') ? overrides.jobNames! : [], jobs: overrides && overrides.hasOwnProperty('jobs') ? overrides.jobs! : [], + kinds: overrides && overrides.hasOwnProperty('kinds') ? overrides.kinds! : [], latestMaterializationByPartition: overrides && overrides.hasOwnProperty('latestMaterializationByPartition') ? overrides.latestMaterializationByPartition! diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py b/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py index eeba5dbbebdc2..4b1ddaa1660d5 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/asset_graph.py @@ -33,6 +33,7 @@ ) from dagster._core.snap.node import GraphDefSnap, OpDefSnap from dagster._core.storage.batch_asset_record_loader import BatchAssetRecordLoader +from dagster._core.storage.tags import KIND_PREFIX from dagster._core.utils import is_valid_email from dagster._core.workspace.permissions import Permissions from dagster._utils.caching_instance_queryer import CachingInstanceQueryer @@ -300,6 +301,7 @@ class GrapheneAssetNode(graphene.ObjectType): partitionStats = graphene.Field(GraphenePartitionStats) metadata_entries = non_null_list(GrapheneMetadataEntry) tags = non_null_list(GrapheneDefinitionTag) + kinds = non_null_list(graphene.String) op = graphene.Field(GrapheneSolidDefinition) opName = graphene.String() opNames = non_null_list(graphene.String) @@ -1214,6 +1216,16 @@ def resolve_tags(self, _graphene_info: ResolveInfo) -> Sequence[GrapheneDefiniti for key, value in (self._external_asset_node.tags or {}).items() ] + def resolve_kinds(self, _graphene_info: ResolveInfo) -> Sequence[str]: + if self._external_asset_node.compute_kind: + return [self._external_asset_node.compute_kind] + + return [ + key[len(KIND_PREFIX) :] + for key in (self._external_asset_node.tags or {}).keys() + if key.startswith(KIND_PREFIX) + ] + def resolve_op( self, _graphene_info: ResolveInfo ) -> Optional[Union[GrapheneSolidDefinition, GrapheneCompositeSolidDefinition]]: diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_all_snapshot_ids.ambr b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_all_snapshot_ids.ambr index 859cb4a2d9ecb..20c7b4bf16758 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_all_snapshot_ids.ambr +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_all_snapshot_ids.ambr @@ -489,259 +489,7 @@ "scalar_kind": null, "type_param_keys": null }, - "Shape.35d3e42b53e66506c5867f04644849cd03763bc6": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{\"hanging_op\": {}, \"my_op\": {}, \"never_runs_op\": {}}", - "description": null, - "is_required": false, - "name": "ops", - "type_key": "Shape.811a60b4c43530c3d6100304f377dbd2d3045291" - } - ], - "given_name": null, - "key": "Shape.35d3e42b53e66506c5867f04644849cd03763bc6", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.44f24ac55059da1634e84af6c1bf7e0ed332251c": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": false, - "default_value_as_json_str": null, - "description": "[DEPRECATED]", - "is_required": false, - "name": "marker_to_close", - "type_key": "String" - }, - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{\"enabled\": {}}", - "description": "Whether retries are enabled or not. By default, retries are enabled.", - "is_required": false, - "name": "retries", - "type_key": "Selector.1bfb167aea90780aa679597800c71bd8c65ed0b2" - } - ], - "given_name": null, - "key": "Shape.44f24ac55059da1634e84af6c1bf7e0ed332251c", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.44f2a71367507edd1b8e64f739222c4312b3691b": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{}", - "description": null, - "is_required": false, - "name": "config", - "type_key": "Shape.18b2faaf1efd505374f7f25fcb61ed59bd5be851" - } - ], - "given_name": null, - "key": "Shape.44f2a71367507edd1b8e64f739222c4312b3691b", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.4b53b73df342381d0d05c5f36183dc99cb9676e2": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": false, - "default_value_as_json_str": null, - "description": null, - "is_required": true, - "name": "path", - "type_key": "String" - } - ], - "given_name": null, - "key": "Shape.4b53b73df342381d0d05c5f36183dc99cb9676e2", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.62edccaf30696e25335ae92685bdc41e204e30e6": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{}", - "description": null, - "is_required": false, - "name": "config", - "type_key": "Shape.da39a3ee5e6b4b0d3255bfef95601890afd80709" - } - ], - "given_name": null, - "key": "Shape.62edccaf30696e25335ae92685bdc41e204e30e6", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.743e47901855cb245064dd633e217bfcb49a11a7": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": false, - "default_value_as_json_str": null, - "description": null, - "is_required": false, - "name": "config", - "type_key": "Any" - } - ], - "given_name": null, - "key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.7493b137e48f8d4b013bce61e92617ff1bc51f7f": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{}", - "description": null, - "is_required": false, - "name": "dummy_io_manager", - "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" - }, - { - "__class__": "ConfigFieldSnap", - "default_provided": false, - "default_value_as_json_str": null, - "description": null, - "is_required": true, - "name": "hanging_asset_resource", - "type_key": "Shape.b13a6c5637084590cc1538f9522324bfeb4b46b3" - }, - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{\"config\": {}}", - "description": "Built-in filesystem IO manager that stores and retrieves values using pickling.", - "is_required": false, - "name": "io_manager", - "type_key": "Shape.44f2a71367507edd1b8e64f739222c4312b3691b" - } - ], - "given_name": null, - "key": "Shape.7493b137e48f8d4b013bce61e92617ff1bc51f7f", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.811a60b4c43530c3d6100304f377dbd2d3045291": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{}", - "description": null, - "is_required": false, - "name": "hanging_op", - "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" - }, - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{}", - "description": null, - "is_required": false, - "name": "my_op", - "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" - }, - { - "__class__": "ConfigFieldSnap", - "default_provided": true, - "default_value_as_json_str": "{}", - "description": null, - "is_required": false, - "name": "never_runs_op", - "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" - } - ], - "given_name": null, - "key": "Shape.811a60b4c43530c3d6100304f377dbd2d3045291", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.9a3a315bff2146cca750edbec49c6b4b4d0ce58e": { - "__class__": "ConfigTypeSnap", - "description": null, - "enum_values": null, - "fields": [ - { - "__class__": "ConfigFieldSnap", - "default_provided": false, - "default_value_as_json_str": null, - "description": null, - "is_required": true, - "name": "file", - "type_key": "String" - } - ], - "given_name": null, - "key": "Shape.9a3a315bff2146cca750edbec49c6b4b4d0ce58e", - "kind": { - "__enum__": "ConfigTypeKind.STRICT_SHAPE" - }, - "scalar_kind": null, - "type_param_keys": null - }, - "Shape.a06590e21e4a45c403d7ae55be9d00980112c450": { + "Shape.2a209c237ad14a621990d7f3e008d26ef6a041bf": { "__class__": "ConfigTypeSnap", "description": null, "enum_values": null, @@ -800,6 +548,15 @@ "name": "asset_two", "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" }, + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{\"config\": {}}", + "description": null, + "is_required": false, + "name": "asset_with_compute_storage_kinds", + "type_key": "Shape.62edccaf30696e25335ae92685bdc41e204e30e6" + }, { "__class__": "ConfigFieldSnap", "default_provided": true, @@ -1061,6 +818,15 @@ "name": "middle_static_partitioned_asset_2", "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" }, + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{\"config\": {}}", + "description": null, + "is_required": false, + "name": "multi_asset_with_kinds", + "type_key": "Shape.62edccaf30696e25335ae92685bdc41e204e30e6" + }, { "__class__": "ConfigFieldSnap", "default_provided": true, @@ -1252,14 +1018,92 @@ } ], "given_name": null, - "key": "Shape.a06590e21e4a45c403d7ae55be9d00980112c450", + "key": "Shape.2a209c237ad14a621990d7f3e008d26ef6a041bf", "kind": { "__enum__": "ConfigTypeKind.STRICT_SHAPE" }, "scalar_kind": null, "type_param_keys": null }, - "Shape.b13a6c5637084590cc1538f9522324bfeb4b46b3": { + "Shape.35d3e42b53e66506c5867f04644849cd03763bc6": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{\"hanging_op\": {}, \"my_op\": {}, \"never_runs_op\": {}}", + "description": null, + "is_required": false, + "name": "ops", + "type_key": "Shape.811a60b4c43530c3d6100304f377dbd2d3045291" + } + ], + "given_name": null, + "key": "Shape.35d3e42b53e66506c5867f04644849cd03763bc6", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.44f24ac55059da1634e84af6c1bf7e0ed332251c": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": false, + "default_value_as_json_str": null, + "description": "[DEPRECATED]", + "is_required": false, + "name": "marker_to_close", + "type_key": "String" + }, + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{\"enabled\": {}}", + "description": "Whether retries are enabled or not. By default, retries are enabled.", + "is_required": false, + "name": "retries", + "type_key": "Selector.1bfb167aea90780aa679597800c71bd8c65ed0b2" + } + ], + "given_name": null, + "key": "Shape.44f24ac55059da1634e84af6c1bf7e0ed332251c", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.44f2a71367507edd1b8e64f739222c4312b3691b": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "config", + "type_key": "Shape.18b2faaf1efd505374f7f25fcb61ed59bd5be851" + } + ], + "given_name": null, + "key": "Shape.44f2a71367507edd1b8e64f739222c4312b3691b", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.4b53b73df342381d0d05c5f36183dc99cb9676e2": { "__class__": "ConfigTypeSnap", "description": null, "enum_values": null, @@ -1270,19 +1114,42 @@ "default_value_as_json_str": null, "description": null, "is_required": true, + "name": "path", + "type_key": "String" + } + ], + "given_name": null, + "key": "Shape.4b53b73df342381d0d05c5f36183dc99cb9676e2", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.62edccaf30696e25335ae92685bdc41e204e30e6": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, "name": "config", - "type_key": "Shape.9a3a315bff2146cca750edbec49c6b4b4d0ce58e" + "type_key": "Shape.da39a3ee5e6b4b0d3255bfef95601890afd80709" } ], "given_name": null, - "key": "Shape.b13a6c5637084590cc1538f9522324bfeb4b46b3", + "key": "Shape.62edccaf30696e25335ae92685bdc41e204e30e6", "kind": { "__enum__": "ConfigTypeKind.STRICT_SHAPE" }, "scalar_kind": null, "type_param_keys": null }, - "Shape.d1be0bdc7af85b71653ea0993b88d427e55e62c7": { + "Shape.669d959211c2a41db35d48daeb2c4afa5df7f552": { "__class__": "ConfigTypeSnap", "description": null, "enum_values": null, @@ -1308,11 +1175,11 @@ { "__class__": "ConfigFieldSnap", "default_provided": true, - "default_value_as_json_str": "{\"asset_1\": {}, \"asset_1_my_check\": {}, \"asset_2\": {}, \"asset_3\": {}, \"asset_one\": {}, \"asset_two\": {}, \"asset_yields_observation\": {}, \"bar\": {}, \"baz\": {}, \"check_in_op_asset\": {}, \"downstream_asset\": {}, \"downstream_dynamic_partitioned_asset\": {}, \"downstream_static_partitioned_asset\": {}, \"downstream_time_partitioned_asset\": {}, \"downstream_weekly_partitioned_asset\": {}, \"dynamic_in_multipartitions_fail\": {}, \"dynamic_in_multipartitions_success\": {}, \"executable_asset\": {}, \"fail_partition_materialization\": {}, \"first_asset\": {}, \"foo\": {}, \"foo_bar\": {}, \"fresh_diamond_bottom\": {}, \"fresh_diamond_left\": {}, \"fresh_diamond_right\": {}, \"fresh_diamond_top\": {}, \"grouped_asset_1\": {}, \"grouped_asset_2\": {}, \"grouped_asset_4\": {}, \"hanging_asset\": {}, \"hanging_graph\": {\"ops\": {\"hanging_op\": {}, \"my_op\": {}, \"never_runs_op\": {}}}, \"hanging_partition_asset\": {}, \"integers_asset\": {}, \"middle_static_partitioned_asset_1\": {}, \"middle_static_partitioned_asset_2\": {}, \"multi_run_backfill_policy_asset\": {}, \"multipartitions_1\": {}, \"multipartitions_2\": {}, \"multipartitions_fail\": {}, \"never_runs_asset\": {}, \"no_multipartitions_1\": {}, \"output_then_hang_asset\": {}, \"single_run_backfill_policy_asset\": {}, \"subsettable_checked_multi_asset\": {\"config\": {}}, \"typed_asset\": {}, \"typed_multi_asset\": {\"config\": {}}, \"unconnected\": {}, \"ungrouped_asset_3\": {}, \"ungrouped_asset_5\": {}, \"unpartitioned_upstream_of_partitioned\": {}, \"untyped_asset\": {}, \"upstream_daily_partitioned_asset\": {}, \"upstream_dynamic_partitioned_asset\": {}, \"upstream_static_partitioned_asset\": {}, \"upstream_time_partitioned_asset\": {}, \"yield_partition_materialization\": {}}", + "default_value_as_json_str": "{\"asset_1\": {}, \"asset_1_my_check\": {}, \"asset_2\": {}, \"asset_3\": {}, \"asset_one\": {}, \"asset_two\": {}, \"asset_with_compute_storage_kinds\": {\"config\": {}}, \"asset_yields_observation\": {}, \"bar\": {}, \"baz\": {}, \"check_in_op_asset\": {}, \"downstream_asset\": {}, \"downstream_dynamic_partitioned_asset\": {}, \"downstream_static_partitioned_asset\": {}, \"downstream_time_partitioned_asset\": {}, \"downstream_weekly_partitioned_asset\": {}, \"dynamic_in_multipartitions_fail\": {}, \"dynamic_in_multipartitions_success\": {}, \"executable_asset\": {}, \"fail_partition_materialization\": {}, \"first_asset\": {}, \"foo\": {}, \"foo_bar\": {}, \"fresh_diamond_bottom\": {}, \"fresh_diamond_left\": {}, \"fresh_diamond_right\": {}, \"fresh_diamond_top\": {}, \"grouped_asset_1\": {}, \"grouped_asset_2\": {}, \"grouped_asset_4\": {}, \"hanging_asset\": {}, \"hanging_graph\": {\"ops\": {\"hanging_op\": {}, \"my_op\": {}, \"never_runs_op\": {}}}, \"hanging_partition_asset\": {}, \"integers_asset\": {}, \"middle_static_partitioned_asset_1\": {}, \"middle_static_partitioned_asset_2\": {}, \"multi_asset_with_kinds\": {\"config\": {}}, \"multi_run_backfill_policy_asset\": {}, \"multipartitions_1\": {}, \"multipartitions_2\": {}, \"multipartitions_fail\": {}, \"never_runs_asset\": {}, \"no_multipartitions_1\": {}, \"output_then_hang_asset\": {}, \"single_run_backfill_policy_asset\": {}, \"subsettable_checked_multi_asset\": {\"config\": {}}, \"typed_asset\": {}, \"typed_multi_asset\": {\"config\": {}}, \"unconnected\": {}, \"ungrouped_asset_3\": {}, \"ungrouped_asset_5\": {}, \"unpartitioned_upstream_of_partitioned\": {}, \"untyped_asset\": {}, \"upstream_daily_partitioned_asset\": {}, \"upstream_dynamic_partitioned_asset\": {}, \"upstream_static_partitioned_asset\": {}, \"upstream_time_partitioned_asset\": {}, \"yield_partition_materialization\": {}}", "description": "Configure runtime parameters for ops or assets.", "is_required": false, "name": "ops", - "type_key": "Shape.a06590e21e4a45c403d7ae55be9d00980112c450" + "type_key": "Shape.2a209c237ad14a621990d7f3e008d26ef6a041bf" }, { "__class__": "ConfigFieldSnap", @@ -1325,7 +1192,158 @@ } ], "given_name": null, - "key": "Shape.d1be0bdc7af85b71653ea0993b88d427e55e62c7", + "key": "Shape.669d959211c2a41db35d48daeb2c4afa5df7f552", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.743e47901855cb245064dd633e217bfcb49a11a7": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": false, + "default_value_as_json_str": null, + "description": null, + "is_required": false, + "name": "config", + "type_key": "Any" + } + ], + "given_name": null, + "key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.7493b137e48f8d4b013bce61e92617ff1bc51f7f": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "dummy_io_manager", + "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" + }, + { + "__class__": "ConfigFieldSnap", + "default_provided": false, + "default_value_as_json_str": null, + "description": null, + "is_required": true, + "name": "hanging_asset_resource", + "type_key": "Shape.b13a6c5637084590cc1538f9522324bfeb4b46b3" + }, + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{\"config\": {}}", + "description": "Built-in filesystem IO manager that stores and retrieves values using pickling.", + "is_required": false, + "name": "io_manager", + "type_key": "Shape.44f2a71367507edd1b8e64f739222c4312b3691b" + } + ], + "given_name": null, + "key": "Shape.7493b137e48f8d4b013bce61e92617ff1bc51f7f", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.811a60b4c43530c3d6100304f377dbd2d3045291": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "hanging_op", + "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" + }, + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "my_op", + "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" + }, + { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "never_runs_op", + "type_key": "Shape.743e47901855cb245064dd633e217bfcb49a11a7" + } + ], + "given_name": null, + "key": "Shape.811a60b4c43530c3d6100304f377dbd2d3045291", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.9a3a315bff2146cca750edbec49c6b4b4d0ce58e": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": false, + "default_value_as_json_str": null, + "description": null, + "is_required": true, + "name": "file", + "type_key": "String" + } + ], + "given_name": null, + "key": "Shape.9a3a315bff2146cca750edbec49c6b4b4d0ce58e", + "kind": { + "__enum__": "ConfigTypeKind.STRICT_SHAPE" + }, + "scalar_kind": null, + "type_param_keys": null + }, + "Shape.b13a6c5637084590cc1538f9522324bfeb4b46b3": { + "__class__": "ConfigTypeSnap", + "description": null, + "enum_values": null, + "fields": [ + { + "__class__": "ConfigFieldSnap", + "default_provided": false, + "default_value_as_json_str": null, + "description": null, + "is_required": true, + "name": "config", + "type_key": "Shape.9a3a315bff2146cca750edbec49c6b4b4d0ce58e" + } + ], + "given_name": null, + "key": "Shape.b13a6c5637084590cc1538f9522324bfeb4b46b3", "kind": { "__enum__": "ConfigTypeKind.STRICT_SHAPE" }, @@ -1610,6 +1628,16 @@ "solid_name": "asset_two", "tags": {} }, + { + "__class__": "SolidInvocationSnap", + "input_dep_snaps": [], + "is_dynamic_mapped": false, + "solid_def_name": "asset_with_compute_storage_kinds", + "solid_name": "asset_with_compute_storage_kinds", + "tags": { + "dagster/compute_kind": "python" + } + }, { "__class__": "SolidInvocationSnap", "input_dep_snaps": [], @@ -2081,6 +2109,14 @@ "solid_name": "middle_static_partitioned_asset_2", "tags": {} }, + { + "__class__": "SolidInvocationSnap", + "input_dep_snaps": [], + "is_dynamic_mapped": false, + "solid_def_name": "multi_asset_with_kinds", + "solid_name": "multi_asset_with_kinds", + "tags": {} + }, { "__class__": "SolidInvocationSnap", "input_dep_snaps": [], @@ -2384,7 +2420,7 @@ "name": "io_manager" } ], - "root_config_key": "Shape.d1be0bdc7af85b71653ea0993b88d427e55e62c7" + "root_config_key": "Shape.669d959211c2a41db35d48daeb2c4afa5df7f552" } ], "name": "__ASSET_JOB", @@ -2671,6 +2707,43 @@ "required_resource_keys": [], "tags": {} }, + { + "__class__": "SolidDefSnap", + "config_field_snap": { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "config", + "type_key": "Shape.da39a3ee5e6b4b0d3255bfef95601890afd80709" + }, + "description": null, + "input_def_snaps": [], + "name": "asset_with_compute_storage_kinds", + "output_def_snaps": [ + { + "__class__": "OutputDefSnap", + "dagster_type_key": "Nothing", + "description": null, + "is_dynamic": false, + "is_required": true, + "name": "third_kinds_key" + }, + { + "__class__": "OutputDefSnap", + "dagster_type_key": "Nothing", + "description": null, + "is_dynamic": false, + "is_required": true, + "name": "fourth_kinds_key" + } + ], + "required_resource_keys": [], + "tags": { + "dagster/compute_kind": "python" + } + }, { "__class__": "SolidDefSnap", "config_field_snap": { @@ -3612,6 +3685,41 @@ "required_resource_keys": [], "tags": {} }, + { + "__class__": "SolidDefSnap", + "config_field_snap": { + "__class__": "ConfigFieldSnap", + "default_provided": true, + "default_value_as_json_str": "{}", + "description": null, + "is_required": false, + "name": "config", + "type_key": "Shape.da39a3ee5e6b4b0d3255bfef95601890afd80709" + }, + "description": null, + "input_def_snaps": [], + "name": "multi_asset_with_kinds", + "output_def_snaps": [ + { + "__class__": "OutputDefSnap", + "dagster_type_key": "Nothing", + "description": null, + "is_dynamic": false, + "is_required": true, + "name": "first_kinds_key" + }, + { + "__class__": "OutputDefSnap", + "dagster_type_key": "Nothing", + "description": null, + "is_dynamic": false, + "is_required": true, + "name": "second_kinds_key" + } + ], + "required_resource_keys": [], + "tags": {} + }, { "__class__": "SolidDefSnap", "config_field_snap": { @@ -33845,7 +33953,7 @@ 'd9f6d85793df3d9df94d4aedb21bb659c1202bda' # --- # name: test_all_snapshot_ids[1] - '69f92df54bd8f367056abb6329513abf99b9c316' + '4c3b777fc2ef247b093eecf6cd5404452fc31ea3' # --- # name: test_all_snapshot_ids[20] ''' diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_assets.ambr b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_assets.ambr index d7a3fdeff8891..4751ca477a314 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_assets.ambr +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_assets.ambr @@ -197,6 +197,14 @@ ]), }), }), + dict({ + 'id': 'test.test_repo.["first_kinds_key"]', + 'key': dict({ + 'path': list([ + 'first_kinds_key', + ]), + }), + }), dict({ 'id': 'test.test_repo.["foo"]', 'key': dict({ @@ -213,6 +221,14 @@ ]), }), }), + dict({ + 'id': 'test.test_repo.["fourth_kinds_key"]', + 'key': dict({ + 'path': list([ + 'fourth_kinds_key', + ]), + }), + }), dict({ 'id': 'test.test_repo.["fresh_diamond_bottom"]', 'key': dict({ @@ -389,6 +405,14 @@ ]), }), }), + dict({ + 'id': 'test.test_repo.["second_kinds_key"]', + 'key': dict({ + 'path': list([ + 'second_kinds_key', + ]), + }), + }), dict({ 'id': 'test.test_repo.["single_run_backfill_policy_asset"]', 'key': dict({ @@ -405,6 +429,14 @@ ]), }), }), + dict({ + 'id': 'test.test_repo.["third_kinds_key"]', + 'key': dict({ + 'path': list([ + 'third_kinds_key', + ]), + }), + }), dict({ 'id': 'test.test_repo.["two"]', 'key': dict({ @@ -643,6 +675,11 @@ 'freshnessPolicy': None, 'id': 'test.test_repo.["first_asset"]', }), + dict({ + 'freshnessInfo': None, + 'freshnessPolicy': None, + 'id': 'test.test_repo.["first_kinds_key"]', + }), dict({ 'freshnessInfo': None, 'freshnessPolicy': None, @@ -653,6 +690,11 @@ 'freshnessPolicy': None, 'id': 'test.test_repo.["foo_bar"]', }), + dict({ + 'freshnessInfo': None, + 'freshnessPolicy': None, + 'id': 'test.test_repo.["fourth_kinds_key"]', + }), dict({ 'freshnessInfo': dict({ 'currentMinutesLate': 0.0, @@ -769,6 +811,11 @@ 'freshnessPolicy': None, 'id': 'test.test_repo.["output_then_hang_asset"]', }), + dict({ + 'freshnessInfo': None, + 'freshnessPolicy': None, + 'id': 'test.test_repo.["second_kinds_key"]', + }), dict({ 'freshnessInfo': None, 'freshnessPolicy': None, @@ -779,6 +826,11 @@ 'freshnessPolicy': None, 'id': 'test.test_repo.["str_asset"]', }), + dict({ + 'freshnessInfo': None, + 'freshnessPolicy': None, + 'id': 'test.test_repo.["third_kinds_key"]', + }), dict({ 'freshnessInfo': None, 'freshnessPolicy': None, diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_solids.ambr b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_solids.ambr index 4f5b8229fbae4..e1cc2414b3fd1 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_solids.ambr +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/__snapshots__/test_solids.ambr @@ -307,6 +307,22 @@ }), ]), }), + dict({ + '__typename': 'UsedSolid', + 'definition': dict({ + 'name': 'asset_with_compute_storage_kinds', + }), + 'invocations': list([ + dict({ + 'pipeline': dict({ + 'name': '__ASSET_JOB', + }), + 'solidHandle': dict({ + 'handleID': 'asset_with_compute_storage_kinds', + }), + }), + ]), + }), dict({ '__typename': 'UsedSolid', 'definition': dict({ @@ -1363,6 +1379,22 @@ }), ]), }), + dict({ + '__typename': 'UsedSolid', + 'definition': dict({ + 'name': 'multi_asset_with_kinds', + }), + 'invocations': list([ + dict({ + 'pipeline': dict({ + 'name': '__ASSET_JOB', + }), + 'solidHandle': dict({ + 'handleID': 'multi_asset_with_kinds', + }), + }), + ]), + }), dict({ '__typename': 'UsedSolid', 'definition': dict({ diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/repo.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/repo.py index d647547580521..574e65d89bfb6 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/repo.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/repo.py @@ -1758,6 +1758,31 @@ def fresh_diamond_bottom(fresh_diamond_left, fresh_diamond_right): return fresh_diamond_left + fresh_diamond_right +@multi_asset( + specs=[ + AssetSpec( + key="first_kinds_key", tags={"dagster/kind/python": "", "dagster/kind/airflow": ""} + ), + AssetSpec(key="second_kinds_key", tags={"dagster/kind/python": ""}), + ], +) +def multi_asset_with_kinds(): + return 1 + + +@multi_asset( + specs=[ + AssetSpec(key="third_kinds_key", tags={"dagster/storage_kind": "snowflake"}), + AssetSpec( + key="fourth_kinds_key", + ), + ], + compute_kind="python", +) +def asset_with_compute_storage_kinds(): + return 1 + + fresh_diamond_assets_job = define_asset_job( "fresh_diamond_assets_job", AssetSelection.assets(fresh_diamond_bottom).upstream() ) @@ -2071,6 +2096,8 @@ def define_assets(): ungrouped_asset_3, grouped_asset_4, ungrouped_asset_5, + multi_asset_with_kinds, + asset_with_compute_storage_kinds, ] diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py index aebb22d9a6584..8bd73f86729fd 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_assets.py @@ -721,6 +721,36 @@ } """ + +GET_TAGS = """ + query AssetNodeQuery($assetKey: AssetKeyInput!) { + assetNodeOrError(assetKey: $assetKey) { + ...on AssetNode { + assetKey { + path + } + tags { + key + value + } + } + } + } +""" + +GET_KINDS = """ + query AssetNodeQuery($assetKey: AssetKeyInput!) { + assetNodeOrError(assetKey: $assetKey) { + ...on AssetNode { + assetKey { + path + } + kinds + } + } + } +""" + GET_TARGETING_INSTIGATORS = """ query AssetNodeQuery($assetKey: AssetKeyInput!) { assetNodeOrError(assetKey: $assetKey) { @@ -2563,6 +2593,63 @@ def test_auto_materialize_policy(self, graphql_context: WorkspaceRequestContext) assert len(fresh_diamond_bottom) == 1 assert fresh_diamond_bottom[0]["autoMaterializePolicy"]["policyType"] == "LAZY" + def test_tags(self, graphql_context: WorkspaceRequestContext): + result = execute_dagster_graphql( + graphql_context, + GET_TAGS, + variables={ + "assetKey": {"path": ["second_kinds_key"]}, + }, + ) + + second_kinds_key = result.data["assetNodeOrError"] + assert second_kinds_key["tags"] == [{"key": "dagster/kind/python", "value": ""}] + + def test_kinds(self, graphql_context: WorkspaceRequestContext): + result = execute_dagster_graphql( + graphql_context, + GET_KINDS, + variables={ + "assetKey": {"path": ["first_kinds_key"]}, + }, + ) + + first_kinds_key = result.data["assetNodeOrError"] + assert set(first_kinds_key["kinds"]) == {"python", "airflow"} + + result = execute_dagster_graphql( + graphql_context, + GET_KINDS, + variables={ + "assetKey": {"path": ["second_kinds_key"]}, + }, + ) + + second_kinds_key = result.data["assetNodeOrError"] + assert set(second_kinds_key["kinds"]) == {"python"} + + result = execute_dagster_graphql( + graphql_context, + GET_KINDS, + variables={ + "assetKey": {"path": ["third_kinds_key"]}, + }, + ) + + third_kinds_key = result.data["assetNodeOrError"] + assert set(third_kinds_key["kinds"]) == {"python"} + + result = execute_dagster_graphql( + graphql_context, + GET_KINDS, + variables={ + "assetKey": {"path": ["fourth_kinds_key"]}, + }, + ) + + fourth_kinds_key = result.data["assetNodeOrError"] + assert set(fourth_kinds_key["kinds"]) == {"python"} + def test_has_asset_checks(self, graphql_context: WorkspaceRequestContext): result = execute_dagster_graphql(graphql_context, HAS_ASSET_CHECKS) diff --git a/python_modules/dagster/dagster/_core/definitions/asset_graph.py b/python_modules/dagster/dagster/_core/definitions/asset_graph.py index c75210656c3e7..33477e5996e47 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_graph.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_graph.py @@ -79,6 +79,10 @@ def metadata(self) -> ArbitraryMetadataMapping: def tags(self) -> Mapping[str, str]: return self._spec.tags + @property + def kinds(self) -> AbstractSet[str]: + return self._spec.kinds or set() + @property def owners(self) -> Sequence[str]: return self._spec.owners diff --git a/python_modules/dagster/dagster/_core/definitions/asset_spec.py b/python_modules/dagster/dagster/_core/definitions/asset_spec.py index 1ec983e4ad4e0..b088a17483e88 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_spec.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_spec.py @@ -1,6 +1,6 @@ from enum import Enum from functools import cached_property -from typing import TYPE_CHECKING, Any, Iterable, Mapping, NamedTuple, Optional, Sequence +from typing import TYPE_CHECKING, Any, Iterable, Mapping, NamedTuple, Optional, Sequence, Set import dagster._check as check from dagster._annotations import PublicAttr, experimental_param @@ -17,6 +17,8 @@ validate_asset_owner, validate_tags_strict, ) +from dagster._core.errors import DagsterInvalidDefinitionError +from dagster._core.storage.tags import KIND_PREFIX from dagster._serdes.serdes import whitelist_for_serdes from dagster._utils.internal_init import IHasInternalInit @@ -150,6 +152,10 @@ def __new__( for owner in owners: validate_asset_owner(owner, key) + kind_tags = {tag_key for tag_key in (tags or {}).keys() if tag_key.startswith(KIND_PREFIX)} + if kind_tags is not None and len(kind_tags) > 2: + raise DagsterInvalidDefinitionError("Assets can have at most two kinds currently.") + return super().__new__( cls, key=key, @@ -225,3 +231,7 @@ def auto_materialize_policy(self) -> Optional[AutoMaterializePolicy]: if self.automation_condition else None ) + + @cached_property + def kinds(self) -> Set[str]: + return {tag[len(KIND_PREFIX) :] for tag in self.tags if tag.startswith(KIND_PREFIX)} diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py b/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py index 344380ed182c6..b56d0fe280c2c 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/decorator_assets_definition_builder.py @@ -284,6 +284,11 @@ def for_multi_asset( if args.asset_out_map and args.specs: raise DagsterInvalidDefinitionError("Must specify only outs or specs but not both.") + if args.compute_kind and args.specs and any(spec.kinds for spec in args.specs): + raise DagsterInvalidDefinitionError( + "Can not specify compute_kind on both the @multi_asset and kinds on AssetSpecs." + ) + if args.specs: check.invariant( args.decorator_name == "@multi_asset", "Only hit this code path in multi_asset." diff --git a/python_modules/dagster/dagster/_core/definitions/utils.py b/python_modules/dagster/dagster/_core/definitions/utils.py index 48437b7819722..97cad78c18c79 100644 --- a/python_modules/dagster/dagster/_core/definitions/utils.py +++ b/python_modules/dagster/dagster/_core/definitions/utils.py @@ -226,7 +226,14 @@ def normalize_tags( # Inspired by allowed Kubernetes labels: # https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set -VALID_DEFINITION_TAG_KEY_REGEX_STR = r"^([A-Za-z0-9_.-]{1,63}/)?[A-Za-z0-9_.-]{1,63}$" + +# We allow in some cases for users to specify multi-level namespaces for tags, +# right now we only allow this for the `dagster/kind` namespace, which is how asset kinds are +# encoded under the hood. +VALID_NESTED_NAMESPACES_TAG_KEYS = r"dagster/kind/" +VALID_DEFINITION_TAG_KEY_REGEX_STR = ( + r"^([A-Za-z0-9_.-]{1,63}/|" + VALID_NESTED_NAMESPACES_TAG_KEYS + r")?[A-Za-z0-9_.-]{1,63}$" +) VALID_DEFINITION_TAG_KEY_REGEX = re.compile(VALID_DEFINITION_TAG_KEY_REGEX_STR) VALID_DEFINITION_TAG_KEY_EXPLANATION = ( "Allowed characters: alpha-numeric, '_', '-', '.'. " diff --git a/python_modules/dagster/dagster/_core/storage/tags.py b/python_modules/dagster/dagster/_core/storage/tags.py index 727aa0b029a78..b7c7923efe7d5 100644 --- a/python_modules/dagster/dagster/_core/storage/tags.py +++ b/python_modules/dagster/dagster/_core/storage/tags.py @@ -5,6 +5,8 @@ SYSTEM_TAG_PREFIX = "dagster/" HIDDEN_TAG_PREFIX = ".dagster/" +KIND_PREFIX = f"{SYSTEM_TAG_PREFIX}kind/" + REPOSITORY_LABEL_TAG = f"{HIDDEN_TAG_PREFIX}repository" SCHEDULE_NAME_TAG = f"{SYSTEM_TAG_PREFIX}schedule_name" diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py index e4e7ce33a4663..092cac24fffb3 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py @@ -2290,6 +2290,42 @@ def assets(): ... def assets(): ... +def test_asset_spec_with_kinds() -> None: + @multi_asset(specs=[AssetSpec("asset1", tags={"dagster/kind/python": ""})]) + def assets(): ... + + assert assets.specs_by_key[AssetKey("asset1")].kinds == {"python"} + + with pytest.raises( + DagsterInvalidDefinitionError, match="Assets can have at most two kinds currently." + ): + + @multi_asset( + specs=[ + AssetSpec( + "asset1", + tags={ + "dagster/kind/python": "", + "dagster/kind/snowflake": "", + "dagster/kind/bigquery": "", + }, + ) + ] + ) + def assets2(): ... + + with pytest.raises( + DagsterInvalidDefinitionError, + match="Can not specify compute_kind on both the @multi_asset and kinds on AssetSpecs.", + ): + + @multi_asset( + compute_kind="my_compute_kind", + specs=[AssetSpec("asset1", tags={"dagster/kind/python": ""})], + ) + def assets3(): ... + + def test_asset_out_with_tags(): @multi_asset(outs={"asset1": AssetOut(tags={"a": "b"})}) def assets(): ... diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_utils.py b/python_modules/dagster/dagster_tests/definitions_tests/test_utils.py index a52b6493a6cb1..5c0f47227dbe0 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_utils.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_utils.py @@ -9,6 +9,15 @@ from dagster._core.errors import DagsterInvariantViolationError +def test_is_valid_definition_tag_key_kinds() -> None: + assert is_valid_definition_tag_key("dagster/kind/foo") is True + assert is_valid_definition_tag_key("dagster/kind/foo.bar") is True + assert is_valid_definition_tag_key("dagster/kind/") is False + assert is_valid_definition_tag_key("dagster/kind/" + "a" * 63) is True + + assert is_valid_definition_tag_key("dragster/kind/foo") is False + + def test_is_valid_definition_tag_key(): assert is_valid_definition_tag_key("abc") is True assert is_valid_definition_tag_key("abc.xhz") is True