kelp.models.jsonschema¶
Reference for the jsonschema configuration model.
pydantic-model
¶
Bases: BaseModel
Base model for json schema generation.
Show JSON schema:
{
"$defs": {
"AbacMatchColumn": {
"description": "A MATCH COLUMNS clause item.",
"properties": {
"condition": {
"description": "hasTag/hasTagValue condition expression",
"title": "Condition",
"type": "string"
},
"alias": {
"description": "Alias used in USING COLUMNS",
"title": "Alias",
"type": "string"
}
},
"required": [
"condition",
"alias"
],
"title": "AbacMatchColumn",
"type": "object"
},
"AbacPolicy": {
"description": "ABAC policy definition for Unity Catalog.",
"properties": {
"name": {
"description": "Policy name",
"title": "Name",
"type": "string"
},
"securable_type": {
"description": "Target securable type",
"enum": [
"CATALOG",
"SCHEMA",
"TABLE"
],
"title": "Securable Type",
"type": "string"
},
"securable_name": {
"description": "Target securable fully qualified name",
"title": "Securable Name",
"type": "string"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Policy comment",
"title": "Description"
},
"mode": {
"description": "Policy operation mode",
"enum": [
"ROW_FILTER",
"COLUMN_MASK"
],
"title": "Mode",
"type": "string"
},
"udf_name": {
"description": "UDF name referenced by policy",
"title": "Udf Name",
"type": "string"
},
"target_column": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Target column for COLUMN MASK policies",
"title": "Target Column"
},
"principals_to": {
"description": "Principals in TO clause",
"items": {
"type": "string"
},
"title": "Principals To",
"type": "array"
},
"principals_except": {
"description": "Principals in EXCEPT clause",
"items": {
"type": "string"
},
"title": "Principals Except",
"type": "array"
},
"for_tables_when": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional FOR TABLES WHEN expression",
"title": "For Tables When"
},
"match_columns": {
"description": "MATCH COLUMNS clauses",
"items": {
"$ref": "#/$defs/AbacMatchColumn"
},
"title": "Match Columns",
"type": "array"
},
"using_columns": {
"description": "USING COLUMNS aliases",
"items": {
"type": "string"
},
"title": "Using Columns",
"type": "array"
},
"meta": {
"additionalProperties": true,
"description": "Generic user-defined metadata for filtering and grouping",
"title": "Meta",
"type": "object"
}
},
"required": [
"name",
"securable_type",
"securable_name",
"mode",
"udf_name"
],
"title": "AbacPolicy",
"type": "object"
},
"Column": {
"description": "Column definition for a model.",
"properties": {
"name": {
"description": "Column name",
"title": "Name",
"type": "string"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Human-readable description of the column",
"title": "Description"
},
"data_type": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "SQL data type of the column",
"title": "Data Type"
},
"nullable": {
"default": true,
"description": "Whether the column allows NULL values",
"title": "Nullable",
"type": "boolean"
},
"generated": {
"anyOf": [
{
"discriminator": {
"mapping": {
"expression": "#/$defs/GeneratedExpressionColumnConfig",
"identity": "#/$defs/GeneratedIdentityColumnConfig"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/$defs/GeneratedIdentityColumnConfig"
},
{
"$ref": "#/$defs/GeneratedExpressionColumnConfig"
}
]
},
{
"type": "null"
}
],
"default": null,
"description": "Configuration for generated columns (identity or expression based)",
"title": "Generated"
},
"tags": {
"additionalProperties": {
"type": "string"
},
"description": "Metadata tags for the column",
"title": "Tags",
"type": "object"
}
},
"required": [
"name"
],
"title": "Column",
"type": "object"
},
"ColumnPolicyRule": {
"description": "Policy rules for column-level governance checks.\n\nAttributes:\n require_description: Each column must have a non-empty description.\n require_tags: Specific tag keys that must be present on each column.\n require_any_tag: At least one tag must exist on each column.\n naming_pattern: Regex pattern that column names must match.\n naming_patterns_by_type: Naming patterns for specific data types.\n not_: Negate checks for this rule block (YAML key: ``not``).\n severity: Severity when this rule is violated.",
"properties": {
"require_description": {
"default": false,
"description": "Each column must have a non-empty description",
"title": "Require Description",
"type": "boolean"
},
"require_tags": {
"description": "Specific tag keys that must be present on each column",
"items": {
"type": "string"
},
"title": "Require Tags",
"type": "array"
},
"require_any_tag": {
"default": false,
"description": "At least one tag must exist on each column",
"title": "Require Any Tag",
"type": "boolean"
},
"naming_pattern": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Regex pattern that all column names must match",
"title": "Naming Pattern"
},
"naming_patterns_by_type": {
"description": "Regex patterns for column names by data type",
"items": {
"$ref": "#/$defs/NamingPattern"
},
"title": "Naming Patterns By Type",
"type": "array"
},
"not": {
"default": false,
"description": "Negate checks for this rule block",
"title": "Not",
"type": "boolean"
},
"severity": {
"$ref": "#/$defs/PolicySeverity",
"default": "warn",
"description": "Severity level when a rule is violated"
}
},
"title": "ColumnPolicyRule",
"type": "object"
},
"DQXQuality": {
"properties": {
"engine": {
"const": "dqx",
"description": "Quality engine type",
"title": "Engine",
"type": "string"
},
"level": {
"default": "row",
"description": "Level at which quality is enforced",
"enum": [
"row",
"table"
],
"title": "Level",
"type": "string"
},
"sdp_expect_level": {
"default": "warn",
"description": "Action for quality violations: warn, fail, drop, or deactivate",
"enum": [
"warn",
"fail",
"drop",
"deactivate"
],
"title": "Sdp Expect Level",
"type": "string"
},
"sdp_quarantine": {
"default": false,
"description": "Whether to quarantine rows failing quality checks",
"title": "Sdp Quarantine",
"type": "boolean"
},
"checks": {
"description": "Quality check configurations",
"items": {
"additionalProperties": true,
"type": "object"
},
"title": "Checks",
"type": "array"
}
},
"required": [
"engine"
],
"title": "DQXQuality",
"type": "object"
},
"ForeignKeyConstraint": {
"properties": {
"name": {
"description": "Constraint name",
"title": "Name",
"type": "string"
},
"type": {
"default": "foreign_key",
"description": "Constraint type identifier",
"title": "Type",
"type": "string"
},
"columns": {
"description": "List of local column names",
"items": {
"type": "string"
},
"title": "Columns",
"type": "array"
},
"reference_table": {
"description": "Fully qualified name of the referenced table",
"title": "Reference Table",
"type": "string"
},
"reference_columns": {
"description": "List of column names in the referenced table",
"items": {
"type": "string"
},
"title": "Reference Columns",
"type": "array"
}
},
"required": [
"name",
"reference_table"
],
"title": "ForeignKeyConstraint",
"type": "object"
},
"FunctionEnvironment": {
"description": "Environment clause for Python functions.",
"properties": {
"dependencies": {
"description": "Python package dependencies and wheel references",
"items": {
"type": "string"
},
"title": "Dependencies",
"type": "array"
},
"environment_version": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Python environment version (Databricks currently supports 'None')",
"title": "Environment Version"
}
},
"title": "FunctionEnvironment",
"type": "object"
},
"FunctionParameter": {
"description": "Function parameter definition.",
"properties": {
"name": {
"description": "Parameter name",
"title": "Name",
"type": "string"
},
"data_type": {
"description": "Parameter SQL data type",
"title": "Data Type",
"type": "string"
},
"default_expression": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional SQL default expression",
"title": "Default Expression"
},
"comment": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional parameter comment",
"title": "Comment"
}
},
"required": [
"name",
"data_type"
],
"title": "FunctionParameter",
"type": "object"
},
"FunctionReturnColumn": {
"description": "Column definition for table-valued function return schema.",
"properties": {
"name": {
"description": "Return column name",
"title": "Name",
"type": "string"
},
"data_type": {
"description": "Return column SQL data type",
"title": "Data Type",
"type": "string"
},
"comment": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional return column comment",
"title": "Comment"
}
},
"required": [
"name",
"data_type"
],
"title": "FunctionReturnColumn",
"type": "object"
},
"GeneratedExpressionColumnConfig": {
"properties": {
"type": {
"const": "expression",
"description": "Column type identifier",
"title": "Type",
"type": "string"
},
"expression": {
"description": "SQL expression used to generate the column value",
"title": "Expression",
"type": "string"
}
},
"required": [
"type",
"expression"
],
"title": "GeneratedExpressionColumnConfig",
"type": "object"
},
"GeneratedIdentityColumnConfig": {
"properties": {
"type": {
"const": "identity",
"description": "Column type identifier",
"title": "Type",
"type": "string"
},
"as_default": {
"default": false,
"description": "Generated as default (True) or always (False)",
"title": "As Default",
"type": "boolean"
},
"start_with": {
"default": 1,
"description": "Starting value for the identity sequence",
"title": "Start With",
"type": "integer"
},
"increment_by": {
"default": 1,
"description": "Increment step for the identity sequence",
"title": "Increment By",
"type": "integer"
}
},
"required": [
"type"
],
"title": "GeneratedIdentityColumnConfig",
"type": "object"
},
"KelpFunction": {
"description": "Function definition in Unity Catalog.\n\nThis model validates function metadata and body source location, while leaving\nSQL/Python body parsing to Databricks at execution time.",
"properties": {
"name": {
"description": "Function name",
"title": "Name",
"type": "string"
},
"catalog": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Unity Catalog name",
"title": "Catalog"
},
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Schema name",
"title": "Schema"
},
"language": {
"default": "SQL",
"description": "Function implementation language",
"enum": [
"SQL",
"PYTHON"
],
"title": "Language",
"type": "string"
},
"temporary": {
"default": false,
"description": "Whether the function is temporary (session scoped)",
"title": "Temporary",
"type": "boolean"
},
"if_not_exists": {
"default": false,
"description": "Create only if function does not already exist",
"title": "If Not Exists",
"type": "boolean"
},
"or_replace": {
"default": true,
"description": "Use CREATE OR REPLACE semantics",
"title": "Or Replace",
"type": "boolean"
},
"deterministic": {
"anyOf": [
{
"type": "boolean"
},
{
"type": "null"
}
],
"default": null,
"description": "Deterministic characteristic",
"title": "Deterministic"
},
"data_access": {
"anyOf": [
{
"enum": [
"CONTAINS SQL",
"READS SQL DATA"
],
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional SQL data access characteristic",
"title": "Data Access"
},
"default_collation": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Default collation for SQL functions",
"title": "Default Collation"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Function comment",
"title": "Description"
},
"parameters": {
"description": "Function parameter definitions",
"items": {
"$ref": "#/$defs/FunctionParameter"
},
"title": "Parameters",
"type": "array"
},
"returns_data_type": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Scalar return data type",
"title": "Returns Data Type"
},
"returns_table": {
"description": "Table-valued return columns (for RETURNS TABLE)",
"items": {
"$ref": "#/$defs/FunctionReturnColumn"
},
"title": "Returns Table",
"type": "array"
},
"body": {
"default": "",
"description": "Function body text, normalized from inline content or file source",
"title": "Body",
"type": "string"
},
"body_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Optional path to external .sql/.py body file, relative to project root",
"title": "Body Path"
},
"environment": {
"anyOf": [
{
"$ref": "#/$defs/FunctionEnvironment"
},
{
"type": "null"
}
],
"default": null,
"description": "Python environment declaration"
},
"tags": {
"additionalProperties": {
"type": "string"
},
"description": "Metadata tags",
"title": "Tags",
"type": "object"
},
"meta": {
"additionalProperties": true,
"description": "Generic user-defined metadata for filtering and grouping",
"title": "Meta",
"type": "object"
}
},
"required": [
"name"
],
"title": "KelpFunction",
"type": "object"
},
"MetricView": {
"description": "Metric View definition for Databricks.\n\nAttributes:\n name: The name of the metric view.\n catalog: The catalog where the metric view will be created.\n schema_: The schema where the metric view will be created.\n description: Optional description of the metric view.\n definition: The metric view definition as a dictionary. This should contain\n the full metric view specification including dimensions, metrics,\n and the underlying table.\n tags: Optional tags for the metric view.\n origin_file_path: Path to the source YAML file (internal use).\n raw_config: Preserve original, unparsed config (including placeholder vars).",
"properties": {
"name": {
"title": "Name",
"type": "string"
},
"catalog": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Catalog"
},
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Schema"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Description"
},
"definition": {
"additionalProperties": true,
"description": "The metric view definition including dimensions, metrics, and source table",
"title": "Definition",
"type": "object"
},
"tags": {
"additionalProperties": {
"type": "string"
},
"title": "Tags",
"type": "object"
},
"meta": {
"additionalProperties": true,
"description": "Generic user-defined metadata for filtering and grouping",
"title": "Meta",
"type": "object"
}
},
"required": [
"name"
],
"title": "MetricView",
"type": "object"
},
"Model": {
"description": "Model definition in Unity Catalog.\n\nRepresents a metadata model in Kelp with configuration that maps to\nDatabricks Unity Catalog objects.",
"properties": {
"table_type": {
"$ref": "#/$defs/TableType",
"default": "managed",
"description": "Type of table: managed, external, view, streaming_table, etc."
},
"catalog": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Unity Catalog name",
"title": "Catalog"
},
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Schema/database name",
"title": "Schema"
},
"name": {
"description": "Model name",
"title": "Name",
"type": "string"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Human-readable description of the model",
"title": "Description"
},
"spark_conf": {
"additionalProperties": true,
"description": "Spark configuration properties",
"title": "Spark Conf",
"type": "object"
},
"table_properties": {
"additionalProperties": true,
"description": "Databricks table properties",
"title": "Table Properties",
"type": "object"
},
"path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Physical path for external tables or custom locations",
"title": "Path"
},
"partition_cols": {
"description": "List of column names for partitioning",
"items": {
"type": "string"
},
"title": "Partition Cols",
"type": "array"
},
"cluster_by_auto": {
"default": false,
"description": "Enable automatic clustering optimization",
"title": "Cluster By Auto",
"type": "boolean"
},
"cluster_by": {
"description": "List of column names for explicit clustering (max 4)",
"items": {
"type": "string"
},
"maxItems": 4,
"title": "Cluster By",
"type": "array"
},
"row_filter": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "SQL expression to filter rows based on security policies",
"title": "Row Filter"
},
"columns": {
"description": "Column definitions for the model",
"items": {
"$ref": "#/$defs/Column"
},
"title": "Columns",
"type": "array"
},
"quality": {
"anyOf": [
{
"discriminator": {
"mapping": {
"dqx": "#/$defs/DQXQuality",
"sdp": "#/$defs/SDPQuality"
},
"propertyName": "engine"
},
"oneOf": [
{
"$ref": "#/$defs/SDPQuality"
},
{
"$ref": "#/$defs/DQXQuality"
}
]
},
{
"type": "null"
}
],
"default": null,
"description": "Data quality configuration using SDPQuality or DQXQuality",
"title": "Quality"
},
"constraints": {
"description": "Constraints like primary key or foreign key",
"items": {
"anyOf": [
{
"$ref": "#/$defs/PrimaryKeyConstraint"
},
{
"$ref": "#/$defs/ForeignKeyConstraint"
}
]
},
"title": "Constraints",
"type": "array"
},
"tags": {
"additionalProperties": {
"type": "string"
},
"description": "Metadata tags for the model",
"title": "Tags",
"type": "object"
},
"meta": {
"additionalProperties": true,
"description": "Generic user-defined metadata for filtering and grouping",
"title": "Meta",
"type": "object"
}
},
"required": [
"name"
],
"title": "Model",
"type": "object"
},
"ModelPolicyRule": {
"description": "Policy rules for table-level governance checks.\n\nAttributes:\n require_description: Table must have a non-empty description.\n require_tags: Specific tag keys that must be present on the table.\n require_any_tag: At least one tag must exist on the table.\n require_constraints: Constraint types that must be defined (e.g. \"primary_key\").\n naming_pattern: Regex pattern that table names must match.\n has_columns: Column names that must be present in the table.\n has_table_property: Table properties that must exist (partial match, allows extra keys).\n has_quality_check: Table must have quality checks configured.\n not_: Negate checks for this rule block (YAML key: ``not``).\n severity: Severity when this rule is violated.",
"properties": {
"require_description": {
"default": false,
"description": "Table must have a non-empty description",
"title": "Require Description",
"type": "boolean"
},
"require_tags": {
"description": "Specific tag keys that must be present on the table",
"items": {
"type": "string"
},
"title": "Require Tags",
"type": "array"
},
"require_any_tag": {
"default": false,
"description": "At least one tag must exist on the table",
"title": "Require Any Tag",
"type": "boolean"
},
"require_constraints": {
"description": "Constraint types that must be defined (e.g. 'primary_key', 'foreign_key')",
"items": {
"type": "string"
},
"title": "Require Constraints",
"type": "array"
},
"naming_pattern": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Regex pattern that table names must match",
"title": "Naming Pattern"
},
"has_columns": {
"description": "Column names that must be present in the table",
"items": {
"type": "string"
},
"title": "Has Columns",
"type": "array"
},
"has_table_property": {
"additionalProperties": true,
"description": "Table properties that must exist (partial match, allows extra keys)",
"title": "Has Table Property",
"type": "object"
},
"has_quality_check": {
"default": false,
"description": "Table must have quality checks configured",
"title": "Has Quality Check",
"type": "boolean"
},
"not": {
"default": false,
"description": "Negate checks for this rule block",
"title": "Not",
"type": "boolean"
},
"severity": {
"$ref": "#/$defs/PolicySeverity",
"default": "warn",
"description": "Severity level when a rule is violated"
}
},
"title": "ModelPolicyRule",
"type": "object"
},
"NamingPattern": {
"description": "Naming convention rule for a specific data type.\n\nAttributes:\n data_type: SQL data type this pattern applies to (e.g., \"STRING\", \"BOOLEAN\").\n pattern: Regex pattern that column names must match (e.g., \"^(is_|has_).*\").",
"properties": {
"data_type": {
"description": "SQL data type (e.g. STRING, BOOLEAN, INT)",
"title": "Data Type",
"type": "string"
},
"pattern": {
"description": "Regex pattern column names must match",
"title": "Pattern",
"type": "string"
}
},
"required": [
"data_type",
"pattern"
],
"title": "NamingPattern",
"type": "object"
},
"Policy": {
"description": "Policy definition for a specific scope or layer.\n\nRepresents a set of governance rules that apply to tables in specified paths.\n\nAttributes:\n origin_file_path: Path to the source YAML file defining this policy.\n name: Policy name (e.g., \"bronze_layer\", \"data_quality\").\n applies_to: Glob pattern for models this policy applies to (e.g., \"models/bronze/*\").\n Must start with \"models/\" and use forward slashes. Matched against the\n `origin_file_path` of models. See module docstring for examples.\n model: Rules applied at the model level.\n column: Rules applied at the column level.\n raw_config: Original unparsed configuration.",
"properties": {
"name": {
"description": "Policy name",
"title": "Name",
"type": "string"
},
"applies_to": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Glob pattern for models this applies to (e.g. 'models/bronze/*'). Must start with 'models/' and use forward slashes.",
"title": "Applies To"
},
"model": {
"$ref": "#/$defs/ModelPolicyRule",
"description": "Model-level policy rules"
},
"column": {
"$ref": "#/$defs/ColumnPolicyRule",
"description": "Column-level policy rules"
},
"meta": {
"additionalProperties": true,
"description": "Generic user-defined metadata for filtering and grouping",
"title": "Meta",
"type": "object"
}
},
"required": [
"name"
],
"title": "Policy",
"type": "object"
},
"PolicyConfig": {
"description": "Top-level policy execution switch.\n\nPolicy rules are authored in policy files (``kelp_policies``). This model\nintentionally keeps only the global enable/disable flag for applying policy\nchecks during context initialization and via the ``check-policies`` CLI.\n\nAttributes:\n enabled: Master switch to activate policy checks (default: False).\n fast_exit: Stop policy evaluation on first violating policy per model.",
"properties": {
"enabled": {
"default": false,
"description": "Master switch to activate policy checks",
"title": "Enabled",
"type": "boolean"
},
"fast_exit": {
"default": false,
"description": "Stop policy evaluation on first violating policy per model",
"title": "Fast Exit",
"type": "boolean"
}
},
"title": "PolicyConfig",
"type": "object"
},
"PolicySeverity": {
"description": "Severity level for policy violations.\n\nAttributes:\n warn: Log a warning but do not fail.\n error: Raise an error and stop processing.",
"enum": [
"warn",
"error"
],
"title": "PolicySeverity",
"type": "string"
},
"PrimaryKeyConstraint": {
"properties": {
"name": {
"description": "Constraint name",
"title": "Name",
"type": "string"
},
"type": {
"default": "primary_key",
"description": "Constraint type identifier",
"title": "Type",
"type": "string"
},
"columns": {
"description": "List of column names forming the primary key",
"items": {
"type": "string"
},
"title": "Columns",
"type": "array"
}
},
"required": [
"name"
],
"title": "PrimaryKeyConstraint",
"type": "object"
},
"ProjectConfig": {
"description": "Project-level configuration for a kelp project.\n\nAttributes:\n models_path: Path to table/model definitions.\n models: Configuration hierarchy for models.\n metrics_path: Path to metric view definitions.\n metric_views: Configuration hierarchy for metric views.\n functions_path: Path to function definitions.\n functions: Configuration hierarchy for functions.\n abacs_path: Path to ABAC policy definitions.\n abacs: Configuration hierarchy for ABAC policies.\n sources_path: Path to source definitions.\n sources: Configuration hierarchy for sources.\n quarantine_config: Configuration for table quarantine and validation.\n remote_catalog_config: Configuration for remote catalog synchronization.\n runtime_vars: Runtime variables (internal use).\n project_file_path: Path to the project configuration file (internal use).",
"properties": {
"models_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to table/model definitions",
"title": "Models Path"
},
"models": {
"additionalProperties": true,
"description": "Configuration hierarchy for models",
"title": "Models",
"type": "object"
},
"metrics_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to metric view definitions",
"title": "Metrics Path"
},
"metric_views": {
"additionalProperties": true,
"description": "Configuration hierarchy for metric views",
"title": "Metric Views",
"type": "object"
},
"functions_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to function definitions",
"title": "Functions Path"
},
"functions": {
"additionalProperties": true,
"description": "Configuration hierarchy for functions",
"title": "Functions",
"type": "object"
},
"abacs_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to ABAC policy definitions",
"title": "Abacs Path"
},
"abacs": {
"additionalProperties": true,
"description": "Configuration hierarchy for ABAC policies",
"title": "Abacs",
"type": "object"
},
"sources_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to source definitions",
"title": "Sources Path"
},
"sources": {
"additionalProperties": true,
"description": "Configuration hierarchy for sources",
"title": "Sources",
"type": "object"
},
"policies_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to policy definitions",
"title": "Policies Path"
},
"quarantine_config": {
"$ref": "#/$defs/QuarantineConfig",
"description": "Configuration for table quarantine and validation"
},
"remote_catalog_config": {
"$ref": "#/$defs/RemoteCatalogConfig",
"description": "Configuration for remote catalog synchronization"
},
"policy_config": {
"$ref": "#/$defs/PolicyConfig",
"description": "Metadata governance policy configuration"
}
},
"title": "ProjectConfig",
"type": "object"
},
"QuarantineConfig": {
"description": "Configuration for table quarantine and validation.\n\nAttributes:\n quarantine_catalog: Catalog name for quarantined data (defaults to project catalog).\n quarantine_schema: Schema name for quarantined data (defaults to project schema).\n quarantine_prefix: Prefix for quarantine table names.\n quarantine_suffix: Suffix for quarantine table names.\n validation_prefix: Prefix for validation table names.\n validation_suffix: Suffix for validation table names.",
"properties": {
"quarantine_catalog": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Catalog for quarantined data",
"title": "Quarantine Catalog"
},
"quarantine_schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Schema for quarantined data",
"title": "Quarantine Schema"
},
"quarantine_prefix": {
"default": "",
"description": "Prefix for quarantine table names",
"title": "Quarantine Prefix",
"type": "string"
},
"quarantine_suffix": {
"default": "_quarantine",
"description": "Suffix for quarantine table names",
"title": "Quarantine Suffix",
"type": "string"
},
"validation_prefix": {
"default": "",
"description": "Prefix for validation table names",
"title": "Validation Prefix",
"type": "string"
},
"validation_suffix": {
"default": "_validation",
"description": "Suffix for validation table names",
"title": "Validation Suffix",
"type": "string"
}
},
"title": "QuarantineConfig",
"type": "object"
},
"RemoteCatalogConfig": {
"description": "Configuration for remote Databricks catalog synchronization.\n\nAttributes:\n table_tag_mode: How to apply table tags (\"append\", \"replace\", \"managed\").\n managed_table_tags: List of tag keys to manage on tables.\n column_tag_mode: How to apply column tags (\"append\", \"replace\", \"managed\").\n managed_column_tags: List of tag keys to manage on columns.\n table_property_mode: How to apply table properties (\"append\" or \"managed\").\n managed_table_properties: List of property keys to manage on tables.",
"properties": {
"table_tag_mode": {
"default": "replace",
"description": "How to apply table tags to remote catalog",
"enum": [
"append",
"replace",
"managed"
],
"title": "Table Tag Mode",
"type": "string"
},
"managed_table_tags": {
"description": "List of tag keys to manage on tables",
"items": {
"type": "string"
},
"title": "Managed Table Tags",
"type": "array"
},
"column_tag_mode": {
"default": "replace",
"description": "How to apply column tags to remote catalog",
"enum": [
"append",
"replace",
"managed"
],
"title": "Column Tag Mode",
"type": "string"
},
"managed_column_tags": {
"description": "List of tag keys to manage on columns",
"items": {
"type": "string"
},
"title": "Managed Column Tags",
"type": "array"
},
"table_property_mode": {
"default": "append",
"description": "How to apply table properties to remote catalog",
"enum": [
"append",
"managed"
],
"title": "Table Property Mode",
"type": "string"
},
"managed_table_properties": {
"description": "List of property keys to manage on tables",
"items": {
"type": "string"
},
"title": "Managed Table Properties",
"type": "array"
}
},
"title": "RemoteCatalogConfig",
"type": "object"
},
"SDPQuality": {
"properties": {
"engine": {
"const": "sdp",
"description": "Quality engine type",
"title": "Engine",
"type": "string"
},
"level": {
"const": "row",
"default": "row",
"description": "Quality enforcement level",
"title": "Level",
"type": "string"
},
"expect_all": {
"additionalProperties": {
"type": "string"
},
"description": "SQL expressions that must pass",
"title": "Expect All",
"type": "object"
},
"expect_all_or_drop": {
"additionalProperties": {
"type": "string"
},
"description": "SQL expressions; failing rows are dropped",
"title": "Expect All Or Drop",
"type": "object"
},
"expect_all_or_fail": {
"additionalProperties": {
"type": "string"
},
"description": "SQL expressions; job fails if any expression fails",
"title": "Expect All Or Fail",
"type": "object"
},
"expect_all_or_quarantine": {
"additionalProperties": {
"type": "string"
},
"description": "SQL expressions; failing rows are quarantined",
"title": "Expect All Or Quarantine",
"type": "object"
}
},
"required": [
"engine"
],
"title": "SDPQuality",
"type": "object"
},
"Source": {
"description": "Source definition for Kelp projects.\n\nRepresents a data source (volume, table, or raw path) that can be referenced\nin pipelines using kp.source(). Sources provide a single point of configuration\nfor data locations and connection options.\n\nAttributes:\n origin_file_path: Path to the source YAML file defining this source.\n name: Source name (unique identifier).\n source_type: Type of source ('volume', 'table', 'raw_path').\n path: Physical path for volume or raw_path sources (e.g., '/Volumes/catalog/schema/volume').\n catalog: Catalog name for table or volume sources.\n schema_: Schema name for table or volume sources.\n table_name: Name for table or view sources.\n volume_name: Volume name for volume sources (constructs path as /Volumes/catalog/schema/volume_name).\n options: Dictionary of options specific to this source (e.g., format, headers).\n description: Human-readable description of the source.",
"properties": {
"name": {
"description": "Source name (unique identifier)",
"title": "Name",
"type": "string"
},
"source_type": {
"default": "table",
"description": "Type of source: volume, table, or raw_path",
"enum": [
"volume",
"table",
"raw_path"
],
"title": "Source Type",
"type": "string"
},
"path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Physical path for volume or raw_path sources",
"title": "Path"
},
"catalog": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Catalog name for table or volume sources",
"title": "Catalog"
},
"schema": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Schema name for table or volume sources",
"title": "Schema"
},
"table_name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Name for table or view sources",
"title": "Table Name"
},
"volume_name": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Volume name for volume sources (constructs /Volumes/catalog/schema/volume_name)",
"title": "Volume Name"
},
"options": {
"additionalProperties": true,
"description": "Source-specific options (e.g., format, headers, encoding)",
"title": "Options",
"type": "object"
},
"description": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Human-readable description of the source",
"title": "Description"
},
"meta": {
"additionalProperties": true,
"description": "Generic user-defined metadata for filtering and grouping",
"title": "Meta",
"type": "object"
}
},
"required": [
"name"
],
"title": "Source",
"type": "object"
},
"TableType": {
"enum": [
"external",
"external_shallow_clone",
"foreign",
"managed",
"managed_shallow_clone",
"materialized_view",
"metric_view",
"streaming_table",
"view"
],
"title": "TableType",
"type": "string"
}
},
"additionalProperties": false,
"description": "Base model for json schema generation.",
"properties": {
"kelp_project": {
"anyOf": [
{
"$ref": "#/$defs/ProjectConfig"
},
{
"type": "null"
}
],
"default": null,
"description": "Project-level configuration for the Kelp project."
},
"kelp_models": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/Model"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "List of table models defined in the configuration.",
"title": "Kelp Models"
},
"kelp_sources": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/Source"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "List of data sources defined in the configuration.",
"title": "Kelp Sources"
},
"kelp_metric_views": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/MetricView"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "List of metric views defined in the configuration.",
"title": "Kelp Metric Views"
},
"kelp_functions": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/KelpFunction"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "List of functions defined in the configuration.",
"title": "Kelp Functions"
},
"kelp_policies": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/Policy"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "List of policies defined in the configuration.",
"title": "Kelp Policies"
},
"kelp_abacs": {
"anyOf": [
{
"items": {
"$ref": "#/$defs/AbacPolicy"
},
"type": "array"
},
{
"type": "null"
}
],
"default": null,
"description": "List of ABAC policies defined in the configuration.",
"title": "Kelp Abacs"
},
"vars": {
"anyOf": [
{
"additionalProperties": true,
"type": "object"
},
{
"type": "null"
}
],
"default": null,
"description": "Dictionary of variables that can be used in the configuration.",
"title": "Vars"
},
"vars_overwrite": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to a YAML file containing variables that overwrite the ones defined in the project file. May be added to .gitignore to set individual developer vars like schema names.",
"title": "Vars Overwrite"
},
"targets": {
"anyOf": [
{
"additionalProperties": true,
"type": "object"
},
{
"type": "null"
}
],
"default": null,
"description": "List of target configurations for deployment or other purposes.",
"title": "Targets"
},
"targets_path": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Path to an external YAML file or folder containing target configurations. This allows separating target definitions from the main project file.",
"title": "Targets Path"
}
},
"title": "Kelp",
"type": "object"
}
Config:
title:Kelpextra:forbid
Fields:
-
kelp_project(ProjectConfig | None) -
kelp_models(list[Model] | None) -
kelp_sources(list[Source] | None) -
kelp_metric_views(list[MetricView] | None) -
kelp_functions(list[KelpFunction] | None) -
kelp_policies(list[Policy] | None) -
kelp_abacs(list[AbacPolicy] | None) -
vars(dict[str, Any] | None) -
vars_overwrite(str | None) -
targets(dict[str, Any] | None) -
targets_path(str | None)
class-attribute
instance-attribute
¶
pydantic-field
¶
List of data sources defined in the configuration.
pydantic-field
¶
List of metric views defined in the configuration.
pydantic-field
¶
List of functions defined in the configuration.
pydantic-field
¶
Path to a YAML file containing variables that overwrite the ones defined in the project file. May be added to .gitignore to set individual developer vars like schema names.
pydantic-field
¶
List of target configurations for deployment or other purposes.
pydantic-field
¶
Path to an external YAML file or folder containing target configurations. This allows separating target definitions from the main project file.