kelp.models.model

Reference for the core model and column configuration models.

kelp.models.model.TableType

Bases: Enum

EXTERNAL class-attribute instance-attribute

EXTERNAL = 'external'

EXTERNAL_SHALLOW_CLONE class-attribute instance-attribute

EXTERNAL_SHALLOW_CLONE = 'external_shallow_clone'

FOREIGN class-attribute instance-attribute

FOREIGN = 'foreign'

MANAGED class-attribute instance-attribute

MANAGED = 'managed'

MANAGED_SHALLOW_CLONE class-attribute instance-attribute

MANAGED_SHALLOW_CLONE = 'managed_shallow_clone'

MATERIALIZED_VIEW class-attribute instance-attribute

MATERIALIZED_VIEW = 'materialized_view'

METRIC_VIEW class-attribute instance-attribute

METRIC_VIEW = 'metric_view'

STREAMING_TABLE class-attribute instance-attribute

STREAMING_TABLE = 'streaming_table'

VIEW class-attribute instance-attribute

VIEW = 'view'

kelp.models.model.Model pydantic-model

Bases: BaseModel

Model definition in Unity Catalog.

Represents a metadata model in Kelp with configuration that maps to Databricks Unity Catalog objects.

Show JSON schema:

{
  "$defs": {
    "AutoTTLConfig": {
      "properties": {
        "timestamp_column": {
          "description": "Name of the timestamp column used for TTL",
          "title": "Timestamp Column",
          "type": "string"
        },
        "expire_in_days": {
          "description": "Number of days after which data expires",
          "title": "Expire In Days",
          "type": "integer"
        }
      },
      "required": [
        "timestamp_column",
        "expire_in_days"
      ],
      "title": "AutoTTLConfig",
      "type": "object"
    },
    "Column": {
      "description": "Column definition for a model.",
      "properties": {
        "name": {
          "description": "Column name",
          "title": "Name",
          "type": "string"
        },
        "description": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Human-readable description of the column",
          "title": "Description"
        },
        "data_type": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "SQL data type of the column",
          "title": "Data Type"
        },
        "nullable": {
          "default": true,
          "description": "Whether the column allows NULL values",
          "title": "Nullable",
          "type": "boolean"
        },
        "generated": {
          "anyOf": [
            {
              "discriminator": {
                "mapping": {
                  "expression": "#/$defs/GeneratedExpressionColumnConfig",
                  "identity": "#/$defs/GeneratedIdentityColumnConfig"
                },
                "propertyName": "type"
              },
              "oneOf": [
                {
                  "$ref": "#/$defs/GeneratedIdentityColumnConfig"
                },
                {
                  "$ref": "#/$defs/GeneratedExpressionColumnConfig"
                }
              ]
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Configuration for generated columns (identity or expression based)",
          "title": "Generated"
        },
        "tags": {
          "additionalProperties": {
            "type": "string"
          },
          "description": "Metadata tags for the column",
          "title": "Tags",
          "type": "object"
        },
        "meta": {
          "additionalProperties": true,
          "description": "Generic user-defined metadata for filtering and grouping columns",
          "title": "Meta",
          "type": "object"
        }
      },
      "required": [
        "name"
      ],
      "title": "Column",
      "type": "object"
    },
    "DQXQuality": {
      "properties": {
        "engine": {
          "const": "dqx",
          "description": "Quality engine type",
          "title": "Engine",
          "type": "string"
        },
        "level": {
          "default": "row",
          "description": "Level at which quality is enforced",
          "enum": [
            "row",
            "table"
          ],
          "title": "Level",
          "type": "string"
        },
        "sdp_expect_level": {
          "default": "warn",
          "description": "Action for quality violations: warn, fail, drop, or deactivate",
          "enum": [
            "warn",
            "fail",
            "drop",
            "deactivate"
          ],
          "title": "Sdp Expect Level",
          "type": "string"
        },
        "sdp_quarantine": {
          "default": false,
          "description": "Whether to quarantine rows failing quality checks",
          "title": "Sdp Quarantine",
          "type": "boolean"
        },
        "spark_violation_action": {
          "default": "error",
          "description": "Action for quality violations at the Spark level: error (raise exception), ignore (store errors in output), or drop (exclude from output)",
          "enum": [
            "error",
            "ignore",
            "drop"
          ],
          "title": "Spark Violation Action",
          "type": "string"
        },
        "spark_quarantine": {
          "default": false,
          "description": "Whether to write rows failing quality checks to a quarantine table at the Spark level",
          "title": "Spark Quarantine",
          "type": "boolean"
        },
        "checks": {
          "description": "Quality check configurations",
          "items": {
            "additionalProperties": true,
            "type": "object"
          },
          "title": "Checks",
          "type": "array"
        }
      },
      "required": [
        "engine"
      ],
      "title": "DQXQuality",
      "type": "object"
    },
    "ForeignKeyConstraint": {
      "properties": {
        "name": {
          "description": "Constraint name",
          "title": "Name",
          "type": "string"
        },
        "type": {
          "default": "foreign_key",
          "description": "Constraint type identifier",
          "title": "Type",
          "type": "string"
        },
        "columns": {
          "description": "List of local column names",
          "items": {
            "type": "string"
          },
          "title": "Columns",
          "type": "array"
        },
        "reference_table": {
          "description": "Fully qualified name of the referenced table",
          "title": "Reference Table",
          "type": "string"
        },
        "reference_columns": {
          "description": "List of column names in the referenced table",
          "items": {
            "type": "string"
          },
          "title": "Reference Columns",
          "type": "array"
        }
      },
      "required": [
        "name",
        "reference_table"
      ],
      "title": "ForeignKeyConstraint",
      "type": "object"
    },
    "GeneratedExpressionColumnConfig": {
      "properties": {
        "type": {
          "const": "expression",
          "description": "Column type identifier",
          "title": "Type",
          "type": "string"
        },
        "expression": {
          "description": "SQL expression used to generate the column value",
          "title": "Expression",
          "type": "string"
        }
      },
      "required": [
        "type",
        "expression"
      ],
      "title": "GeneratedExpressionColumnConfig",
      "type": "object"
    },
    "GeneratedIdentityColumnConfig": {
      "properties": {
        "type": {
          "const": "identity",
          "description": "Column type identifier",
          "title": "Type",
          "type": "string"
        },
        "as_default": {
          "default": false,
          "description": "Generated as default (True) or always (False)",
          "title": "As Default",
          "type": "boolean"
        },
        "start_with": {
          "default": 1,
          "description": "Starting value for the identity sequence",
          "title": "Start With",
          "type": "integer"
        },
        "increment_by": {
          "default": 1,
          "description": "Increment step for the identity sequence",
          "title": "Increment By",
          "type": "integer"
        }
      },
      "required": [
        "type"
      ],
      "title": "GeneratedIdentityColumnConfig",
      "type": "object"
    },
    "ModelMaterializationConfig": {
      "description": "Configuration controlling how a kelp model is materialized.\n\nAll fields are optional; set only what you need.\n\nArgs:\n    write_mode: How to write the result \u2014 ``'append'``, ``'overwrite'``, or ``'merge'``.\n    When ``None`` (default) the DataFrame\n        is returned without writing.\n    options: Extra DataFrameWriter options\n        (e.g. ``{'mergeSchema': 'true'}``).\n    unique_keys: List of unique key columns for merge operations. Required when ``write_mode='merge'``.\n    target_alias: Alias to use for the target table in merge conditions. Default is ``'target'``.\n    source_alias: Alias to use for the source table in merge conditions. Default is ``'source'``.\n    predicates: Extra predicates to prefilter source and target data before the merge condition is applied.\n    matched_condition: Additional SQL condition to apply to matched rows in a merge.\n    matched_condition_exclude_cols: List of columns to exclude from the matched condition in a merge. Used to prevent certain columns from being evaluated in the matched condition, which can be useful for handling cases where certain columns may have non-deterministic values or should not be considered for matching.\n    matched_condition_include_cols: List of columns to include in the matched condition in a merge. Used to specify a subset of columns to be evaluated in the matched condition, which can be useful for optimizing the merge operation by only considering relevant columns for matching.\n    not_matched_condition: Additional SQL condition to apply to unmatched source rows in a merge.\n    not_matched_by_source_condition: Additional SQL condition to apply to target rows not found in source in a merge.\n    not_matched_by_source_action: Action to take for target rows not found in source in a merge. If ``'delete'`` rows will be deleted, else a update with the provided condition will be performed. Only applicable when ``write_mode='merge'``.\n    matched_update_exclude_cols: List of columns to exclude from updates in a merge. Only applicable when ``write_mode='merge'``. Used to prevent certain columns from being updated during a merge operation.\n    matched_update_include_cols: List of columns to include in updates in a merge. Only applicable when ``write_mode='merge'``. Used to specify a subset of columns to be updated during a merge operation.\n    merge_with_schema_evolution: Whether to enable schema evolution during merge operations. Only applicable when ``write_mode='merge'``.\n    exclude_at_target_cols: List of columns to exclude from the target dataset during a merge operation. Only applicable when ``write_mode='merge'.\n    include_at_target_cols: List of columns to include from the target dataset during a merge operation. Only applicable when ``write_mode='merge'``.\n    sequence_by: List of columns to order source and target datasets by before applying the merge condition. Only applicable when ``write_mode='merge'``.\n    ignore_null_updates: Whether to ignore updates to columns where the source value is null during a merge operation. Only applicable when ``write_mode='merge'``.\n    apply_as_delete: Used for CDC scenarios. Whether to apply a condition as a delete operation instead of an update or insert during a merge. Only applicable when ``write_mode='merge'``.\n    prevent_full_refresh: Whether to prevent full refreshes for this model. If True, attempts to perform a full refresh (e.g. via the ``'full_refresh'`` flag in the materialization context) will\n        be overridden to perform an append or merge instead, and a warning will be logged. This can be useful for protecting critical tables from accidental full refreshes.",
      "properties": {
        "write_mode": {
          "anyOf": [
            {
              "enum": [
                "append",
                "overwrite",
                "merge"
              ],
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "How to write the model: 'append', 'overwrite', or 'merge'. When None the DataFrame is returned without writing.",
          "title": "Write Mode"
        },
        "options": {
          "additionalProperties": true,
          "description": "Extra DataFrameWriter options (e.g. {'mergeSchema': 'true'}).",
          "title": "Options",
          "type": "object"
        },
        "unique_keys": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of unique key columns for merge operations. Required when write_mode='merge'. Used to validate the merge condition and ensure it references a unique key.",
          "title": "Unique Keys"
        },
        "target_alias": {
          "default": "target",
          "description": "Alias to use for the target table in merge conditions. Default is 'target'.",
          "title": "Target Alias",
          "type": "string"
        },
        "source_alias": {
          "default": "source",
          "description": "Alias to use for the source table in merge conditions. Default is 'source'.",
          "title": "Source Alias",
          "type": "string"
        },
        "predicates": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Extra predicates to prefilter source and target data before the merge condition is applied.",
          "title": "Predicates"
        },
        "matched_condition": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Additional SQL condition to apply to matched rows in a merge. ",
          "title": "Matched Condition"
        },
        "matched_condition_exclude_cols": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to exclude from the matched condition in a merge. Used to prevent certain columns from being evaluated in the matched condition, which can be useful for handling cases where certain columns may have non-deterministic values or should not be considered for matching.",
          "title": "Matched Condition Exclude Cols"
        },
        "matched_condition_include_cols": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to include in the matched condition in a merge. Used to specify a subset of columns to be evaluated in the matched condition, which can be useful for optimizing the merge operation by only considering relevant columns for matching.",
          "title": "Matched Condition Include Cols"
        },
        "not_matched_condition": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Additional SQL condition to apply to unmatched source rows in a merge. ",
          "title": "Not Matched Condition"
        },
        "not_matched_by_source_condition": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Additional SQL condition to apply to target rows not found in source in a merge. ",
          "title": "Not Matched By Source Condition"
        },
        "not_matched_by_source_action": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Action to take for target rows not found in source in a merge. If 'delete' rows will be deleted, else a update with the provided condition will be performed. Only applicable when write_mode='merge'. ",
          "title": "Not Matched By Source Action"
        },
        "matched_update_exclude_cols": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to exclude from updates in a merge. Only applicable when write_mode='merge'. Used to prevent certain columns from being updated during a merge operation.",
          "title": "Matched Update Exclude Cols"
        },
        "matched_update_include_cols": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to include in updates in a merge. Only applicable when write_mode='merge'. Used to specify a subset of columns to be updated during a merge operation.",
          "title": "Matched Update Include Cols"
        },
        "merge_with_schema_evolution": {
          "default": true,
          "description": "Whether to enable schema evolution during merge operations. Only applicable when write_mode='merge'. ",
          "title": "Merge With Schema Evolution",
          "type": "boolean"
        },
        "exclude_at_target_cols": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to exclude from the target dataset during a merge operation. Only applicable when write_mode='merge'. ",
          "title": "Exclude At Target Cols"
        },
        "include_at_target_cols": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to include from the target dataset during a merge operation. Only applicable when write_mode='merge'. ",
          "title": "Include At Target Cols"
        },
        "sequence_by": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of columns to order source and target datasets by before applying the merge condition. Only applicable when write_mode='merge'. ",
          "title": "Sequence By"
        },
        "ignore_null_updates": {
          "default": false,
          "description": "Whether to ignore updates to columns where the source value is null during a merge operation. Only applicable when write_mode='merge'. ",
          "title": "Ignore Null Updates",
          "type": "boolean"
        },
        "apply_as_delete": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "Used for CDC scenarios. Whether to apply a condition as a delete operation instead of an update or insert during a merge. Only applicable when write_mode='merge'. ",
          "title": "Apply As Delete"
        },
        "prevent_full_refresh": {
          "default": false,
          "description": "Whether to prevent full refreshes for this model. If True, attempts to perform a full refresh (e.g. via the 'full_refresh' flag in the materialization context) will be overridden to perform an append or merge instead, and a warning will be logged. This can be useful for protecting critical tables from accidental full refreshes.",
          "title": "Prevent Full Refresh",
          "type": "boolean"
        }
      },
      "title": "ModelMaterializationConfig",
      "type": "object"
    },
    "PrimaryKeyConstraint": {
      "properties": {
        "name": {
          "description": "Constraint name",
          "title": "Name",
          "type": "string"
        },
        "type": {
          "default": "primary_key",
          "description": "Constraint type identifier",
          "title": "Type",
          "type": "string"
        },
        "columns": {
          "description": "List of column names forming the primary key",
          "items": {
            "type": "string"
          },
          "title": "Columns",
          "type": "array"
        }
      },
      "required": [
        "name"
      ],
      "title": "PrimaryKeyConstraint",
      "type": "object"
    },
    "SDPQuality": {
      "properties": {
        "engine": {
          "const": "sdp",
          "description": "Quality engine type",
          "title": "Engine",
          "type": "string"
        },
        "level": {
          "const": "row",
          "default": "row",
          "description": "Quality enforcement level",
          "title": "Level",
          "type": "string"
        },
        "expect_all": {
          "additionalProperties": {
            "type": "string"
          },
          "description": "SQL expressions that must pass",
          "title": "Expect All",
          "type": "object"
        },
        "expect_all_or_drop": {
          "additionalProperties": {
            "type": "string"
          },
          "description": "SQL expressions; failing rows are dropped",
          "title": "Expect All Or Drop",
          "type": "object"
        },
        "expect_all_or_fail": {
          "additionalProperties": {
            "type": "string"
          },
          "description": "SQL expressions; job fails if any expression fails",
          "title": "Expect All Or Fail",
          "type": "object"
        },
        "expect_all_or_quarantine": {
          "additionalProperties": {
            "type": "string"
          },
          "description": "SQL expressions; failing rows are quarantined",
          "title": "Expect All Or Quarantine",
          "type": "object"
        }
      },
      "required": [
        "engine"
      ],
      "title": "SDPQuality",
      "type": "object"
    },
    "TableType": {
      "enum": [
        "external",
        "external_shallow_clone",
        "foreign",
        "managed",
        "managed_shallow_clone",
        "materialized_view",
        "metric_view",
        "streaming_table",
        "view"
      ],
      "title": "TableType",
      "type": "string"
    }
  },
  "description": "Model definition in Unity Catalog.\n\nRepresents a metadata model in Kelp with configuration that maps to\nDatabricks Unity Catalog objects.",
  "properties": {
    "table_type": {
      "$ref": "#/$defs/TableType",
      "default": "managed",
      "description": "Type of table: managed, external, view, streaming_table, etc."
    },
    "catalog": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Unity Catalog name",
      "title": "Catalog"
    },
    "schema": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Schema/database name",
      "title": "Schema"
    },
    "name": {
      "description": "Model name",
      "title": "Name",
      "type": "string"
    },
    "description": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Human-readable description of the model",
      "title": "Description"
    },
    "spark_conf": {
      "additionalProperties": true,
      "description": "Spark configuration properties",
      "title": "Spark Conf",
      "type": "object"
    },
    "table_properties": {
      "additionalProperties": true,
      "description": "Databricks table properties",
      "title": "Table Properties",
      "type": "object"
    },
    "path": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Physical path for external tables or custom locations",
      "title": "Path"
    },
    "partition_cols": {
      "description": "List of column names for partitioning",
      "items": {
        "type": "string"
      },
      "title": "Partition Cols",
      "type": "array"
    },
    "cluster_by_auto": {
      "default": false,
      "description": "Enable automatic clustering optimization",
      "title": "Cluster By Auto",
      "type": "boolean"
    },
    "cluster_by": {
      "description": "List of column names for explicit clustering (max 4)",
      "items": {
        "type": "string"
      },
      "maxItems": 4,
      "title": "Cluster By",
      "type": "array"
    },
    "row_filter": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "SQL expression to filter rows based on security policies",
      "title": "Row Filter"
    },
    "columns": {
      "description": "Column definitions for the model",
      "items": {
        "$ref": "#/$defs/Column"
      },
      "title": "Columns",
      "type": "array"
    },
    "quality": {
      "anyOf": [
        {
          "discriminator": {
            "mapping": {
              "dqx": "#/$defs/DQXQuality",
              "sdp": "#/$defs/SDPQuality"
            },
            "propertyName": "engine"
          },
          "oneOf": [
            {
              "$ref": "#/$defs/SDPQuality"
            },
            {
              "$ref": "#/$defs/DQXQuality"
            }
          ]
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Data quality configuration using SDPQuality or DQXQuality",
      "title": "Quality"
    },
    "materialization": {
      "anyOf": [
        {
          "$ref": "#/$defs/ModelMaterializationConfig"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Configuration controlling how the model is materialized."
    },
    "constraints": {
      "description": "Constraints like primary key or foreign key",
      "items": {
        "anyOf": [
          {
            "$ref": "#/$defs/PrimaryKeyConstraint"
          },
          {
            "$ref": "#/$defs/ForeignKeyConstraint"
          }
        ]
      },
      "title": "Constraints",
      "type": "array"
    },
    "auto_ttl": {
      "anyOf": [
        {
          "$ref": "#/$defs/AutoTTLConfig"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Configuration for automatic TTL (Time To Live) based on a timestamp column"
    },
    "tags": {
      "additionalProperties": {
        "type": "string"
      },
      "description": "Metadata tags for the model",
      "title": "Tags",
      "type": "object"
    },
    "meta": {
      "additionalProperties": true,
      "description": "Generic user-defined metadata for filtering and grouping",
      "title": "Meta",
      "type": "object"
    }
  },
  "required": [
    "name"
  ],
  "title": "Model",
  "type": "object"
}

Config:

• validate_by_name: True
• validate_by_alias: True
• serialize_by_alias: True
• use_enum_values: True

Fields:

• origin_file_path (SkipJsonSchema[str] | None)
• table_type (TableType)
• catalog (str | None)
• schema_ (str | None)
• name (str)
• description (str | None)
• spark_conf (dict)
• table_properties (dict)
• path (str | None)
• partition_cols (list[str])
• cluster_by_auto (bool)
• cluster_by (list[str])
• row_filter (str | None)
• columns (list[Column])
• quality (SDPQuality | DQXQuality | None)
• materialization (ModelMaterializationConfig | None)
• constraints (list[PrimaryKeyConstraint | ForeignKeyConstraint])
• auto_ttl (AutoTTLConfig | None)
• tags (dict[str, str])
• meta (dict[str, Any])
• raw_config (SkipJsonSchema[dict])

Validators:

• _serialize_complex_property_values → table_properties
• _validate_catalog_requires_schema

origin_file_path pydantic-field

origin_file_path = None

Path to the source YAML file defining this model

table_type pydantic-field

table_type = MANAGED

Type of table: managed, external, view, streaming_table, etc.

catalog pydantic-field

catalog = None

Unity Catalog name

schema_ pydantic-field

schema_ = None

Schema/database name

name pydantic-field

name

Model name

description pydantic-field

description = None

Human-readable description of the model

spark_conf pydantic-field

spark_conf

Spark configuration properties

table_properties pydantic-field

table_properties

Databricks table properties

path pydantic-field

path = None

Physical path for external tables or custom locations

partition_cols pydantic-field

partition_cols

List of column names for partitioning

cluster_by_auto pydantic-field

cluster_by_auto = False

Enable automatic clustering optimization

cluster_by pydantic-field

cluster_by

List of column names for explicit clustering (max 4)

row_filter pydantic-field

row_filter = None

SQL expression to filter rows based on security policies

columns pydantic-field

columns

Column definitions for the model

quality pydantic-field

quality = None

Data quality configuration using SDPQuality or DQXQuality

materialization pydantic-field

materialization = None

Configuration controlling how the model is materialized.

constraints pydantic-field

constraints

Constraints like primary key or foreign key

auto_ttl pydantic-field

auto_ttl = None

Configuration for automatic TTL (Time To Live) based on a timestamp column

tags pydantic-field

tags

Metadata tags for the model

meta pydantic-field

meta

Generic user-defined metadata for filtering and grouping

raw_config pydantic-field

raw_config

Original unparsed configuration preserving placeholder variables

model_config class-attribute instance-attribute

model_config = ConfigDict(
    validate_by_name=True,
    validate_by_alias=True,
    serialize_by_alias=True,
    use_enum_values=True,
)

deserialize_property_values staticmethod

deserialize_property_values(properties)

Deserialize JSON-encoded property values back to complex types.

Used by the YAML writer to restore the original structure (list, dict) so that the YAML output uses native YAML types instead of JSON strings.

Source code in src/kelp/models/model.py

@staticmethod
def deserialize_property_values(properties: dict) -> dict:
    """Deserialize JSON-encoded property values back to complex types.

    Used by the YAML writer to restore the original structure (list, dict)
    so that the YAML output uses native YAML types instead of JSON strings.
    """
    if not isinstance(properties, dict):
        return properties
    result: dict = {}
    for key, value in properties.items():
        if isinstance(value, str):
            try:
                parsed = json.loads(value)
            except (json.JSONDecodeError, ValueError):
                parsed = value
            if isinstance(parsed, (dict, list)):
                result[key] = parsed
            else:
                result[key] = value
        else:
            result[key] = value
    return result

get_qualified_name

get_qualified_name()

Get the fully qualified model name including database/schema if applicable.

Source code in src/kelp/models/model.py

def get_qualified_name(self) -> str:
    """Get the fully qualified model name including database/schema if applicable."""
    parts = []
    if self.catalog:
        parts.append(self.catalog)
    if self.schema_:
        parts.append(self.schema_)
    parts.append(self.name)
    return ".".join(parts)

kelp.models.model_mat_config.ModelMaterializationConfig pydantic-model

Bases: BaseModel

Configuration controlling how a kelp model is materialized.

All fields are optional; set only what you need.

Parameters:

Name	Type	Description	Default
write_mode		How to write the result — 'append', 'overwrite', or 'merge'.	required
options		Extra DataFrameWriter options (e.g. {'mergeSchema': 'true'}).	required
unique_keys		List of unique key columns for merge operations. Required when write_mode='merge'.	required
target_alias		Alias to use for the target table in merge conditions. Default is 'target'.	required
source_alias		Alias to use for the source table in merge conditions. Default is 'source'.	required
predicates		Extra predicates to prefilter source and target data before the merge condition is applied.	required
matched_condition		Additional SQL condition to apply to matched rows in a merge.	required
matched_condition_exclude_cols		List of columns to exclude from the matched condition in a merge. Used to prevent certain columns from being evaluated in the matched condition, which can be useful for handling cases where certain columns may have non-deterministic values or should not be considered for matching.	required
matched_condition_include_cols		List of columns to include in the matched condition in a merge. Used to specify a subset of columns to be evaluated in the matched condition, which can be useful for optimizing the merge operation by only considering relevant columns for matching.	required
not_matched_condition		Additional SQL condition to apply to unmatched source rows in a merge.	required
not_matched_by_source_condition		Additional SQL condition to apply to target rows not found in source in a merge.	required
not_matched_by_source_action		Action to take for target rows not found in source in a merge. If 'delete' rows will be deleted, else a update with the provided condition will be performed. Only applicable when write_mode='merge'.	required
matched_update_exclude_cols		List of columns to exclude from updates in a merge. Only applicable when write_mode='merge'. Used to prevent certain columns from being updated during a merge operation.	required
matched_update_include_cols		List of columns to include in updates in a merge. Only applicable when write_mode='merge'. Used to specify a subset of columns to be updated during a merge operation.	required
merge_with_schema_evolution		Whether to enable schema evolution during merge operations. Only applicable when write_mode='merge'.	required
exclude_at_target_cols		List of columns to exclude from the target dataset during a merge operation. Only applicable when ``write_mode='merge'.	required
include_at_target_cols		List of columns to include from the target dataset during a merge operation. Only applicable when write_mode='merge'.	required
sequence_by		List of columns to order source and target datasets by before applying the merge condition. Only applicable when write_mode='merge'.	required
ignore_null_updates		Whether to ignore updates to columns where the source value is null during a merge operation. Only applicable when write_mode='merge'.	required
apply_as_delete		Used for CDC scenarios. Whether to apply a condition as a delete operation instead of an update or insert during a merge. Only applicable when write_mode='merge'.	required
prevent_full_refresh		Whether to prevent full refreshes for this model. If True, attempts to perform a full refresh (e.g. via the 'full_refresh' flag in the materialization context) will be overridden to perform an append or merge instead, and a warning will be logged. This can be useful for protecting critical tables from accidental full refreshes.	required

Show JSON schema:

{
  "description": "Configuration controlling how a kelp model is materialized.\n\nAll fields are optional; set only what you need.\n\nArgs:\n    write_mode: How to write the result \u2014 ``'append'``, ``'overwrite'``, or ``'merge'``.\n    When ``None`` (default) the DataFrame\n        is returned without writing.\n    options: Extra DataFrameWriter options\n        (e.g. ``{'mergeSchema': 'true'}``).\n    unique_keys: List of unique key columns for merge operations. Required when ``write_mode='merge'``.\n    target_alias: Alias to use for the target table in merge conditions. Default is ``'target'``.\n    source_alias: Alias to use for the source table in merge conditions. Default is ``'source'``.\n    predicates: Extra predicates to prefilter source and target data before the merge condition is applied.\n    matched_condition: Additional SQL condition to apply to matched rows in a merge.\n    matched_condition_exclude_cols: List of columns to exclude from the matched condition in a merge. Used to prevent certain columns from being evaluated in the matched condition, which can be useful for handling cases where certain columns may have non-deterministic values or should not be considered for matching.\n    matched_condition_include_cols: List of columns to include in the matched condition in a merge. Used to specify a subset of columns to be evaluated in the matched condition, which can be useful for optimizing the merge operation by only considering relevant columns for matching.\n    not_matched_condition: Additional SQL condition to apply to unmatched source rows in a merge.\n    not_matched_by_source_condition: Additional SQL condition to apply to target rows not found in source in a merge.\n    not_matched_by_source_action: Action to take for target rows not found in source in a merge. If ``'delete'`` rows will be deleted, else a update with the provided condition will be performed. Only applicable when ``write_mode='merge'``.\n    matched_update_exclude_cols: List of columns to exclude from updates in a merge. Only applicable when ``write_mode='merge'``. Used to prevent certain columns from being updated during a merge operation.\n    matched_update_include_cols: List of columns to include in updates in a merge. Only applicable when ``write_mode='merge'``. Used to specify a subset of columns to be updated during a merge operation.\n    merge_with_schema_evolution: Whether to enable schema evolution during merge operations. Only applicable when ``write_mode='merge'``.\n    exclude_at_target_cols: List of columns to exclude from the target dataset during a merge operation. Only applicable when ``write_mode='merge'.\n    include_at_target_cols: List of columns to include from the target dataset during a merge operation. Only applicable when ``write_mode='merge'``.\n    sequence_by: List of columns to order source and target datasets by before applying the merge condition. Only applicable when ``write_mode='merge'``.\n    ignore_null_updates: Whether to ignore updates to columns where the source value is null during a merge operation. Only applicable when ``write_mode='merge'``.\n    apply_as_delete: Used for CDC scenarios. Whether to apply a condition as a delete operation instead of an update or insert during a merge. Only applicable when ``write_mode='merge'``.\n    prevent_full_refresh: Whether to prevent full refreshes for this model. If True, attempts to perform a full refresh (e.g. via the ``'full_refresh'`` flag in the materialization context) will\n        be overridden to perform an append or merge instead, and a warning will be logged. This can be useful for protecting critical tables from accidental full refreshes.",
  "properties": {
    "write_mode": {
      "anyOf": [
        {
          "enum": [
            "append",
            "overwrite",
            "merge"
          ],
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "How to write the model: 'append', 'overwrite', or 'merge'. When None the DataFrame is returned without writing.",
      "title": "Write Mode"
    },
    "options": {
      "additionalProperties": true,
      "description": "Extra DataFrameWriter options (e.g. {'mergeSchema': 'true'}).",
      "title": "Options",
      "type": "object"
    },
    "unique_keys": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of unique key columns for merge operations. Required when write_mode='merge'. Used to validate the merge condition and ensure it references a unique key.",
      "title": "Unique Keys"
    },
    "target_alias": {
      "default": "target",
      "description": "Alias to use for the target table in merge conditions. Default is 'target'.",
      "title": "Target Alias",
      "type": "string"
    },
    "source_alias": {
      "default": "source",
      "description": "Alias to use for the source table in merge conditions. Default is 'source'.",
      "title": "Source Alias",
      "type": "string"
    },
    "predicates": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Extra predicates to prefilter source and target data before the merge condition is applied.",
      "title": "Predicates"
    },
    "matched_condition": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Additional SQL condition to apply to matched rows in a merge. ",
      "title": "Matched Condition"
    },
    "matched_condition_exclude_cols": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to exclude from the matched condition in a merge. Used to prevent certain columns from being evaluated in the matched condition, which can be useful for handling cases where certain columns may have non-deterministic values or should not be considered for matching.",
      "title": "Matched Condition Exclude Cols"
    },
    "matched_condition_include_cols": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to include in the matched condition in a merge. Used to specify a subset of columns to be evaluated in the matched condition, which can be useful for optimizing the merge operation by only considering relevant columns for matching.",
      "title": "Matched Condition Include Cols"
    },
    "not_matched_condition": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Additional SQL condition to apply to unmatched source rows in a merge. ",
      "title": "Not Matched Condition"
    },
    "not_matched_by_source_condition": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Additional SQL condition to apply to target rows not found in source in a merge. ",
      "title": "Not Matched By Source Condition"
    },
    "not_matched_by_source_action": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Action to take for target rows not found in source in a merge. If 'delete' rows will be deleted, else a update with the provided condition will be performed. Only applicable when write_mode='merge'. ",
      "title": "Not Matched By Source Action"
    },
    "matched_update_exclude_cols": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to exclude from updates in a merge. Only applicable when write_mode='merge'. Used to prevent certain columns from being updated during a merge operation.",
      "title": "Matched Update Exclude Cols"
    },
    "matched_update_include_cols": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to include in updates in a merge. Only applicable when write_mode='merge'. Used to specify a subset of columns to be updated during a merge operation.",
      "title": "Matched Update Include Cols"
    },
    "merge_with_schema_evolution": {
      "default": true,
      "description": "Whether to enable schema evolution during merge operations. Only applicable when write_mode='merge'. ",
      "title": "Merge With Schema Evolution",
      "type": "boolean"
    },
    "exclude_at_target_cols": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to exclude from the target dataset during a merge operation. Only applicable when write_mode='merge'. ",
      "title": "Exclude At Target Cols"
    },
    "include_at_target_cols": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to include from the target dataset during a merge operation. Only applicable when write_mode='merge'. ",
      "title": "Include At Target Cols"
    },
    "sequence_by": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "List of columns to order source and target datasets by before applying the merge condition. Only applicable when write_mode='merge'. ",
      "title": "Sequence By"
    },
    "ignore_null_updates": {
      "default": false,
      "description": "Whether to ignore updates to columns where the source value is null during a merge operation. Only applicable when write_mode='merge'. ",
      "title": "Ignore Null Updates",
      "type": "boolean"
    },
    "apply_as_delete": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Used for CDC scenarios. Whether to apply a condition as a delete operation instead of an update or insert during a merge. Only applicable when write_mode='merge'. ",
      "title": "Apply As Delete"
    },
    "prevent_full_refresh": {
      "default": false,
      "description": "Whether to prevent full refreshes for this model. If True, attempts to perform a full refresh (e.g. via the 'full_refresh' flag in the materialization context) will be overridden to perform an append or merge instead, and a warning will be logged. This can be useful for protecting critical tables from accidental full refreshes.",
      "title": "Prevent Full Refresh",
      "type": "boolean"
    }
  },
  "title": "ModelMaterializationConfig",
  "type": "object"
}

Fields:

• write_mode (Literal['append', 'overwrite', 'merge'] | None)
• options (dict)
• unique_keys (list[str] | None)
• target_alias (str)
• source_alias (str)
• predicates (str | None)
• matched_condition (str | None)
• matched_condition_exclude_cols (list[str] | None)
• matched_condition_include_cols (list[str] | None)
• not_matched_condition (str | None)
• not_matched_by_source_condition (str | None)
• not_matched_by_source_action (str | None)
• matched_update_exclude_cols (list[str] | None)
• matched_update_include_cols (list[str] | None)
• merge_with_schema_evolution (bool)
• exclude_at_target_cols (list[str] | None)
• include_at_target_cols (list[str] | None)
• sequence_by (list[str] | None)
• ignore_null_updates (bool)
• apply_as_delete (str | None)
• prevent_full_refresh (bool)

write_mode pydantic-field

write_mode = None

How to write the model: 'append', 'overwrite', or 'merge'. When None the DataFrame is returned without writing.

options pydantic-field

options

Extra DataFrameWriter options (e.g. {'mergeSchema': 'true'}).

unique_keys pydantic-field

unique_keys = None

List of unique key columns for merge operations. Required when write_mode='merge'. Used to validate the merge condition and ensure it references a unique key.

target_alias pydantic-field

target_alias = 'target'

Alias to use for the target table in merge conditions. Default is 'target'.

source_alias pydantic-field

source_alias = 'source'

Alias to use for the source table in merge conditions. Default is 'source'.

predicates pydantic-field

predicates = None

Extra predicates to prefilter source and target data before the merge condition is applied.

matched_condition pydantic-field

matched_condition = None

Additional SQL condition to apply to matched rows in a merge.

matched_condition_exclude_cols pydantic-field

matched_condition_exclude_cols = None

List of columns to exclude from the matched condition in a merge. Used to prevent certain columns from being evaluated in the matched condition, which can be useful for handling cases where certain columns may have non-deterministic values or should not be considered for matching.

matched_condition_include_cols pydantic-field

matched_condition_include_cols = None

List of columns to include in the matched condition in a merge. Used to specify a subset of columns to be evaluated in the matched condition, which can be useful for optimizing the merge operation by only considering relevant columns for matching.

not_matched_condition pydantic-field

not_matched_condition = None

Additional SQL condition to apply to unmatched source rows in a merge.

not_matched_by_source_condition pydantic-field

not_matched_by_source_condition = None

Additional SQL condition to apply to target rows not found in source in a merge.

not_matched_by_source_action pydantic-field

not_matched_by_source_action = None

Action to take for target rows not found in source in a merge. If 'delete' rows will be deleted, else a update with the provided condition will be performed. Only applicable when write_mode='merge'.

matched_update_exclude_cols pydantic-field

matched_update_exclude_cols = None

List of columns to exclude from updates in a merge. Only applicable when write_mode='merge'. Used to prevent certain columns from being updated during a merge operation.

matched_update_include_cols pydantic-field

matched_update_include_cols = None

List of columns to include in updates in a merge. Only applicable when write_mode='merge'. Used to specify a subset of columns to be updated during a merge operation.

merge_with_schema_evolution pydantic-field

merge_with_schema_evolution = True

Whether to enable schema evolution during merge operations. Only applicable when write_mode='merge'.

exclude_at_target_cols pydantic-field

exclude_at_target_cols = None

List of columns to exclude from the target dataset during a merge operation. Only applicable when write_mode='merge'.

include_at_target_cols pydantic-field

include_at_target_cols = None

List of columns to include from the target dataset during a merge operation. Only applicable when write_mode='merge'.

sequence_by pydantic-field

sequence_by = None

List of columns to order source and target datasets by before applying the merge condition. Only applicable when write_mode='merge'.

ignore_null_updates pydantic-field

ignore_null_updates = False

Whether to ignore updates to columns where the source value is null during a merge operation. Only applicable when write_mode='merge'.

apply_as_delete pydantic-field

apply_as_delete = None

Used for CDC scenarios. Whether to apply a condition as a delete operation instead of an update or insert during a merge. Only applicable when write_mode='merge'.

prevent_full_refresh pydantic-field

prevent_full_refresh = False

Whether to prevent full refreshes for this model. If True, attempts to perform a full refresh (e.g. via the 'full_refresh' flag in the materialization context) will be overridden to perform an append or merge instead, and a warning will be logged. This can be useful for protecting critical tables from accidental full refreshes.

kelp.models.model.Column pydantic-model

Bases: BaseModel

Column definition for a model.

Show JSON schema:

{
  "$defs": {
    "GeneratedExpressionColumnConfig": {
      "properties": {
        "type": {
          "const": "expression",
          "description": "Column type identifier",
          "title": "Type",
          "type": "string"
        },
        "expression": {
          "description": "SQL expression used to generate the column value",
          "title": "Expression",
          "type": "string"
        }
      },
      "required": [
        "type",
        "expression"
      ],
      "title": "GeneratedExpressionColumnConfig",
      "type": "object"
    },
    "GeneratedIdentityColumnConfig": {
      "properties": {
        "type": {
          "const": "identity",
          "description": "Column type identifier",
          "title": "Type",
          "type": "string"
        },
        "as_default": {
          "default": false,
          "description": "Generated as default (True) or always (False)",
          "title": "As Default",
          "type": "boolean"
        },
        "start_with": {
          "default": 1,
          "description": "Starting value for the identity sequence",
          "title": "Start With",
          "type": "integer"
        },
        "increment_by": {
          "default": 1,
          "description": "Increment step for the identity sequence",
          "title": "Increment By",
          "type": "integer"
        }
      },
      "required": [
        "type"
      ],
      "title": "GeneratedIdentityColumnConfig",
      "type": "object"
    }
  },
  "description": "Column definition for a model.",
  "properties": {
    "name": {
      "description": "Column name",
      "title": "Name",
      "type": "string"
    },
    "description": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Human-readable description of the column",
      "title": "Description"
    },
    "data_type": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "SQL data type of the column",
      "title": "Data Type"
    },
    "nullable": {
      "default": true,
      "description": "Whether the column allows NULL values",
      "title": "Nullable",
      "type": "boolean"
    },
    "generated": {
      "anyOf": [
        {
          "discriminator": {
            "mapping": {
              "expression": "#/$defs/GeneratedExpressionColumnConfig",
              "identity": "#/$defs/GeneratedIdentityColumnConfig"
            },
            "propertyName": "type"
          },
          "oneOf": [
            {
              "$ref": "#/$defs/GeneratedIdentityColumnConfig"
            },
            {
              "$ref": "#/$defs/GeneratedExpressionColumnConfig"
            }
          ]
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Configuration for generated columns (identity or expression based)",
      "title": "Generated"
    },
    "tags": {
      "additionalProperties": {
        "type": "string"
      },
      "description": "Metadata tags for the column",
      "title": "Tags",
      "type": "object"
    },
    "meta": {
      "additionalProperties": true,
      "description": "Generic user-defined metadata for filtering and grouping columns",
      "title": "Meta",
      "type": "object"
    }
  },
  "required": [
    "name"
  ],
  "title": "Column",
  "type": "object"
}

Fields:

• name (str)
• description (str | None)
• data_type (str | None)
• nullable (bool)
• generated (GeneratedIdentityColumnConfig | GeneratedExpressionColumnConfig | None)
• tags (dict[str, str])
• meta (dict[str, Any])

name pydantic-field

name

Column name

description pydantic-field

description = None

Human-readable description of the column

data_type pydantic-field

data_type = None

SQL data type of the column

nullable pydantic-field

nullable = True

Whether the column allows NULL values

generated pydantic-field

generated = None

Configuration for generated columns (identity or expression based)

tags pydantic-field

tags

Metadata tags for the column

meta pydantic-field

meta

Generic user-defined metadata for filtering and grouping columns

kelp.models.model.GeneratedIdentityColumnConfig pydantic-model

Bases: BaseModel

Show JSON schema:

{
  "properties": {
    "type": {
      "const": "identity",
      "description": "Column type identifier",
      "title": "Type",
      "type": "string"
    },
    "as_default": {
      "default": false,
      "description": "Generated as default (True) or always (False)",
      "title": "As Default",
      "type": "boolean"
    },
    "start_with": {
      "default": 1,
      "description": "Starting value for the identity sequence",
      "title": "Start With",
      "type": "integer"
    },
    "increment_by": {
      "default": 1,
      "description": "Increment step for the identity sequence",
      "title": "Increment By",
      "type": "integer"
    }
  },
  "required": [
    "type"
  ],
  "title": "GeneratedIdentityColumnConfig",
  "type": "object"
}

Fields:

• type (Literal['identity'])
• as_default (bool)
• start_with (int)
• increment_by (int)

type pydantic-field

type

Column type identifier

as_default pydantic-field

as_default = False

Generated as default (True) or always (False)

start_with pydantic-field

start_with = 1

Starting value for the identity sequence

increment_by pydantic-field

increment_by = 1

Increment step for the identity sequence

kelp.models.model.GeneratedExpressionColumnConfig pydantic-model

Bases: BaseModel

Show JSON schema:

{
  "properties": {
    "type": {
      "const": "expression",
      "description": "Column type identifier",
      "title": "Type",
      "type": "string"
    },
    "expression": {
      "description": "SQL expression used to generate the column value",
      "title": "Expression",
      "type": "string"
    }
  },
  "required": [
    "type",
    "expression"
  ],
  "title": "GeneratedExpressionColumnConfig",
  "type": "object"
}

Fields:

• type (Literal['expression'])
• expression (str)

type pydantic-field

type

Column type identifier

expression pydantic-field

expression

SQL expression used to generate the column value

kelp.models.model.Constraint pydantic-model

Bases: BaseModel

Show JSON schema:

{
  "properties": {
    "name": {
      "description": "Constraint name",
      "title": "Name",
      "type": "string"
    }
  },
  "required": [
    "name"
  ],
  "title": "Constraint",
  "type": "object"
}

Fields:

• name (str)

name pydantic-field

name

Constraint name

kelp.models.model.PrimaryKeyConstraint pydantic-model

Bases: Constraint

Show JSON schema:

{
  "properties": {
    "name": {
      "description": "Constraint name",
      "title": "Name",
      "type": "string"
    },
    "type": {
      "default": "primary_key",
      "description": "Constraint type identifier",
      "title": "Type",
      "type": "string"
    },
    "columns": {
      "description": "List of column names forming the primary key",
      "items": {
        "type": "string"
      },
      "title": "Columns",
      "type": "array"
    }
  },
  "required": [
    "name"
  ],
  "title": "PrimaryKeyConstraint",
  "type": "object"
}

Fields:

• name (str)
• type (str)
• columns (list[str])

type pydantic-field

type = 'primary_key'

Constraint type identifier

columns pydantic-field

columns

List of column names forming the primary key

name pydantic-field

name

Constraint name

kelp.models.model.ForeignKeyConstraint pydantic-model

Bases: Constraint

Show JSON schema:

{
  "properties": {
    "name": {
      "description": "Constraint name",
      "title": "Name",
      "type": "string"
    },
    "type": {
      "default": "foreign_key",
      "description": "Constraint type identifier",
      "title": "Type",
      "type": "string"
    },
    "columns": {
      "description": "List of local column names",
      "items": {
        "type": "string"
      },
      "title": "Columns",
      "type": "array"
    },
    "reference_table": {
      "description": "Fully qualified name of the referenced table",
      "title": "Reference Table",
      "type": "string"
    },
    "reference_columns": {
      "description": "List of column names in the referenced table",
      "items": {
        "type": "string"
      },
      "title": "Reference Columns",
      "type": "array"
    }
  },
  "required": [
    "name",
    "reference_table"
  ],
  "title": "ForeignKeyConstraint",
  "type": "object"
}

Fields:

• name (str)
• type (str)
• columns (list[str])
• reference_table (str)
• reference_columns (list[str])

type pydantic-field

type = 'foreign_key'

Constraint type identifier

columns pydantic-field

columns

List of local column names

reference_table pydantic-field

reference_table

Fully qualified name of the referenced table

reference_columns pydantic-field

reference_columns

List of column names in the referenced table

name pydantic-field

name

Constraint name

kelp.models.model.Quality pydantic-model

Bases: BaseModel

Show JSON schema:

{
  "properties": {
    "engine": {
      "description": "Quality engine type",
      "title": "Engine",
      "type": "string"
    },
    "level": {
      "default": "row",
      "description": "Level at which quality is enforced",
      "enum": [
        "row",
        "table"
      ],
      "title": "Level",
      "type": "string"
    }
  },
  "required": [
    "engine"
  ],
  "title": "Quality",
  "type": "object"
}

Fields:

• engine (str)
• level (Literal['row', 'table'])

engine pydantic-field

engine

Quality engine type

level pydantic-field

level = 'row'

Level at which quality is enforced

kelp.models.model.SDPQuality pydantic-model

Bases: Quality

Show JSON schema:

{
  "properties": {
    "engine": {
      "const": "sdp",
      "description": "Quality engine type",
      "title": "Engine",
      "type": "string"
    },
    "level": {
      "const": "row",
      "default": "row",
      "description": "Quality enforcement level",
      "title": "Level",
      "type": "string"
    },
    "expect_all": {
      "additionalProperties": {
        "type": "string"
      },
      "description": "SQL expressions that must pass",
      "title": "Expect All",
      "type": "object"
    },
    "expect_all_or_drop": {
      "additionalProperties": {
        "type": "string"
      },
      "description": "SQL expressions; failing rows are dropped",
      "title": "Expect All Or Drop",
      "type": "object"
    },
    "expect_all_or_fail": {
      "additionalProperties": {
        "type": "string"
      },
      "description": "SQL expressions; job fails if any expression fails",
      "title": "Expect All Or Fail",
      "type": "object"
    },
    "expect_all_or_quarantine": {
      "additionalProperties": {
        "type": "string"
      },
      "description": "SQL expressions; failing rows are quarantined",
      "title": "Expect All Or Quarantine",
      "type": "object"
    }
  },
  "required": [
    "engine"
  ],
  "title": "SDPQuality",
  "type": "object"
}

Fields:

• engine (Literal['sdp'])
• level (Literal['row'])
• expect_all (dict[str, str])
• expect_all_or_drop (dict[str, str])
• expect_all_or_fail (dict[str, str])
• expect_all_or_quarantine (dict[str, str])

engine pydantic-field

engine

Quality engine type

level pydantic-field

level = 'row'

Quality enforcement level

expect_all pydantic-field

expect_all

SQL expressions that must pass

expect_all_or_drop pydantic-field

expect_all_or_drop

SQL expressions; failing rows are dropped

expect_all_or_fail pydantic-field

expect_all_or_fail

SQL expressions; job fails if any expression fails

expect_all_or_quarantine pydantic-field

expect_all_or_quarantine

SQL expressions; failing rows are quarantined

kelp.models.model.DQXQuality pydantic-model

Bases: Quality

Show JSON schema:

{
  "properties": {
    "engine": {
      "const": "dqx",
      "description": "Quality engine type",
      "title": "Engine",
      "type": "string"
    },
    "level": {
      "default": "row",
      "description": "Level at which quality is enforced",
      "enum": [
        "row",
        "table"
      ],
      "title": "Level",
      "type": "string"
    },
    "sdp_expect_level": {
      "default": "warn",
      "description": "Action for quality violations: warn, fail, drop, or deactivate",
      "enum": [
        "warn",
        "fail",
        "drop",
        "deactivate"
      ],
      "title": "Sdp Expect Level",
      "type": "string"
    },
    "sdp_quarantine": {
      "default": false,
      "description": "Whether to quarantine rows failing quality checks",
      "title": "Sdp Quarantine",
      "type": "boolean"
    },
    "spark_violation_action": {
      "default": "error",
      "description": "Action for quality violations at the Spark level: error (raise exception), ignore (store errors in output), or drop (exclude from output)",
      "enum": [
        "error",
        "ignore",
        "drop"
      ],
      "title": "Spark Violation Action",
      "type": "string"
    },
    "spark_quarantine": {
      "default": false,
      "description": "Whether to write rows failing quality checks to a quarantine table at the Spark level",
      "title": "Spark Quarantine",
      "type": "boolean"
    },
    "checks": {
      "description": "Quality check configurations",
      "items": {
        "additionalProperties": true,
        "type": "object"
      },
      "title": "Checks",
      "type": "array"
    }
  },
  "required": [
    "engine"
  ],
  "title": "DQXQuality",
  "type": "object"
}

Fields:

• level (Literal['row', 'table'])
• engine (Literal['dqx'])
• sdp_expect_level (Literal['warn', 'fail', 'drop', 'deactivate'])
• sdp_quarantine (bool)
• spark_violation_action (Literal['error', 'ignore', 'drop'])
• spark_quarantine (bool)
• checks (list[dict])

engine pydantic-field

engine

Quality engine type

sdp_expect_level pydantic-field

sdp_expect_level = 'warn'

Action for quality violations: warn, fail, drop, or deactivate

sdp_quarantine pydantic-field

sdp_quarantine = False

Whether to quarantine rows failing quality checks

spark_violation_action pydantic-field

spark_violation_action = 'error'

Action for quality violations at the Spark level: error (raise exception), ignore (store errors in output), or drop (exclude from output)

spark_quarantine pydantic-field

spark_quarantine = False

Whether to write rows failing quality checks to a quarantine table at the Spark level

checks pydantic-field

checks

Quality check configurations

level pydantic-field

level = 'row'

Level at which quality is enforced