Example source: dataflows/transform_patterns.json

Source revision: 75f65139e26eb7079b7a897c877de97859c0120b

This page is the generated source; it shows the complete readable projection of the canonical file.

raw · View repository source

{
  "dataflows": [
    {
      "name": "transform_schema_and_values",
      "stage": "transform",
      "source": {"connection_name": "input", "table": "orders"},
      "destination": {"connection_name": "output", "table": "orders_clean", "load_type": "overwrite"},
      "transform": {
        "schema_hints": [{"column_name": "order_id", "data_type": "long"}],
        "value_rules": [{"operation": "trim", "columns": ["category"]}, {"operation": "case", "columns": ["category"], "mode": "lower"}],
        "additional_columns": [{"column": "source_system", "expression": "'example'"}],
        "hash_columns": [{"target_column": "order_hash", "columns": ["order_id"], "algorithm": "sha256"}],
        "deduplicate_columns": ["order_id"],
        "latest_data_columns": ["updated_at"],
        "masking_rules": [{"method": "partial", "columns": ["phone"], "keep_end": 4}],
        "rename_columns": {"category": "order_category"},
        "select_columns": ["order_id", "category", "order_hash", "source_system"]
      }
    }
  ]
}