xnuinside / simple-ddl-parser

Simple DDL Parser to parse SQL (HQL, TSQL, AWS Redshift, BigQuery, Snowflake and other dialects) ddl files to json/python dict with full information about columns: types, defaults, primary keys, etc. & table properties, types, domains, etc.
MIT License
175 stars 40 forks source link

fix Snowflake FILE_FORMAT options #221

Closed dmaresma closed 6 months ago

dmaresma commented 6 months ago

Support the Stage and File format options of snowflake for external table ddl statement following the snowflake requirements https://docs.snowflake.com/en/sql-reference/sql/create-file-format


def test_virtual_column_table():
    ddl = """
    create or replace table if not exists TABLE_DATA_SRC.EXT_PAYLOAD_MANIFEST_WEB (
       id bigint,
       derived bigint as (id * 10)
       )
    location = @ADL_Azure_Storage_Account_Container_Name/
    auto_refresh = false
    file_format = (TYPE=JSON NULL_IF=('field') DATE_FORMAT=AUTO TRIM_SPACE=TRUE)
    stage_file_format = (TYPE=JSON NULL_IF=())
    ;
    """
    result_ext_table = DDLParser(ddl, normalize_names=True, debug=True).run(
        output_mode="snowflake"
    )

    expected_ext_table = [
        {
            "alter": {},
            "checks": [],
            "clone": None,
            "columns": [
                {
                    "name": "id",
                    "type": "bigint",
                    "size": None,
                    "references": None,
                    "unique": False,
                    "nullable": True,
                    "default": None,
                    "check": None,
                },
                {
                    "name": "derived",
                    "type": "bigint",
                    "size": None,
                    "references": None,
                    "unique": False,
                    "nullable": True,
                    "default": None,
                    "check": None,
                    "generated" : {"as" : "id * 10" }
                }
            ],
            "index": [],
            "partitioned_by" : [],
            "primary_key": [],
            "primary_key_enforced": None,
            "auto_refresh" : False,
            "schema": "TABLE_DATA_SRC",
            "table_name": "EXT_PAYLOAD_MANIFEST_WEB",
            "tablespace": None,
            "replace" : True,
            "if_not_exists": True,
            "location" : "@ADL_Azure_Storage_Account_Container_Name/",
            "file_format": ['TYPE=JSON', "NULL_IF=('field')",'DATE_FORMAT=AUTO','TRIM_SPACE=TRUE'],
            'stage_file_format': ['TYPE=JSON','NULL_IF=()']
            }
    ]

    assert result_ext_table == expected_ext_table
xnuinside commented 6 months ago

merged & released @ 0.31.3