[Feature request]: Spark version 3.5.0

[eruizalo]

Feature suggestion Spark 3.5.0 has been released. We should:

• [x] Update sbt to be this version the default version #435
• [x] Update CI #435
• [x] Update docs #435

• [ ] Add new sql functions:

  • From functions

    • [ ] def count_min_sketch(e: Column,eps: Column, confidence: Column, seed: Column): Column
    • [ ] def first_value(e: Column): Column
    • [ ] def first_value(e: Column, ignoreNulls: Column): Column
    • [ ] def hll_sketch_agg(e: Column, lgConfigK: Column): Column
    • [ ] def hll_sketch_agg(e: Column, lgConfigK: Int): Column
    • [ ] def hll_sketch_agg(columnName: String, lgConfigK: Int): Column
    • [ ] def hll_sketch_agg(e: Column): Column
    • [ ] def hll_sketch_agg(columnName: String): Column
    • [ ] def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column
    • [ ] def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column
    • [ ] def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column
    • [ ] def hll_union_agg(e: Column): Column
    • [ ] def hll_union_agg(columnName: String): Column
    • [ ] def last_value(e: Column): Column
    • [ ] def last_value(e: Column, ignoreNulls: Column): Column
    • [ ] def percentile(e: Column, percentage: Column): Column
    • [ ] def percentile(e: Column,percentage: Column,frequency: Column): Column
    • [ ] def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column
    • [ ] def std(e: Column): Column
    • [ ] def regr_avgx(y: Column, x: Column): Column
    • [ ] def regr_avgy(y: Column, x: Column): Column
    • [ ] def regr_count(y: Column, x: Column): Column
    • [ ] def regr_intercept(y: Column, x: Column): Column
    • [ ] def regr_r2(y: Column, x: Column): Column
    • [ ] def regr_slope(y: Column, x: Column): Column
    • [ ] def regr_sxx(y: Column, x: Column): Column
    • [ ] def regr_sxy(y: Column, x: Column): Column
    • [ ] def regr_syy(y: Column, x: Column): Column
    • [ ] def any_value(e: Column): Column
    • [ ] def any_value(e: Column, ignoreNulls: Column): Column
    • [ ] def count_if(e: Column): Column
    • [ ] def histogram_numeric(e: Column, nBins: Column): Column
    • [ ] def every(e: Column): Column
    • [ ] def bool_and(e: Column): Column
    • [ ] def some(e: Column): Column
    • [ ] def any(e: Column): Column
    • [ ] def bool_or(e: Column): Column
    • [ ] def bit_and(e: Column): Column
    • [ ] def bit_or(e: Column): Column
    • [ ] def bit_xor(e: Column): Column
    • [ ] def named_struct(cols: Column*): Column
    • [ ] def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column
    • [ ] def str_to_map(text: Column, pairDelim: Column): Column
    • [ ] def str_to_map(text: Column): Column
    • [ ] def try_add(left: Column, right: Column): Column
    • [ ] def try_avg(e: Column): Column
    • [ ] def try_divide(dividend: Column, divisor: Column): Column
    • [ ] def try_multiply(left: Column, right: Column): Column
    • [ ] def try_subtract(left: Column, right: Column): Column
    • [ ] def try_sum(e: Column): Column
    • [ ] def bit_count(e: Column): Column
    • [ ] def bit_get(e: Column, pos: Column): Column
    • [ ] def getbit(e: Column, pos: Column): Column
    • [ ] def ceiling(e: Column, scale: Column): Column
    • [ ] def ceiling(e: Column): Column
    • [ ] def e(): Column
    • [ ] def ln(e: Column): Column
    • [ ] def negative(e: Column): Column
    • [ ] def pi(): Column
    • [ ] def positive(e: Column): Column
    • [ ] def power(l: Column, r: Column): Column
    • [ ] def sign(e: Column): Column
    • [ ] def width_bucket(v: Column, min: Column, max: Column, numBucket: Column): Column
    • [ ] def current_catalog(): Column
    • [ ] def current_database(): Column
    • [ ] def current_schema(): Column
    • [ ] def current_user(): Column
    • [ ] def hll_sketch_estimate(c: Column): Column
    • [ ] def hll_sketch_estimate(columnName: String): Column
    • [ ] def hll_union(c1: Column, c2: Column): Column
    • [ ] def hll_union(columnName1: String, columnName2: String): Column
    • [ ] def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column
    • [ ] def hll_union(columnName1: String, columnName2: String, allowDifferentLgConfigK: Boolean): Column
    • [ ] def user(): Column
    • [ ] def uuid(): Column
    • [ ] def aes_encrypt(input: Column,key: Column,mode: Column,padding: Column,iv: Column,aad: Column): Column
    • [ ] def aes_encrypt(input: Column,key: Column,mode: Column,padding: Column,iv: Column): Column
    • [ ] def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column
    • [ ] def aes_encrypt(input: Column, key: Column, mode: Column): Column
    • [ ] def aes_encrypt(input: Column, key: Column): Column
    • [ ] def aes_decrypt(input: Column,key: Column,mode: Column,padding: Column,aad: Column): Column
    • [ ] def aes_decrypt(input: Column,key: Column,mode: Column,padding: Column): Column
    • [ ] def aes_decrypt(input: Column, key: Column, mode: Column): Column
    • [ ] def aes_decrypt(input: Column, key: Column): Column
    • [ ] def try_aes_decrypt(input: Column,key: Column,mode: Column,padding: Column,aad: Column): Column
    • [ ] def try_aes_decrypt(input: Column,key: Column,mode: Column,padding: Column): Column
    • [ ] def try_aes_decrypt(input: Column, key: Column, mode: Column): Column
    • [ ] def try_aes_decrypt(input: Column, key: Column): Column
    • [ ] def sha(col: Column): Column
    • [ ] def input_file_block_length(): Column
    • [ ] def input_file_block_start(): Column
    • [ ] def reflect(cols: Column*): Column
    • [ ] def java_method(cols: Column*): Column
    • [ ] def version(): Column
    • [ ] def typeof(col: Column): Column
    • [ ] def stack(cols: Column*): Column
    • [ ] def random(seed: Column): Column
    • [ ] def random(): Column
    • [ ] def bitmap_bucket_number(col: Column): Column
    • [ ] def bitmap_bit_position(col: Column): Column
    • [ ] def bitmap_construct_agg(col: Column): Column
    • [ ] def bitmap_count(col: Column): Column
    • [ ] def bitmap_or_agg(col: Column): Column
    • [ ] def len(e: Column): Column
    • [ ] def levenshtein(l: Column, r: Column, threshold: Int): Column
    • [ ] def rlike(str: Column, regexp: Column): Column
    • [ ] def regexp(str: Column, regexp: Column): Column
    • [ ] def regexp_like(str: Column, regexp: Column): Column
    • [ ] def regexp_count(str: Column, regexp: Column): Column
    • [ ] def regexp_extract_all(str: Column, regexp: Column): Column
    • [ ] def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column
    • [ ] def regexp_substr(str: Column, regexp: Column): Column
    • [ ] def regexp_instr(str: Column, regexp: Column): Column
    • [ ] def regexp_instr(str: Column, regexp: Column, idx: Column): Column
    • [ ] def to_binary(e: Column, format: Column): Column
    • [ ] def to_binary(e: Column): Column
    • [ ] def to_char(e: Column, format: Column): Column
    • [ ] def to_varchar(e: Column, format: Column): Column
    • [ ] def to_number(e: Column, format: Column): Column
    • [ ] def replace(src: Column, search: Column, replace: Column): Column
    • [ ] def replace(src: Column, search: Column): Column
    • [ ] def split_part(str: Column, delimiter: Column, partNum: Column): Column
    • [ ] def substr(str: Column, pos: Column, len: Column): Column
    • [ ] def substr(str: Column, pos: Column): Column
    • [ ] def parse_url(url: Column, partToExtract: Column, key: Column): Column
    • [ ] def parse_url(url: Column, partToExtract: Column): Column
    • [ ] def printf(format: Column, arguments: Column*): Column
    • [ ] def url_decode(str: Column): Column
    • [ ] def url_encode(str: Column): Column
    • [ ] def position(substr: Column, str: Column, start: Column): Column
    • [ ] def position(substr: Column, str: Column): Column
    • [ ] def endswith(str: Column, suffix: Column): Column
    • [ ] def startswith(str: Column, prefix: Column): Column
    • [ ] def char(n: Column): Column
    • [ ] def btrim(str: Column): Column
    • [ ] def btrim(str: Column, trim: Column): Column
    • [ ] def try_to_binary(e: Column, format: Column): Column
    • [ ] def try_to_binary(e: Column): Column
    • [ ] def try_to_number(e: Column, format: Column): Column
    • [ ] def char_length(str: Column): Column
    • [ ] def character_length(str: Column): Column
    • [ ] def chr(n: Column): Column
    • [ ] def contains(left: Column, right: Column): Column
    • [ ] def elt(inputs: Column*): Column
    • [ ] def find_in_set(str: Column, strArray: Column): Column
    • [ ] def like(str: Column, pattern: Column, escapeChar: Column): Column
    • [ ] def like(str: Column, pattern: Column): Column
    • [ ] def ilike(str: Column, pattern: Column, escapeChar: Column): Column
    • [ ] def ilike(str: Column, pattern: Column): Column
    • [ ] def lcase(str: Column): Column
    • [ ] def ucase(str: Column): Column
    • [ ] def left(str: Column, len: Column): Column
    • [ ] def right(str: Column, len: Column): Column
    • [ ] def curdate(): Column
    • [ ] def current_timezone(): Column
    • [ ] def now(): Column
    • [ ] def dateadd(start: Column, days: Column): Column
    • [ ] def date_diff(end: Column, start: Column): Column
    • [ ] def date_from_unix_date(days: Column): Column
    • [ ] def day(e: Column): Column
    • [ ] def extract(field: Column, source: Column): Column
    • [ ] def date_part(field: Column, source: Column): Column
    • [ ] def datepart(field: Column, source: Column): Column
    • [ ] def weekday(e: Column): Column
    • [ ] def try_to_timestamp(s: Column, format: Column): Column
    • [ ] def try_to_timestamp(s: Column): Column
    • [ ] def unix_date(e: Column): Column
    • [ ] def unix_micros(e: Column): Column
    • [ ] def unix_millis(e: Column): Column
    • [ ] def unix_seconds(e: Column): Column
    • [ ] def timestamp_millis(e: Column): Column
    • [ ] def timestamp_micros(e: Column): Column
    • [ ] def to_timestamp_ltz(timestamp: Column, format: Column): Column
    • [ ] def to_timestamp_ltz(timestamp: Column): Column
    • [ ] def to_timestamp_ntz(timestamp: Column, format: Column): Column
    • [ ] def to_timestamp_ntz(timestamp: Column): Column
    • [ ] def to_unix_timestamp(e: Column, format: Column): Column
    • [ ] def to_unix_timestamp(e: Column): Column
    • [ ] def try_element_at(column: Column, value: Column): Column
    • [ ] def array_prepend(column: Column, element: Any): Column
    • [ ] def reduce(expr: Column,initialValue: Column,merge: (Column, Column)
    • [ ] def reduce(expr: Column, initialValue: Column, merge: (Column, Column)
    • [ ] def json_array_length(jsonArray: Column): Column
    • [ ] def json_object_keys(json: Column): Column
    • [ ] def mask(input: Column): Column
    • [ ] def mask(input: Column, upperChar: Column): Column
    • [ ] def mask(input: Column, upperChar: Column, lowerChar: Column): Column
    • [ ] def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column
    • [ ] def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column, otherChar: Column): Column
    • [ ] def cardinality(e: Column): Column
    • [ ] def array_size(e: Column): Column
    • [ ] def array_agg(e: Column): Column
    • [ ] def xpath(x: Column, p: Column): Column
    • [ ] def xpath_boolean(x: Column, p: Column): Column
    • [ ] def xpath_double(x: Column, p: Column): Column
    • [ ] def xpath_number(x: Column, p: Column): Column
    • [ ] def xpath_float(x: Column, p: Column): Column
    • [ ] def xpath_int(x: Column, p: Column): Column
    • [ ] def xpath_long(x: Column, p: Column): Column
    • [ ] def xpath_short(x: Column, p: Column): Column
    • [ ] def xpath_string(x: Column, p: Column): Column
    • [ ] def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column
    • [ ] def convert_timezone(targetTz: Column, sourceTs: Column): Column
    • [ ] def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column
    • [ ] def make_dt_interval(days: Column, hours: Column, mins: Column): Column
    • [ ] def make_dt_interval(days: Column, hours: Column): Column
    • [ ] def make_dt_interval(days: Column): Column
    • [ ] def make_dt_interval(): Column
    • [ ] def make_interval(years: Column,months: Column,weeks: Column,days: Column,hours: Column,mins: Column,secs: Column): Column
    • [ ] def make_interval(years: Column,months: Column,weeks: Column,days: Column,hours: Column,mins: Column): Column
    • [ ] def make_interval(years: Column,months: Column,weeks: Column,days: Column,hours: Column): Column
    • [ ] def make_interval(years: Column,months: Column,weeks: Column,days: Column): Column
    • [ ] def make_interval(years: Column, months: Column, weeks: Column): Column
    • [ ] def make_interval(years: Column, months: Column): Column
    • [ ] def make_interval(years: Column): Column
    • [ ] def make_interval(): Column
    • [ ] def make_timestamp(years: Column,months: Column,days: Column,hours: Column,mins: Column,secs: Column,timezone: Column): Column
    • [ ] def make_timestamp(years: Column,months: Column,days: Column,hours: Column,mins: Column,secs: Column): Column
    • [ ] def make_timestamp_ltz(years: Column,months: Column,days: Column,hours: Column,mins: Column,secs: Column,timezone: Column): Column
    • [ ] def make_timestamp_ltz(years: Column,months: Column,days: Column,hours: Column,mins: Column,secs: Column): Column
    • [ ] def make_timestamp_ntz(years: Column,months: Column,days: Column,hours: Column,mins: Column,secs: Column): Column
    • [ ] def make_ym_interval(years: Column, months: Column): Column
    • [ ] def make_ym_interval(years: Column): Column
    • [ ] def make_ym_interval(): Column
    • [ ] def ifnull(col1: Column, col2: Column): Column
    • [ ] def isnotnull(col: Column): Column
    • [ ] def equal_null(col1: Column, col2: Column): Column
    • [ ] def nullif(col1: Column, col2: Column): Column
    • [ ] def nvl(col1: Column, col2: Column): Column
    • [ ] def nvl2(col1: Column, col2: Column, col3: Column): Column
    • [ ] def call_function(funcName: String, cols: Column*): Column

  • From column

    • It seems no new column functions were added
• [x] Close N/A as "won't do" and relate this PR with the new one

Take a look at #307 or #323 if you need help, as it seems to be very similar

Current behaviour

N/A

[Amalicia]

Hey @eruizalo @alfonsorr 👋

It's been a while 😁

Any chance I can get a version of 3.5.x released? Our infrastructure recently upgraded and ideally I'd like to have the properly supported Doric version.

Happy to help in any way I can of course!

[alfonsorr]

Hello @Amalicia

Yup, it's been a while as you say

It's true that we don't have a 3.5.x version, we tested against it in CI but we didn't make a release for a long time. I will try to create one soon. But if you are in a hurry, the 3.4 version of Doric should work fine with spark 3.5

[alfonsorr]

Hello again @Amalicia

We have great news!!! We have published a new release into maven central with 3.5 support. It doesn't have a lot of new features but it is tested against 3.5 to be sure all is ok.