Check Remove duplicates functionality from SparkIcebergUtils

[Neuw84]

spark .sql(""" INSERT OVERWRITE employee SELECT employee_id, start_date, first(team),first(role),first(address),first(name) FROM employee WHERE cast(start_date as date) = '2020-07-01' -- here we remove from a predefined day GROUP BY employee_id, start_date """ ) .show();