timestampdiff support

the timestampdiff operation currently fails to do missing reinterpret implementation error.

Here’s a reproducer:

from dask_sql import Context
from dask import dataframe as dd
import pandas as pd

c = Context()
df = pd.DataFrame({"dt":["2002-06-05 00:00:00", "2005-09-01 00:00:00", "2003-12-03 00:00:00"]})
df2 = pd.DataFrame({"dt2":["2002-06-07 00:00:00", "2002-06-05 00:00:00", "2002-06-05 00:00:00"]})
c.create_table("df", df)
c.create_table("df2", df2)

query = "SELECT timestampdiff(DAY, CAST(dt AS TIMESTAMP),  CAST(dt2 AS TIMESTAMP)) FROM df, df2"
c.sql(query)

Error with:

NotImplementedError                       Traceback (most recent call last)
/tmp/ipykernel_64215/3992071282.py in <module>
     10 
     11 query = "SELECT timestampdiff(DAY, CAST(dt AS TIMESTAMP),  CAST(dt2 AS TIMESTAMP)) FROM df, df2"
---> 12 c.sql(query)

lib/python3.8/site-packages/dask_sql/context.py in sql(self, sql, return_futures, dataframes)
    421         rel, select_names, _ = self._get_ral(sql)
    422 
--> 423         dc = RelConverter.convert(rel, context=self)
    424 
    425         if dc is None:

lib/python3.8/site-packages/dask_sql/physical/rel/convert.py in convert(cls, rel, context)
     54             f"Processing REL {rel} using {plugin_instance.__class__.__name__}..."
     55         )
---> 56         df = plugin_instance.convert(rel, context=context)
     57         logger.debug(f"Processed REL {rel} into {LoggableDataFrame(df)}")
     58         return df

lib/python3.8/site-packages/dask_sql/physical/rel/logical/project.py in convert(self, rel, context)
     51             else:
     52                 random_name = new_temporary_column(df)
---> 53                 new_columns[random_name] = RexConverter.convert(
     54                     expr, dc, context=context
     55                 )

lib/python3.8/site-packages/dask_sql/physical/rex/convert.py in convert(cls, rex, dc, context)
     60         )
     61 
---> 62         df = plugin_instance.convert(rex, dc, context=context)
     63         logger.debug(f"Processed REX {rex} into {LoggableDataFrame(df)}")
     64         return df

lib/python3.8/site-packages/dask_sql/physical/rex/core/call.py in convert(self, rex, dc, context)
    769     ) -> SeriesOrScalar:
    770         # Prepare the operands by turning the RexNodes into python expressions
--> 771         operands = [
    772             RexConverter.convert(o, dc, context=context) for o in rex.getOperands()
    773         ]

lib/python3.8/site-packages/dask_sql/physical/rex/core/call.py in <listcomp>(.0)
    770         # Prepare the operands by turning the RexNodes into python expressions
    771         operands = [
--> 772             RexConverter.convert(o, dc, context=context) for o in rex.getOperands()
    773         ]
    774 

lib/python3.8/site-packages/dask_sql/physical/rex/convert.py in convert(cls, rex, dc, context)
     60         )
     61 
---> 62         df = plugin_instance.convert(rex, dc, context=context)
     63         logger.debug(f"Processed REX {rex} into {LoggableDataFrame(df)}")
     64         return df

lib/python3.8/site-packages/dask_sql/physical/rex/core/call.py in convert(self, rex, dc, context)
    769     ) -> SeriesOrScalar:
    770         # Prepare the operands by turning the RexNodes into python expressions
--> 771         operands = [
    772             RexConverter.convert(o, dc, context=context) for o in rex.getOperands()
    773         ]

lib/python3.8/site-packages/dask_sql/physical/rex/core/call.py in <listcomp>(.0)
    770         # Prepare the operands by turning the RexNodes into python expressions
    771         operands = [
--> 772             RexConverter.convert(o, dc, context=context) for o in rex.getOperands()
    773         ]
    774 

lib/python3.8/site-packages/dask_sql/physical/rex/convert.py in convert(cls, rex, dc, context)
     60         )
     61 
---> 62         df = plugin_instance.convert(rex, dc, context=context)
     63         logger.debug(f"Processed REX {rex} into {LoggableDataFrame(df)}")
     64         return df

lib/python3.8/site-packages/dask_sql/physical/rex/core/call.py in convert(self, rex, dc, context)
    783                 operation = context.schema[schema_name].functions[operator_name]
    784             except KeyError:  # pragma: no cover
--> 785                 raise NotImplementedError(f"{operator_name} not (yet) implemented")
    786 
    787         logger.debug(

NotImplementedError: reinterpret not (yet) implemented

[jdye64]

@rajagurunath I think it would be good if you could open a WIP PR. That would allow us to discuss a little more and also take advantage of CI for testing theories and changes.

[ayushdg]

I have been looking into this a bit more and updating #293 locally to work with the latest branch. On further inspection it looks like the main issue with month and year comes from calcite assuming a different time_interval for the date subtraction vs the other time units. Currently we handle the SqlDatetimeSubtractionOperator the same as any regular subtraction operation but I think we might need a way to extract the time units for this operation as well from calcite based on the docs here.

I’ll look into this further to see how to get that from calcite.