DatabaseError: Execution failed on sql: SELECT name FROM sqlite_master WHERE type='table' AND name=?;

from impala.dbapi import connect
import pandas as pd


conn = connect(host='localhost', port=10000,
               database='default',user='406449', password='password!',
               auth_mechanism='PLAIN')

sql_select ="select userid, movieid from u_data_new limit 10"
df_result = pd.read_sql(sql_select, conn)                                   # works well

df_result.to_sql('new_table', conn)    # raise error

DatabaseError: Execution failed on sql: SELECT name FROM sqlite_master WHERE type='table' AND name=?;
Error while compiling statement: FAILED: ParseException line 1:59 cannot recognize input near ''new_table'' ';' '<EOF>' in expression specification
unable to rollback

I installed hadoop, hive, hiveserver2 on my local mac. I can create hive engine. With this engine, i can use pandas read_sql(). but i can’t use to_sql() though both method use same engine object.

In my opinion, there is parsing error while handling parameters.

Below is full error messages.

Thank you for this cool project impyla

---------------------------------------------------------------------------
HiveServer2Error                          Traceback (most recent call last)
/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1400             else:
-> 1401                 cur.execute(*args)
   1402             return cur

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in execute(self, operation, parameters, configuration)
    301         self.execute_async(operation, parameters=parameters,
--> 302                            configuration=configuration)
    303         log.debug('Waiting for query to finish')

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in execute_async(self, operation, parameters, configuration)
    342 
--> 343         self._execute_async(op)
    344 

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in _execute_async(self, operation_fn)
    361         self._debug_log_state()
--> 362         operation_fn()
    363         self._last_operation_active = True

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in op()
    339                                       configuration,
--> 340                                       async=True)
    341             self._last_operation = op

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in execute(self, statement, configuration, async)
   1026                                    runAsync=async)
-> 1027         return self._operation('ExecuteStatement', req)
   1028 

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in _operation(self, kind, request)
    956     def _operation(self, kind, request):
--> 957         resp = self._rpc(kind, request)
    958         return self._get_operation(resp.operationHandle)

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in _rpc(self, func_name, request)
    924         self._log_response(func_name, response)
--> 925         err_if_rpc_not_ok(response)
    926         return response

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in err_if_rpc_not_ok(resp)
    703             resp.status.statusCode != TStatusCode.STILL_EXECUTING_STATUS):
--> 704         raise HiveServer2Error(resp.status.errorMessage)
    705 

HiveServer2Error: Error while compiling statement: FAILED: ParseException line 1:59 cannot recognize input near ''new_table'' ';' '<EOF>' in expression specification

During handling of the above exception, another exception occurred:

NotSupportedError                         Traceback (most recent call last)
/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1404             try:
-> 1405                 self.con.rollback()
   1406             except Exception:  # pragma: no cover

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in rollback(self)
     77         # PEP 249
---> 78         raise NotSupportedError
     79 

NotSupportedError: 

During handling of the above exception, another exception occurred:

DatabaseError                             Traceback (most recent call last)
<ipython-input-20-c60572db3bb2> in <module>()
----> 1 df_result.to_sql('new_table', conn)

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/core/generic.py in to_sql(self, name, con, flavor, schema, if_exists, index, index_label, chunksize, dtype)
   1199         sql.to_sql(self, name, con, flavor=flavor, schema=schema,
   1200                    if_exists=if_exists, index=index, index_label=index_label,
-> 1201                    chunksize=chunksize, dtype=dtype)
   1202 
   1203     def to_pickle(self, path):

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in to_sql(frame, name, con, flavor, schema, if_exists, index, index_label, chunksize, dtype)
    468     pandas_sql.to_sql(frame, name, if_exists=if_exists, index=index,
    469                       index_label=index_label, schema=schema,
--> 470                       chunksize=chunksize, dtype=dtype)
    471 
    472 

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in to_sql(self, frame, name, if_exists, index, index_label, schema, chunksize, dtype)
   1500                             if_exists=if_exists, index_label=index_label,
   1501                             dtype=dtype)
-> 1502         table.create()
   1503         table.insert(chunksize)
   1504 

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in create(self)
    582 
    583     def create(self):
--> 584         if self.exists():
    585             if self.if_exists == 'fail':
    586                 raise ValueError("Table '%s' already exists." % self.name)

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in exists(self)
    570 
    571     def exists(self):
--> 572         return self.pd_sql.has_table(self.name, self.schema)
    573 
    574     def sql_schema(self):

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in has_table(self, name, schema)
   1512                  "WHERE type='table' AND name=%s;") % wld
   1513 
-> 1514         return len(self.execute(query, [name, ]).fetchall()) > 0
   1515 
   1516     def get_table(self, table_name, schema=None):

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1407                 ex = DatabaseError("Execution failed on sql: %s\n%s\nunable"
   1408                                    " to rollback" % (args[0], exc))
-> 1409                 raise_with_traceback(ex)
   1410 
   1411             ex = DatabaseError(

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/compat/__init__.py in raise_with_traceback(exc, traceback)
    338         if traceback == Ellipsis:
    339             _, _, traceback = sys.exc_info()
--> 340         raise exc.with_traceback(traceback)
    341 else:
    342     # this version of raise is a syntax error in Python 3

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/pandas/io/sql.py in execute(self, *args, **kwargs)
   1403         except Exception as exc:
   1404             try:
-> 1405                 self.con.rollback()
   1406             except Exception:  # pragma: no cover
   1407                 ex = DatabaseError("Execution failed on sql: %s\n%s\nunable"

/Users/hyundai/.pyenv/versions/3.6.0/envs/py3/lib/python3.6/site-packages/impala/hiveserver2.py in rollback(self)
     76         """Impala doesn't support transactions; raises NotSupportedError"""
     77         # PEP 249
---> 78         raise NotSupportedError
     79 
     80     def cursor(self, user=None, configuration=None, convert_types=True,

DatabaseError: Execution failed on sql: SELECT name FROM sqlite_master WHERE type='table' AND name=?;
Error while compiling statement: FAILED: ParseException line 1:59 cannot recognize input near ''new_table'' ';' '<EOF>' in expression specification
unable to rollback

[kourou1034]

the ‘conn’ in df.to_sql is sqlalchemy engine. As explained at https://github.com/cloudera/impyla/issues/214, you can try this

[shacker]

I’m also blocked by this error. Upgrading to snowflake-connector 2.8.2 (the latest) did not solve it.