Dataset download fails when reading boolean column with nulls
See original GitHub issueReproducible code:
from cartoframes.auth import set_default_context
from cartoframes import Dataset
set_default_context(
base_url='https://eschbacher.carto.com',
api_key='default_public'
)
df = Dataset.from_table('seattle_collisions').download()
StackTrace
--------------------------------------------------------------------------- ValueError Traceback (most recent call last) <ipython-input-33-30c7f7d520dd> in <module> ----> 1 df = Dataset.from_table('seattle_collisions').download()~/git/CartoDB/cartoframes/cartoframes/dataset.py in download(self, limit, decode_geom, retry_times) 131 table_columns = self.get_table_columns() 132 query = self._get_read_query(table_columns, limit) –> 133 self.df = self.cc.fetch(query, decode_geom=decode_geom) 134 return self.df 135
~/git/CartoDB/cartoframes/cartoframes/context.py in fetch(self, query, decode_geom) 525 false_values=[‘f’], 526 index_col=‘cartodb_id’ if ‘cartodb_id’ in df_types else False, –> 527 converters={‘the_geom’: lambda x: _decode_geom(x) if decode_geom else x}) 528 529 if decode_geom:
~/.local/share/virtualenvs/cartoframes-pRWzcuWT/lib/python3.7/site-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision) 700 skip_blank_lines=skip_blank_lines) 701 –> 702 return _read(filepath_or_buffer, kwds) 703 704 parser_f.name = name
~/.local/share/virtualenvs/cartoframes-pRWzcuWT/lib/python3.7/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds) 433 434 try: –> 435 data = parser.read(nrows) 436 finally: 437 parser.close()
~/.local/share/virtualenvs/cartoframes-pRWzcuWT/lib/python3.7/site-packages/pandas/io/parsers.py in read(self, nrows) 1137 def read(self, nrows=None): 1138 nrows = _validate_integer(‘nrows’, nrows) -> 1139 ret = self._engine.read(nrows) 1140 1141 # May alter columns / col_dict
~/.local/share/virtualenvs/cartoframes-pRWzcuWT/lib/python3.7/site-packages/pandas/io/parsers.py in read(self, nrows) 1993 def read(self, nrows=None): 1994 try: -> 1995 data = self._reader.read(nrows) 1996 except StopIteration: 1997 if self._first_chunk:
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.read()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_low_memory()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_rows()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._convert_column_data()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._convert_tokens()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._convert_with_dtype()
ValueError: Bool column has NA values in column 28
Not sure of the cause yet, but I have a feeling that pandas bool columns cannot have null values.
Issue Analytics
- State:
- Created 4 years ago
- Comments:13 (13 by maintainers)
Hi @andy-esch,
I have created here a PoC with the implementation of the solution: https://github.com/CartoDB/cartoframes/pull/760. It would be nice if you can try it 😃
Fixed in #760