StatisticsGen does not recognize missing fields

For some reason StatisticsGen is not recognizing the missing values in my file. I was expecting that StatisticsGen recognizes the missing fields in the last records. When I generate the statistics using generate_statistics_from_csv directly works perfectly fine.

# Loading the Files
input_config = example_gen_pb2.Input(splits=[
    example_gen_pb2.Input.Split(name='train', pattern='train/*'),
    example_gen_pb2.Input.Split(name='eval', pattern='eval/*')
])

example_gen = CsvExampleGen(
    input_base=_data_root,
    input_config=input_config)

context.run(example_gen, enable_cache=False)

# Generating Stats
statistics_gen = StatisticsGen(examples=example_gen.outputs['examples'])
context.run(statistics_gen, enable_cache=False)

# Showing the Stats
context.show(statistics_gen.outputs['statistics'])

File being used:

index,inputSurname,label
0,BALLADARAS NATE RAE,0
1,LABRANCHE TRACIE SURIANO,0
2,VENTURES LLC             TIERNAN RE,1
3,CHOU                     ABC,1
4,JENSEN DARREN RANEE,0
5,VANDERMOLEN DEBORA PATRICIA,0
6,ZAMBRANO YANGFANG SESE,0
7,IMAGE LLC                DENTAL,1
8,OFFICE                   S BRUCE             LAW,1
9,,

[aondra17]

I had same problem, and this worked as solution (i had only int and “missing”):

import csv
import tensorflow as tf
from tqdm import tqdm

def x(a) :
    if not a:
        a = tf.train.Feature()
    else:
        
        a = tf.train.Feature(int64_list=tf.train.Int64List(value=[int(a)]))   
    return  a


original_data_file = "./data/data.csv"
tfrecords_filename = "./tfR/consumer-complaints.tfrecords"
tf_record_writer = tf.io.TFRecordWriter(tfrecords_filename)

with open(original_data_file) as csv_file:
    reader = csv.DictReader(csv_file, delimiter=",", quotechar='"')
    for row in tqdm(reader):
        #row = clean_rows(row)
        example = tf.train.Example(
            features=tf.train.Features(
                feature={
                    "index": x(row["index"]),
                    "inputSurname": x(row["inputSurname"]),
                    "label":x(row["label"]),
                    
                }
            )
        )
        tf_record_writer.write(example.SerializeToString())
    tf_record_writer.close()

[google-ml-butler[bot]]

Are you satisfied with the resolution of your issue? Yes No