Memory leak found on 1.4.4

I found a memory leak problem on version 1.4.4 You can use the segment code below in order to reproduce easily. Maybe you need install lib psutil - it helps collecting memory info

def create_kafka_producer(**kwargs):
    bootstrap_servers = kwargs.pop(
        'bootstrap_servers', None) or os.environ.get('KAFKA_SERVER')
    assert bootstrap_servers is not None, 'bootstrap_servers should not be none'
    producer = KafkaProducer(
        bootstrap_servers=bootstrap_servers,
        client_id="test_leak_memory",
        retries=2,
        batch_size=200 * 1024,
        max_request_size=3 * 1024 * 1024
    )
    return producer

topic = 'test_mem_leak'
dummy = {
    'parcel_id': time.time(),
    'chosen_carrier': "{}_{}".format(randint(0, 10), randint(0, 5)),
    'event_count': 1,
    'imported_date': '',
    'should_send_kick_off_msg': False,
    'scraping_parameters': {}
}
dummy_str = json.dumps(dummy)
dummy_byte = str(dummy_str).encode()
producer = create_kafka_producer()

def on_send_success_cb(msg, record_metadata):
    print("msg={}".format(msg))

def produce_msg(limit=2000):
    for _ in range(limit):
        producer.send(topic, value=dummy_byte).add_callback(functools.partial(on_send_success, dummy))
    producer.flush()

def produce_msg_without_leak_mem(limit=2000):
    """"if you pass function and its params to future.add_callback directly, it's ok. Although I found another partial is created inside"""
    for _ in range(limit):
        producer.send(topic, value=dummy_byte).add_callback(on_send_success, dummy)
    producer.flush()

if __name__ == '__main__':
    monitor_logger.info("Start test_leak_memory at {}".format(datetime.now()))
    start_rss = psutil.Process(os.getpid()).memory_info().rss
    for _ in range(10):
        produce_msg(limit=2000)
        time.sleep(5)
    end_rss = psutil.Process(os.getpid()).memory_info().rss
    gc.collect()
    gc_rss = psutil.Process(os.getpid()).memory_info().rss
    monitor_logger.info("Produce success {} msgs".format(max_offset - min_offset))
    monitor_logger.info("Total consumed memory: {} KB".format((end_rss - start_rss) / 1024.0))
    monitor_logger.info("After gc.collect - Total consumed memory: {} KB".format((gc_rss - start_rss) / 1024.0))

Call produce_msg several times, we got memory increases overtime. But take a look produce_msg_without_leak_mem method, if you pass function and its params to future.add_callback directly, it’s ok. Although I found another partial is created inside.

[kingnand]

Thanks for your support, I closed this issue now, will reopen as I collect more proper information that helps reproduce easier

[tvoinarovskyi]

No change after a few hours even with prints in the handler. Not sure if I can help any more without a proper reproduce script.