python minio client may cause thread exhaust

• python: 3.5
• minio-client: 2.2.6
• minio-server info: VERSION 2017-10-27T18:59:02Z MEMORY Used: 7.5 MB | Allocated: 50 GB | Used-Heap: 7.5 MB | Allocated-Heap: 29 MB PLATFORM Host: 6cbd0abfd258 | OS: linux | Arch: amd64 RUNTIME Version: go1.9.2 | CPUs: 12

I use python script to create img and save record into db, and use minio python client upload to minio server.

minio_client = Minio('192.168.1.144:9000',
                     access_key='',
                     secret_key='',
                     secure=False)


def upload(file_path, key_name):
  try:
    minio_client.fput_object('intellid',
                             key_name,
                             file_path,
                             content_type='image/png',
                             metadata={'Content-Disposition': 'attachment'})
  except ResponseError as err:
    logging.exception(file_path)
    logging.exception('upload failed!')


def main():
  main_id = threading.get_ident()
  db = default_session()
  gens = db.query(Generation).filter(Generation.status == None).limit(5).all()

  processed = 0
  while gens and len(gens) != 0:
    for gen in gens:
      compositions = copy.deepcopy(gen.composition['compositions'])
      for composition in compositions:
        composition['img_path'] = path.join(data_home, path.basename(
          db.query(Img).filter(Img.guid == composition[
            'guid']).first().oss))

      out_img = compose(gen.composition['width'], gen.composition['height'],
                        gen.composition['linkType'], compositions)

      key_name = 'ori/' + str(uuid.uuid4()) + '.png'
      key_name_thumb = 'thumb/' + str(uuid.uuid4()) + '.png'

      tmp = path.join(prod_out, key_name)
      thumb_tmp = path.join(prod_out, key_name_thumb)

      out_img.save(tmp)
      res = post_proc(tmp, thumb_tmp, 300)

      guid = uuid.uuid1().int >> 65
      while True:
        img = db.query(Img).filter(Img.guid == guid).first()
        if not img:
          break
        guid = uuid.uuid1().int >> 65
      minio_oss = minio_oss_prefix + key_name
      minio_oss_thumb = minio_oss_prefix + key_name_thumb

      new_img = Img(guid=guid, oss=minio_oss, thumb_oss=minio_oss_thumb,
                    file_type='PNG', img_type='AI', attr=res)
      print("before upload acitve_thread {}.".format(threading.active_count()))
      upload(tmp, key_name)
      upload(thumb_tmp, key_name_thumb)
      print("after upload acitve_thread {}.".format(threading.active_count()))
      db.add(new_img)
      db.commit()
      gen.guid = guid
      gen.status = 0
      processed += 1
      db.commit()
      print("processed {}.".format(processed))
    # end for
    gens = db.query(Generation).filter(Generation.status == None).limit(
      5).all()
  # end while


if __name__ == '__main__':
  main()

partial log print: before upload acitve_thread 319. after upload acitve_thread 322. processed 107. before upload acitve_thread 322. after upload acitve_thread 325. processed 108. before upload acitve_thread 325. after upload acitve_thread 328. processed 109.

problem: the minio client will create ThreadPool with 3 thread (code location: minio/api.py line 1511). And these thread (minio.thread_pool.Worker) will nerver exit. After every iteration in loop, the active thread number will increase 3. Finally the total threads in current process will exceed the linux limit(2048 in ulimit -a).

I’m a novice in minio. appreciation for any help. thanks.

[nnehdi-zz]

Thank You, I will send it to you soon.

[harshavardhana]

This is excellent debugging. Thanks for doing that feel free to send PR @nnehdi