undefined symbol: cudaRuntimeGetVersion Error

Device setup:

NVIDIA-SMI 510.47.03    Driver Version: 510.47.03    CUDA Version: 11.6

nvidia-smi --query-gpu=gpu_name,gpu_bus_id,vbios_version --format=csv
name, pci.bus_id, vbios_version
Tesla T4, 00000000:00:1B.0, 90.04.96.00.02
Tesla T4, 00000000:00:1C.0, 90.04.96.00.02
Tesla T4, 00000000:00:1D.0, 90.04.96.00.02
Tesla T4, 00000000:00:1E.0, 90.04.96.00.02

Code:

In [1]: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
In [2]: import torch
In [3]: model_name = "t5-3b-sharded"
In [4]: model_id=f"ybelkada/{model_name}"
In [5]: model_8bit = AutoModelForSeq2SeqLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True)

StackTrace:

AttributeError                            Traceback (most recent call last)
<ipython-input-6-f8bf7f53fe60> in <module>
----> 1 model_8bit = AutoModelForSeq2SeqLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True)

~/anaconda3/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    461         elif type(config) in cls._model_mapping.keys():
    462             model_class = _get_model_class(config, cls._model_mapping)
--> 463             return model_class.from_pretrained(
    464                 pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs
    465             )

~/anaconda3/lib/python3.8/site-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
   2231
   2232         if load_in_8bit:
-> 2233             from .utils.bitsandbytes import get_keys_to_not_convert, replace_8bit_linear
   2234
   2235             logger.info("Detected 8-bit loading: activating 8-bit loading for this model")

~/anaconda3/lib/python3.8/site-packages/transformers/utils/bitsandbytes.py in <module>
      8     import torch.nn as nn
      9
---> 10     import bitsandbytes as bnb
     11
     12 if is_accelerate_available():

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/__init__.py in <module>
      4 # LICENSE file in the root directory of this source tree.
      5
----> 6 from .autograd._functions import (
      7     MatmulLtState,
      8     bmm_cublas,

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/autograd/_functions.py in <module>
      3
      4 import torch
----> 5 import bitsandbytes.functional as F
      6
      7 from dataclasses import dataclass

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/functional.py in <module>
     11 from torch import Tensor
     12
---> 13 from .cextension import COMPILED_WITH_CUDA, lib
     14 from functools import reduce  # Required in Python 3
     15

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/cextension.py in <module>
    111
    112
--> 113 lib = CUDASetup.get_instance().lib
    114 try:
    115     if lib is None and torch.cuda.is_available():

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/cextension.py in get_instance(cls)
    107         if cls._instance is None:
    108             cls._instance = cls.__new__(cls)
--> 109             cls._instance.initialize()
    110         return cls._instance
    111

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/cextension.py in initialize(self)
     57
     58         from .cuda_setup.main import evaluate_cuda_setup
---> 59         binary_name, cudart_path, cuda, cc, cuda_version_string = evaluate_cuda_setup()
     60         self.cudart_path = cudart_path
     61         self.cuda = cuda

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py in evaluate_cuda_setup()
    123     cuda = get_cuda_lib_handle()
    124     cc = get_compute_capability(cuda)
--> 125     cuda_version_string = get_cuda_version(cuda, cudart_path)
    126
    127     failure = False

~/anaconda3/lib/python3.8/site-packages/bitsandbytes/cuda_setup/main.py in get_cuda_version(cuda, cudart_path)
     43
     44     version = ctypes.c_int()
---> 45     check_cuda_result(cuda, cudart.cudaRuntimeGetVersion(ctypes.byref(version)))
     46     version = int(version.value)
     47     major = version//1000

~/anaconda3/lib/python3.8/ctypes/__init__.py in __getattr__(self, name)
    392         if name.startswith('__') and name.endswith('__'):
    393             raise AttributeError(name)
--> 394         func = self.__getitem__(name)
    395         setattr(self, name, func)
    396         return func

~/anaconda3/lib/python3.8/ctypes/__init__.py in __getitem__(self, name_or_ordinal)
    397
    398     def __getitem__(self, name_or_ordinal):
--> 399         func = self._FuncPtr((name_or_ordinal, self))
    400         if not isinstance(name_or_ordinal, int):
    401             func.__name__ = name_or_ordinal

AttributeError: /home/hchoi/anaconda3/bin/python: undefined symbol: cudaRuntimeGetVersion

[RamonBeast]

Do you think there is a workaround for those that are using pip with virtualenv instead of conda? libcudart.so is already part of the cuda_runtime, so I’m wondering if the entire cudatoolkit is required or we can get away with just the runtime pkg.

[tshadley]

Was able to workaround this with setenv LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu which is where my libcudart.so is (Ubuntu 22.04). Not sure why it misses that.