ModuleNotFoundError: No module named 'fairseq.data.token_block_utils_fast'

I’m trying to reproduce results in https://github.com/pytorch/fairseq/blob/master/examples/language_model/README.md for Russian (transformer_lm.wmt19.ru). Unfoortunately when I run the code:

import torch

# List available models
torch.hub.list('pytorch/fairseq')  # [..., 'transformer_lm.wmt19.en', ...]

# Load an LM trained on
en_lm = torch.hub.load('pytorch/fairseq',
                                       'transformer_lm.wmt19.ru', tokenizer='moses', bpe='fastbpe')

# Sample from the language model
en_lm.sample('Barack Obama', beam=1, sampling=True, sampling_topk=10, temperature=0.8)

It fails with error:

loading archive file https://dl.fbaipublicfiles.com/fairseq/models/lm/wmt19.ru.tar.bz2 from cache at /home/alx/.cache/torch/pytorch_fairseq/29ba20dab200b0132a327b03b8db9551bc9a7b14963ee3e1fb5636f83023523b.21ea13fab90519145b4248e8a1411920238a7c2d1723c5e456bc0404d6e6ec08
| dictionary: 31232 types
Namespace(activation_fn='relu', adaptive_input=False, adaptive_input_cutoff=None, adaptive_input_factor=4, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, adaptive_softmax_factor=4, add_bos_token=False, arch='transformer_lm_gbw', attention_dropout=0.1, bpe='fastbpe', bpe_codes='/home/alx/.cache/torch/pytorch_fairseq/29ba20dab200b0132a327b03b8db9551bc9a7b14963ee3e1fb5636f83023523b.21ea13fab90519145b4248e8a1411920238a7c2d1723c5e456bc0404d6e6ec08/bpecodes', bucket_cap_mb=25, char_embedder_highway_layers=2, character_embedding_dim=4, character_embeddings=False, character_filters='[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]', clip_norm=0.1, cpu=False, criterion='cross_entropy', data='/home/alx/.cache/torch/pytorch_fairseq/29ba20dab200b0132a327b03b8db9551bc9a7b14963ee3e1fb5636f83023523b.21ea13fab90519145b4248e8a1411920238a7c2d1723c5e456bc0404d6e6ec08', ddp_backend='c10d', decoder_attention_heads=16, decoder_embed_dim=1024, decoder_ffn_embed_dim=6144, decoder_input_dim=1024, decoder_layerdrop=0, decoder_layers=20, decoder_layers_to_keep=None, decoder_learned_pos=False, decoder_normalize_before=True, decoder_output_dim=1024, device_id=0, distributed_backend='nccl', distributed_init_method='tcp://learnfair1169:55498', distributed_port=55498, distributed_rank=0, distributed_world_size=32, dropout=0.1, fix_batches_to_gpus=False, fp16=True, fp16_init_scale=128, fp16_scale_tolerance=0.0, fp16_scale_window=None, future_target=False, keep_interval_updates=1, keep_last_epochs=-1, layernorm_embedding=False, lazy_load=False, log_format='json', log_interval=1000, lr=[5e-05], lr_period_updates=959000.0, lr_scheduler='cosine', lr_shrink=0.6, max_epoch=0, max_lr=1.0, max_sentences=None, max_sentences_valid=None, max_source_positions=512, max_target_positions=512, max_tokens=2048, max_update=975000, memory_efficient_fp16=False, min_loss_scale=0.0001, min_lr=1e-09, momentum=0.99, moses_no_dash_splits=False, moses_no_escape=False, no_decoder_final_norm=True, no_epoch_checkpoints=True, no_progress_bar=False, no_save=False, no_scale_embedding=False, no_token_positional_embeddings=False, num_workers=0, optimizer='nag', optimizer_overrides='{}', output_dictionary_size=-1, past_target=False, raw_text=False, relu_dropout=0.1, reset_lr_scheduler=False, reset_optimizer=False, restore_file='checkpoint_last.pt', sample_break_mode='eos', save_dir='/checkpoint/nng/lm/wmt/ru_news.mxup975000.mlr1.0.tmult2.per959000.csn.lrs0.6.wrm16000.int1e-07.nag.lr5e-05.clp0.1.lyr20.hd16.drp0.1.ffn6144.at_d0.1.rl_d0.1.i1024.m1024.o1024.mxtk2048.tps512.seed1.bm=eos.ngpu32', save_interval=1, save_interval_updates=959000, seed=1, self_target=False, sentence_avg=False, share_decoder_input_output_embed=False, skip_invalid_size_inputs_valid_test=True, t_mult=2.0, task='language_modeling', tensorboard_logdir='/checkpoint/nng/tensorboard_logs/2019-03-22/ru_news.mxup975000.mlr1.0.tmult2.per959000.csn.lrs0.6.wrm16000.int1e-07.nag.lr5e-05.clp0.1.lyr20.hd16.drp0.1.ffn6144.at_d0.1.rl_d0.1.i1024.m1024.o1024.mxtk2048.tps512.seed1.bm=eos.ngpu32', threshold_loss_scale=None, tie_adaptive_proj=False, tie_adaptive_weights=False, tokenizer='moses', tokens_per_sample=512, train_subset='train', update_freq=[1], user_dir=None, valid_subset='valid', validate_interval=1, warmup_init_lr=1e-07, warmup_updates=16000, weight_decay=0.0)
Loading codes from /home/alx/.cache/torch/pytorch_fairseq/29ba20dab200b0132a327b03b8db9551bc9a7b14963ee3e1fb5636f83023523b.21ea13fab90519145b4248e8a1411920238a7c2d1723c5e456bc0404d6e6ec08/bpecodes ...
Read 24000 codes from the codes file.
Traceback (most recent call last):
  File "/home/alx/.cache/torch/hub/pytorch_fairseq_master/fairseq/data/token_block_dataset.py", line 46, in __init__
    from fairseq.data.token_block_utils_fast import (
ModuleNotFoundError: No module named 'fairseq.data.token_block_utils_fast'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "fairseq_transformer.py", line 13, in <module>
    en_lm.sample('Barack Obama', beam=1, sampling=True, sampling_topk=10, temperature=0.8)
  File "/home/alx/.cache/torch/hub/pytorch_fairseq_master/fairseq/hub_utils.py", line 107, in sample
    hypo = self.generate(input, beam, verbose, **kwargs)[0]['tokens']
  File "/home/alx/.cache/torch/hub/pytorch_fairseq_master/fairseq/hub_utils.py", line 111, in generate
    sample = self._build_sample(tokens)
  File "/home/alx/.cache/torch/hub/pytorch_fairseq_master/fairseq/hub_utils.py", line 183, in _build_sample
    dataset = self.task.build_dataset_for_inference([src_tokens], [src_tokens.numel()])
  File "/home/alx/.cache/torch/hub/pytorch_fairseq_master/fairseq/tasks/language_modeling.py", line 208, in build_dataset_for_inference
    include_targets=False,
  File "/home/alx/.cache/torch/hub/pytorch_fairseq_master/fairseq/data/token_block_dataset.py", line 52, in __init__
    'Please build Cython components with: `pip install --editable .` '
ImportError: Please build Cython components with: `pip install --editable .` or `python setup.py build_ext --inplace`

By the way I installed fairseq with pip install --editable ., but error still occurs.

Does anyone know how to fix this?

[Jantory]

@Erifai Hi, I encountered the same issue as yours when using colab for loading the PTM. I struggled a lot and found the following solution feasible. I hope this can help you.

!pip install cython -U !git clone https://github.com/pytorch/fairseq.git %cd fairseq !pip install --quiet --editable . !pip install --quiet sentencepiece cd …

Best, Zeyu

[Erifai]

hello guys i’m trying to load camembert model and i’m having the same issue

here’s what m trying to do (jupyter notebook)

any one can help please ? (it’s been a week that m trying to solve this)

import torch
import numpy as np
import hydra
import omegaconf
import cython
import tensorboardX
#### Loading CamemBERT model
camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0', force_reload=True)

---

ERROR :

ModuleNotFoundError                       Traceback (most recent call last)
~/.cache\torch\hub\pytorch_fairseq_main\hubconf.py in <module>
     48 try:
---> 49     import fairseq.data.token_block_utils_fast  # noqa
     50 except ImportError:

ModuleNotFoundError: No module named 'fairseq.data.token_block_utils_fast'

During handling of the above exception, another exception occurred:

DistutilsClassError                       Traceback (most recent call last)
~\Anaconda3\lib\site-packages\setuptools\sandbox.py in save_modules()
    155     with ExceptionSaver() as saved_exc:
--> 156         yield saved
    157 

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in setup_context(setup_dir)
    197                             __import__('setuptools')
--> 198                             yield
    199 

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in run_setup(setup_script, args)
    258                 ns = dict(__file__=setup_script, __name__='__main__')
--> 259                 _execfile(setup_script, ns)
    260         except SystemExit as v:

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in _execfile(filename, globals, locals)
     45     code = compile(script, filename, 'exec')
---> 46     exec(code, globals, locals)
     47 

~/.cache\torch\hub\pytorch_fairseq_main\setup.py in <module>
    281         }
--> 282         do_setup(package_data)
    283     finally:

~/.cache\torch\hub\pytorch_fairseq_main\setup.py in do_setup(package_data)
    189 def do_setup(package_data):
--> 190     setup(
    191         name="fairseq",

~\Anaconda3\lib\site-packages\setuptools\__init__.py in setup(**attrs)
    154     _install_setup_requires(attrs)
--> 155     return distutils.core.setup(**attrs)
    156 

~\Anaconda3\lib\site-packages\setuptools\_distutils\core.py in setup(**attrs)
    134     try:
--> 135         ok = dist.parse_command_line()
    136     except DistutilsArgError as msg:

~\Anaconda3\lib\site-packages\setuptools\_distutils\dist.py in parse_command_line(self)
    483         while args:
--> 484             args = self._parse_command_opts(parser, args)
    485             if args is None:            # user asked for help (and got it)

~\Anaconda3\lib\site-packages\setuptools\dist.py in _parse_command_opts(self, parser, args)
   1083 
-> 1084         nargs = _Distribution._parse_command_opts(self, parser, args)
   1085 

~\Anaconda3\lib\site-packages\setuptools\_distutils\dist.py in _parse_command_opts(self, parser, args)
    546         if not issubclass(cmd_class, Command):
--> 547             raise DistutilsClassError(
    548                 "command class %s must subclass Command" % cmd_class)

DistutilsClassError: command class <class 'torch.utils.cpp_extension.BuildExtension'> must subclass Command

During handling of the above exception, another exception occurred:

DistutilsClassError                       Traceback (most recent call last)
<ipython-input-15-63fa60b7fe1a> in <module>
      6 import tensorboardX
      7 # Loading CamemBERT model
----> 8 camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')#, force_reload=True)

~\Anaconda3\lib\site-packages\torch\hub.py in load(repo_or_dir, model, source, force_reload, verbose, skip_validation, *args, **kwargs)
    397         repo_or_dir = _get_cache_or_reload(repo_or_dir, force_reload, verbose, skip_validation)
    398 
--> 399     model = _load_local(repo_or_dir, model, *args, **kwargs)
    400     return model
    401 

~\Anaconda3\lib\site-packages\torch\hub.py in _load_local(hubconf_dir, model, *args, **kwargs)
    423 
    424     hubconf_path = os.path.join(hubconf_dir, MODULE_HUBCONF)
--> 425     hub_module = import_module(MODULE_HUBCONF, hubconf_path)
    426 
    427     entry = _load_entry_from_hubconf(hub_module, model)

~\Anaconda3\lib\site-packages\torch\hub.py in import_module(name, path)
     74     module = importlib.util.module_from_spec(spec)
     75     assert isinstance(spec.loader, Loader)
---> 76     spec.loader.exec_module(module)
     77     return module
     78 

~\Anaconda3\lib\importlib\_bootstrap_external.py in exec_module(self, module)

~\Anaconda3\lib\importlib\_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

~/.cache\torch\hub\pytorch_fairseq_main\hubconf.py in <module>
     54         from setuptools import sandbox
     55 
---> 56         sandbox.run_setup(
     57             os.path.join(os.path.dirname(__file__), "setup.py"),
     58             ["build_ext", "--inplace"],

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in run_setup(setup_script, args)
    260         except SystemExit as v:
    261             if v.args and v.args[0]:
--> 262                 raise
    263             # Normal exit, just return
    264 

~\Anaconda3\lib\contextlib.py in __exit__(self, type, value, traceback)
    129                 value = type()
    130             try:
--> 131                 self.gen.throw(type, value, traceback)
    132             except StopIteration as exc:
    133                 # Suppress StopIteration *unless* it's the same exception that

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in setup_context(setup_dir)
    196                             # ensure setuptools commands are available
    197                             __import__('setuptools')
--> 198                             yield
    199 
    200 

~\Anaconda3\lib\contextlib.py in __exit__(self, type, value, traceback)
    129                 value = type()
    130             try:
--> 131                 self.gen.throw(type, value, traceback)
    132             except StopIteration as exc:
    133                 # Suppress StopIteration *unless* it's the same exception that

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in save_modules()
    167     _clear_modules(del_modules)
    168 
--> 169     saved_exc.resume()
    170 
    171 

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in resume(self)
    141 
    142         type, exc = map(pickle.loads, self._saved)
--> 143         raise exc.with_traceback(self._tb)
    144 
    145 

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in save_modules()
    154     saved = sys.modules.copy()
    155     with ExceptionSaver() as saved_exc:
--> 156         yield saved
    157 
    158     sys.modules.update(saved)

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in setup_context(setup_dir)
    196                             # ensure setuptools commands are available
    197                             __import__('setuptools')
--> 198                             yield
    199 
    200 

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in run_setup(setup_script, args)
    257             with DirectorySandbox(setup_dir):
    258                 ns = dict(__file__=setup_script, __name__='__main__')
--> 259                 _execfile(setup_script, ns)
    260         except SystemExit as v:
    261             if v.args and v.args[0]:

~\Anaconda3\lib\site-packages\setuptools\sandbox.py in _execfile(filename, globals, locals)
     44         locals = globals
     45     code = compile(script, filename, 'exec')
---> 46     exec(code, globals, locals)
     47 
     48 

~/.cache\torch\hub\pytorch_fairseq_main\setup.py in <module>
    280             )
    281         }
--> 282         do_setup(package_data)
    283     finally:
    284         if "build_ext" not in sys.argv[1:] and os.path.islink(fairseq_examples):

~/.cache\torch\hub\pytorch_fairseq_main\setup.py in do_setup(package_data)
    188 
    189 def do_setup(package_data):
--> 190     setup(
    191         name="fairseq",
    192         version=version,

~\Anaconda3\lib\site-packages\setuptools\__init__.py in setup(**attrs)
    153     logging.configure()
    154     _install_setup_requires(attrs)
--> 155     return distutils.core.setup(**attrs)
    156 
    157 

~\Anaconda3\lib\site-packages\setuptools\_distutils\core.py in setup(**attrs)
    133     # SystemExit to suppress tracebacks.
    134     try:
--> 135         ok = dist.parse_command_line()
    136     except DistutilsArgError as msg:
    137         raise SystemExit(gen_usage(dist.script_name) + "\nerror: %s" % msg)

~\Anaconda3\lib\site-packages\setuptools\_distutils\dist.py in parse_command_line(self)
    482             return
    483         while args:
--> 484             args = self._parse_command_opts(parser, args)
    485             if args is None:            # user asked for help (and got it)
    486                 return

~\Anaconda3\lib\site-packages\setuptools\dist.py in _parse_command_opts(self, parser, args)
   1082             command = args[0]
   1083 
-> 1084         nargs = _Distribution._parse_command_opts(self, parser, args)
   1085 
   1086         # Handle commands that want to consume all remaining arguments

~\Anaconda3\lib\site-packages\setuptools\_distutils\dist.py in _parse_command_opts(self, parser, args)
    545         # to be sure that the basic "command" interface is implemented.
    546         if not issubclass(cmd_class, Command):
--> 547             raise DistutilsClassError(
    548                 "command class %s must subclass Command" % cmd_class)
    549 

DistutilsClassError: command class <class 'torch.utils.cpp_extension.BuildExtension'> must subclass Command