Windows permission error

I have a permission error while using Parallel(n_jobs=2) on windows:

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\site-packages\joblib\disk.py:122: UserWarning: Unable to delete folder xxxx\AppData\Local\Temp\joblib_memmapping_folder_15364_9553521538 after 5 tentatives.
  .format(folder_path, RM_SUBDIRS_N_RETRY))
---------------------------------------------------------------------------
PermissionError                           Traceback (most recent call last)
~\Desktop\all_repos\gitlab_projects\wireless_optim\code\complete_experiment.py in <module>()
    422         simulator_data = np.concatenate([simulator_data, new_simulator_data],
    423                                         axis=0)
--> 424         n_admissible_actions.append(n_admissible_actions_d)
    425
    426 if 0:

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\site-packages\joblib\parallel.py in __exit__(self, exc_type, exc_value, traceback)
    664
    665     def __exit__(self, exc_type, exc_value, traceback):
--> 666         self._terminate_backend()
    667         self._managed_backend = False
    668

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\site-packages\joblib\parallel.py in _terminate_backend(self)
    694     def _terminate_backend(self):
    695         if self._backend is not None:
--> 696             self._backend.terminate()
    697
    698     def _dispatch(self, batch):

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\site-packages\joblib\_parallel_backends.py in terminate(self)
    528             # in latter calls but we free as much memory as we can by deleting
    529             # the shared memory
--> 530             delete_folder(self._workers._temp_folder)
    531             self._workers = None
    532

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\site-packages\joblib\disk.py in delete_folder(folder_path, onerror)
    113             while True:
    114                 try:
--> 115                     shutil.rmtree(folder_path, False, None)
    116                     break
    117                 except (OSError, WindowsError):

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\shutil.py in rmtree(path, ignore_errors, onerror)
    492             os.close(fd)
    493     else:
--> 494         return _rmtree_unsafe(path, onerror)
    495
    496 # Allow introspection of whether or not the hardening against symlink

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\shutil.py in _rmtree_unsafe(path, onerror)
    387                 os.unlink(fullname)
    388             except OSError:
--> 389                 onerror(os.unlink, fullname, sys.exc_info())
    390     try:
    391         os.rmdir(path)

~\AppData\Local\Continuum\anaconda3\envs\wireless_optim\lib\shutil.py in _rmtree_unsafe(path, onerror)
    385         else:
    386             try:
--> 387                 os.unlink(fullname)
    388             except OSError:
    389                 onerror(os.unlink, fullname, sys.exc_info())

PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'xxxx\\AppData\\Local\\Temp\\joblib_memmapping_folder_15364_9553521538\\15364-3031207306352-94b02d62d9b44d709a9a405235589ead.pkl'

Is this related to the fact that removing a memmap can fail on Windows as said in the doc? I can try to have a small reproducible example.

I’m using joblib 0.13. (FWIW this code is working on Ubuntu 16.04.5 LTS)

[albertcthomas]

@robintwhite for now, if this is possible in your case, you can disable memmaping by setting max_nbytes=None in Parallel

[albertcthomas]

Thanks for the details. For now I’m using Parallel with max_nbytes=None.