sfaira breaks when Universe is initialised from read-only python installation

I guess we need to control the caching a bit more tightly so that this is written to a location that can always assumed to be writable (like /tmp or so)

OSError                                   Traceback (most recent call last)
~/.ch_tmp/ipykernel_20042/80063267.py in <module>
      1 import sfaira
      2 
----> 3 univ = sfaira.data.Universe(
      4             data_path=f"/storage/groups/ml01/datasets/projects/20200101_Various_SfairaDataRepository_leander/raw/",
      5             meta_path=f"/storage/groups/ml01/datasets/projects/20200101_Various_SfairaDataRepository_leander/meta/",

/opt/python/lib/python3.8/site-packages/sfaira/data/dataloaders/super_group.py in __init__(self, data_path, meta_path, cache_path, exclude_databases)
     29         """
     30         dsgs = [
---> 31             DatasetSuperGroupLoaders(
     32                 data_path=data_path,
     33                 meta_path=meta_path,

/opt/python/lib/python3.8/site-packages/sfaira/data/dataloaders/loaders/super_group.py in __init__(self, data_path, meta_path, cache_path)
     37                     if path_dsg is not None:
     38                         try:
---> 39                             dsg = DatasetGroupDirectoryOriented(
     40                                 file_base=path_dsg,
     41                                 data_path=data_path,

/opt/python/lib/python3.8/site-packages/sfaira/data/dataloaders/base/dataset_group.py in __init__(self, file_base, data_path, meta_path, cache_path)
    745                             for x in sample_fns:
    746                                 datasets_f.append(
--> 747                                     DatasetFound(
    748                                         data_path=data_path,
    749                                         meta_path=meta_path,

/opt/python/lib/python3.8/site-packages/sfaira/data/dataloaders/loaders/d10_1016_j_cell_2017_09_004/human_isletoflangerhans_2017_smartseq2_enge_001.py in __init__(self, **kwargs)
     18 
     19         self.author = "Enge"
---> 20         self.disease = "healthy"
     21         self.doi_journal = "10.1016/j.cell.2017.09.004"
     22         self.doi_preprint = "10.1101/108043"

/opt/python/lib/python3.8/site-packages/sfaira/data/dataloaders/base/dataset.py in disease(self, x)
   1611     @disease.setter
   1612     def disease(self, x: str):
-> 1613         x = self._value_protection(attr="disease", allowed=self.ontology_container_sfaira.disease,
   1614                                    attempted=x)
   1615         self._disease = x

/opt/python/lib/python3.8/site-packages/sfaira/consts/ontologies.py in disease(self)
    120     def disease(self):
    121         if self._disease is None:
--> 122             self._disease = OntologyMondo(branch=DEFAULT_MONDO)
    123         return self._disease
    124 

/opt/python/lib/python3.8/site-packages/sfaira/versions/metadata/base.py in __init__(self, branch, recache, **kwargs)
    955     ):
    956         # Latest release also available from url="http://purl.obolibrary.org/obo/mondo.obo".
--> 957         obofile = cached_load_file(
    958             url=f"https://raw.githubusercontent.com/monarch-initiative/mondo/{branch}/mondo-lastbuild.obo",
    959             ontology_cache_dir="mondo",

/opt/python/lib/python3.8/site-packages/sfaira/versions/metadata/base.py in cached_load_file(url, ontology_cache_dir, ontology_cache_fn, recache)
     38         # Download if necessary:
     39         if not os.path.isfile(obofile) or recache:
---> 40             os.makedirs(name=ontology_cache_dir, exist_ok=True)
     41 
     42             def download_file():

/opt/python/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    211     if head and tail and not path.exists(head):
    212         try:
--> 213             makedirs(head, exist_ok=exist_ok)
    214         except FileExistsError:
    215             # Defeats race condition when another thread created the path

/opt/python/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    211     if head and tail and not path.exists(head):
    212         try:
--> 213             makedirs(head, exist_ok=exist_ok)
    214         except FileExistsError:
    215             # Defeats race condition when another thread created the path

/opt/python/lib/python3.8/os.py in makedirs(name, mode, exist_ok)
    221             return
    222     try:
--> 223         mkdir(name, mode)
    224     except OSError:
    225         # Cannot rely on checking for EEXIST, since the operating system

OSError: [Errno 30] Read-only file system: '/opt/python/lib/python3.8/site-packages/cache'

[le-ander]

@davidsebfischer are you willing to drop this then?

I think it is desirable to keep the caches in the sfaira installation so that people can easily copy them there if they work on an offline machine and check this out if there are issues with downloads. We can discuss whether this outweighs the issue above.

we could have a ~/.sfaira_caches/ where caches are stored by default that can be copied over of needed?

[le-ander]

hmm, my feeling is that writing to the sfaira installation directory might be bad practise when sfaira is not installed as editable (ie. when just using pip install sraira). @Zethson what’s your opinion on this? Anyway, for container use this definitely leads to problems (it also does when a server provides a preinstalled python installed with sfaira system wide). in these cases the installation directory cannot be assumed as writable.