race condition on post multiple files to the same folder

[Han Hu]

Here is my model definition

class Item(models.Model):
    name = models.CharField(max_length=256)

def item_id_path(instance, filename):
    return '{}/{}'.format(instance.item.id, filename)


# Create your models here.
class File(models.Model):
    file = models.FileField(upload_to=item_id_path)
    timestamp = models.DateTimeField(auto_now_add=True)
    item = models.ForeignKey(Item, on_delete=models.CASCADE)

    def __str__(self):
        return self.file.name

The application posts multiples file that belong to the same item. But I run into race condition in making directories. My server stores all the related files in a separate folder (named by the item id) together.

In django/core/files/storage.py _save method it checks the existence of uploading directory. When multiple files for the same item are uploaded the first request checks the existence of the directory and is going to create one, then it switches to the second file upload. It also finds that the directory doesn't exist yet and it is going to do the same directory creation. The second file upload will fail because the first file upload has already created the folder. It fails at the last line of the following code snippet in storage.py.

def _save(self, name, content):
    full_path = self.path(name)

    # Create any intermediate directories that do not exist.
    directory = os.path.dirname(full_path)
    if not os.path.exists(directory):
        try:
            if self.directory_permissions_mode is not None:
                # os.makedirs applies the global umask, so we reset it,
                # for consistency with file_permissions_mode behavior.
                old_umask = os.umask(0)
                try:
                    os.makedirs(directory, self.directory_permissions_mode)
                finally:
                    os.umask(old_umask)
            else:
                os.makedirs(directory)

what approach can I take to avoid this race condition? 

I guess changing the source code from os.makedirs(directory) to os.makedirs(directory, exist_ok=True) could work.

 File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/rest_framework/mixins.py", line 21, in create

    self.perform_create(serializer)

  File "/Users/han/projects/django_maplus_server/rest_api/views.py", line 27, in perform_create

    return super(FileViewSet, self).perform_create(serializer)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/rest_framework/mixins.py", line 26, in perform_create

    serializer.save()

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/rest_framework/serializers.py", line 214, in save

    self.instance = self.create(validated_data)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/rest_framework/serializers.py", line 940, in create

    instance = ModelClass.objects.create(**validated_data)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method

    return getattr(self.get_queryset(), name)(*args, **kwargs)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/query.py", line 413, in create

    obj.save(force_insert=True, using=self.db)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py", line 717, in save

    force_update=force_update, update_fields=update_fields)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py", line 747, in save_base

    updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py", line 830, in _save_table

    result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/base.py", line 868, in _do_insert

    using=using, raw=raw)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/manager.py", line 82, in manager_method

    return getattr(self.get_queryset(), name)(*args, **kwargs)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/query.py", line 1133, in _insert

    return query.get_compiler(using=using).execute_sql(return_id)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1284, in execute_sql

    for sql, params in self.as_sql():

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1237, in as_sql

    for obj in self.query.objs

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1237, in <listcomp>

    for obj in self.query.objs

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1236, in <listcomp>

    [self.prepare_value(field, self.pre_save_val(field, obj)) for field in fields]

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/sql/compiler.py", line 1188, in pre_save_val

    return field.pre_save(obj, add=True)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/fields/files.py", line 288, in pre_save

    file.save(file.name, file.file, save=False)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/db/models/fields/files.py", line 87, in save

    self.name = self.storage.save(name, content, max_length=self.field.max_length)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/core/files/storage.py", line 49, in save

    return self._save(name, content)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/site-packages/django/core/files/storage.py", line 236, in _save

    os.makedirs(directory)

  File "/Users/han/anaconda3/envs/django/lib/python3.6/os.py", line 220, in makedirs

    mkdir(name, mode)

FileExistsError: [Errno 17] File exists: 

[Simon Charette]

Hello there,

I think you just hit #29890[0], a bug present in Django in 2.0+ and fixed in Django 2.1.3

which should be released today[1].

Simply updating to 2.1.3 when it's released should address your issue.

Cheers,

Simon

[0] https://code.djangoproject.com/ticket/29890

[1] https://docs.djangoproject.com/en/2.1/releases/2.1.3/