Question: Sitemaps without items raise ValueError on callable lastmod. Description When sitemap contains not items, but supports returning lastmod for an item, it fails with a ValueError: Traceback (most recent call last): File "/usr/local/lib/python3.10/site-", line 55, in inner response = getresponse(request) File "/usr/local/lib/python3.10/site-", line 197, in getresponse response = wrappedcallback(request, callbackargs, callbackkwargs) File "/usr/local/lib/python3.10/site-", line 133, in wrappedview response = viewfunc(request, args, kwargs) File "/usr/local/lib/python3.10/site-", line 34, in inner response = func(request, args, *kwargs) File "/usr/local/lib/python3.10/site-", line 76, in index sitelastmod = site.getlatestlastmod() File "/usr/local/lib/python3.10/site-", line 170, in getlatestlastmod return max([self.lastmod(item) for item in ()]) Exception Type: ValueError at Exception Value: max() arg is an empty sequence Something like this might be a solution: def getlatestlastmod(self): if not hasattr(self, "lastmod"): return None if callable(self.lastmod): try: return max([self.lastmod(item) for item in ()]) except TypeError: except (TypeError, ValueError): return None else: return self.lastmod

return None         if callable(self.lastmod):             try:                return max([self.lastmod(item) for item in self.items()])                return max([self.lastmod(item) for item in self.items()], default=None)             except TypeError:                 return None         else:


Question: model.refreshfromdb() doesn't clear cached generic foreign keys Description (last modified by pascal chambon) In my code, Users have a generic foreign key like so: controlledentitycontenttype = models.ForeignKey(ContentType, blank=True, null=True, ondelete=models.CASCADE) controlledentityobjectid = models.PositiveIntegerField(blank=True, null=True) controlledentity = GenericForeignKey("controlledentitycontenttype", "controlledentityobjectid") However, in unit-tests, when I refresh a user instance, the controlledentity relation isn't cleared from cache, as can be seen here with IDs: oldcontrolledentity = authenticateduser.controlledentity authenticateduser.refreshfromdb() newcontrolledentity = authenticateduser.controlledentity assert id(oldcontrolledentity) != id(newcontrolledentity) # FAILS And this leads to subtle bugs like non-transitive equalities in tests : assert authenticateduser.controlledentity == fixtures.client1project2organization3 assert fixtures.client1project2organization3.getpricingplan() == pricingplan assert authenticateduser.controlledentity.getpricingplan() == pricingplan # FAILS Calling "authenticateduser.controlledentity.refreshfromdb()" solved this particular bug, but "authenticateduser.refreshfromdb()" isn't enough. Tested under Django3.2.13 but the code of refreshfromdb() hasn't changed since then in Git's main branch (except few cosmetic adjustments on code format). I'm a bit lost in the code of refreshfromdb(), but I've just seen that the generic relation appears once in this loop, but is not considered as "cached" in the if() branch. for field in #print("%% CLEARING RELATED FIELD", field) if field.iscached(self): #print("%% DONE") # not called field.deletecachedvalue(self)

if field.is_cached(self):                 field.delete_cached_value(self) <ins>        # Clear cached private relations.        for field in self._meta.private_fields:            if field.is_relation and field.is_cached(self):                field.delete_cached_value(self)</ins>         self._state.db = db_instance._state.db      async def arefresh_from_db(self, using=None, fields=None):


Question: Migration changing ManyToManyField target to 'self' doesn't work correctly Description Steps to reproduce: Create Models: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Bar', blank=True) Migrate: makemigrations app migrate Change type of the ManyToManyField to Foo: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Foo', blank=True) Migrate (see above) In the admin page, navigate to "add Foo", click save You should see an OperationalError, "no such column: appfoobar.fromfooid"

super().alter_field(model, old_field, new_field, strict=strict)      def _remake_table(        self, model, create_field=None, delete_field=None, alter_field=None        self, model, create_field=None, delete_field=None, alter_fields=None     ):         """         Shortcut to transform a model from old_model into new_model


Question: Migration changing ManyToManyField target to 'self' doesn't work correctly Description Steps to reproduce: Create Models: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Bar', blank=True) Migrate: makemigrations app migrate Change type of the ManyToManyField to Foo: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Foo', blank=True) Migrate (see above) In the admin page, navigate to "add Foo", click save You should see an OperationalError, "no such column: appfoobar.fromfooid"

# If any of the new or altered fields is introducing a new PK,         # remove the old one         restore_pk_field = None        if getattr(create_field, "primary_key", False) or (            alter_field and getattr(alter_field[1], "primary_key", False)        alter_fields = alter_fields or []        if getattr(create_field, "primary_key", False) or any(            getattr(new_field, "primary_key", False) for _, new_field in alter_fields         ):             for name, field in list(body.items()):                if field.primary_key and not (                if field.primary_key and not any(                     # Do not remove the old primary key when an altered field                     # that introduces a primary key is the same field.                    alter_field                    and name == alter_field[1].name                    name == new_field.name                    for _, new_field in alter_fields                 ):                     field.primary_key = False                     restore_pk_field = field


Question: Migration changing ManyToManyField target to 'self' doesn't work correctly Description Steps to reproduce: Create Models: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Bar', blank=True) Migrate: makemigrations app migrate Change type of the ManyToManyField to Foo: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Foo', blank=True) Migrate (see above) In the admin page, navigate to "add Foo", click save You should see an OperationalError, "no such column: appfoobar.fromfooid"

self.effective_default(create_field),                 )         # Add in any altered fields        if alter_field:        for alter_field in alter_fields:             old_field, new_field = alter_field             body.pop(old_field.name, None)             mapping.pop(old_field.column, None)


Question: Migration changing ManyToManyField target to 'self' doesn't work correctly Description Steps to reproduce: Create Models: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Bar', blank=True) Migrate: makemigrations app migrate Change type of the ManyToManyField to Foo: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Foo', blank=True) Migrate (see above) In the admin page, navigate to "add Foo", click save You should see an OperationalError, "no such column: appfoobar.fromfooid"

)             )         # Alter by remaking table        self._remake_table(model, alter_field=(old_field, new_field))        self._remake_table(model, alter_fields=[(old_field, new_field)])         # Rebuild tables with FKs pointing to this field.         old_collation = old_db_params.get("collation")         new_collation = new_db_params.get("collation")


Question: Migration changing ManyToManyField target to 'self' doesn't work correctly Description Steps to reproduce: Create Models: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Bar', blank=True) Migrate: makemigrations app migrate Change type of the ManyToManyField to Foo: class Bar(models.Model): pass class Foo(models.Model): bar = models.ManyToManyField('Foo', blank=True) Migrate (see above) In the admin page, navigate to "add Foo", click save You should see an OperationalError, "no such column: appfoobar.fromfooid"

# propagate this altering.             self._remake_table(                 old_field.remote_field.through,                alter_field=(                    # The field that points to the target model is needed, so                    # we can tell alter_field to change it - this is                    # m2m_reverse_field_name() (as opposed to m2m_field_name(),                    # which points to our model).                    old_field.remote_field.through._meta.get_field(                        old_field.m2m_reverse_field_name()                alter_fields=[                    (                        # The field that points to the target model is needed,                        # so that table can be remade with the new m2m field -                        # this is m2m_reverse_field_name().                        old_field.remote_field.through._meta.get_field(                            old_field.m2m_reverse_field_name()                        ),                        new_field.remote_field.through._meta.get_field(                            new_field.m2m_reverse_field_name()                        ),                     ),                    new_field.remote_field.through._meta.get_field(                        new_field.m2m_reverse_field_name()                    (                        # The field that points to the model itself is needed,                        # so that table can be remade with the new self field -                        # this is m2m_field_name().                        old_field.remote_field.through._meta.get_field(                            old_field.m2m_field_name()                        ),                        new_field.remote_field.through._meta.get_field(                            new_field.m2m_field_name()                        ),                     ),                ),                ],             )             return


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

# Add max_length for all CharFields.         if field_type == "CharField" and row.display_size:            field_params["max_length"] = int(row.display_size)            if (size := int(row.display_size)) and size > 0:                field_params["max_length"] = size          if field_type in {"CharField", "TextField"} and row.collation:             field_params["db_collation"] = row.collationdiff --git a/django/db/backends/base/features.py b/django/db/backends/base/features.py-- a/django/db/backends/base/features.py++ b/django/db/backends/base/features.py


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

# Set to (exception, message) if null characters in text are disallowed.     prohibits_null_characters_in_text_exception = None     # Does the backend support unlimited character columns?    supports_unlimited_charfield = False     # Collation names for use by the Django test suite.     test_collations = {         "ci": None,  # Case-insensitive.diff --git a/django/db/backends/postgresql/base.py b/django/db/backends/postgresql/base.py-- a/django/db/backends/postgresql/base.py++ b/django/db/backends/postgresql/base.py


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

from .schema import DatabaseSchemaEditor  # NOQA isort:skip  <ins>def _get_varchar_column(data):    if data["max_length"] is None:        return "varchar"    return "varchar(%(max_length)s)" % data</ins> class DatabaseWrapper(BaseDatabaseWrapper):     vendor = "postgresql"     display_name = "PostgreSQL"


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

"BigAutoField": "bigint",         "BinaryField": "bytea",         "BooleanField": "boolean",        "CharField": "varchar(%(max_length)s)",        "CharField": _get_varchar_column,         "DateField": "date",         "DateTimeField": "timestamp with time zone",         "DecimalField": "numeric(%(max_digits)s, %(decimal_places)s)",diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py-- a/django/db/backends/postgresql/features.py++ b/django/db/backends/postgresql/features.py


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

has_bit_xor = property(operator.attrgetter("is_postgresql_14"))     supports_covering_spgist_indexes = property(operator.attrgetter("is_postgresql_14"))    supports_unlimited_charfield = Truediff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py-- a/django/db/models/fields/__init__.py++ b/django/db/models/fields/__init__.py


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

# exactly which wacky database column type you want to use.         data = self.db_type_parameters(connection)         try:            return connection.data_types[self.get_internal_type()] % data            column_type = connection.data_types[self.get_internal_type()]         except KeyError:             return None        else:            # column_type is either a single-parameter function or a string.            if callable(column_type):                return column_type(data)            return column_type % data      def rel_db_type(self, connection):         """


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

class CharField(Field):    description = _("String (up to %(max_length)s)")     def __init__(self, *args, db_collation=None, **kwargs):         super().__init__(*args, **kwargs)         self.db_collation = db_collation         if self.max_length is not None:             self.validators.append(validators.MaxLengthValidator(self.max_length))     @property    def description(self):        if self.max_length is not None:            return _("String (up to %(max_length)s)")        else:            return _("String (unlimited)")     def check(self, **kwargs):         databases = kwargs.get("databases") or []         return [


Question: Cannot define CharField with unlimited length Description Model validation throws an error on CharField with a null maxlength: class Test(Model): charfield = CharField(maxlength=None) One or more models did not validate: "charfield": CharFields require a "maxlength" attribute that is a positive integer. CharField should allow maxlength=None, which intuitively means there is no maximum length. This is a perfectly valid use case. Postgres, for example, supports varchar/text columns without a length limit, but Django appears to have no way to define such a column in a model class. The model validation code looks like this (): if isinstance(f, models.CharField): try: maxlength = int(f.maxlength) if maxlength <= 0: e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) except (ValueError, TypeError): e.add(opts, '"%s": CharFields require a "maxlength" attribute that is a positive integer.' % f.name) It should be changed to something this: if isinstance(f, models.CharField) and f.maxlength is not None: ... The FileField does not happen to throw this error because it is not a derivative of CharField. However, the SQL generated for FileField is not correct when max_length=None, so that would need to be addressed as well.

def _check_max_length_attribute(self, **kwargs):         if self.max_length is None:<ins>            if (                connection.features.supports_unlimited_charfield                or "supports_unlimited_charfield"                in self.model._meta.required_db_features            ):                return []</ins>             return [                 checks.Error(                     "CharFields must define a 'max_length' attribute.",


Question: BaseForm.getitem() does unneeded work in the happy path Description I noticed that in the "happy path," BaseForm.getitem() does unneeded work:  It can just return self.boundfieldscache[name] at the beginning and handle KeyError, instead of accessing followed by checking for the presence of name in self.boundfieldscache before doing so each time.

def __getitem__(self, name):         """Return a BoundField with the given name."""<del>        try:            return self._bound_fields_cache[name]        except KeyError:            pass</del>         try:             field = self.fields[name]         except KeyError:


Question: BaseForm.getitem() does unneeded work in the happy path Description I noticed that in the "happy path," BaseForm.getitem() does unneeded work:  It can just return self.boundfieldscache[name] at the beginning and handle KeyError, instead of accessing followed by checking for the presence of name in self.boundfieldscache before doing so each time.

", ".join(sorted(self.fields)),                 )             )        bound_field = field.get_bound_field(self, name)        self._bound_fields_cache[name] = bound_field        return bound_field        if name not in self._bound_fields_cache:            self._bound_fields_cache[name] = field.get_bound_field(self, name)        return self._bound_fields_cache[name]      @property     def errors(self):


Question: Mitigate the BREACH attack Description (last modified by Nick Pope) The BREACH attack () was published in 2013. The Django project responded soon after () suggesting users to basically stop using gzip. CSRF masking was implemented in 2016 (). In April 2022, a paper called "Heal The Breach" was published, suggesting a mitigation that does not depend on masking specific tokens or injecting data into HTML. It is rather a generic and effective mitigation. It suggests adding randomness to the compressed response by injecting random bytes in the gzip filename field of the gzip stream:  Telling users to disable gzip is not great for bandwidth consumption. I propose that Django should implement "Heal The Breach" with sensible default.

on the Accept-Encoding header.     """ <ins>    max_random_bytes = 100</ins>     def process_response(self, request, response):         # It's not worth attempting to compress really short responses.         if not response.streaming and len(response.content) < 200:


Question: Mitigate the BREACH attack Description (last modified by Nick Pope) The BREACH attack () was published in 2013. The Django project responded soon after () suggesting users to basically stop using gzip. CSRF masking was implemented in 2016 (). In April 2022, a paper called "Heal The Breach" was published, suggesting a mitigation that does not depend on masking specific tokens or injecting data into HTML. It is rather a generic and effective mitigation. It suggests adding randomness to the compressed response by injecting random bytes in the gzip filename field of the gzip stream:  Telling users to disable gzip is not great for bandwidth consumption. I propose that Django should implement "Heal The Breach" with sensible default.

if response.streaming:             # Delete the `Content-Length` header for streaming content, because             # we won't know the compressed size until we stream it.            response.streaming_content = compress_sequence(response.streaming_content)            response.streaming_content = compress_sequence(                response.streaming_content,                max_random_bytes=self.max_random_bytes,            )             del response.headers["Content-Length"]         else:             # Return the compressed content only if it's actually shorter.            compressed_content = compress_string(response.content)            compressed_content = compress_string(                response.content,                max_random_bytes=self.max_random_bytes,            )             if len(compressed_content) >= len(response.content):                 return response             response.content = compressed_contentdiff --git a/django/utils/text.py b/django/utils/text.py-- a/django/utils/text.py++ b/django/utils/text.py


Question: Mitigate the BREACH attack Description (last modified by Nick Pope) The BREACH attack () was published in 2013. The Django project responded soon after () suggesting users to basically stop using gzip. CSRF masking was implemented in 2016 (). In April 2022, a paper called "Heal The Breach" was published, suggesting a mitigation that does not depend on masking specific tokens or injecting data into HTML. It is rather a generic and effective mitigation. It suggests adding randomness to the compressed response by injecting random bytes in the gzip filename field of the gzip stream:  Telling users to disable gzip is not great for bandwidth consumption. I propose that Django should implement "Heal The Breach" with sensible default.

import gzip import reimport secrets import unicodedata from gzip import GzipFile from gzip import compress as gzip_compress


Question: Mitigate the BREACH attack Description (last modified by Nick Pope) The BREACH attack () was published in 2013. The Django project responded soon after () suggesting users to basically stop using gzip. CSRF masking was implemented in 2016 (). In April 2022, a paper called "Heal The Breach" was published, suggesting a mitigation that does not depend on masking specific tokens or injecting data into HTML. It is rather a generic and effective mitigation. It suggests adding randomness to the compressed response by injecting random bytes in the gzip filename field of the gzip stream:  Telling users to disable gzip is not great for bandwidth consumption. I propose that Django should implement "Heal The Breach" with sensible default.

return "".join(char2number.get(c, c) for c in phone.lower())  def compress_string(s):    return gzip_compress(s, compresslevel=6, mtime=0)def _get_random_filename(max_random_bytes):    return b"a" * secrets.randbelow(max_random_bytes)def compress_string(s, *, max_random_bytes=None):    compressed_data = gzip_compress(s, compresslevel=6, mtime=0)    if not max_random_bytes:        return compressed_data    compressed_view = memoryview(compressed_data)    header = bytearray(compressed_view[:10])    header[3] = gzip.FNAME    filename = _get_random_filename(max_random_bytes) + b"\x00"    return bytes(header) + filename + compressed_view[10:]   class StreamingBuffer(BytesIO):


Question: Mitigate the BREACH attack Description (last modified by Nick Pope) The BREACH attack () was published in 2013. The Django project responded soon after () suggesting users to basically stop using gzip. CSRF masking was implemented in 2016 (). In April 2022, a paper called "Heal The Breach" was published, suggesting a mitigation that does not depend on masking specific tokens or injecting data into HTML. It is rather a generic and effective mitigation. It suggests adding randomness to the compressed response by injecting random bytes in the gzip filename field of the gzip stream:  Telling users to disable gzip is not great for bandwidth consumption. I propose that Django should implement "Heal The Breach" with sensible default.

# Like compress_string, but for iterators of strings.def compress_sequence(sequence):def compress_sequence(sequence, *, max_random_bytes=None):     buf = StreamingBuffer()    with GzipFile(mode="wb", compresslevel=6, fileobj=buf, mtime=0) as zfile:    filename = _get_random_filename(max_random_bytes) if max_random_bytes else None    with GzipFile(        filename=filename, mode="wb", compresslevel=6, fileobj=buf, mtime=0    ) as zfile:         # Output headers...         yield buf.read()         for item in sequence:


Question: QuerySet.bulkcreate() crashes on mixed case columns in uniquefields/updatefields. Description Not sure exactly how to phrase this, but when I I'm calling bulkupdate on the manager for a class with dbcolumn set on fields the SQL is invalid. Ellipses indicate other fields excluded for clarity. class ActivityBlackListed(models.Model): """ Originally sourced from ActivityBlackListed in /home/josh/PNDSInterimMIS- (13 records) """ class Meta: dbtable = "ActivityBlackListed" blacklistid = models.IntegerField(primarykey=True, dbcolumn="BlacklistID") sectorid = models.IntegerField(null=True, blank=True, dbcolumn="SectorID") ... qs.bulkcreate(instances, updateconflicts=True, updatefields=["sectorid", ...], uniquefields=["blacklistid"]) The "INSERT" code does take into account the dbcolumns INSERT INTO "ActivityBlackListed" ("BlacklistID",...) VALUES (%s, ...), The code which is generated for "ON CONFLICT" uses the field name and not the dbcolumn which leads to a syntax error 'ON CONFLICT("blacklistid") DO UPDATE SET "sectorid" = EXCLUDED."sectorid", ... PostgreSQL returns ERROR: column "blacklistid" does not exist at character 1508 What should be generated is I think: 'ON CONFLICT("BlacklistID") DO UPDATE SET "SectorID" = EXCLUDED."SectorID", ...

"Unique fields that can trigger the upsert must be provided."                 )             # Updating primary keys and non-concrete fields is forbidden.<del>            update_fields = [self.model._meta.get_field(name) for name in update_fields]</del>             if any(not f.concrete or f.many_to_many for f in update_fields):                 raise ValueError(                     "bulk_create() can only be used with concrete fields in "


Question: QuerySet.bulkcreate() crashes on mixed case columns in uniquefields/updatefields. Description Not sure exactly how to phrase this, but when I I'm calling bulkupdate on the manager for a class with dbcolumn set on fields the SQL is invalid. Ellipses indicate other fields excluded for clarity. class ActivityBlackListed(models.Model): """ Originally sourced from ActivityBlackListed in /home/josh/PNDSInterimMIS- (13 records) """ class Meta: dbtable = "ActivityBlackListed" blacklistid = models.IntegerField(primarykey=True, dbcolumn="BlacklistID") sectorid = models.IntegerField(null=True, blank=True, dbcolumn="SectorID") ... qs.bulkcreate(instances, updateconflicts=True, updatefields=["sectorid", ...], uniquefields=["blacklistid"]) The "INSERT" code does take into account the dbcolumns INSERT INTO "ActivityBlackListed" ("BlacklistID",...) VALUES (%s, ...), The code which is generated for "ON CONFLICT" uses the field name and not the dbcolumn which leads to a syntax error 'ON CONFLICT("blacklistid") DO UPDATE SET "sectorid" = EXCLUDED."sectorid", ... PostgreSQL returns ERROR: column "blacklistid" does not exist at character 1508 What should be generated is I think: 'ON CONFLICT("BlacklistID") DO UPDATE SET "SectorID" = EXCLUDED."SectorID", ...

"update_fields."                 )             if unique_fields:<del>                unique_fields = [                    self.model._meta.get_field(name) for name in unique_fields                ]</del>                 if any(not f.concrete or f.many_to_many for f in unique_fields):                     raise ValueError(                         "bulk_create() can only be used with concrete fields "


Question: QuerySet.bulkcreate() crashes on mixed case columns in uniquefields/updatefields. Description Not sure exactly how to phrase this, but when I I'm calling bulkupdate on the manager for a class with dbcolumn set on fields the SQL is invalid. Ellipses indicate other fields excluded for clarity. class ActivityBlackListed(models.Model): """ Originally sourced from ActivityBlackListed in /home/josh/PNDSInterimMIS- (13 records) """ class Meta: dbtable = "ActivityBlackListed" blacklistid = models.IntegerField(primarykey=True, dbcolumn="BlacklistID") sectorid = models.IntegerField(null=True, blank=True, dbcolumn="SectorID") ... qs.bulkcreate(instances, updateconflicts=True, updatefields=["sectorid", ...], uniquefields=["blacklistid"]) The "INSERT" code does take into account the dbcolumns INSERT INTO "ActivityBlackListed" ("BlacklistID",...) VALUES (%s, ...), The code which is generated for "ON CONFLICT" uses the field name and not the dbcolumn which leads to a syntax error 'ON CONFLICT("blacklistid") DO UPDATE SET "sectorid" = EXCLUDED."sectorid", ... PostgreSQL returns ERROR: column "blacklistid" does not exist at character 1508 What should be generated is I think: 'ON CONFLICT("BlacklistID") DO UPDATE SET "SectorID" = EXCLUDED."SectorID", ...

if unique_fields:             # Primary key is allowed in unique_fields.             unique_fields = [                opts.pk.name if name == "pk" else name for name in unique_fields                self.model._meta.get_field(opts.pk.name if name == "pk" else name)                for name in unique_fields             ]        if update_fields:            update_fields = [self.model._meta.get_field(name) for name in update_fields]         on_conflict = self._check_bulk_create_options(             ignore_conflicts,             update_conflicts,diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py-- a/django/db/models/sql/compiler.py++ b/django/db/models/sql/compiler.py


Question: QuerySet.bulkcreate() crashes on mixed case columns in uniquefields/updatefields. Description Not sure exactly how to phrase this, but when I I'm calling bulkupdate on the manager for a class with dbcolumn set on fields the SQL is invalid. Ellipses indicate other fields excluded for clarity. class ActivityBlackListed(models.Model): """ Originally sourced from ActivityBlackListed in /home/josh/PNDSInterimMIS- (13 records) """ class Meta: dbtable = "ActivityBlackListed" blacklistid = models.IntegerField(primarykey=True, dbcolumn="BlacklistID") sectorid = models.IntegerField(null=True, blank=True, dbcolumn="SectorID") ... qs.bulkcreate(instances, updateconflicts=True, updatefields=["sectorid", ...], uniquefields=["blacklistid"]) The "INSERT" code does take into account the dbcolumns INSERT INTO "ActivityBlackListed" ("BlacklistID",...) VALUES (%s, ...), The code which is generated for "ON CONFLICT" uses the field name and not the dbcolumn which leads to a syntax error 'ON CONFLICT("blacklistid") DO UPDATE SET "sectorid" = EXCLUDED."sectorid", ... PostgreSQL returns ERROR: column "blacklistid" does not exist at character 1508 What should be generated is I think: 'ON CONFLICT("BlacklistID") DO UPDATE SET "SectorID" = EXCLUDED."SectorID", ...

on_conflict_suffix_sql = self.connection.ops.on_conflict_suffix_sql(             fields,             self.query.on_conflict,            self.query.update_fields,            self.query.unique_fields,            (f.column for f in self.query.update_fields),            (f.column for f in self.query.unique_fields),         )         if (             self.returning_fields


Question: QuerySet.bulkcreate() crashes on "pk" in uniquefields. Description (last modified by Mariusz Felisiak) QuerySet.bulkcreate() crashes on "pk" in uniquefields which should be allowed. File "", line 89, in _execute return (sql, params) column "pk" does not exist LINE 1: ...S (3127, 3, 3, 'c'), (3128, 4, 4, 'd') ON CONFLICT("pk") DO ... Bug in .

"update_fields."                 )             if unique_fields:                # Primary key is allowed in unique_fields.                 unique_fields = [                    self.model._meta.get_field(name)                    for name in unique_fields                    if name != "pk"                    self.model._meta.get_field(name) for name in unique_fields                 ]                 if any(not f.concrete or f.many_to_many for f in unique_fields):                     raise ValueError(


Question: QuerySet.bulkcreate() crashes on "pk" in uniquefields. Description (last modified by Mariusz Felisiak) QuerySet.bulkcreate() crashes on "pk" in uniquefields which should be allowed. File "", line 89, in _execute return (sql, params) column "pk" does not exist LINE 1: ...S (3127, 3, 3, 'c'), (3128, 4, 4, 'd') ON CONFLICT("pk") DO ... Bug in .

raise ValueError("Can't bulk create a multi-table inherited model")         if not objs:             return objs<ins>        opts = self.model._meta        if unique_fields:            # Primary key is allowed in unique_fields.            unique_fields = [                opts.pk.name if name == "pk" else name for name in unique_fields            ]</ins>         on_conflict = self._check_bulk_create_options(             ignore_conflicts,             update_conflicts,


Question: QuerySet.bulkcreate() crashes on "pk" in uniquefields. Description (last modified by Mariusz Felisiak) QuerySet.bulkcreate() crashes on "pk" in uniquefields which should be allowed. File "", line 89, in _execute return (sql, params) column "pk" does not exist LINE 1: ...S (3127, 3, 3, 'c'), (3128, 4, 4, 'd') ON CONFLICT("pk") DO ... Bug in .

unique_fields,         )         self._for_write = True<del>        opts = self.model._meta</del>         fields = opts.concrete_fields         objs = list(objs)         self._prepare_for_bulk_create(objs)


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

from django.db.models import AutoField, DateField, DateTimeField, Field, sql from django.db.models.constants import LOOKUP_SEP, OnConflict from django.db.models.deletion import Collectorfrom django.db.models.expressions import Case, F, Ref, Value, Whenfrom django.db.models.expressions import Case, F, Value, When from django.db.models.functions import Cast, Trunc from django.db.models.query_utils import FilteredRelation, Q from django.db.models.sql.constants import CURSOR, GET_ITERATOR_CHUNK_SIZE


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

raise TypeError("Complex aggregates require an alias")             kwargs[arg.default_alias] = arg         query = self.query.chain()        for (alias, aggregate_expr) in kwargs.items():            query.add_annotation(aggregate_expr, alias, is_summary=True)            annotation = query.annotations[alias]            if not annotation.contains_aggregate:                raise TypeError("%s is not an aggregate expression" % alias)            for expr in annotation.get_source_expressions():                if (                    expr.contains_aggregate                    and isinstance(expr, Ref)                    and expr.refs in kwargs                ):                    name = expr.refs                    raise exceptions.FieldError(                        "Cannot compute %s('%s'): '%s' is an aggregate"                        % (annotation.name, name, name)                    )        return query.get_aggregation(self.db, kwargs)        return self.query.chain().get_aggregation(self.db, kwargs)      async def aaggregate(self, *args, **kwargs):         return await sync_to_async(self.aggregate)(*args, **kwargs)


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

clone.query.add_annotation(                     annotation,                     alias,                    is_summary=False,                     select=select,                 )         for alias, annotation in clone.query.annotations.items():diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py-- a/django/db/models/sql/query.py++ b/django/db/models/sql/query.py


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

alias = None         return target.get_col(alias, field)     def get_aggregation(self, using, added_aggregate_names):    def get_aggregation(self, using, aggregate_exprs):         """         Return the dictionary with the values of the existing aggregations.         """        if not self.annotation_select:        if not aggregate_exprs:             return {}        existing_annotations = {            alias: annotation            for alias, annotation in self.annotations.items()            if alias not in added_aggregate_names        }        aggregates = {}        for alias, aggregate_expr in aggregate_exprs.items():            self.check_alias(alias)            aggregate = aggregate_expr.resolve_expression(                self, allow_joins=True, reuse=None, summarize=True            )            if not aggregate.contains_aggregate:                raise TypeError("%s is not an aggregate expression" % alias)            aggregates[alias] = aggregate         # Existing usage of aggregation can be determined by the presence of         # selected aggregates but also by filters against aliased aggregates.         _, having, qualify = self.where.split_having_qualify()         has_existing_aggregation = (             any(                 getattr(annotation, "contains_aggregate", True)                for annotation in existing_annotations.values()                for annotation in self.annotations.values()             )             or having         )


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

# filtering against window functions is involved as it                     # requires complex realising.                     annotation_mask = set()                    for name in added_aggregate_names:                        annotation_mask.add(name)                        annotation_mask |= inner_query.annotations[name].get_refs()                    for aggregate in aggregates.values():                        annotation_mask |= aggregate.get_refs()                     inner_query.set_annotation_mask(annotation_mask)             # Remove any aggregates marked for reduction from the subquery and            # move them to the outer AggregateQuery. This requires making sure            # all columns referenced by the aggregates are selected in the            # subquery. It is achieved by retrieving all column references from            # the aggregates, explicitly selecting them if they are not            # already, and making sure the aggregates are repointed to            # referenced to them.            # Add aggregates to the outer AggregateQuery. This requires making            # sure all columns referenced by the aggregates are selected in the            # inner query. It is achieved by retrieving all column references            # by the aggregates, explicitly selecting them in the inner query,            # and making sure the aggregates are repointed to them.             col_refs = {}            for alias, expression in list(inner_query.annotation_select.items()):                if not expression.is_summary:                    continue                annotation_select_mask = inner_query.annotation_select_mask            for alias, aggregate in aggregates.items():                 replacements = {}                for col in self._gen_cols([expression], resolve_refs=False):                for col in self._gen_cols([aggregate], resolve_refs=False):                     if not (col_ref := col_refs.get(col)):                         index = len(col_refs) + 1                         col_alias = f"__col{index}"


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

inner_query.annotations[col_alias] = col                         inner_query.append_annotation_mask([col_alias])                     replacements[col] = col_ref                outer_query.annotations[alias] = expression.replace_expressions(                outer_query.annotations[alias] = aggregate.replace_expressions(                     replacements                 )                del inner_query.annotations[alias]                annotation_select_mask.remove(alias)                # Make sure the annotation_select wont use cached results.                inner_query.set_annotation_mask(inner_query.annotation_select_mask)             if (                 inner_query.select == ()                 and not inner_query.default_cols


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

self.select = ()             self.default_cols = False             self.extra = {}            if existing_annotations:            if self.annotations:                 # Inline reference to existing annotations and mask them as                 # they are unnecessary given only the summarized aggregations                 # are requested.                 replacements = {                     Ref(alias, annotation): annotation                    for alias, annotation in existing_annotations.items()                    for alias, annotation in self.annotations.items()                 }                for name in added_aggregate_names:                    self.annotations[name] = self.annotations[name].replace_expressions(                        replacements                    )                self.set_annotation_mask(added_aggregate_names)                self.annotations = {                    alias: aggregate.replace_expressions(replacements)                    for alias, aggregate in aggregates.items()                }            else:                self.annotations = aggregates            self.set_annotation_mask(aggregates)          empty_set_result = [             expression.empty_result_set_value


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

Perform a COUNT() query using the current filter constraints.         """         obj = self.clone()        obj.add_annotation(Count("*"), alias="__count", is_summary=True)        return obj.get_aggregation(using, ["__count"])["__count"]        return obj.get_aggregation(using, {"__count": Count("*")})["__count"]      def has_filters(self):         return self.where


Question: Django subtly produces incorrect query when the same keyword appears in both aggregate() and annotate() Description I have the following query: Model.objects.annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) Initially, I was running this query on SQLite and Django was producing a result (i.e. 0). When I switched to MySQL this query crushed with the following exception: Traceback (most recent call last): File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) MySQLdb.exceptions.OperationalError: (1054, "Unknown column 'foo' in 'field list'") The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 15, in <moduleret2 = ('mysql').annotate(foo=F('column')).aggregate(foo=Sum(F('foo'))) File "", line 384, in aggregate return query.getaggregation(, kwargs) File "", line 502, in getaggregation result = compiler.executesql(SINGLE) File "", line 1151, in executesql cursor.execute(sql, params) File "", line 100, in execute return super().execute(sql, params) File "", line 68, in execute return self.executewithwrappers(sql, params, many=False, executor=self.execute) File "", line 77, in executewithwrappers return executor(sql, params, many, context) File "", line 86, in execute return (sql, params) File "", line 90, in exit raise djexcvalue.withtraceback(traceback) from excvalue File "", line 86, in execute return (sql, params) File "", line 74, in execute return (query, args) File "", line 209, in execute res = (query) File "", line 315, in query (q) File "", line 239, in query (self, query) (1054, "Unknown column 'foo' in 'field list'") After examining the SQL query produced by Django, I realized that the query is not correct. Specifically, Django produced the following SQL query: SELECT SUM() AS FROM Instead of SELECT SUM() FROM (SELECT . AS FROM ) subquery The issue appears when a keyword in aggregate function is the same with one of the keywords in annotate function. I initially thought that Django does not have any restriction on the keywords used in aggregate (i.e. a keyword in aggregate can be the same with the name of a column from the inspected model). If you think that aggregate should not conflict with the annotate, then Django should produce a warning as running this query in SQLite subtly produces an incorrect result.

"semicolons, or SQL comments."             )     def add_annotation(self, annotation, alias, is_summary=False, select=True):    def add_annotation(self, annotation, alias, select=True):         """Add a single annotation expression to the Query."""         self.check_alias(alias)        annotation = annotation.resolve_expression(            self, allow_joins=True, reuse=None, summarize=is_summary        )        annotation = annotation.resolve_expression(self, allow_joins=True, reuse=None)         if select:             self.append_annotation_mask([alias])         else:


Question: UserCreationForm should save data from ManyToMany form fields Description When using with a custom User model which has ManyToManyField fields, the data in all related form fields (e.g. a ModelMultipleChoiceField) is not saved. This is because unlike its parent class , (commit=True) omits to call self.save_m2m(). This has been discussed on the #django-developers mailing list  and I'm ready to work on a PR.

user.set_password(self.cleaned_data["password1"])         if commit:             user.save()<ins>            if hasattr(self, "save_m2m"):                self.save_m2m()</ins>         return user


Question: Deprecate passing positional arguments to Signer. Description We discovered a vulnerability in one of our applications recently which was caused by an inaccurate instantiation of The developer intended to use the user's email address as the salt for the Signing instance but instead caused it to be used as the key. Here's an example code block that demonstrates the problem: signer = Signer(self.context['request']) signeddata = signer.signobject(dict( licensenumber='...', productid='...', device_count='...' )) In our case, this signed data was then being used to verify a later request and generate an active license. This meant that an attacker could feasibly generate their own licenses if they realised that their email address was the key. The fix for this was to add salt= in front of the email variable. It occurred to us that this is a relatively easy mistake to make and could be avoided if the signature of was changed thusly: def init(self, key=None, sep=':', salt=None, algorithm=None): def init(self, , key=None, sep=':', salt=None, algorithm=None): That is, adding a after self to force the developer to name the parameters.

import datetime import json import timeimport warnings import zlib  from django.conf import settings from django.utils.crypto import constant_time_compare, salted_hmacfrom django.utils.deprecation import RemovedInDjango51Warning from django.utils.encoding import force_bytes from django.utils.module_loading import import_string from django.utils.regex_helper import _lazy_re_compile


Question: Deprecate passing positional arguments to Signer. Description We discovered a vulnerability in one of our applications recently which was caused by an inaccurate instantiation of The developer intended to use the user's email address as the salt for the Signing instance but instead caused it to be used as the key. Here's an example code block that demonstrates the problem: signer = Signer(self.context['request']) signeddata = signer.signobject(dict( licensenumber='...', productid='...', device_count='...' )) In our case, this signed data was then being used to verify a later request and generate an active license. This meant that an attacker could feasibly generate their own licenses if they realised that their email address was the key. The fix for this was to add salt= in front of the email variable. It occurred to us that this is a relatively easy mistake to make and could be avoided if the signature of was changed thusly: def init(self, key=None, sep=':', salt=None, algorithm=None): def init(self, , key=None, sep=':', salt=None, algorithm=None): That is, adding a after self to force the developer to name the parameters.

The serializer is expected to return a bytestring.     """    return TimestampSigner(key, salt=salt).sign_object(    return TimestampSigner(key=key, salt=salt).sign_object(         obj, serializer=serializer, compress=compress     )


Question: Deprecate passing positional arguments to Signer. Description We discovered a vulnerability in one of our applications recently which was caused by an inaccurate instantiation of The developer intended to use the user's email address as the salt for the Signing instance but instead caused it to be used as the key. Here's an example code block that demonstrates the problem: signer = Signer(self.context['request']) signeddata = signer.signobject(dict( licensenumber='...', productid='...', device_count='...' )) In our case, this signed data was then being used to verify a later request and generate an active license. This meant that an attacker could feasibly generate their own licenses if they realised that their email address was the key. The fix for this was to add salt= in front of the email variable. It occurred to us that this is a relatively easy mistake to make and could be avoided if the signature of was changed thusly: def init(self, key=None, sep=':', salt=None, algorithm=None): def init(self, , key=None, sep=':', salt=None, algorithm=None): That is, adding a after self to force the developer to name the parameters.

The serializer is expected to accept a bytestring.     """    return TimestampSigner(key, salt=salt, fallback_keys=fallback_keys).unsign_object(    return TimestampSigner(        key=key, salt=salt, fallback_keys=fallback_keys    ).unsign_object(         s,         serializer=serializer,         max_age=max_age,


Question: Deprecate passing positional arguments to Signer. Description We discovered a vulnerability in one of our applications recently which was caused by an inaccurate instantiation of The developer intended to use the user's email address as the salt for the Signing instance but instead caused it to be used as the key. Here's an example code block that demonstrates the problem: signer = Signer(self.context['request']) signeddata = signer.signobject(dict( licensenumber='...', productid='...', device_count='...' )) In our case, this signed data was then being used to verify a later request and generate an active license. This meant that an attacker could feasibly generate their own licenses if they realised that their email address was the key. The fix for this was to add salt= in front of the email variable. It occurred to us that this is a relatively easy mistake to make and could be avoided if the signature of was changed thusly: def init(self, key=None, sep=':', salt=None, algorithm=None): def init(self, , key=None, sep=':', salt=None, algorithm=None): That is, adding a after self to force the developer to name the parameters.

class Signer:    # RemovedInDjango51Warning: When the deprecation ends, replace with:    # def __init__(    #   self, *, key=None, sep=":", salt=None, algorithm=None, fallback_keys=None    # ):     def __init__(         self,        *args,         key=None,         sep=":",         salt=None,


Question: Deprecate passing positional arguments to Signer. Description We discovered a vulnerability in one of our applications recently which was caused by an inaccurate instantiation of The developer intended to use the user's email address as the salt for the Signing instance but instead caused it to be used as the key. Here's an example code block that demonstrates the problem: signer = Signer(self.context['request']) signeddata = signer.signobject(dict( licensenumber='...', productid='...', device_count='...' )) In our case, this signed data was then being used to verify a later request and generate an active license. This meant that an attacker could feasibly generate their own licenses if they realised that their email address was the key. The fix for this was to add salt= in front of the email variable. It occurred to us that this is a relatively easy mistake to make and could be avoided if the signature of was changed thusly: def init(self, key=None, sep=':', salt=None, algorithm=None): def init(self, , key=None, sep=':', salt=None, algorithm=None): That is, adding a after self to force the developer to name the parameters.

else settings.SECRET_KEY_FALLBACKS         )         self.sep = sep        if _SEP_UNSAFE.match(self.sep):            raise ValueError(                "Unsafe Signer separator: %r (cannot be empty or consist of "                "only A-z0-9-_=)" % sep,            )         self.salt = salt or "%s.%s" % (             self.__class__.__module__,             self.__class__.__name__,         )         self.algorithm = algorithm or "sha256"        # RemovedInDjango51Warning.        if args:            warnings.warn(                f"Passing positional arguments to {self.__class__.__name__} is "                f"deprecated.",                RemovedInDjango51Warning,                stacklevel=2,            )            for arg, attr in zip(                args, ["key", "sep", "salt", "algorithm", "fallback_keys"]            ):                if arg or attr == "sep":                    setattr(self, attr, arg)        if _SEP_UNSAFE.match(self.sep):            raise ValueError(                "Unsafe Signer separator: %r (cannot be empty or consist of "                "only A-z0-9-_=)" % sep,            )      def signature(self, value, key=None):         key = key or self.key


Question: Drop support for SQLite < 3.21.0 Description SQLite 3.15+ supports functions in partial indexes. SQLite 3.20+ can defer constraint checks and supports PRAGMA foreign key checks. Ubuntu Xenial ships with SQLite 3.22.0 (which will still by supported by Django) and will EOL in April 2023. Debian Buster ships with 3.27.2 and will EOL in June 2024. Python 3.7 ships with 3.21.0. SQLite 3.21.0 was released in October 2017. SQLite version support seems like a similar situation as GEOS libraries which we generally support about 5 years after released.

determine if rows with invalid references were entered while constraint         checks were off.         """        if self.features.supports_pragma_foreign_key_check:            with self.cursor() as cursor:                if table_names is None:                    violations = cursor.execute("PRAGMA foreign_key_check").fetchall()                else:                    violations = chain.from_iterable(                        cursor.execute(                            "PRAGMA foreign_key_check(%s)"                            % self.ops.quote_name(table_name)                        ).fetchall()                        for table_name in table_names                    )                # See https://www.sqlite.org/pragma.html#pragma_foreign_key_check                for (                    table_name,                    rowid,                    referenced_table_name,                    foreign_key_index,                ) in violations:                    foreign_key = cursor.execute(                        "PRAGMA foreign_key_list(%s)" % self.ops.quote_name(table_name)                    ).fetchall()[foreign_key_index]                    column_name, referenced_column_name = foreign_key[3:5]                    primary_key_column_name = self.introspection.get_primary_key_column(                        cursor, table_name                    )                    primary_key_value, bad_value = cursor.execute(                        "SELECT %s, %s FROM %s WHERE rowid = %%s"                        % (                            self.ops.quote_name(primary_key_column_name),                            self.ops.quote_name(column_name),                            self.ops.quote_name(table_name),                        ),                        (rowid,),                    ).fetchone()                    raise IntegrityError(                        "The row in table '%s' with primary key '%s' has an "                        "invalid foreign key: %s.%s contains a value '%s' that "                        "does not have a corresponding value in %s.%s."                        % (                            table_name,                            primary_key_value,                            table_name,                            column_name,                            bad_value,                            referenced_table_name,                            referenced_column_name,                        )                    )        else:            with self.cursor() as cursor:                if table_names is None:                    table_names = self.introspection.table_names(cursor)                for table_name in table_names:                    primary_key_column_name = self.introspection.get_primary_key_column(                        cursor, table_name                    )                    if not primary_key_column_name:                        continue                    relations = self.introspection.get_relations(cursor, table_name)                    for column_name, (                        referenced_column_name,        with self.cursor() as cursor:            if table_names is None:                violations = cursor.execute("PRAGMA foreign_key_check").fetchall()            else:                violations = chain.from_iterable(                    cursor.execute(                        "PRAGMA foreign_key_check(%s)" % self.ops.quote_name(table_name)                    ).fetchall()                    for table_name in table_names                )            # See https://www.sqlite.org/pragma.html#pragma_foreign_key_check            for (                table_name,                rowid,                referenced_table_name,                foreign_key_index,            ) in violations:                foreign_key = cursor.execute(                    "PRAGMA foreign_key_list(%s)" % self.ops.quote_name(table_name)                ).fetchall()[foreign_key_index]                column_name, referenced_column_name = foreign_key[3:5]                primary_key_column_name = self.introspection.get_primary_key_column(                    cursor, table_name                )                primary_key_value, bad_value = cursor.execute(                    "SELECT %s, %s FROM %s WHERE rowid = %%s"                    % (                        self.ops.quote_name(primary_key_column_name),                        self.ops.quote_name(column_name),                        self.ops.quote_name(table_name),                    ),                    (rowid,),                ).fetchone()                raise IntegrityError(                    "The row in table '%s' with primary key '%s' has an "                    "invalid foreign key: %s.%s contains a value '%s' that "                    "does not have a corresponding value in %s.%s."                    % (                        table_name,                        primary_key_value,                        table_name,                        column_name,                        bad_value,                         referenced_table_name,                    ) in relations.items():                        cursor.execute(                            """                            SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING                            LEFT JOIN `%s` as REFERRED                            ON (REFERRING.`%s` = REFERRED.`%s`)                            WHERE REFERRING.`%s` IS NOT NULL AND REFERRED.`%s` IS NULL                            """                            % (                                primary_key_column_name,                                column_name,                                table_name,                                referenced_table_name,                                column_name,                                referenced_column_name,                                column_name,                                referenced_column_name,                            )                        )                        for bad_row in cursor.fetchall():                            raise IntegrityError(                                "The row in table '%s' with primary key '%s' has an "                                "invalid foreign key: %s.%s contains a value '%s' that "                                "does not have a corresponding value in %s.%s."                                % (                                    table_name,                                    bad_row[0],                                    table_name,                                    column_name,                                    bad_row[1],                                    referenced_table_name,                                    referenced_column_name,                                )                            )                        referenced_column_name,                    )                )      def is_usable(self):         return Truediff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py-- a/django/db/backends/sqlite3/features.py++ b/django/db/backends/sqlite3/features.py


Question: Drop support for SQLite < 3.21.0 Description SQLite 3.15+ supports functions in partial indexes. SQLite 3.20+ can defer constraint checks and supports PRAGMA foreign key checks. Ubuntu Xenial ships with SQLite 3.22.0 (which will still by supported by Django) and will EOL in April 2023. Debian Buster ships with 3.27.2 and will EOL in June 2024. Python 3.7 ships with 3.21.0. SQLite 3.21.0 was released in October 2017. SQLite version support seems like a similar situation as GEOS libraries which we generally support about 5 years after released.

class DatabaseFeatures(BaseDatabaseFeatures):    minimum_database_version = (3, 9)    minimum_database_version = (3, 21)     test_db_allows_multiple_connections = False     supports_unspecified_pk = True     supports_timezones = False


Question: Drop support for SQLite < 3.21.0 Description SQLite 3.15+ supports functions in partial indexes. SQLite 3.20+ can defer constraint checks and supports PRAGMA foreign key checks. Ubuntu Xenial ships with SQLite 3.22.0 (which will still by supported by Django) and will EOL in April 2023. Debian Buster ships with 3.27.2 and will EOL in June 2024. Python 3.7 ships with 3.21.0. SQLite 3.21.0 was released in October 2017. SQLite version support seems like a similar situation as GEOS libraries which we generally support about 5 years after released.

# Is "ALTER TABLE ... DROP COLUMN" supported?     can_alter_table_drop_column = Database.sqlite_version_info >= (3, 35, 5)     supports_parentheses_in_compound = False    # Deferred constraint checks can be emulated on SQLite < 3.20 but not in a    # reasonably performant way.    supports_pragma_foreign_key_check = Database.sqlite_version_info >= (3, 20, 0)    can_defer_constraint_checks = supports_pragma_foreign_key_check    supports_functions_in_partial_indexes = Database.sqlite_version_info >= (3, 15, 0)    can_defer_constraint_checks = True     supports_over_clause = Database.sqlite_version_info >= (3, 25, 0)     supports_frame_range_fixed_distance = Database.sqlite_version_info >= (3, 28, 0)     supports_aggregate_filter_clause = Database.sqlite_version_info >= (3, 30, 1)


Question: Choose which items are displayed per language in Sitemap Description The current implementation of Sitemap is : if we use i18n, then we display a cartesian product between some items and some languages. There is no way to use the provided i18n automation if we want to display some items depending on the language (for instance non-translated blog articles). I precise in my case, urls are translated, so given a language the url may not exist or raise an error.

return attr(item)         return attr <ins>    def get_languages_for_item(self, item):        """Languages for which this item is displayed."""        return self._languages()</ins>     def _languages(self):         if self.languages is not None:             return self.languages


Question: Choose which items are displayed per language in Sitemap Description The current implementation of Sitemap is : if we use i18n, then we display a cartesian product between some items and some languages. There is no way to use the provided i18n automation if we want to display some items depending on the language (for instance non-translated blog articles). I precise in my case, urls are translated, so given a language the url may not exist or raise an error.

# This is necessary to paginate with all languages already considered.             items = [                 (item, lang_code)                for lang_code in self._languages()                 for item in self.items()                for lang_code in self.get_languages_for_item(item)             ]             return items         return self.items()


Question: Choose which items are displayed per language in Sitemap Description The current implementation of Sitemap is : if we use i18n, then we display a cartesian product between some items and some languages. There is no way to use the provided i18n automation if we want to display some items depending on the language (for instance non-translated blog articles). I precise in my case, urls are translated, so given a language the url may not exist or raise an error.

}              if self.i18n and self.alternates:                for lang_code in self._languages():                item_languages = self.get_languages_for_item(item[0])                for lang_code in item_languages:                     loc = f"{protocol}://{domain}{self._location(item, lang_code)}"                     url_info["alternates"].append(                         {


Question: Choose which items are displayed per language in Sitemap Description The current implementation of Sitemap is : if we use i18n, then we display a cartesian product between some items and some languages. There is no way to use the provided i18n automation if we want to display some items depending on the language (for instance non-translated blog articles). I precise in my case, urls are translated, so given a language the url may not exist or raise an error.

"lang_code": lang_code,                         }                     )                if self.x_default:                if self.x_default and settings.LANGUAGE_CODE in item_languages:                     lang_code = settings.LANGUAGE_CODE                     loc = f"{protocol}://{domain}{self._location(item, lang_code)}"                     loc = loc.replace(f"/{lang_code}/", "/", 1)


Question: FileBasedCache haskey is susceptible to race conditions Description (last modified by Marti Raudsepp) I received the exception from Django's cache framework: FileNotFoundError: [Errno 2] No such file or directory: '/app/var/cache/.djcache' [...] File "", line 229, in getorset (key, default, timeout=timeout, version=version) File "", line 26, in add if (key, version): File "", line 94, in haskey with open(fname, "rb") as f: The code is: def haskey(self, key, version=None): fname = self.keytofile(key, version) if (fname): with open(fname, "rb") as f: return not self.isexpired(f) return False Between the exists() check and open(), it's possible for the file to be deleted. In fact, the is_expired() method itself deletes the file if it finds it to be expired. So if many threads race to read an expired cache at once, it's not that unlikely to hit this window.

def has_key(self, key, version=None):         fname = self._key_to_file(key, version)        if os.path.exists(fname):        try:             with open(fname, "rb") as f:                 return not self._is_expired(f)        return False        except FileNotFoundError:            return False      def _cull(self):         """


Question: QuerySet.selectrelated() with multiple filtered relations to the OneToOneField sets the last one. Description   should use partial, just like the remotesetter

)                 get_related_klass_infos(klass_info, next_klass_infos)             def local_setter(obj, from_obj):            def local_setter(final_field, obj, from_obj):                 # Set a reverse fk object when relation is non-empty.                 if from_obj:                     final_field.remote_field.set_cached_value(from_obj, obj)


Question: QuerySet.selectrelated() with multiple filtered relations to the OneToOneField sets the last one. Description   should use partial, just like the remotesetter

"model": model,                         "field": final_field,                         "reverse": True,                        "local_setter": local_setter,                        "local_setter": partial(local_setter, final_field),                         "remote_setter": partial(remote_setter, name),                         "from_parent": from_parent,                     }


Question: migrate management command does not respect database parameter when adding Permissions. Description (last modified by Vasanth) When invoking migrate with a database parameter, the migration runs successfully. However, there seems to be a DB read request that runs after the migration. This call does not respect the db param and invokes the db router . When naming the db as a parameter, all DB calls in the context of the migrate command are expected to use the database specified. I came across this as I am currently using a thread-local variable to get the active DB with a custom DB router for a multi-tenant service . Minimal example Setup the custom middleware and custom DB Router as show below. Then run any DB migration. We see that "read {}" is being printed before the exception message. Ideally none of this code must be called as the DB was specified during management command. from threading import local from import settings localstate = local() class InvalidTenantException(Exception): pass class TenantSubdomainMiddleware: def init(self, getresponse): self.getresponse = getresponse def call(self, request): ## Get Subdomain host = request.gethost().split(":")[0] localstate.subdomain = ( # We assume single level of subdomain : # HOSTIP : used to for local dev. host if host in settings.HOSTIP else (".")[0] ) response = self.getresponse(request) return response class TenantDatabaseRouter: def defaultdb(self): subdomain = getattr(localstate, "subdomain", None) if subdomain is not None and subdomain in settings.TENANTMAP: dbname = settings.TENANTMAP[localstate.subdomain] return dbname else: raise InvalidTenantException() def dbforread(self, model, hints): print("read", hints) return self.defaultdb() def dbforwrite(self, model, hints): print("write", hints) return self.defaultdb() def allowrelation(self, obj1, obj2, hints): return None def allowmigrate(self, db, applabel, modelname=None, hints): return None MIDDLEWARE = [ "utils.tenantdbrouter.TenantSubdomainMiddleware", "django.middleware.security.SecurityMiddleware", ... ] TENANTMAP = {"localhost":"default", "tenant1":"default"} DATABASEROUTERS = ["utils.tenantdbrouter.TenantDatabaseRouter"]

.values_list("content_type", "codename")     )     perms = [        Permission(codename=codename, name=name, content_type=ct)        for ct, (codename, name) in searched_perms        if (ct.pk, codename) not in all_perms    ]    perms = []    for ct, (codename, name) in searched_perms:        if (ct.pk, codename) not in all_perms:            permission = Permission()            permission._state.db = using            permission.codename = codename            permission.name = name            permission.content_type = ct            perms.append(permission)     Permission.objects.using(using).bulk_create(perms)     if verbosity >= 2:         for perm in perms: