[Django] #36338: Grouping on JSONField subkey throws tuple index out of range error

[Django]

#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------
Reporter: Marc DEBUREAUX | Type: Bug
Status: new | Component: Database
| layer (models, ORM)
Version: 5.2 | Severity: Normal
Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------
Have this simple model:

{{{
class Link(models.Model):
item = models.ForeignKey("Item", on_delete=models.CASCADE)
extra = models.JSONField(blank=True, null=True)
}}}

And have this simple query:

{{{
Link.objects.filter(item_id=1).values("extra__key").annotate(count=Count("id"))
}}}

This throws the following stacktrace error since 5.2 release:


{{{
---------------------------------------------------------------------------
IndexError Traceback (most recent call
last)
File ~/python3.12/site-packages/IPython/core/formatters.py:770, in
PlainTextFormatter.__call__(self, obj)
763 stream = StringIO()
764 printer = pretty.RepresentationPrinter(stream, self.verbose,
765 self.max_width, self.newline,
766 max_seq_length=self.max_seq_length,
767 singleton_pprinters=self.singleton_printers,
768 type_pprinters=self.type_printers,
769 deferred_pprinters=self.deferred_printers)
--> 770 printer.pretty(obj)
771 printer.flush()
772 return stream.getvalue()

File ~/python3.12/site-packages/IPython/lib/pretty.py:411, in
RepresentationPrinter.pretty(self, obj)
400 return meth(obj, self, cycle)
401 if (
402 cls is not object
403 # check if cls defines __repr__
(...) 409 and callable(_safe_getattr(cls,
"__repr__", None))
410 ):
--> 411 return _repr_pprint(obj, self, cycle)
413 return _default_pprint(obj, self, cycle)
414 finally:

File ~/python3.12/site-packages/IPython/lib/pretty.py:786, in
_repr_pprint(obj, p, cycle)
784 """A pprint that just redirects to the normal repr function."""
785 # Find newlines and replace them with p.break_()
--> 786 output = repr(obj)
787 lines = output.splitlines()
788 with p.group():

File ~/python3.12/site-packages/django/db/models/query.py:360, in
QuerySet.__repr__(self)
359 def __repr__(self):
--> 360 data = list(self[: REPR_OUTPUT_SIZE + 1])
361 if len(data) > REPR_OUTPUT_SIZE:
362 data[-1] = "...(remaining elements truncated)..."

File ~/python3.12/site-packages/django/db/models/query.py:384, in
QuerySet.__iter__(self)
369 def __iter__(self):
370 """
371 The queryset iterator protocol uses three nested iterators in
the
372 default case:
(...) 382 - Responsible for turning the rows into model
objects.
383 """
--> 384 self._fetch_all()
385 return iter(self._result_cache)

File ~/python3.12/site-packages/django/db/models/query.py:1935, in
QuerySet._fetch_all(self)
1933 def _fetch_all(self):
1934 if self._result_cache is None:
-> 1935 self._result_cache = list(self._iterable_class(self))
1936 if self._prefetch_related_lookups and not self._prefetch_done:
1937 self._prefetch_related_objects()

File ~/python3.12/site-packages/django/db/models/query.py:216, in
ValuesIterable.__iter__(self)
210 names = [
211 *query.extra_select,
212 *query.values_select,
213 *query.annotation_select,
214 ]
215 indexes = range(len(names))
--> 216 for row in compiler.results_iter(
217 chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size
218 ):
219 yield {names[i]: row[i] for i in indexes}

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1571, in
SQLCompiler.results_iter(self, results, tuple_expected, chunked_fetch,
chunk_size)
1569 """Return an iterator over the results from executing this
query."""
1570 if results is None:
-> 1571 results = self.execute_sql(
1572 MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size
1573 )
1574 fields = [s[0] for s in self.select[0 : self.col_count]]
1575 converters = self.get_converters(fields)

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:1609, in
SQLCompiler.execute_sql(self, result_type, chunked_fetch, chunk_size)
1607 result_type = result_type or NO_RESULTS
1608 try:
-> 1609 sql, params = self.as_sql()
1610 if not sql:
1611 raise EmptyResultSet

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:765, in
SQLCompiler.as_sql(self, with_limits, with_col_aliases)
763 try:
764 combinator = self.query.combinator
--> 765 extra_select, order_by, group_by = self.pre_sql_setup(
766 with_col_aliases=with_col_aliases or bool(combinator),
767 )
768 for_update_part = None
769 # Is a LIMIT/OFFSET clause needed?

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:85, in
SQLCompiler.pre_sql_setup(self, with_col_aliases)
79 def pre_sql_setup(self, with_col_aliases=False):
80 """
81 Do any necessary class setup immediately prior to producing
SQL. This
82 is for things that can't necessarily be done in __init__
because we
83 might not have all the pieces in place at that time.
84 """
---> 85 self.setup_query(with_col_aliases=with_col_aliases)
86 order_by = self.get_order_by()
87 self.where, self.having, self.qualify =
self.query.where.split_having_qualify(
88 must_group_by=self.query.group_by is not None
89 )

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:74, in
SQLCompiler.setup_query(self, with_col_aliases)
72 if all(self.query.alias_refcount[a] == 0 for a in
self.query.alias_map):
73 self.query.get_initial_alias()
---> 74 self.select, self.klass_info, self.annotation_col_map =
self.get_select(
75 with_col_aliases=with_col_aliases,
76 )
77 self.col_count = len(self.select)

File ~/python3.12/site-packages/django/db/models/sql/compiler.py:286, in
SQLCompiler.get_select(self, with_col_aliases)
284 # Reference to a column.
285 elif isinstance(expression, int):
--> 286 expression = cols[expression]
287 # ColPairs cannot be aliased.
288 if isinstance(expression, ColPairs):
}}}
--
Ticket URL: <https://code.djangoproject.com/ticket/36338>
Django <https://code.djangoproject.com/>
The Web framework for perfectionists with deadlines.

[Django]

#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------

Reporter: Marc DEBUREAUX | Owner: (none)
Type: Bug | Status: new
Component: Database layer | Version: 5.2
(models, ORM) |
Severity: Normal | Resolution:

Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------

Description changed by Marc DEBUREAUX:

Old description:

New description:

Have this simple model:

{{{
class Link(models.Model):
item = models.ForeignKey("Item", on_delete=models.CASCADE)
extra = models.JSONField(blank=True, null=True)
}}}

And have this simple query:

{{{

Link.objects.all().values("extra__key",
"item_id").annotate(count=Count("id"))

--
Ticket URL: <https://code.djangoproject.com/ticket/36338#comment:1>

[Django]

#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------
Reporter: Marc DEBUREAUX | Owner: (none)
Type: Bug | Status: new
Component: Database layer | Version: 5.2
(models, ORM) |
Severity: Normal | Resolution:
Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------

Comment (by Marc DEBUREAUX):

It oddly works if the JSON subkey is the only group clause though, but
crash when used along with any other group clause.
--
Ticket URL: <https://code.djangoproject.com/ticket/36338#comment:2>

[Django]

#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------
Reporter: Marc DEBUREAUX | Owner: (none)
Type: Bug | Status: new
Component: Database layer | Version: 5.2
(models, ORM) |
Severity: Normal | Resolution:
Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------

Description changed by Marc DEBUREAUX:

Old description:

> Have this simple model:
>
> {{{
> class Link(models.Model):
> item = models.ForeignKey("Item", on_delete=models.CASCADE)
> extra = models.JSONField(blank=True, null=True)
> }}}
>
> And have this simple query:
>
> {{{

New description:

**EDIT:** After further investigation, it seems the issue only happens if
the JSONField key is the first item of the group clause, it works fine if
it is alone or if another non-JSON field group clause is positioned first.

--
Ticket URL: <https://code.djangoproject.com/ticket/36338#comment:3>

[Django]

#36338: Grouping on JSONField subkey throws tuple index out of range error
-------------------------------------+-------------------------------------
Reporter: Marc DEBUREAUX | Owner: (none)

Type: Bug | Status: closed

Component: Database layer | Version: 5.2
(models, ORM) |

Severity: Normal | Resolution: duplicate

Keywords: jsonfield | Triage Stage:
| Unreviewed
Has patch: 0 | Needs documentation: 0
Needs tests: 0 | Patch needs improvement: 0
Easy pickings: 0 | UI/UX: 0
-------------------------------------+-------------------------------------

Changes (by Simon Charette):

* resolution: => duplicate
* status: new => closed

Comment:

Duplicate of #36292 (''Annotating an aggregate function over a group
including annotations or transforms followed by a column references
crashes with `IndexError`'') fixed by
543e17c4405dfdac4f18759fc78b190406d14239 in Django 5.2.1 meant to be
released on May 7th.
--
Ticket URL: <https://code.djangoproject.com/ticket/36338#comment:4>