I'm seeing errors in the Celery queue when requesting reports:
2025-05-16 12:32:01,740 ERROR [notify] Notify Exception: Celery task failure
Traceback (most recent call last):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/celery/app/trace.py", line 453, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/celery/app/trace.py", line 736, in __protected_call__
return self.run(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/tasks.py", line 42, in export_all_rows_task
file = report.excel_response
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/generic.py", line 714, in excel_response
export_from_tables(self.export_table, file, self.export_format)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/export.py", line 43, in export_from_tables
writer.write(rows_by_sheet)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/writers.py", line 220, in write
for i, row in enumerate(table):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/export.py", line 338, in <genexpr>
(name, (cls(row) for row in coerce_to_list_of_rows(rows)))
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/reports/r1_master_data.py", line 48, in rows
for row in self._food_data.rows:
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/memoized.py", line 20, in _memoized
cache[key] = value = fn(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 439, in rows
for row in self._get_all_rows():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 422, in _get_all_rows
for group in self._get_grouped_rows():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 408, in _get_grouped_rows
for row in self._ucr.get_data():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/sqlreport.py", line 272, in get_data
data = self._get_data(start=start, limit=limit)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/sqlreport.py", line 288, in _get_data
return qc.resolve(session.connection(), self.filter_values)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlagg/base.py", line 285, in resolve
result = qm.execute(self.connection, filter_values or {})
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlagg/base.py", line 95, in execute
return connection.execute(query, **filter_values).fetchall()
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
ret = self._execute_context(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
util.raise_(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
billiard.einfo.ExceptionWithTraceback:
"""
Traceback (most recent call last):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
psycopg2.errors.UndefinedTable: relation "ucr_intake4earth-study_food_consumption_i_dd9bba7f" does not exist
LINE 2: FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f"
^
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/celery/app/trace.py", line 453, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/celery/app/trace.py", line 736, in __protected_call__
return self.run(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/tasks.py", line 42, in export_all_rows_task
file = report.excel_response
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/generic.py", line 714, in excel_response
export_from_tables(self.export_table, file, self.export_format)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/export.py", line 43, in export_from_tables
writer.write(rows_by_sheet)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/writers.py", line 220, in write
for i, row in enumerate(table):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/export.py", line 338, in <genexpr>
(name, (cls(row) for row in coerce_to_list_of_rows(rows)))
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/reports/r1_master_data.py", line 48, in rows
for row in self._food_data.rows:
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/memoized.py", line 20, in _memoized
cache[key] = value = fn(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 439, in rows
for row in self._get_all_rows():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 422, in _get_all_rows
for group in self._get_grouped_rows():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 408, in _get_grouped_rows
for row in self._ucr.get_data():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/sqlreport.py", line 272, in get_data
data = self._get_data(start=start, limit=limit)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/sqlreport.py", line 288, in _get_data
return qc.resolve(session.connection(), self.filter_values)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlagg/base.py", line 285, in resolve
result = qm.execute(self.connection, filter_values or {})
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlagg/base.py", line 95, in execute
return connection.execute(query, **filter_values).fetchall()
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
ret = self._execute_context(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
util.raise_(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.ProgrammingError: (psycopg2.errors.UndefinedTable) relation "ucr_intake4earth-study_food_consumption_i_dd9bba7f" does not exist
LINE 2: FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f"
^
[SQL: SELECT doc_id AS doc_id, inserted_at AS inserted_at, unique_respondent_id AS unique_respondent_id, location_id AS location_id, respondent_id AS respondent_id, recall_case_id AS recall_case_id, opened_by_username AS opened_by_username, owner_name AS owner_name, visit_date AS visit_date, opened_on AS opened_on, recall_status AS recall_status, gender AS gender, age_years_calculated AS age_years_calculated, age_months_calculated AS age_months_calculated, pregnant AS pregnant, breastfeeding AS breastfeeding, urban_rural AS urban_rural, supplements AS supplements, food_code AS food_code, food_name AS food_name, recipe_name AS recipe_name, food_type AS food_type, food_status AS food_status, base_term_food_code AS base_term_food_code, eating_time AS eating_time, time_block AS time_block, already_reported_food AS already_reported_food, already_reported_food_case_id AS already_reported_food_case_id, already_reported_recipe AS already_reported_recipe, already_reported_recipe_case_id AS already_reported_recipe_case_id, already_reported_recipe_name AS already_reported_recipe_name, is_ingredient AS is_ingredient, recipe_case_id AS recipe_case_id, short_name AS short_name, food_base_term AS food_base_term, tag_1 AS tag_1, other_tag_1 AS other_tag_1, tag_2 AS tag_2, other_tag_2 AS other_tag_2, tag_3 AS tag_3, other_tag_3 AS other_tag_3, tag_4 AS tag_4, other_tag_4 AS other_tag_4, tag_5 AS tag_5, other_tag_5 AS other_tag_5, tag_6 AS tag_6, other_tag_6 AS other_tag_6, tag_7 AS tag_7, other_tag_7 AS other_tag_7, tag_8 AS tag_8, other_tag_8 AS other_tag_8, tag_9 AS tag_9, other_tag_9 AS other_tag_9, tag_10 AS tag_10, other_tag_10 AS other_tag_10, conv_method_code AS conv_method_code, conv_method_desc AS conv_method_desc, conv_option_code AS conv_option_code, conv_option_desc AS conv_option_desc, measurement_amount AS measurement_amount, conv_units AS conv_units, portions AS portions, nsr_conv_method_code_post_cooking AS nsr_conv_method_code_post_cooking, nsr_conv_method_desc_post_cooking AS nsr_conv_method_desc_post_cooking, nsr_conv_option_code_post_cooking AS nsr_conv_option_code_post_cooking, nsr_conv_option_desc_post_cooking AS nsr_conv_option_desc_post_cooking, nsr_measurement_amount_post_cooking AS nsr_measurement_amount_post_cooking, nsr_consumed_cooked_fraction AS nsr_consumed_cooked_fraction
FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f"
WHERE visit_date >= %s AND visit_date <= %s GROUP BY doc_id]
[parameters: ('2024-11-10 00:00:00', '2025-05-14 00:00:00')]
(Background on this error at: http://sqlalche.me/e/13/f405)
"""
2025-05-16 12:32:01,740 ERROR [celery.app.trace] Task corehq.apps.reports.tasks.export_all_rows_task[66779297-bb39-468f-8126-9fc3756c7ba4] raised unexpected: ProgrammingError('(psycopg2.errors.UndefinedTable) relation "ucr_intake4earth-study_food_consumption_i_dd9bba7f" does not exist\nLINE 2: FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f" \n ^\n')
Traceback (most recent call last):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
psycopg2.errors.UndefinedTable: relation "ucr_intake4earth-study_food_consumption_i_dd9bba7f" does not exist
LINE 2: FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f"
^
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/celery/app/trace.py", line 453, in trace_task
R = retval = fun(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/celery/app/trace.py", line 736, in __protected_call__
return self.run(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/tasks.py", line 42, in export_all_rows_task
file = report.excel_response
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/generic.py", line 714, in excel_response
export_from_tables(self.export_table, file, self.export_format)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/export.py", line 43, in export_from_tables
writer.write(rows_by_sheet)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/writers.py", line 220, in write
for i, row in enumerate(table):
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/ex-submodules/couchexport/export.py", line 338, in <genexpr>
(name, (cls(row) for row in coerce_to_list_of_rows(rows)))
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/reports/r1_master_data.py", line 48, in rows
for row in self._food_data.rows:
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/memoized.py", line 20, in _memoized
cache[key] = value = fn(*args, **kwargs)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 439, in rows
for row in self._get_all_rows():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 422, in _get_all_rows
for group in self._get_grouped_rows():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/custom/inddex/food.py", line 408, in _get_grouped_rows
for row in self._ucr.get_data():
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/sqlreport.py", line 272, in get_data
data = self._get_data(start=start, limit=limit)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/corehq/apps/reports/sqlreport.py", line 288, in _get_data
return qc.resolve(session.connection(), self.filter_values)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlagg/base.py", line 285, in resolve
result = qm.execute(self.connection, filter_values or {})
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlagg/base.py", line 95, in execute
return connection.execute(query, **filter_values).fetchall()
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1124, in _execute_clauseelement
ret = self._execute_context(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1316, in _execute_context
self._handle_dbapi_exception(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1510, in _handle_dbapi_exception
util.raise_(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1276, in _execute_context
self.dialect.do_execute(
File "/home/cchq/www/monolith/releases/2025-05-15_03.39/python_env-3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 608, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.ProgrammingError: (psycopg2.errors.UndefinedTable) relation "ucr_intake4earth-study_food_consumption_i_dd9bba7f" does not exist
LINE 2: FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f"
^
[SQL: SELECT doc_id AS doc_id, inserted_at AS inserted_at, unique_respondent_id AS unique_respondent_id, location_id AS location_id, respondent_id AS respondent_id, recall_case_id AS recall_case_id, opened_by_username AS opened_by_username, owner_name AS owner_name, visit_date AS visit_date, opened_on AS opened_on, recall_status AS recall_status, gender AS gender, age_years_calculated AS age_years_calculated, age_months_calculated AS age_months_calculated, pregnant AS pregnant, breastfeeding AS breastfeeding, urban_rural AS urban_rural, supplements AS supplements, food_code AS food_code, food_name AS food_name, recipe_name AS recipe_name, food_type AS food_type, food_status AS food_status, base_term_food_code AS base_term_food_code, eating_time AS eating_time, time_block AS time_block, already_reported_food AS already_reported_food, already_reported_food_case_id AS already_reported_food_case_id, already_reported_recipe AS already_reported_recipe, already_reported_recipe_case_id AS already_reported_recipe_case_id, already_reported_recipe_name AS already_reported_recipe_name, is_ingredient AS is_ingredient, recipe_case_id AS recipe_case_id, short_name AS short_name, food_base_term AS food_base_term, tag_1 AS tag_1, other_tag_1 AS other_tag_1, tag_2 AS tag_2, other_tag_2 AS other_tag_2, tag_3 AS tag_3, other_tag_3 AS other_tag_3, tag_4 AS tag_4, other_tag_4 AS other_tag_4, tag_5 AS tag_5, other_tag_5 AS other_tag_5, tag_6 AS tag_6, other_tag_6 AS other_tag_6, tag_7 AS tag_7, other_tag_7 AS other_tag_7, tag_8 AS tag_8, other_tag_8 AS other_tag_8, tag_9 AS tag_9, other_tag_9 AS other_tag_9, tag_10 AS tag_10, other_tag_10 AS other_tag_10, conv_method_code AS conv_method_code, conv_method_desc AS conv_method_desc, conv_option_code AS conv_option_code, conv_option_desc AS conv_option_desc, measurement_amount AS measurement_amount, conv_units AS conv_units, portions AS portions, nsr_conv_method_code_post_cooking AS nsr_conv_method_code_post_cooking, nsr_conv_method_desc_post_cooking AS nsr_conv_method_desc_post_cooking, nsr_conv_option_code_post_cooking AS nsr_conv_option_code_post_cooking, nsr_conv_option_desc_post_cooking AS nsr_conv_option_desc_post_cooking, nsr_measurement_amount_post_cooking AS nsr_measurement_amount_post_cooking, nsr_consumed_cooked_fraction AS nsr_consumed_cooked_fraction
FROM "ucr_intake4earth-study_food_consumption_i_dd9bba7f"
WHERE visit_date >= %s AND visit_date <= %s GROUP BY doc_id]
[parameters: ('2024-11-10 00:00:00', '2025-05-14 00:00:00')]
(Background on this error at: http://sqlalche.me/e/13/f405)
Any ideas what could be causing this?
For reference, we've upgraded to 0d2334b for HQ (13 May) but we were unable to upgrade to Elasticsearch 6 so we left that changelog off for now.
It's worth noting that these are custom reports. Is there a way I can force rebuild of these custom reports data source?