Code Deploymet Error

sirajhassan · November 2, 2023, 1:14pm

Hi,

Code deployment (cchq deploy) is failed due to the following error.

Any suggestion please?

Thank you,

TASK [deploy_hq : Migrate databases] *****************************************************************************************************************************
failed: [172.19.3.38] (item=migrate_multi --noinput) => {"ansible_loop_var": "item", "changed": true, "cmd": ["./manage.py", "migrate_multi", "--noinput"], "delta": "0:00:26.983196", "end": "2023-11-02 12:59:46.098516", "item": "migrate_multi --noinput", "msg": "non-zero return code", "rc": 1, "start": "2023-11-02 12:59:19.115320", "stderr": "Traceback (most recent call last):\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute\n return self.cursor.execute(sql)\npsycopg2.errors.SyntaxError: syntax error at or near "FUNCTION"\nLINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...\n ^\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate_multi.py", line 43, in migrate_db\n call_command('migrate', *args, **call_options)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/init.py", line 181, in call_command\n return command.execute(*args, **defaults)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 398, in execute\n output = self.handle(*args, **options)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped\n res = handle_func(*args, **kwargs)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate.py", line 17, in handle\n result = super().handle(*args, **options)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped\n res = handle_func(*args, **kwargs)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/commands/migrate.py", line 244, in handle\n post_migrate_state = executor.migrate(\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 117, in migrate\n state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards\n state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 227, in apply_migration\n state = migration.apply(state, schema_editor)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/migration.py", line 126, in apply\n operation.database_forwards(self.app_label, schema_editor, old_state, project_state)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 105, in database_forwards\n self._run_sql(schema_editor, self.sql)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 130, in _run_sql\n schema_editor.execute(statement, params=None)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/base/schema.py", line 145, in execute\n cursor.execute(sql, params)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/sentry_sdk/integrations/django/init.py", line 582, in execute\n return real_execute(self, sql, params)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute\n return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers\n return executor(sql, params, many, context)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute\n return self.cursor.execute(sql, params)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/utils.py", line 90, in exit\n raise dj_exc_value.with_traceback(traceback) from exc_value\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute\n return self.cursor.execute(sql)\ndjango.db.utils.ProgrammingError: syntax error at or near "FUNCTION"\nLINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...\n ^\n\n2023-11-02T12:59:43Z <Greenlet at 0x7f796d3cdae0: migrate_db('default')> failed with ProgrammingError", "stderr_lines": ["Traceback (most recent call last):", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute", " return self.cursor.execute(sql)", "psycopg2.errors.SyntaxError: syntax error at or near "FUNCTION"", "LINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...", " ^", "", "", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate_multi.py", line 43, in migrate_db", " call_command('migrate', *args, **call_options)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/init.py", line 181, in call_command", " return command.execute(*args, **defaults)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 398, in execute", " output = self.handle(*args, **options)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped", " res = handle_func(*args, **kwargs)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate.py", line 17, in handle", " result = super().handle(*args, **options)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped", " res = handle_func(*args, **kwargs)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/commands/migrate.py", line 244, in handle", " post_migrate_state = executor.migrate(", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 117, in migrate", " state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards", " state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 227, in apply_migration", " state = migration.apply(state, schema_editor)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/migration.py", line 126, in apply", " operation.database_forwards(self.app_label, schema_editor, old_state, project_state)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 105, in database_forwards", " self._run_sql(schema_editor, self.sql)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 130, in _run_sql", " schema_editor.execute(statement, params=None)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/base/schema.py", line 145, in execute", " cursor.execute(sql, params)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/sentry_sdk/integrations/django/init.py", line 582, in execute", " return real_execute(self, sql, params)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute", " return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers", " return executor(sql, params, many, context)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute", " return self.cursor.execute(sql, params)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/utils.py", line 90, in exit", " raise dj_exc_value.with_traceback(traceback) from exc_value", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute", " return self.cursor.execute(sql)", "django.db.utils.ProgrammingError: syntax error at or near "FUNCTION"", "LINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...", " ^", "", "2023-11-02T12:59:43Z <Greenlet at 0x7f796d3cdae0: migrate_db('default')> failed with ProgrammingError"], "stdout": "\nThe following databases will be migrated:\n * default\n * auditcare\n * p1\n * p2\n * p3\n * p4\n * p5\n * p6\n * p7\n * p8\n * proxy\n * synclogs\n\n\nThe following databases will be skipped:\n * ucr\n\n\n======================= Error During Migration =======================\n<Greenlet at 0x7f796d3cdae0: migrate_db('default')>\nTraceback (most recent call last):\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute\n return self.cursor.execute(sql)\npsycopg2.errors.SyntaxError: syntax error at or near "FUNCTION"\nLINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...\n ^\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate_multi.py", line 66, in handle\n job.get()\n File "src/gevent/greenlet.py", line 805, in gevent._gevent_cgreenlet.Greenlet.get\n File "src/gevent/greenlet.py", line 373, in gevent._gevent_cgreenlet.Greenlet._raise_exception\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/gevent/_compat.py", line 49, in reraise\n raise value.with_traceback(tb)\n File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate_multi.py", line 43, in migrate_db\n call_command('migrate', *args, **call_options)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/init.py", line 181, in call_command\n return command.execute(*args, **defaults)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 398, in execute\n output = self.handle(*args, **options)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped\n res = handle_func(*args, **kwargs)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate.py", line 17, in handle\n result = super().handle(*args, **options)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped\n res = handle_func(*args, **kwargs)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/commands/migrate.py", line 244, in handle\n post_migrate_state = executor.migrate(\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 117, in migrate\n state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards\n state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 227, in apply_migration\n state = migration.apply(state, schema_editor)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/migration.py", line 126, in apply\n operation.database_forwards(self.app_label, schema_editor, old_state, project_state)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 105, in database_forwards\n self._run_sql(schema_editor, self.sql)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 130, in _run_sql\n schema_editor.execute(statement, params=None)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/base/schema.py", line 145, in execute\n cursor.execute(sql, params)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/sentry_sdk/integrations/django/init.py", line 582, in execute\n return real_execute(self, sql, params)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute\n return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers\n return executor(sql, params, many, context)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute\n return self.cursor.execute(sql, params)\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/utils.py", line 90, in exit\n raise dj_exc_value.with_traceback(traceback) from exc_value\n File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute\n return self.cursor.execute(sql)\ndjango.db.utils.ProgrammingError: syntax error at or near "FUNCTION"\nLINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...\n ^", "stdout_lines": ["", "The following databases will be migrated:", " * default", " * auditcare", " * p1", " * p2", " * p3", " * p4", " * p5", " * p6", " * p7", " * p8", " * proxy", " * synclogs", "", "", "The following databases will be skipped:", " * ucr", "", "", "======================= Error During Migration =======================", "<Greenlet at 0x7f796d3cdae0: migrate_db('default')>", "Traceback (most recent call last):", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute", " return self.cursor.execute(sql)", "psycopg2.errors.SyntaxError: syntax error at or near "FUNCTION"", "LINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...", " ^", "", "", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate_multi.py", line 66, in handle", " job.get()", " File "src/gevent/greenlet.py", line 805, in gevent._gevent_cgreenlet.Greenlet.get", " File "src/gevent/greenlet.py", line 373, in gevent._gevent_cgreenlet.Greenlet._raise_exception", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/gevent/_compat.py", line 49, in reraise", " raise value.with_traceback(tb)", " File "src/gevent/greenlet.py", line 908, in gevent._gevent_cgreenlet.Greenlet.run", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate_multi.py", line 43, in migrate_db", " call_command('migrate', *args, **call_options)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/init.py", line 181, in call_command", " return command.execute(*args, **defaults)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 398, in execute", " output = self.handle(*args, **options)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped", " res = handle_func(*args, **kwargs)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/corehq/sql_db/management/commands/migrate.py", line 17, in handle", " result = super().handle(*args, **options)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/base.py", line 89, in wrapped", " res = handle_func(*args, **kwargs)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/core/management/commands/migrate.py", line 244, in handle", " post_migrate_state = executor.migrate(", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 117, in migrate", " state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards", " state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py", line 227, in apply_migration", " state = migration.apply(state, schema_editor)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/migration.py", line 126, in apply", " operation.database_forwards(self.app_label, schema_editor, old_state, project_state)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 105, in database_forwards", " self._run_sql(schema_editor, self.sql)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py", line 130, in _run_sql", " schema_editor.execute(statement, params=None)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/base/schema.py", line 145, in execute", " cursor.execute(sql, params)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/sentry_sdk/integrations/django/init.py", line 582, in execute", " return real_execute(self, sql, params)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute", " return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers", " return executor(sql, params, many, context)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute", " return self.cursor.execute(sql, params)", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/utils.py", line 90, in exit", " raise dj_exc_value.with_traceback(traceback) from exc_value", " File "/home/cchq/www/echis/releases/2023-11-02_12.49/python_env/lib/python3.9/site-packages/django/db/backends/utils.py", line 82, in _execute", " return self.cursor.execute(sql)", "django.db.utils.ProgrammingError: syntax error at or near "FUNCTION"", "LINE 26: FOR EACH ROW EXECUTE FUNCTION set_default_re...", " ^"]}

mkangia · November 3, 2023, 5:59pm

Hi Siraj,

Can you share which version of postgres you are using?

sirajhassan · November 3, 2023, 6:01pm

Hi @mkangia,

We are using PostgreSQL 10.

mkangia · November 3, 2023, 6:14pm

Hi @sirajhassan

Looks like you are running an unsupported version of postgres.
Please upgrade using steps shared in
https://commcare-cloud.readthedocs.io/en/latest/changelog/0059-postgres-v14-upgrade.html

Generally, it's important to keep an eye on About this changelog — CommCareHQ Deployment documentation and do recommended updates.

sirajhassan · November 3, 2023, 6:26pm

Thank you Manish. Will reach out to you.

sirajhassan · November 8, 2023, 10:50am

Hi @mkangia,

Able to deploy the code after upgrading the PostgreSQL to 14.

However, Django log is displaying the following error message after the upgrade.

Thank you,

2023-11-08 10:45:16,681 ERROR [django.request] Internal Server Error: /a/fmoh-echis/receiver/secure/bc4edc58e56a4708a69a0fc30722f477/
Traceback (most recent call last):
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
psycopg2.errors.DataCorrupted: invalid page in block 78924 of relation base/16403/25631


The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/form_processor/models/cases.py", line 793, in save
    index.save(update_fields=update_fields)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/base.py", line 739, in save
    self.save_base(using=using, force_insert=force_insert,
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/base.py", line 776, in save_base
    updated = self._save_table(
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/base.py", line 881, in _save_table
    results = self._do_insert(cls._base_manager, using, fields, returning_fields, raw)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/base.py", line 919, in _do_insert
    return manager._insert(
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/manager.py", line 85, in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/query.py", line 1270, in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/models/sql/compiler.py", line 1416, in execute_sql
    cursor.execute(sql, params)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/sentry_sdk/integrations/django/__init__.py", line 582, in execute
    return real_execute(self, sql, params)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/backends/utils.py", line 66, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/backends/utils.py", line 75, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/utils.py", line 90, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/db/backends/utils.py", line 84, in _execute
    return self.cursor.execute(sql, params)
django.db.utils.InternalError: invalid page in block 78924 of relation base/16403/25631


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/core/handlers/exception.py", line 47, in inner
    response = get_response(request)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/core/handlers/base.py", line 181, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/sentry_sdk/integrations/django/views.py", line 85, in sentry_wrapped_callback
    return callback(request, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/views/decorators/csrf.py", line 54, in wrapped_view
    return view_func(*args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/django/views/decorators/http.py", line 40, in inner
    return func(request, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/decorators.py", line 677, in wrapped_view
    return view_func(request, domain, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/receiverwrapper/views.py", line 427, in secure_post
    return decorated_view(request, domain, app_id=app_id)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/ota/utils.py", line 207, in _inner
    response = f(request, domain, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/decorators.py", line 277, in safe_fn_with_domain
    return _inner(request, domain, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/decorators.py", line 596, in _inner
    return fn(request, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/auth.py", line 142, in wrapper
    return view(request, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/decorators.py", line 480, in _inner
    return fn(request, domain, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/decorators.py", line 273, in _inner
    return fn(request, domain, *args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/domain/decorators.py", line 402, in wrapped_view
    return view_func(*args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/receiverwrapper/views.py", line 361, in _secure_post_basic
    return _process_form(
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/apps/receiverwrapper/views.py", line 158, in _process_form
    result = submission_post.run()
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/form_processor/submission_post.py", line 345, in run
    openrosa_kwargs['error_message'] = self.save_processed_models(case_db, xforms,
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/python_env-3.9/lib/python3.9/site-packages/ddtrace/tracer.py", line 985, in func_wrapper
    return f(*args, **kwargs)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/form_processor/submission_post.py", line 426, in save_processed_models
    self.interface.save_processed_models(
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/form_processor/interfaces/processor.py", line 148, in save_processed_models
    return self.processor.save_processed_models(
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/form_processor/backends/sql/processor.py", line 118, in save_processed_models
    case.save(with_tracked_models=True)
  File "/home/cchq/www/echis/releases/2023-11-07_23.12/corehq/form_processor/models/cases.py", line 804, in save
    raise CaseSaveError(e)
corehq.form_processor.exceptions.CaseSaveError: invalid page in block 78924 of relation base/16403/25631