I'm currently on CCHQ revision 82d3b84 from 14 September 2023 and cannot deploy any of the newer releases as I keep running into an issue when the database migration step is run. The error I get is:
TASK [Run run_migrations] *****************************************************************************************************************************
TASK [deploy_hq : Migrate databases] ******************************************************************************************************************
failed: [10.3.0.4] (item=migrate_multi --noinput) => {"ansible_loop_var": "item", "changed": true, "cmd": ["./manage.py", "migrate_multi", "--noinput"]
, "delta": "0:00:19.406221", "end": "2023-10-28 20:44:43.449036", "item": "migrate_multi --noinput", "msg": "non-zero return code", "rc": 1, "start": "
2023-10-28 20:44:24.042815", "stderr": "Traceback (most recent call last):
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/p
ython3.9/site-packages/django/db/backends/utils.py\", line 82, in _execute
return self.cursor.execute(sql)
psycopg2.errors.InsufficientPrivilege:
permission denied for language plproxy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File \"src/gevent/greenlet.py\", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/cor
ehq/sql_db/management/commands/migrate_multi.py\", line 43, in migrate_db
call_command('migrate', *args, **call_options)
File \"/home/cchq/www/
monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/__init__.py\", line 181, in call_command
return c
ommand.execute(*args, **defaults)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/manage
ment/base.py\", line 398, in execute
output = self.handle(*args, **options)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env
/lib/python3.9/site-packages/django/core/management/base.py\", line 89, in wrapped
res = handle_func(*args, **kwargs)
File \"/home/cchq/www/mon
olith/releases/2023-10-28_20.35/corehq/sql_db/management/commands/migrate.py\", line 17, in handle
result = super().handle(*args, **options)
Fi
le \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\", line 89, in wrapped
res = handle_func(*args, **kwargs)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/man
agement/commands/migrate.py\", line 244, in handle
post_migrate_state = executor.migrate(
File \"/home/cchq/www/monolith/releases/2023-10-28_20
.35/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 117, in migrate
state = self._migrate_all_forwards(state, plan
, full_plan, fake=fake, fake_initial=fake_initial)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/d
jango/db/migrations/executor.py\", line 147, in _migrate_all_forwards
state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_
initial)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 22
7, in apply_migration
state = migration.apply(state, schema_editor)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/pyt
hon3.9/site-packages/django/db/migrations/migration.py\", line 126, in apply
operation.database_forwards(self.app_label, schema_editor, old_state,
project_state)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/operations.py\", line 39, in database_forwards
super(Run
SqlLazy, self).database_forwards(app_label, schema_editor, from_state, to_state)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env
/lib/python3.9/site-packages/django/db/migrations/operations/special.py\", line 105, in database_forwards
self._run_sql(schema_editor, self.sql)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py\", line 13
0, in _run_sql
schema_editor.execute(statement, params=None)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/
site-packages/django/db/backends/base/schema.py\", line 145, in execute
cursor.execute(sql, params)
File \"/home/cchq/www/monolith/releases/202
3-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 66, in execute
return self._execute_with_wrappers(sql, pa
rams, many=False, executor=self._execute)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/
backends/utils.py\", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File \"/home/cchq/www/monolith/releases/202
3-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 84, in _execute
return self.cursor.execute(sql, params)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/utils.py\", line 90, in __exit__
raise
dj_exc_value.with_traceback(traceback) from exc_value
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packag
es/django/db/backends/utils.py\", line 82, in _execute
return self.cursor.execute(sql)
django.db.utils.ProgrammingError: permission denied for la
nguage plproxy
2023-10-28T20:44:41Z <Greenlet at 0x7fe680056590: migrate_db('proxy')> failed with ProgrammingError", "stderr_lines": ["Traceback (mo
st recent call last):", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\
", line 82, in _execute", " return self.cursor.execute(sql)", "psycopg2.errors.InsufficientPrivilege: permission denied for language plproxy", "", "
", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File \"src/gevent/greenlet.py\"
, line 906, in gevent._gevent_cgreenlet.Greenlet.run", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/management/commands/mi
grate_multi.py\", line 43, in migrate_db", " call_command('migrate', *args, **call_options)", " File \"/home/cchq/www/monolith/releases/2023-10-28_
20.35/python_env/lib/python3.9/site-packages/django/core/management/__init__.py\", line 181, in call_command", " return command.execute(*args, **def
aults)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\", line 398,
in execute", " output = self.handle(*args, **options)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-p
ackages/django/core/management/base.py\", line 89, in wrapped", " res = handle_func(*args, **kwargs)", " File \"/home/cchq/www/monolith/releases/20
23-10-28_20.35/corehq/sql_db/management/commands/migrate.py\", line 17, in handle", " result = super().handle(*args, **options)", " File \"/home/cc
hq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\", line 89, in wrapped", " res = han
dle_func(*args, **kwargs)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/c
ommands/migrate.py\", line 244, in handle", " post_migrate_state = executor.migrate(", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/p
ython_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 117, in migrate", " state = self._migrate_all_forwards(state, plan, f
ull_plan, fake=fake, fake_initial=fake_initial)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/dj
ango/db/migrations/executor.py\", line 147, in _migrate_all_forwards", " state = self.apply_migration(state, migration, fake=fake, fake_initial=fake
_initial)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line
227, in apply_migration", " state = migration.apply(state, schema_editor)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/
lib/python3.9/site-packages/django/db/migrations/migration.py\", line 126, in apply", " operation.database_forwards(self.app_label, schema_editor, o
ld_state, project_state)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/operations.py\", line 39, in database_forwards", "
super(RunSqlLazy, self).database_forwards(app_label, schema_editor, from_state, to_state)", " File \"/home/cchq/www/monolith/releases/2023-10-28_2
0.35/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py\", line 105, in database_forwards", " self._run_sql(schema_ed
itor, self.sql)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/operations/sp
ecial.py\", line 130, in _run_sql", " schema_editor.execute(statement, params=None)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/py
thon_env/lib/python3.9/site-packages/django/db/backends/base/schema.py\", line 145, in execute", " cursor.execute(sql, params)", " File \"/home/cch
q/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 66, in execute", " return self._
execute_with_wrappers(sql, params, many=False, executor=self._execute)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/pyt
hon3.9/site-packages/django/db/backends/utils.py\", line 75, in _execute_with_wrappers", " return executor(sql, params, many, context)", " File \"/
home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 84, in _execute", " retu
rn self.cursor.execute(sql, params)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/util
s.py\", line 90, in __exit__", " raise dj_exc_value.with_traceback(traceback) from exc_value", " File \"/home/cchq/www/monolith/releases/2023-10-28
_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 82, in _execute", " return self.cursor.execute(sql)", "django.db.u
tils.ProgrammingError: permission denied for language plproxy", "", "2023-10-28T20:44:41Z <Greenlet at 0x7fe680056590: migrate_db('proxy')> failed with
ProgrammingError"], "stdout": "
The following databases will be migrated:
* default
* p1
* p2
* proxy
* synclogs
The following databas
es will be skipped:
* ucr
======================= Error During Migration =======================
<Greenlet at 0x7fe680056590: migrate_db('proxy
')>
Traceback (most recent call last):
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/ba
ckends/utils.py\", line 82, in _execute
return self.cursor.execute(sql)
psycopg2.errors.InsufficientPrivilege: permission denied for language plproxy
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File \"/home/cchq/www/monolith/
releases/2023-10-28_20.35/corehq/sql_db/management/commands/migrate_multi.py\", line 66, in handle
job.get()
File \"src/gevent/greenlet.py\", l
ine 803, in gevent._gevent_cgreenlet.Greenlet.get
File \"src/gevent/greenlet.py\", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/gevent/_compat.py\", line 65, in reraise
raise v
alue.with_traceback(tb)
File \"src/gevent/greenlet.py\", line 906, in gevent._gevent_cgreenlet.Greenlet.run
File \"/home/cchq/www/monolith/releas
es/2023-10-28_20.35/corehq/sql_db/management/commands/migrate_multi.py\", line 43, in migrate_db
call_command('migrate', *args, **call_options)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/__init__.py\", line 181, in cal
l_command
return command.execute(*args, **defaults)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-pack
ages/django/core/management/base.py\", line 398, in execute
output = self.handle(*args, **options)
File \"/home/cchq/www/monolith/releases/2023
-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\", line 89, in wrapped
res = handle_func(*args, **kwargs)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/management/commands/migrate.py\", line 17, in handle
result = super().handle(
*args, **options)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\",
line 89, in wrapped
res = handle_func(*args, **kwargs)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-p
ackages/django/core/management/commands/migrate.py\", line 244, in handle
post_migrate_state = executor.migrate(
File \"/home/cchq/www/monolith
/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 117, in migrate
state = self._migrate_a
ll_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/py
thon3.9/site-packages/django/db/migrations/executor.py\", line 147, in _migrate_all_forwards
state = self.apply_migration(state, migration, fake=f
ake, fake_initial=fake_initial)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations
/executor.py\", line 227, in apply_migration
state = migration.apply(state, schema_editor)
File \"/home/cchq/www/monolith/releases/2023-10-28_2
0.35/python_env/lib/python3.9/site-packages/django/db/migrations/migration.py\", line 126, in apply
operation.database_forwards(self.app_label, sc
hema_editor, old_state, project_state)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/operations.py\", line 39, in database_
forwards
super(RunSqlLazy, self).database_forwards(app_label, schema_editor, from_state, to_state)
File \"/home/cchq/www/monolith/releases/2023
-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py\", line 105, in database_forwards
self._run_sql(sch
ema_editor, self.sql)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/migrations/operation
s/special.py\", line 130, in _run_sql
schema_editor.execute(statement, params=None)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/py
thon_env/lib/python3.9/site-packages/django/db/backends/base/schema.py\", line 145, in execute
cursor.execute(sql, params)
File \"/home/cchq/ww
w/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 66, in execute
return self._execut
e_with_wrappers(sql, params, many=False, executor=self._execute)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/s
ite-packages/django/db/backends/utils.py\", line 75, in _execute_with_wrappers
return executor(sql, params, many, context)
File \"/home/cchq/ww
w/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 84, in _execute
return self.cursor
.execute(sql, params)
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/utils.py\", line 90,
in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/li
b/python3.9/site-packages/django/db/backends/utils.py\", line 82, in _execute
return self.cursor.execute(sql)
django.db.utils.ProgrammingError: permission denied for language plproxy", "stdout_lines": ["", "The following databases will be migrated:", " * default", " * p1", " * p2", " * proxy", "
* synclogs", "", "", "The following databases will be skipped:", " * ucr", "", "", "======================= Error During Migration ===================
====", "<Greenlet at 0x7fe680056590: migrate_db('proxy')>", "Traceback (most recent call last):", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 82, in _execute", " return self.cursor.execute(sql)", "psycopg2.errors.InsufficientPrivilege: permission denied for language plproxy", "", "", "The above exception was the direct cause of the following exception:", "", "Traceback (most recent call last):", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/management/commands/migrate_multi.py\
", line 66, in handle", " job.get()", " File \"src/gevent/greenlet.py\", line 803, in gevent._gevent_cgreenlet.Greenlet.get", " File \"src/gevent/
greenlet.py\", line 371, in gevent._gevent_cgreenlet.Greenlet._raise_exception", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env
/lib/python3.9/site-packages/gevent/_compat.py\", line 65, in reraise", " raise value.with_traceback(tb)", " File \"src/gevent/greenlet.py\", line
906, in gevent._gevent_cgreenlet.Greenlet.run", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/corehq/sql_db/management/commands/migrate_m
ulti.py\", line 43, in migrate_db", " call_command('migrate', *args, **call_options)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/p
ython_env/lib/python3.9/site-packages/django/core/management/__init__.py\", line 181, in call_command", " return command.execute(*args, **defaults)"
, " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\", line 398, in exe
cute", " output = self.handle(*args, **options)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages
/django/core/management/base.py\", line 89, in wrapped", " res = handle_func(*args, **kwargs)", " File \"/home/cchq/www/monolith/releases/2023-10-2
8_20.35/corehq/sql_db/management/commands/migrate.py\", line 17, in handle", " result = super().handle(*args, **options)", " File \"/home/cchq/www/
monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/base.py\", line 89, in wrapped", " res = handle_fun
c(*args, **kwargs)", " File \"/home/cchq/www/monolith/releases/2023-10-28_20.35/python_env/lib/python3.9/site-packages/django/core/management/commands
/migrate.py\", line 244, in handle", " post_migrate_state = executor.migrate(", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 117, in migrate", " state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 147, in _migrate_all_forwards", " state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/migrations/executor.py\", line 227, in apply_migration", " state = migration.apply(state, schema_editor)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/migrations/migration.py\", line 126, in apply", " operation.database_forwards(self.app_label, schema_editor, old_state, project_state)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/corehq/sql_db/operations.py\", line 39, in database_forwards", " super(RunSqlLazy, self).database_forwards(app_label, schema_editor, from_state, to_state)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py\", line 105, in database_forwards", " self._run_sql(schema_editor, self.sql)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/migrations/operations/special.py\", line 130, in _run_sql", " schema_editor.execute(statement, params=None)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/backends/base/schema.py\", line 145, in execute", " cursor.execute(sql, params)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 66, in execute", " return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 75, in _execute_with_wrappers", " return executor(sql, params, many, context)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 84, in _execute", " return self.cursor.execute(sql, params)", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/utils.py\", line 90, in __exit__", " raise dj_exc_value.with_traceback(traceback) from exc_value", " File \"/home/cchq/www/monolith/releases/2023-10-28_18.54/python_env/lib/python3.9/site-packages/django/db/backends/utils.py\", line 82, in _execute", " return self.cursor.execute(sql)", "django.db.utils.ProgrammingError: permission denied for language plproxy"]}