Unable to delete mobile user from specific location

Hello,

I was trying to delete(detach) the mobile user from the wrongly placed location, as shown below

The following error displayed after click on update location membership

2022-08-17 15:57:45,793 ERROR [django.request] Internal Server Error: /a/fmoh-echis/settings/locations/fddeef5f386147b18e212c83ba8512f1/
Traceback (most recent call last):
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/blobs/s3db.py", line 175, in maybe_not_found
yield
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/blobs/s3db.py", line 99, in get
resp = self._s3_bucket().Object(key).get()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/boto3/resources/factory.py", line 580, in do_action
response = action(self, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/boto3/resources/action.py", line 88, in call
response = getattr(parent.meta.client, operation_name)(*args, **params)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/botocore/client.py", line 508, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/botocore/client.py", line 915, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.errorfactory.NoSuchKey: An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist.

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/form_processor/models/forms.py", line 674, in get_xml
return self.get_attachment('form.xml')
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/form_processor/models/attachment.py", line 258, in get_attachment
with attachment.open() as content:
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/blobs/models.py", line 107, in open
return db.get(meta=self)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/ex-submodules/dimagi/utils/retry.py", line 31, in retry
return func(*args, **kw)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/blobs/s3db.py", line 99, in get
resp = self._s3_bucket().Object(key).get()
File "/usr/lib/python3.9/contextlib.py", line 137, in exit
self.gen.throw(typ, value, traceback)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/blobs/s3db.py", line 181, in maybe_not_found
raise throw
corehq.blobs.exceptions.NotFound: 7ca7d0f9090240baa51f658dd16897d5

Traceback (most recent call last):
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/core/handlers/exception.py", line 47, in inner
response = get_response(request)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/core/handlers/base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/sentry_sdk/integrations/django/views.py", line 67, in sentr
y_wrapped_callback
return callback(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/views/generic/base.py", line 70, in view
return self.dispatch(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/hqwebapp/decorators.py", line 70, in _wrapped
return view_func(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/utils/decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/views.py", line 160, in new_fn
return view_fn(request, domain, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/utils/decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 111, in _inner
return call_view()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 84, in call_view
def call_view(): return view_func(req, domain_name, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django_prbac/decorators.py", line 38, in wrapped
return fn(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 111, in _inner
return call_view()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 84, in call_view
def call_view(): return view_func(req, domain_name, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/users/decorators.py", line 45, in _inner
return view_func(request, domain, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/permissions.py", line 337, in _inner
return view_fn(request, domain, loc_id, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/views.py", line 700, in dispatch
return super(EditLocationView, self).dispatch(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/utils/decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 111, in _inner
return call_view()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 84, in call_view
def call_view(): return view_func(req, domain_name, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/domain/decorators.py", line 181, in dispatch
return super(LoginAndDomainMixin, self).dispatch(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/views/generic/base.py", line 98, in dispatch
return handler(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/django/utils/decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/views.py", line 96, in func_wrapper
return func(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/views.py", line 834, in post
return self.users_form_post(request, *args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/views.py", line 810, in users_form_post
self.users_form.save()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/forms.py", line 510, in save
self.unassign_users(to_remove)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/locations/forms.py", line 482, in unassign_users
CommCareUser.wrap(doc).unset_location_by_id(self.location.location_id, fall_back_to_next=True)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/users/models.py", line 2057, in unset_location_by_id
self.unset_location(fall_back_to_next)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/users/models.py", line 2035, in unset_location
self.set_location(SQLLocation.objects.get(location_id=new_primary_location_id))
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/users/models.py", line 1995, in set_location
self.create_location_delegates([location])
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/users/models.py", line 2153, in create_location_delegates
self.clear_location_delegates()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/apps/users/models.py", line 2146, in clear_location_delegates
safe_hard_delete(mapping)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/ex-submodules/casexml/apps/case/cleanup.py", line 95, in safe_hard_delete
case_updates = get_case_updates(form)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/ex-submodules/casexml/apps/case/xform.py", line 245, in get_case_updates
[case_update_from_block(cb) for cb in extract_case_blocks(xform)],
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/ex-submodules/casexml/apps/case/xform.py", line 200, in extract_case_blocks
form = doc.form_data
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/memoized.py", line 20, in _memoized
cache[key] = value = fn(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/form_processor/models/forms.py", line 593, in form_data
xml = self.get_xml()
File "/home/cchq/www/echis/releases/2022-08-14_17.31/python_env-3.9/lib/python3.9/site-packages/memoized.py", line 20, in _memoized
cache[key] = value = fn(*args, **kwargs)
File "/home/cchq/www/echis/releases/2022-08-14_17.31/corehq/form_processor/models/forms.py", line 676, in get_xml
raise MissingFormXml(self.form_id)
corehq.form_processor.exceptions.MissingFormXml: 170101f9fa1f41699e367cbf8fbcd307

This is kind of a weird error. When assigning users to locations, a "delegate" case is created (case type user-owner-mapping-case), for use in some particular app workflows. What's happening here is you are attempting to remove a user from a location, which triggers a hard deletion of this case. In processing this deletion, it looks up the forms which modify that case, but one of those forms is missing XML.

I'm not certain why this XML is missing - it could be a problem with your blobdb (S3, riak, or similar), where that form XML is stored. You can find the case ID in question by running this command in a shell session:

>>> CommCareUser.get_by_username('USERNAME@DOMAIN.commcarehq.org').get_location_map_case().case_id

That might help get started troubleshooting.