django db timeout in epoll mode

source code

class EventWorker():
    @classmethod
    def get_task(cls):
        """Get first waiting task with FIFO order

        :returns: task
        :rtype: dict or None
        """
        try:
            event_task = EventTask.objects.all().exclude(status__in=COMPLETE_STATUS).latest()
        except:
            logger.debug('get no task, continue...')
            return None
        return event_task

    @classmethod
    def start(cls):
        while True:
            task = cls.get_task()
            if not task:
                time.sleep(1)
                continue
            
            logger.info('get task {}, will start it.'.format(task))
            task.status = STATUS_IN_PROGRESS
            task.started_at = datetime.datetime.now()
            task.save()
            task_serializers = EventTaskSerializer(task)
            worker = EventHandler(task_serializers.data)
            worker.start()
            time.sleep(1)

EventWorker.start()

log

Traceback (most recent call last):
  File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
    return self.cursor.execute(sql, params)
psycopg2.OperationalError: server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.

The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 185, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python3.5/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/viewsets.py", line 83, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 477, in dispatch
    response = self.handle_exception(exc)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 437, in handle_exception
    self.raise_uncaught_exception(exc)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 474, in dispatch
    response = handler(request, *args, **kwargs)
  File "/app/instant_jobs/views.py", line 56, in create
    job_client.create_job(serializer.validated_data)
  File "/app/alauda_job/job_client.py", line 16, in create_job
    return self._scheduler.create_job(data)
  File "/app/alauda_job/kubernetes.py", line 57, in create_job
    event_task.save()
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 806, in save
    force_update=force_update, update_fields=update_fields)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 836, in save_base
    updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 903, in _save_table
    forced_update)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 953, in _do_update
    return filtered._update(values) > 0
  File "/usr/local/lib/python3.5/site-packages/django/db/models/query.py", line 661, in _update
    return query.get_compiler(self.db).execute_sql(CURSOR)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 1191, in execute_sql
    cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 876, in execute_sql
    cursor.execute(sql, params)
  File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
    return self.cursor.execute(sql, params)
  File "/usr/local/lib/python3.5/site-packages/django/db/utils.py", line 94, in __exit__
    six.reraise(dj_exc_type, dj_exc_value, traceback)
  File "/usr/local/lib/python3.5/site-packages/django/utils/six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
    return self.cursor.execute(sql, params)
django.db.utils.OperationalError: server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
2018-03-27 05:04:38,653 [WARNING][MainThread][mekansm.contrib.django.middleware:27] the failed request info:
{
    "user": "None",
    "remote_host": null,
    "path": "/v1/instant-jobs/",
    "method": "POST",
    "remote_addr": ""
}
2018-03-27 05:04:38,653 [ERROR][MainThread][django_config.middleware:16] Exception was not wrapped: server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.
2018-03-27 05:04:38,654 [ERROR][MainThread][django_config.middleware:17] Traceback (most recent call last):
  File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
    return self.cursor.execute(sql, params)
psycopg2.OperationalError: server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.

The above exception was the direct cause of the following exception:
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/site-packages/django/core/handlers/base.py", line 185, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python3.5/site-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/viewsets.py", line 83, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 477, in dispatch
    response = self.handle_exception(exc)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 437, in handle_exception
    self.raise_uncaught_exception(exc)
  File "/usr/local/lib/python3.5/site-packages/rest_framework/views.py", line 474, in dispatch
    response = handler(request, *args, **kwargs)
  File "/app/instant_jobs/views.py", line 56, in create
    job_client.create_job(serializer.validated_data)
  File "/app/alauda_job/job_client.py", line 16, in create_job
    return self._scheduler.create_job(data)
  File "/app/alauda_job/kubernetes.py", line 57, in create_job
    event_task.save()
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 806, in save
    force_update=force_update, update_fields=update_fields)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 836, in save_base
    updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 903, in _save_table
    forced_update)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/base.py", line 953, in _do_update
    return filtered._update(values) > 0
  File "/usr/local/lib/python3.5/site-packages/django/db/models/query.py", line 661, in _update
    return query.get_compiler(self.db).execute_sql(CURSOR)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 1191, in execute_sql
    cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
  File "/usr/local/lib/python3.5/site-packages/django/db/models/sql/compiler.py", line 876, in execute_sql
    cursor.execute(sql, params)
  File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
    return self.cursor.execute(sql, params)
  File "/usr/local/lib/python3.5/site-packages/django/db/utils.py", line 94, in __exit__
    six.reraise(dj_exc_type, dj_exc_value, traceback)
  File "/usr/local/lib/python3.5/site-packages/django/utils/six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "/usr/local/lib/python3.5/site-packages/django/db/backends/utils.py", line 65, in execute
    return self.cursor.execute(sql, params)
django.db.utils.OperationalError: server closed the connection unexpectedly
	This probably means the server terminated abnormally
	before or while processing the request.

Django中的db不是持久链接,会因链接超时断开链接报错,可以使用装饰器来解决。

optimaze

def reconnect_database_on_error(func):
    """
    Decorator to reconnect to the database if the connection is closed in Django
    refer to jakiro code and https://stackoverflow.com/questions/4447497/django-how-to-reconnect-after-databaseerror-query-timeout
    """

    def process_connection_exception(*args, **kwargs):
        result = None
        try:
            result = func(*args, **kwargs)
        except (DatabaseError, OperationalError) as err:
            logger.info('[EventTask] Database connection error: {}'.format(err))
            connection.connection.close()
            connection.connection = None
        except ObjectDoesNotExist as ne:
            logger.info('[EventTask] {}'.format(ne))
        except Exception as exc:
            logger.error('[EventTask] Some other exception: {}'.format(exc))
            # return exc
        return result
    return process_connection_exception

@reconnect_database_on_error
    def handle_event(cls):
        """
        Get first waiting task with FIFO order and promise consistency
        """
        found = False
        with transaction.atomic():
            event_task = EventTask.objects.select_for_update().exclude(status__in=COMPLETE_STATUS).latest()
            if not event_task:
                # logger.info('get no k8s event task, will try again')
                return found

            logger.info('get task {}, will start it.'.format(event_task))
            event_task.status = STATUS_IN_PROGRESS
            event_task.started_at = datetime.datetime.now()
            event_task.save()
            worker = EventHandler(event_task)
            worker.start()
            found = True

        return found
comments powered by Disqus