kallithea Changeset - 1f92cded1bd2

Changeset - 1f92cded1bd2

Parent rev.

Child rev.

[Not reviewed]

default

0 5 0

Mads Kiilerich (mads) - 5 years ago 2020-12-29 22:07:41
mads@kiilerich.com

celery: move Whoosh indexing task to whoosh library where it belongs

Avoid bundling everything from many different layers in one big task library.

This is more feasible now when we don't need kallithea.CELERY_APP set at import
time.

5 files changed with 18 insertions and 14 deletions:

kallithea/controllers/admin/settings.py

kallithea/lib/celery_app.py

kallithea/lib/indexers/daemon.py

kallithea/model/async_tasks.py

scripts/deps.py

0 comments (0 inline, 0 general)

kallithea/controllers/admin/settings.py

➞

Show inline comments

@@ @@ -33,12 +33,13 @@ from formencode import htmlfill @@
 from tg import config, request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPFound
 import kallithea
 import kallithea.lib.indexers.daemon
 from kallithea.controllers import base
 from kallithea.lib import webutils
 from kallithea.lib.auth import HasPermissionAnyDecorator, LoginRequired
 from kallithea.lib.utils import repo2db_mapper, set_app_settings
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.vcs import VCSError
@@ @@ -375,13 +376,13 @@ class SettingsController(base.BaseContro @@
     @HasPermissionAnyDecorator('hg.admin')
     def settings_search(self):
         c.active = 'search'
         if request.POST:
             repo_location = self._get_hg_ui_settings()['paths_root_path']
             full_index = request.POST.get('full_index', False)
-            async_tasks.whoosh_index(repo_location, full_index)
+            kallithea.lib.indexers.daemon.whoosh_index(repo_location, full_index)
             webutils.flash(_('Whoosh reindex task scheduled'), category='success')
             raise HTTPFound(location=url('admin_settings_search'))
         defaults = db.Setting.get_app_settings()
         defaults.update(self._get_hg_ui_settings())

kallithea/lib/celery_app.py

➞

Show inline comments

@@ @@ -15,13 +15,16 @@ mandatory settings. @@
 """
 import logging
 class CeleryConfig(object):
     imports = ['kallithea.model.async_tasks']
     imports = [
         'kallithea.lib.indexers.daemon',
         'kallithea.model.async_tasks',
+    ]
     task_always_eager = False
 list_config_names = {'imports', 'accept_content'}
 desupported = set([

kallithea/lib/indexers/daemon.py

➞

Show inline comments

@@ @@ -29,15 +29,17 @@ Original author and date, and relevant c @@
 import logging
 import os
 import traceback
 from shutil import rmtree
 from time import mktime
 from tg import config
 from whoosh.index import create_in, exists_in, open_dir
 from whoosh.qparser import QueryParser
 from kallithea.lib import celerylib
 from kallithea.lib.conf import INDEX_EXTENSIONS, INDEX_FILENAMES
 from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA
 from kallithea.lib.utils2 import safe_str
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, NodeDoesNotExistError, RepositoryError
 from kallithea.model import db
 from kallithea.model.scm import ScmModel
@@ @@ -441,6 +443,15 @@ class WhooshIndexingDaemon(object): @@
     def run(self, full_index=False):
         """Run daemon"""
         if full_index or self.initial:
             self.build_indexes()
         else:
             self.update_indexes()
 @celerylib.task
 @celerylib.locked_task
 def whoosh_index(repo_location, full_index):
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location) \
                          .run(full_index=full_index)

kallithea/model/async_tasks.py

➞

Show inline comments

@@ @@ -38,33 +38,23 @@ from time import mktime @@
 import celery.utils.log
 from tg import config
 import kallithea
 from kallithea.lib import celerylib, conf, ext_json, hooks
 from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
 from kallithea.lib.utils2 import asbool, ascii_bytes
 from kallithea.lib.vcs.utils import author_email, author_name
 from kallithea.model import db, meta, repo, userlog
-__all__ = ['whoosh_index', 'get_commits_stats', 'send_email']
 __all__ = ['get_commits_stats', 'send_email']
 log = celery.utils.log.get_task_logger(__name__)
 @celerylib.task
 @celerylib.locked_task
 def whoosh_index(repo_location, full_index):
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location) \
                          .run(full_index=full_index)
 def _author_username(author):
     """Return the username of the user identified by the email part of the 'author' string,
     default to the name or email.
     Kind of similar to h.person() ."""
     email = author_email(author)
     if email:

scripts/deps.py

➞

Show inline comments

@@ @@ -156,13 +156,12 @@ shown_modules = normal_modules | top_mod @@
 known_violations = set([
 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'),  # needs base&facade
 ('kallithea.lib.utils', 'kallithea.model'),  # clean up utils
 ('kallithea.lib.utils', 'kallithea.model.db'),
 ('kallithea.lib.utils', 'kallithea.model.scm'),
 ('kallithea.model.async_tasks', 'kallithea.lib.hooks'),
 ('kallithea.model.async_tasks', 'kallithea.lib.indexers'),
 ('kallithea.model.async_tasks', 'kallithea.model'),
 ('kallithea.model', 'kallithea.lib.auth'),  # auth.HasXXX
 ('kallithea.model', 'kallithea.lib.auth_modules'),  # validators
 ('kallithea.model', 'kallithea.lib.hooks'),  # clean up hooks
 ('kallithea.model', 'kallithea.model.scm'),
 ('kallithea.model.scm', 'kallithea.lib.hooks'),

0 comments (0 inline, 0 general)