Changeset - aa5cd5e44621
[Not reviewed]
default
0 3 0
Mads Kiilerich (mads) - 5 years ago 2021-01-01 18:04:16
mads@kiilerich.com
Grafted from: e2421a84e1f4
celery: always set kallithea.CELERY_APP to a Celery app - it is lazy until it actually is used
3 files changed with 4 insertions and 13 deletions:
0 comments (0 inline, 0 general)
kallithea/__init__.py
Show inline comments
 
@@ -9,54 +9,56 @@
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea
 
~~~~~~~~~
 

	
 
Kallithea, a web based repository management system.
 

	
 
Versioning implementation: http://www.python.org/dev/peps/pep-0386/
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 9, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, (C) 2014 Bradley M. Kuhn, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import platform
 
import sys
 

	
 
import celery
 

	
 

	
 
if sys.version_info < (3, 6):
 
    raise Exception('Kallithea requires python 3.6 or later')
 

	
 
VERSION = (0, 6, 99)
 
BACKENDS = {
 
    'hg': 'Mercurial repository',
 
    'git': 'Git repository',
 
}
 

	
 
CELERY_APP = None  # set to Celery app instance if using Celery
 
CELERY_APP = celery.Celery()  # needed at import time but is lazy and can be configured later
 

	
 
CONFIG = {}  # set to tg.config when TG app is initialized and calls app_cfg
 

	
 
# URL prefix for non repository related links - must start with `/`
 
ADMIN_PREFIX = '/_admin'
 
URL_SEP = '/'
 

	
 
# Linked module for extensions
 
EXTENSIONS = {}
 

	
 
__version__ = '.'.join(str(each) for each in VERSION)
 
__platform__ = platform.system()
 
__license__ = 'GPLv3'
 
__py_version__ = sys.version_info
 
__author__ = "Various Authors"
 
__url__ = 'https://kallithea-scm.org/'
 

	
 
is_windows = __platform__ in ['Windows']
 
is_unix = not is_windows
kallithea/config/app_cfg.py
Show inline comments
 
@@ -115,49 +115,49 @@ def setup_configuration(app):
 
        alembic_cfg = alembic.config.Config()
 
        alembic_cfg.set_main_option('script_location', 'kallithea:alembic')
 
        alembic_cfg.set_main_option('sqlalchemy.url', dbconf)
 
        script_dir = ScriptDirectory.from_config(alembic_cfg)
 
        available_heads = sorted(script_dir.get_heads())
 

	
 
        engine = create_engine(dbconf)
 
        with engine.connect() as conn:
 
            context = MigrationContext.configure(conn)
 
            current_heads = sorted(str(s) for s in context.get_current_heads())
 
        if current_heads != available_heads:
 
            log.error('Failed to run Kallithea:\n\n'
 
                      'The database version does not match the Kallithea version.\n'
 
                      'Please read the documentation on how to upgrade or downgrade the database.\n'
 
                      'Current database version id(s): %s\n'
 
                      'Expected database version id(s): %s\n'
 
                      'If you are a developer and you know what you are doing, you can add `ignore_alembic_revision = True` '
 
                      'to your .ini file to skip the check.\n' % (' '.join(current_heads), ' '.join(available_heads)))
 
            sys.exit(1)
 

	
 
    # store some globals into kallithea
 
    kallithea.DEFAULT_USER_ID = db.User.get_default_user().user_id
 

	
 
    if asbool(config.get('use_celery')):
 
        kallithea.CELERY_APP = celery_app.make_app()
 
        kallithea.CELERY_APP.config_from_object(celery_app.make_celery_config(config))
 
    kallithea.CONFIG = config
 

	
 
    load_extensions(root_path=config['here'])
 

	
 
    set_app_settings(config)
 

	
 
    instance_id = kallithea.CONFIG.get('instance_id', '*')
 
    if instance_id == '*':
 
        instance_id = '%s-%s' % (platform.uname()[1], os.getpid())
 
        kallithea.CONFIG['instance_id'] = instance_id
 

	
 
    # update kallithea.CONFIG with the meanwhile changed 'config'
 
    kallithea.CONFIG.update(config)
 

	
 
    # configure vcs and indexer libraries (they are supposed to be independent
 
    # as much as possible and thus avoid importing tg.config or
 
    # kallithea.CONFIG).
 
    set_vcs_config(kallithea.CONFIG)
 
    set_indexer_config(kallithea.CONFIG)
 

	
 
    check_git_version()
 

	
 
    kallithea.model.meta.Session.remove()
 

	
kallithea/lib/celery_app.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the Kallithea ini file but must be converted to an
 
entirely different format before Celery can use it.
 

	
 
We read the configuration from tg.config at module import time. This module can
 
thus not be imported in global scope but must be imported on demand in function
 
scope after tg.config has been initialized.
 

	
 
To make sure that the config really has been initialized, we check one of the
 
mandatory settings.
 
"""
 

	
 
import logging
 

	
 
import celery
 
import tg
 

	
 

	
 
class CeleryConfig(object):
 
    imports = ['kallithea.model.async_tasks']
 
    task_always_eager = False
 

	
 
list_config_names = {'imports', 'accept_content'}
 

	
 

	
 
desupported = set([
 
    'broker.url',
 
    'celery.accept.content',
 
    'celery.always.eager',
 
    'celery.amqp.task.result.expires',
 
    'celeryd.concurrency',
 
    'celeryd.max.tasks.per.child',
 
    'celery.result.backend',  # Note: the .ini template used this instead of 'celery.result_backend' in 0.6
 
    'celery.result.dburi',
 
    'celery.result.serialier',
 
    'celery.result.serializer',
 
    'celery.send.task.error.emails',
 
    'celery.task_always_eager',  # still a valid configuration in celery, but not supported in Kallithea
 
    'celery.task.serializer',
 
])
 

	
 
@@ -51,32 +48,24 @@ def make_celery_config(config):
 
    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
 

	
 
    celery_config = CeleryConfig()
 

	
 
    for config_key, config_value in sorted(config.items()):
 
        if config_key in desupported and config_value:
 
            log.error('Celery configuration setting %r is no longer supported', config_key)
 
        parts = config_key.split('.', 1)
 
        if parts[0] == 'celery' and len(parts) == 2:  # Celery 4 config key
 
            celery_key = parts[1]
 
        else:
 
            continue
 
        if not isinstance(config_value, str):
 
            continue
 
        if celery_key in list_config_names:
 
            celery_value = config_value.split()
 
        elif config_value.isdigit():
 
            celery_value = int(config_value)
 
        elif config_value.lower() in ['true', 'false']:
 
            celery_value = config_value.lower() == 'true'
 
        else:
 
            celery_value = config_value
 
        setattr(celery_config, celery_key, celery_value)
 
    return celery_config
 

	
 

	
 
def make_app():
 
    """Create celery app from the TurboGears configuration file"""
 
    app = celery.Celery()
 
    celery_config = make_celery_config(tg.config)
 
    app.config_from_object(celery_config)
 
    return app
0 comments (0 inline, 0 general)