Changeset - 341e4bb9e227
[Not reviewed]
default
1 10 1
Mads Kiilerich (mads) - 5 years ago 2020-11-07 18:42:58
mads@kiilerich.com
Grafted from: b741ae580d67
celery: move async tasks to model - they both use model and are used by model, so they must be pretty much the same
11 files changed with 77 insertions and 82 deletions:
0 comments (0 inline, 0 general)
kallithea/controllers/admin/settings.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.controllers.admin.settings
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
settings controller for Kallithea admin
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Jul 14, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import logging
 
import traceback
 

	
 
import formencode
 
from formencode import htmlfill
 
from tg import config, request
 
from tg import tmpl_context as c
 
from tg.i18n import ugettext as _
 
from webob.exc import HTTPFound
 

	
 
import kallithea
 
from kallithea.lib import webutils
 
from kallithea.lib.auth import HasPermissionAnyDecorator, LoginRequired
 
from kallithea.lib.base import BaseController, render
 
from kallithea.lib.celerylib import tasks
 
from kallithea.lib.utils import repo2db_mapper, set_app_settings
 
from kallithea.lib.utils2 import safe_str
 
from kallithea.lib.vcs import VCSError
 
from kallithea.lib.webutils import url
 
from kallithea.model import db, meta
 
from kallithea.model import async_tasks, db, meta
 
from kallithea.model.forms import ApplicationSettingsForm, ApplicationUiSettingsForm, ApplicationVisualisationForm
 
from kallithea.model.notification import EmailNotificationModel
 
from kallithea.model.scm import ScmModel
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class SettingsController(BaseController):
 

	
 
    @LoginRequired(allow_default_user=True)
 
    def _before(self, *args, **kwargs):
 
        super(SettingsController, self)._before(*args, **kwargs)
 

	
 
    def _get_hg_ui_settings(self):
 
        ret = db.Ui.query().all()
 

	
 
        settings = {}
 
        for each in ret:
 
            k = each.ui_section + '_' + each.ui_key
 
            v = each.ui_value
 
            if k == 'paths_/':
 
                k = 'paths_root_path'
 

	
 
            k = k.replace('.', '_')
 

	
 
            if each.ui_section in ['hooks', 'extensions']:
 
                v = each.ui_active
 

	
 
            settings[k] = v
 
        return settings
 

	
 
    @HasPermissionAnyDecorator('hg.admin')
 
    def settings_vcs(self):
 
        c.active = 'vcs'
 
        if request.POST:
 
            application_form = ApplicationUiSettingsForm()()
 
            try:
 
                form_result = application_form.to_python(dict(request.POST))
 
            except formencode.Invalid as errors:
 
                return htmlfill.render(
 
                     render('admin/settings/settings.html'),
 
                     defaults=errors.value,
 
                     errors=errors.error_dict or {},
 
                     prefix_error=False,
 
                     encoding="UTF-8",
 
                     force_defaults=False)
 

	
 
@@ -256,156 +255,156 @@ class SettingsController(BaseController)
 
                    ('clone_ssh_tmpl', 'clone_ssh_tmpl', 'unicode'),
 
                ]
 
                for setting, form_key, type_ in settings:
 
                    db.Setting.create_or_update(setting, form_result[form_key], type_)
 

	
 
                meta.Session().commit()
 
                set_app_settings(config)
 
                webutils.flash(_('Updated visualisation settings'),
 
                        category='success')
 

	
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                webutils.flash(_('Error occurred during updating '
 
                          'visualisation settings'),
 
                        category='error')
 

	
 
            raise HTTPFound(location=url('admin_settings_visual'))
 

	
 
        defaults = db.Setting.get_app_settings()
 
        defaults.update(self._get_hg_ui_settings())
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
 

	
 
    @HasPermissionAnyDecorator('hg.admin')
 
    def settings_email(self):
 
        c.active = 'email'
 
        if request.POST:
 
            test_email = request.POST.get('test_email')
 
            test_email_subj = 'Kallithea test email'
 
            test_body = ('Kallithea Email test, '
 
                               'Kallithea version: %s' % c.kallithea_version)
 
            if not test_email:
 
                webutils.flash(_('Please enter email address'), category='error')
 
                raise HTTPFound(location=url('admin_settings_email'))
 

	
 
            test_email_txt_body = EmailNotificationModel() \
 
                .get_email_tmpl(EmailNotificationModel.TYPE_DEFAULT,
 
                                'txt', body=test_body)
 
            test_email_html_body = EmailNotificationModel() \
 
                .get_email_tmpl(EmailNotificationModel.TYPE_DEFAULT,
 
                                'html', body=test_body)
 

	
 
            recipients = [test_email] if test_email else None
 

	
 
            tasks.send_email(recipients, test_email_subj,
 
            async_tasks.send_email(recipients, test_email_subj,
 
                             test_email_txt_body, test_email_html_body)
 

	
 
            webutils.flash(_('Send email task created'), category='success')
 
            raise HTTPFound(location=url('admin_settings_email'))
 

	
 
        defaults = db.Setting.get_app_settings()
 
        defaults.update(self._get_hg_ui_settings())
 

	
 
        c.ini = kallithea.CONFIG
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
 

	
 
    @HasPermissionAnyDecorator('hg.admin')
 
    def settings_hooks(self):
 
        c.active = 'hooks'
 
        if request.POST:
 
            if c.visual.allow_custom_hooks_settings:
 
                ui_key = request.POST.get('new_hook_ui_key')
 
                ui_value = request.POST.get('new_hook_ui_value')
 

	
 
                hook_id = request.POST.get('hook_id')
 

	
 
                try:
 
                    ui_key = ui_key and ui_key.strip()
 
                    if ui_key in (x.ui_key for x in db.Ui.get_custom_hooks()):
 
                        webutils.flash(_('Hook already exists'), category='error')
 
                    elif ui_key in (x.ui_key for x in db.Ui.get_builtin_hooks()):
 
                        webutils.flash(_('Builtin hooks are read-only. Please use another hook name.'), category='error')
 
                    elif ui_value and ui_key:
 
                        db.Ui.create_or_update_hook(ui_key, ui_value)
 
                        webutils.flash(_('Added new hook'), category='success')
 
                    elif hook_id:
 
                        db.Ui.delete(hook_id)
 
                        meta.Session().commit()
 

	
 
                    # check for edits
 
                    update = False
 
                    _d = request.POST.dict_of_lists()
 
                    for k, v, ov in zip(_d.get('hook_ui_key', []),
 
                                        _d.get('hook_ui_value_new', []),
 
                                        _d.get('hook_ui_value', [])):
 
                        if v != ov:
 
                            db.Ui.create_or_update_hook(k, v)
 
                            update = True
 

	
 
                    if update:
 
                        webutils.flash(_('Updated hooks'), category='success')
 
                    meta.Session().commit()
 
                except Exception:
 
                    log.error(traceback.format_exc())
 
                    webutils.flash(_('Error occurred during hook creation'),
 
                            category='error')
 

	
 
                raise HTTPFound(location=url('admin_settings_hooks'))
 

	
 
        defaults = db.Setting.get_app_settings()
 
        defaults.update(self._get_hg_ui_settings())
 

	
 
        c.hooks = db.Ui.get_builtin_hooks()
 
        c.custom_hooks = db.Ui.get_custom_hooks()
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
 

	
 
    @HasPermissionAnyDecorator('hg.admin')
 
    def settings_search(self):
 
        c.active = 'search'
 
        if request.POST:
 
            repo_location = self._get_hg_ui_settings()['paths_root_path']
 
            full_index = request.POST.get('full_index', False)
 
            tasks.whoosh_index(repo_location, full_index)
 
            async_tasks.whoosh_index(repo_location, full_index)
 
            webutils.flash(_('Whoosh reindex task scheduled'), category='success')
 
            raise HTTPFound(location=url('admin_settings_search'))
 

	
 
        defaults = db.Setting.get_app_settings()
 
        defaults.update(self._get_hg_ui_settings())
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
 

	
 
    @HasPermissionAnyDecorator('hg.admin')
 
    def settings_system(self):
 
        c.active = 'system'
 

	
 
        defaults = db.Setting.get_app_settings()
 
        defaults.update(self._get_hg_ui_settings())
 

	
 
        c.ini = kallithea.CONFIG
 
        server_info = db.Setting.get_server_info()
 
        for key, val in server_info.items():
 
            setattr(c, key, val)
 

	
 
        return htmlfill.render(
 
            render('admin/settings/settings.html'),
 
            defaults=defaults,
 
            encoding="UTF-8",
 
            force_defaults=False)
kallithea/controllers/summary.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.controllers.summary
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
Summary controller for Kallithea
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Apr 18, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import calendar
 
import itertools
 
import logging
 
import traceback
 
from datetime import date, timedelta
 
from time import mktime
 

	
 
from beaker.cache import cache_region
 
from tg import request
 
from tg import tmpl_context as c
 
from tg.i18n import ugettext as _
 
from webob.exc import HTTPBadRequest
 

	
 
from kallithea.lib import ext_json, webutils
 
from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 
from kallithea.lib.base import BaseRepoController, jsonify, render
 
from kallithea.lib.celerylib.tasks import get_commits_stats
 
from kallithea.lib.conf import ALL_EXTS, ALL_READMES, LANGUAGES_EXTENSIONS_MAP
 
from kallithea.lib.markup_renderer import MarkupRenderer
 
from kallithea.lib.page import Page
 
from kallithea.lib.utils2 import safe_int, safe_str
 
from kallithea.lib.vcs.backends.base import EmptyChangeset
 
from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, NodeDoesNotExistError
 
from kallithea.lib.vcs.nodes import FileNode
 
from kallithea.model import db
 
from kallithea.model import async_tasks, db
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 
README_FILES = [''.join([x[0][0], x[1][0]]) for x in
 
                    sorted(list(itertools.product(ALL_READMES, ALL_EXTS)),
 
                           key=lambda y:y[0][1] + y[1][1])]
 

	
 

	
 
class SummaryController(BaseRepoController):
 

	
 
    def __get_readme_data(self, db_repo):
 
        repo_name = db_repo.repo_name
 
        log.debug('Looking for README file')
 

	
 
        @cache_region('long_term_file', '_get_readme_from_cache')
 
        def _get_readme_from_cache(*_cache_keys):  # parameters are not really used - only as caching key
 
            readme_data = None
 
            readme_file = None
 
            try:
 
                # gets the landing revision! or tip if fails
 
                cs = db_repo.get_landing_changeset()
 
                if isinstance(cs, EmptyChangeset):
 
                    raise EmptyRepositoryError()
 
                renderer = MarkupRenderer()
 
                for f in README_FILES:
 
                    try:
 
                        readme = cs.get_node(f)
 
                        if not isinstance(readme, FileNode):
 
                            continue
 
                        readme_file = f
 
                        log.debug('Found README file `%s` rendering...',
 
                                  readme_file)
 
                        readme_data = renderer.render(safe_str(readme.content),
 
                                                      filename=f)
 
                        break
 
                    except NodeDoesNotExistError:
 
                        continue
 
            except ChangesetError:
 
                log.error(traceback.format_exc())
 
                pass
 
            except EmptyRepositoryError:
 
                pass
 

	
 
            return readme_data, readme_file
 

	
 
        kind = 'README'
 
        return _get_readme_from_cache(repo_name, kind, c.db_repo.changeset_cache.get('raw_id'))
 
@@ -164,50 +163,50 @@ class SummaryController(BaseRepoControll
 
    @HasRepoPermissionLevelDecorator('read')
 
    def statistics(self, repo_name):
 
        if c.db_repo.enable_statistics:
 
            c.show_stats = True
 
            c.no_data_msg = _('No data ready yet')
 
        else:
 
            c.show_stats = False
 
            c.no_data_msg = _('Statistics are disabled for this repository')
 

	
 
        td = date.today() + timedelta(days=1)
 
        td_1m = td - timedelta(days=calendar.monthrange(td.year, td.month)[1])
 
        td_1y = td - timedelta(days=365)
 

	
 
        ts_min_m = mktime(td_1m.timetuple())
 
        ts_min_y = mktime(td_1y.timetuple())
 
        ts_max_y = mktime(td.timetuple())
 
        c.ts_min = ts_min_m
 
        c.ts_max = ts_max_y
 

	
 
        stats = db.Statistics.query() \
 
            .filter(db.Statistics.repository == c.db_repo) \
 
            .scalar()
 
        c.stats_percentage = 0
 
        if stats and stats.languages:
 
            c.commit_data = ext_json.loads(stats.commit_activity)
 
            c.overview_data = ext_json.loads(stats.commit_activity_combined)
 

	
 
            lang_stats_d = ext_json.loads(stats.languages)
 
            lang_stats = [(x, {"count": y,
 
                               "desc": LANGUAGES_EXTENSIONS_MAP.get(x, '?')})
 
                          for x, y in lang_stats_d.items()]
 
            lang_stats.sort(key=lambda k: (-k[1]['count'], k[0]))
 
            c.trending_languages = lang_stats[:10]
 

	
 
            last_rev = stats.stat_on_revision + 1
 
            c.repo_last_rev = c.db_repo_scm_instance.count() \
 
                if c.db_repo_scm_instance.revisions else 0
 
            if last_rev == 0 or c.repo_last_rev == 0:
 
                pass
 
            else:
 
                c.stats_percentage = '%.2f' % ((float((last_rev)) /
 
                                                c.repo_last_rev) * 100)
 
        else:
 
            c.commit_data = {}
 
            c.overview_data = ([[ts_min_y, 0], [ts_max_y, 10]])
 
            c.trending_languages = []
 

	
 
        recurse_limit = 500  # don't recurse more than 500 times when parsing
 
        get_commits_stats(c.db_repo.repo_name, ts_min_y, ts_max_y, recurse_limit)
 
        async_tasks.get_commits_stats(c.db_repo.repo_name, ts_min_y, ts_max_y, recurse_limit)
 
        return render('summary/statistics.html')
kallithea/lib/celerypylons/__init__.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
"""
 
Kallithea wrapper of Celery
 

	
 
The Celery configuration is in the Kallithea ini file but must be converted to an
 
entirely different format before Celery can use it.
 

	
 
We read the configuration from tg.config at module import time. This module can
 
thus not be imported in global scope but must be imported on demand in function
 
scope after tg.config has been initialized.
 

	
 
To make sure that the config really has been initialized, we check one of the
 
mandatory settings.
 
"""
 

	
 
import logging
 

	
 
import celery
 
import tg
 

	
 
import kallithea
 

	
 

	
 
class CeleryConfig(object):
 
    imports = ['kallithea.lib.celerylib.tasks']
 
    imports = ['kallithea.model.async_tasks']
 
    task_always_eager = False
 

	
 
# map from Kallithea .ini Celery 3 config names to Celery 4 config names
 
celery3_compat = {
 
    'broker.url': 'broker_url',
 
    'celery.accept.content': 'accept_content',
 
    'celery.always.eager': 'task_always_eager',
 
    'celery.amqp.task.result.expires': 'result_expires',
 
    'celeryd.concurrency': 'worker_concurrency',
 
    'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child',
 
    #'celery.imports' ends up unchanged
 
    'celery.result.backend': 'result_backend',
 
    'celery.result.serializer': 'result_serializer',
 
    'celery.task.serializer': 'task_serializer',
 
}
 

	
 
list_config_names = """imports accept_content""".split()
 

	
 

	
 
desupported = set([
 
    'celery.result.dburi',
 
    'celery.result.serialier',
 
    'celery.send.task.error.emails',
 
])
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def make_celery_config(config):
 
    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
 

	
 
    celery_config = CeleryConfig()
 

	
 
    for config_key, config_value in sorted(config.items()):
 
        if config_key in desupported and config_value:
 
            log.error('Celery configuration setting %r is no longer supported', config_key)
 
        celery_key = celery3_compat.get(config_key)
 
        parts = config_key.split('.', 1)
 
        if celery_key:  # explicit Celery 3 backwards compatibility
 
            pass
 
        elif parts[0] == 'celery' and len(parts) == 2:  # Celery 4 config key
 
            celery_key = parts[1]
 
        else:
 
            continue
 
        if not isinstance(config_value, str):
 
            continue
 
        if celery_key in list_config_names:
kallithea/model/async_tasks.py
Show inline comments
 
file renamed from kallithea/lib/celerylib/tasks.py to kallithea/model/async_tasks.py
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.lib.celerylib.tasks
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
kallithea.model.async_tasks
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
Kallithea task modules, containing all task that suppose to be run
 
by celery daemon
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Oct 6, 2010
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import email.message
 
import email.utils
 
import os
 
import smtplib
 
import time
 
import traceback
 
from collections import OrderedDict
 
from operator import itemgetter
 
from time import mktime
 

	
 
import celery.utils.log
 
from tg import config
 

	
 
import kallithea
 
import kallithea.lib.helpers as h
 
from kallithea.lib import celerylib, conf, ext_json, hooks
 
from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
 
from kallithea.lib.utils2 import asbool, ascii_bytes
 
from kallithea.lib.vcs.utils import author_email
 
from kallithea.model import db, userlog
 
from kallithea.model.repo import RepoModel
 
from kallithea.model import db, repo, userlog
 

	
 

	
 
__all__ = ['whoosh_index', 'get_commits_stats', 'send_email']
 

	
 

	
 
log = celery.utils.log.get_task_logger(__name__)
 

	
 

	
 
@celerylib.task
 
@celerylib.locked_task
 
@celerylib.dbsession
 
def whoosh_index(repo_location, full_index):
 
    celerylib.get_session() # initialize database connection
 

	
 
    index_location = config['index_dir']
 
    WhooshIndexingDaemon(index_location=index_location,
 
                         repo_location=repo_location) \
 
                         .run(full_index=full_index)
 

	
 

	
 
# for js data compatibility cleans the key for person from '
 
def akc(k):
 
    return h.person(k).replace('"', '')
 

	
 

	
 
@celerylib.task
 
@celerylib.dbsession
 
def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
 
    DBS = celerylib.get_session()
 
    lockkey = celerylib.__get_lockkey('get_commits_stats', repo_name, ts_min_y,
 
                            ts_max_y)
 
    lockkey_path = config.get('cache_dir') or config['app_conf']['cache_dir']  # Backward compatibility for TurboGears < 2.4
 

	
 
    log.info('running task with lockkey %s', lockkey)
 

	
 
    try:
 
        lock = celerylib.DaemonLock(os.path.join(lockkey_path, lockkey))
 

	
 
        co_day_auth_aggr = {}
 
        commits_by_day_aggregate = {}
 
        repo = db.Repository.get_by_repo_name(repo_name)
 
        if repo is None:
 
        db_repo = db.Repository.get_by_repo_name(repo_name)
 
        if db_repo is None:
 
            return True
 

	
 
        repo = repo.scm_instance
 
        repo_size = repo.count()
 
        scm_repo = db_repo.scm_instance
 
        repo_size = scm_repo.count()
 
        # return if repo have no revisions
 
        if repo_size < 1:
 
            lock.release()
 
            return True
 

	
 
        skip_date_limit = True
 
        parse_limit = int(config.get('commit_parse_limit'))
 
        last_rev = None
 
        last_cs = None
 
        timegetter = itemgetter('time')
 

	
 
        dbrepo = DBS.query(db.Repository) \
 
            .filter(db.Repository.repo_name == repo_name).scalar()
 
        cur_stats = DBS.query(db.Statistics) \
 
            .filter(db.Statistics.repository == dbrepo).scalar()
 

	
 
        if cur_stats is not None:
 
            last_rev = cur_stats.stat_on_revision
 

	
 
        if last_rev == repo.get_changeset().revision and repo_size > 1:
 
        if last_rev == scm_repo.get_changeset().revision and repo_size > 1:
 
            # pass silently without any work if we're not on first revision or
 
            # current state of parsing revision(from db marker) is the
 
            # last revision
 
            lock.release()
 
            return True
 

	
 
        if cur_stats:
 
            commits_by_day_aggregate = OrderedDict(ext_json.loads(
 
                                        cur_stats.commit_activity_combined))
 
            co_day_auth_aggr = ext_json.loads(cur_stats.commit_activity)
 

	
 
        log.debug('starting parsing %s', parse_limit)
 

	
 
        last_rev = last_rev + 1 if last_rev and last_rev >= 0 else 0
 
        log.debug('Getting revisions from %s to %s',
 
             last_rev, last_rev + parse_limit
 
        )
 
        for cs in repo[last_rev:last_rev + parse_limit]:
 
        for cs in scm_repo[last_rev:last_rev + parse_limit]:
 
            log.debug('parsing %s', cs)
 
            last_cs = cs  # remember last parsed changeset
 
            tt = cs.date.timetuple()
 
            k = mktime(tt[:3] + (0, 0, 0, 0, 0, 0))
 

	
 
            if akc(cs.author) in co_day_auth_aggr:
 
                try:
 
                    l = [timegetter(x) for x in
 
                         co_day_auth_aggr[akc(cs.author)]['data']]
 
                    time_pos = l.index(k)
 
                except ValueError:
 
                    time_pos = None
 

	
 
                if time_pos is not None and time_pos >= 0:
 
                    datadict = \
 
                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
 

	
 
                    datadict["commits"] += 1
 
                    datadict["added"] += len(cs.added)
 
                    datadict["changed"] += len(cs.changed)
 
                    datadict["removed"] += len(cs.removed)
 

	
 
                else:
 
                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 

	
 
                        datadict = {"time": k,
 
                                    "commits": 1,
 
                                    "added": len(cs.added),
 
                                    "changed": len(cs.changed),
 
                                    "removed": len(cs.removed),
 
                                   }
 
                        co_day_auth_aggr[akc(cs.author)]['data'] \
 
                            .append(datadict)
 

	
 
            else:
 
                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                    co_day_auth_aggr[akc(cs.author)] = {
 
                                        "label": akc(cs.author),
 
                                        "data": [{"time": k,
 
                                                 "commits": 1,
 
                                                 "added": len(cs.added),
 
                                                 "changed": len(cs.changed),
 
                                                 "removed": len(cs.removed),
 
                                                 }],
 
                                        "schema": ["commits"],
 
                                        }
 

	
 
            # gather all data by day
 
            if k in commits_by_day_aggregate:
 
                commits_by_day_aggregate[k] += 1
 
            else:
 
                commits_by_day_aggregate[k] = 1
 

	
 
        overview_data = sorted(commits_by_day_aggregate.items(),
 
                               key=itemgetter(0))
 

	
 
        if not co_day_auth_aggr:
 
            co_day_auth_aggr[akc(repo.contact)] = {
 
                "label": akc(repo.contact),
 
            co_day_auth_aggr[akc(scm_repo.contact)] = {
 
                "label": akc(scm_repo.contact),
 
                "data": [0, 1],
 
                "schema": ["commits"],
 
            }
 

	
 
        stats = cur_stats if cur_stats else db.Statistics()
 
        stats.commit_activity = ascii_bytes(ext_json.dumps(co_day_auth_aggr))
 
        stats.commit_activity_combined = ascii_bytes(ext_json.dumps(overview_data))
 

	
 
        log.debug('last revision %s', last_rev)
 
        leftovers = len(repo.revisions[last_rev:])
 
        leftovers = len(scm_repo.revisions[last_rev:])
 
        log.debug('revisions to parse %s', leftovers)
 

	
 
        if last_rev == 0 or leftovers < parse_limit:
 
            log.debug('getting code trending stats')
 
            stats.languages = ascii_bytes(ext_json.dumps(__get_codes_stats(repo_name)))
 

	
 
        try:
 
            stats.repository = dbrepo
 
            stats.stat_on_revision = last_cs.revision if last_cs else 0
 
            DBS.add(stats)
 
            DBS.commit()
 
        except:
 
            log.error(traceback.format_exc())
 
            DBS.rollback()
 
            lock.release()
 
            return False
 

	
 
        # final release
 
        lock.release()
 

	
 
        # execute another task if celery is enabled
 
        if len(repo.revisions) > 1 and kallithea.CELERY_APP and recurse_limit > 0:
 
        if len(scm_repo.revisions) > 1 and kallithea.CELERY_APP and recurse_limit > 0:
 
            get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit - 1)
 
        elif recurse_limit <= 0:
 
            log.debug('Not recursing - limit has been reached')
 
        else:
 
            log.debug('Not recursing')
 
    except celerylib.LockHeld:
 
        log.info('Task with key %s already running', lockkey)
 
        return 'Task with key %s already running' % lockkey
 

	
 

	
 
@celerylib.task
 
@celerylib.dbsession
 
def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
    """
 
    Sends an email with defined parameters from the .ini files.
 

	
 
    :param recipients: list of recipients, if this is None, the defined email
 
        address from field 'email_to' and all admins is used instead
 
    :param subject: subject of the mail
 
    :param body: plain text body of the mail
 
    :param html_body: html version of body
 
    :param headers: dictionary of prepopulated e-mail headers
 
    :param from_name: full name to be used as sender of this mail - often a
 
    .full_name_or_username value
 
    """
 
    assert isinstance(recipients, list), recipients
 
    if headers is None:
 
        headers = {}
 
    else:
 
        # do not modify the original headers object passed by the caller
 
        headers = headers.copy()
 

	
 
    email_config = config
 
    email_prefix = email_config.get('email_prefix', '')
 
    if email_prefix:
 
        subject = "%s %s" % (email_prefix, subject)
 

	
 
    if not recipients:
 
        # if recipients are not defined we send to email_config + all admins
 
        recipients = [u.email for u in db.User.query()
 
                      .filter(db.User.admin == True).all()]
 
        if email_config.get('email_to') is not None:
 
            recipients += email_config.get('email_to').split(',')
 

	
 
        # If there are still no recipients, there are no admins and no address
 
        # configured in email_to, so return.
 
        if not recipients:
 
            log.error("No recipients specified and no fallback available.")
 
@@ -331,191 +330,191 @@ def send_email(recipients, subject, body
 
        if smtp_use_tls:
 
            smtp_serv.starttls()
 

	
 
        if smtp_auth:
 
            smtp_serv.ehlo()  # populate esmtp_features
 
            smtp_serv.esmtp_features["auth"] = smtp_auth
 

	
 
        if smtp_username and smtp_password is not None:
 
            smtp_serv.login(smtp_username, smtp_password)
 

	
 
        smtp_serv.sendmail(app_email_from, recipients, msg.as_string())
 
        smtp_serv.quit()
 

	
 
        log.info('Mail was sent to: %s' % recipients)
 
    except:
 
        log.error('Mail sending failed')
 
        log.error(traceback.format_exc())
 
        return False
 
    return True
 

	
 

	
 
@celerylib.task
 
@celerylib.dbsession
 
def create_repo(form_data, cur_user):
 
    DBS = celerylib.get_session()
 

	
 
    cur_user = db.User.guess_instance(cur_user)
 

	
 
    owner = cur_user
 
    repo_name = form_data['repo_name']
 
    repo_name_full = form_data['repo_name_full']
 
    repo_type = form_data['repo_type']
 
    description = form_data['repo_description']
 
    private = form_data['repo_private']
 
    clone_uri = form_data.get('clone_uri')
 
    repo_group = form_data['repo_group']
 
    landing_rev = form_data['repo_landing_rev']
 
    copy_fork_permissions = form_data.get('copy_permissions')
 
    copy_group_permissions = form_data.get('repo_copy_permissions')
 
    fork_of = form_data.get('fork_parent_id')
 
    state = form_data.get('repo_state', db.Repository.STATE_PENDING)
 

	
 
    # repo creation defaults, private and repo_type are filled in form
 
    defs = db.Setting.get_default_repo_settings(strip_prefix=True)
 
    enable_statistics = defs.get('repo_enable_statistics')
 
    enable_downloads = defs.get('repo_enable_downloads')
 

	
 
    try:
 
        repo = RepoModel()._create_repo(
 
        db_repo = repo.RepoModel()._create_repo(
 
            repo_name=repo_name_full,
 
            repo_type=repo_type,
 
            description=description,
 
            owner=owner,
 
            private=private,
 
            clone_uri=clone_uri,
 
            repo_group=repo_group,
 
            landing_rev=landing_rev,
 
            fork_of=fork_of,
 
            copy_fork_permissions=copy_fork_permissions,
 
            copy_group_permissions=copy_group_permissions,
 
            enable_statistics=enable_statistics,
 
            enable_downloads=enable_downloads,
 
            state=state
 
        )
 

	
 
        userlog.action_logger(cur_user, 'user_created_repo',
 
                      form_data['repo_name_full'], '')
 

	
 
        DBS.commit()
 
        # now create this repo on Filesystem
 
        RepoModel()._create_filesystem_repo(
 
        repo.RepoModel()._create_filesystem_repo(
 
            repo_name=repo_name,
 
            repo_type=repo_type,
 
            repo_group=db.RepoGroup.guess_instance(repo_group),
 
            clone_uri=clone_uri,
 
        )
 
        repo = db.Repository.get_by_repo_name(repo_name_full)
 
        hooks.log_create_repository(repo.get_dict(), created_by=owner.username)
 
        db_repo = db.Repository.get_by_repo_name(repo_name_full)
 
        hooks.log_create_repository(db_repo.get_dict(), created_by=owner.username)
 

	
 
        # update repo changeset caches initially
 
        repo.update_changeset_cache()
 
        db_repo.update_changeset_cache()
 

	
 
        # set new created state
 
        repo.set_state(db.Repository.STATE_CREATED)
 
        db_repo.set_state(db.Repository.STATE_CREATED)
 
        DBS.commit()
 
    except Exception as e:
 
        log.warning('Exception %s occurred when forking repository, '
 
                    'doing cleanup...' % e)
 
        # rollback things manually !
 
        repo = db.Repository.get_by_repo_name(repo_name_full)
 
        if repo:
 
            db.Repository.delete(repo.repo_id)
 
        db_repo = db.Repository.get_by_repo_name(repo_name_full)
 
        if db_repo:
 
            db.Repository.delete(db_repo.repo_id)
 
            DBS.commit()
 
            RepoModel()._delete_filesystem_repo(repo)
 
            repo.RepoModel()._delete_filesystem_repo(db_repo)
 
        raise
 

	
 
    return True
 

	
 

	
 
@celerylib.task
 
@celerylib.dbsession
 
def create_repo_fork(form_data, cur_user):
 
    """
 
    Creates a fork of repository using interval VCS methods
 

	
 
    :param form_data:
 
    :param cur_user:
 
    """
 
    DBS = celerylib.get_session()
 

	
 
    base_path = kallithea.CONFIG['base_path']
 
    cur_user = db.User.guess_instance(cur_user)
 

	
 
    repo_name = form_data['repo_name']  # fork in this case
 
    repo_name_full = form_data['repo_name_full']
 

	
 
    repo_type = form_data['repo_type']
 
    owner = cur_user
 
    private = form_data['private']
 
    clone_uri = form_data.get('clone_uri')
 
    repo_group = form_data['repo_group']
 
    landing_rev = form_data['landing_rev']
 
    copy_fork_permissions = form_data.get('copy_permissions')
 

	
 
    try:
 
        fork_of = db.Repository.guess_instance(form_data.get('fork_parent_id'))
 

	
 
        RepoModel()._create_repo(
 
        repo.RepoModel()._create_repo(
 
            repo_name=repo_name_full,
 
            repo_type=repo_type,
 
            description=form_data['description'],
 
            owner=owner,
 
            private=private,
 
            clone_uri=clone_uri,
 
            repo_group=repo_group,
 
            landing_rev=landing_rev,
 
            fork_of=fork_of,
 
            copy_fork_permissions=copy_fork_permissions
 
        )
 
        userlog.action_logger(cur_user, 'user_forked_repo:%s' % repo_name_full,
 
                      fork_of.repo_name, '')
 
        DBS.commit()
 

	
 
        source_repo_path = os.path.join(base_path, fork_of.repo_name)
 

	
 
        # now create this repo on Filesystem
 
        RepoModel()._create_filesystem_repo(
 
        repo.RepoModel()._create_filesystem_repo(
 
            repo_name=repo_name,
 
            repo_type=repo_type,
 
            repo_group=db.RepoGroup.guess_instance(repo_group),
 
            clone_uri=source_repo_path,
 
        )
 
        repo = db.Repository.get_by_repo_name(repo_name_full)
 
        hooks.log_create_repository(repo.get_dict(), created_by=owner.username)
 
        db_repo = db.Repository.get_by_repo_name(repo_name_full)
 
        hooks.log_create_repository(db_repo.get_dict(), created_by=owner.username)
 

	
 
        # update repo changeset caches initially
 
        repo.update_changeset_cache()
 
        db_repo.update_changeset_cache()
 

	
 
        # set new created state
 
        repo.set_state(db.Repository.STATE_CREATED)
 
        db_repo.set_state(db.Repository.STATE_CREATED)
 
        DBS.commit()
 
    except Exception as e:
 
        log.warning('Exception %s occurred when forking repository, '
 
                    'doing cleanup...' % e)
 
        # rollback things manually !
 
        repo = db.Repository.get_by_repo_name(repo_name_full)
 
        if repo:
 
            db.Repository.delete(repo.repo_id)
 
        db_repo = db.Repository.get_by_repo_name(repo_name_full)
 
        if db_repo:
 
            db.Repository.delete(db_repo.repo_id)
 
            DBS.commit()
 
            RepoModel()._delete_filesystem_repo(repo)
 
            repo.RepoModel()._delete_filesystem_repo(db_repo)
 
        raise
 

	
 
    return True
 

	
 

	
 
def __get_codes_stats(repo_name):
 
    repo = db.Repository.get_by_repo_name(repo_name).scm_instance
 
    scm_repo = db.Repository.get_by_repo_name(repo_name).scm_instance
 

	
 
    tip = repo.get_changeset()
 
    tip = scm_repo.get_changeset()
 
    code_stats = {}
 

	
 
    for _topnode, _dirnodes, filenodes in tip.walk('/'):
 
        for filenode in filenodes:
 
            ext = filenode.extension.lower()
 
            if ext in conf.LANGUAGES_EXTENSIONS_MAP and not filenode.is_binary:
 
                if ext in code_stats:
 
                    code_stats[ext] += 1
 
                else:
 
                    code_stats[ext] = 1
 

	
 
    return code_stats or {}
kallithea/model/notification.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
"""
 
kallithea.model.notification
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
Model for notifications
 

	
 

	
 
This file was forked by the Kallithea project in July 2014.
 
Original author and date, and relevant copyright and licensing information is below:
 
:created_on: Nov 20, 2011
 
:author: marcink
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
import datetime
 
import logging
 

	
 
from tg import app_globals
 
from tg import tmpl_context as c
 
from tg.i18n import ugettext as _
 

	
 
from kallithea.lib.utils2 import fmt_date
 
from kallithea.model import db
 
from kallithea.model import async_tasks, db
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class NotificationModel(object):
 

	
 
    TYPE_CHANGESET_COMMENT = 'cs_comment'
 
    TYPE_MESSAGE = 'message'
 
    TYPE_MENTION = 'mention' # not used
 
    TYPE_REGISTRATION = 'registration'
 
    TYPE_PULL_REQUEST = 'pull_request'
 
    TYPE_PULL_REQUEST_COMMENT = 'pull_request_comment'
 

	
 
    def create(self, created_by, body, recipients=None,
 
               type_=TYPE_MESSAGE, with_email=True,
 
               email_kwargs=None, repo_name=None):
 
        """
 

	
 
        Creates notification of given type
 

	
 
        :param created_by: int, str or User instance. User who created this
 
            notification
 
        :param body:
 
        :param recipients: list of int, str or User objects, when None
 
            is given send to all admins
 
        :param type_: type of notification
 
        :param with_email: send email with this notification
 
        :param email_kwargs: additional dict to pass as args to email template
 
        """
 
        import kallithea.lib.helpers as h
 
        from kallithea.lib.celerylib import tasks
 
        email_kwargs = email_kwargs or {}
 
        if recipients and not getattr(recipients, '__iter__', False):
 
            raise Exception('recipients must be a list or iterable')
 

	
 
        created_by_obj = db.User.guess_instance(created_by)
 

	
 
        recipients_objs = set()
 
        if recipients:
 
            for u in recipients:
 
                obj = db.User.guess_instance(u)
 
                if obj is not None:
 
                    recipients_objs.add(obj)
 
                else:
 
                    # TODO: inform user that requested operation couldn't be completed
 
                    log.error('cannot email unknown user %r', u)
 
            log.debug('sending notifications %s to %s',
 
                type_, recipients_objs
 
            )
 
        elif recipients is None:
 
            # empty recipients means to all admins
 
            recipients_objs = db.User.query().filter(db.User.admin == True).all()
 
            log.debug('sending notifications %s to admins: %s',
 
                type_, recipients_objs
 
            )
 
        #else: silently skip notification mails?
 

	
 
        if not with_email:
 
            return
 

	
 
        headers = {}
 
        headers['X-Kallithea-Notification-Type'] = type_
 
        if 'threading' in email_kwargs:
 
            headers['References'] = ' '.join('<%s>' % x for x in email_kwargs['threading'])
 

	
 
        # this is passed into template
 
        created_on = fmt_date(datetime.datetime.now())
 
        html_kwargs = {
 
                  'body': h.render_w_mentions(body, repo_name),
 
                  'when': created_on,
 
                  'user': created_by_obj.username,
 
                  }
 

	
 
        txt_kwargs = {
 
                  'body': body,
 
                  'when': created_on,
 
                  'user': created_by_obj.username,
 
                  }
 

	
 
        html_kwargs.update(email_kwargs)
 
        txt_kwargs.update(email_kwargs)
 
        email_subject = EmailNotificationModel() \
 
                            .get_email_description(type_, **txt_kwargs)
 
        email_txt_body = EmailNotificationModel() \
 
                            .get_email_tmpl(type_, 'txt', **txt_kwargs)
 
        email_html_body = EmailNotificationModel() \
 
                            .get_email_tmpl(type_, 'html', **html_kwargs)
 

	
 
        # don't send email to the person who caused the notification, except for
 
        # notifications about new pull requests where the author is explicitly
 
        # added.
 
        rec_mails = set(obj.email for obj in recipients_objs)
 
        if type_ == NotificationModel.TYPE_PULL_REQUEST:
 
            rec_mails.add(created_by_obj.email)
 
        else:
 
            rec_mails.discard(created_by_obj.email)
 

	
 
        # send email with notification to participants
 
        for rec_mail in sorted(rec_mails):
 
            tasks.send_email([rec_mail], email_subject, email_txt_body,
 
            async_tasks.send_email([rec_mail], email_subject, email_txt_body,
 
                     email_html_body, headers,
 
                     from_name=created_by_obj.full_name_or_username)
 

	
 

	
 
class EmailNotificationModel(object):
 

	
 
    TYPE_CHANGESET_COMMENT = NotificationModel.TYPE_CHANGESET_COMMENT
 
    TYPE_MESSAGE = NotificationModel.TYPE_MESSAGE # only used for testing
 
    # NotificationModel.TYPE_MENTION is not used
 
    TYPE_PASSWORD_RESET = 'password_link'
 
    TYPE_REGISTRATION = NotificationModel.TYPE_REGISTRATION
 
    TYPE_PULL_REQUEST = NotificationModel.TYPE_PULL_REQUEST
 
    TYPE_PULL_REQUEST_COMMENT = NotificationModel.TYPE_PULL_REQUEST_COMMENT
 
    TYPE_DEFAULT = 'default'
 

	
 
    def __init__(self):
 
        super(EmailNotificationModel, self).__init__()
 
        self._tmpl_lookup = app_globals.mako_lookup
 
        self.email_types = {
 
            self.TYPE_CHANGESET_COMMENT: 'changeset_comment',
 
            self.TYPE_PASSWORD_RESET: 'password_reset',
 
            self.TYPE_REGISTRATION: 'registration',
 
            self.TYPE_DEFAULT: 'default',
 
            self.TYPE_PULL_REQUEST: 'pull_request',
 
            self.TYPE_PULL_REQUEST_COMMENT: 'pull_request_comment',
 
        }
 
        self._subj_map = {
 
            self.TYPE_CHANGESET_COMMENT: _('[Comment] %(repo_name)s changeset %(short_id)s "%(message_short)s" on %(branch)s by %(cs_author_username)s'),
 
            self.TYPE_MESSAGE: 'Test Message',
 
            # self.TYPE_PASSWORD_RESET
 
            self.TYPE_REGISTRATION: _('New user %(new_username)s registered'),
 
            # self.TYPE_DEFAULT
 
            self.TYPE_PULL_REQUEST: _('[Review] %(repo_name)s PR %(pr_nice_id)s "%(pr_title_short)s" from %(pr_source_branch)s by %(pr_owner_username)s'),
 
            self.TYPE_PULL_REQUEST_COMMENT: _('[Comment] %(repo_name)s PR %(pr_nice_id)s "%(pr_title_short)s" from %(pr_source_branch)s by %(pr_owner_username)s'),
 
        }
 

	
 
    def get_email_description(self, type_, **kwargs):
 
        """
 
        return subject for email based on given type
 
        """
 
        tmpl = self._subj_map[type_]
 
        try:
 
            subj = tmpl % kwargs
 
        except KeyError as e:
 
            log.error('error generating email subject for %r from %s: %s', type_, ', '.join(self._subj_map), e)
 
            raise
 
        # gmail doesn't do proper threading but will ignore leading square
 
        # bracket content ... so that is where we put status info
kallithea/model/repo.py
Show inline comments
 
@@ -360,141 +360,141 @@ class RepoModel(object):
 
                group_perms = db.UserGroupRepoToPerm.query() \
 
                    .filter(db.UserGroupRepoToPerm.repository == repo).all()
 

	
 
                for perm in user_perms:
 
                    db.UserRepoToPerm.create(perm.user, new_repo, perm.permission)
 

	
 
                for perm in group_perms:
 
                    db.UserGroupRepoToPerm.create(perm.users_group, new_repo,
 
                                               perm.permission)
 

	
 
            elif repo_group and copy_group_permissions:
 

	
 
                user_perms = db.UserRepoGroupToPerm.query() \
 
                    .filter(db.UserRepoGroupToPerm.group == repo_group).all()
 

	
 
                group_perms = db.UserGroupRepoGroupToPerm.query() \
 
                    .filter(db.UserGroupRepoGroupToPerm.group == repo_group).all()
 

	
 
                for perm in user_perms:
 
                    perm_name = perm.permission.permission_name.replace('group.', 'repository.')
 
                    perm_obj = db.Permission.get_by_key(perm_name)
 
                    db.UserRepoToPerm.create(perm.user, new_repo, perm_obj)
 

	
 
                for perm in group_perms:
 
                    perm_name = perm.permission.permission_name.replace('group.', 'repository.')
 
                    perm_obj = db.Permission.get_by_key(perm_name)
 
                    db.UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
 

	
 
            else:
 
                self._create_default_perms(new_repo, private)
 

	
 
            # now automatically start following this repository as owner
 
            scm.ScmModel().toggle_following_repo(new_repo.repo_id, owner.user_id)
 
            # we need to flush here, in order to check if database won't
 
            # throw any exceptions, create filesystem dirs at the very end
 
            meta.Session().flush()
 
            return new_repo
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            raise
 

	
 
    def create(self, form_data, cur_user):
 
        """
 
        Create repository using celery tasks
 

	
 
        :param form_data:
 
        :param cur_user:
 
        """
 
        from kallithea.lib.celerylib import tasks
 
        return tasks.create_repo(form_data, cur_user)
 
        from kallithea.model import async_tasks
 
        return async_tasks.create_repo(form_data, cur_user)
 

	
 
    def _update_permissions(self, repo, perms_new=None, perms_updates=None,
 
                            check_perms=True):
 
        if not perms_new:
 
            perms_new = []
 
        if not perms_updates:
 
            perms_updates = []
 

	
 
        # update permissions
 
        for member, perm, member_type in perms_updates:
 
            if member_type == 'user':
 
                # this updates existing one
 
                self.grant_user_permission(
 
                    repo=repo, user=member, perm=perm
 
                )
 
            else:
 
                # check if we have permissions to alter this usergroup's access
 
                if not check_perms or HasUserGroupPermissionLevel('read')(member):
 
                    self.grant_user_group_permission(
 
                        repo=repo, group_name=member, perm=perm
 
                    )
 
            # set new permissions
 
        for member, perm, member_type in perms_new:
 
            if member_type == 'user':
 
                self.grant_user_permission(
 
                    repo=repo, user=member, perm=perm
 
                )
 
            else:
 
                # check if we have permissions to alter this usergroup's access
 
                if not check_perms or HasUserGroupPermissionLevel('read')(member):
 
                    self.grant_user_group_permission(
 
                        repo=repo, group_name=member, perm=perm
 
                    )
 

	
 
    def create_fork(self, form_data, cur_user):
 
        """
 
        Simple wrapper into executing celery task for fork creation
 

	
 
        :param form_data:
 
        :param cur_user:
 
        """
 
        from kallithea.lib.celerylib import tasks
 
        return tasks.create_repo_fork(form_data, cur_user)
 
        from kallithea.model import async_tasks
 
        return async_tasks.create_repo_fork(form_data, cur_user)
 

	
 
    def delete(self, repo, forks=None, fs_remove=True, cur_user=None):
 
        """
 
        Delete given repository, forks parameter defines what do do with
 
        attached forks. Throws AttachedForksError if deleted repo has attached
 
        forks
 

	
 
        :param repo:
 
        :param forks: str 'delete' or 'detach'
 
        :param fs_remove: remove(archive) repo from filesystem
 
        """
 
        if not cur_user:
 
            cur_user = getattr(get_current_authuser(), 'username', None)
 
        repo = db.Repository.guess_instance(repo)
 
        if repo is not None:
 
            if forks == 'detach':
 
                for r in repo.forks:
 
                    r.fork = None
 
            elif forks == 'delete':
 
                for r in repo.forks:
 
                    self.delete(r, forks='delete')
 
            elif [f for f in repo.forks]:
 
                raise AttachedForksError()
 

	
 
            old_repo_dict = repo.get_dict()
 
            try:
 
                meta.Session().delete(repo)
 
                if fs_remove:
 
                    self._delete_filesystem_repo(repo)
 
                else:
 
                    log.debug('skipping removal from filesystem')
 
                hooks.log_delete_repository(old_repo_dict,
 
                                      deleted_by=cur_user)
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                raise
 

	
 
    def grant_user_permission(self, repo, user, perm):
 
        """
 
        Grant permission for user on given repository, or update existing one
 
        if found
 

	
 
        :param repo: Instance of Repository, repository_id, or repository name
 
        :param user: Instance of User, user_id or username
 
        :param perm: Instance of Permission, or permission_name
 
        """
 
        user = db.User.guess_instance(user)
 
        repo = db.Repository.guess_instance(repo)
kallithea/model/user.py
Show inline comments
 
@@ -253,175 +253,174 @@ class UserModel(object):
 
        return 'password' not in managed_fields
 

	
 
    def get_reset_password_token(self, user, timestamp, session_id):
 
        """
 
        The token is a 40-digit hexstring, calculated as a HMAC-SHA1.
 

	
 
        In a traditional HMAC scenario, an attacker is unable to know or
 
        influence the secret key, but can know or influence the message
 
        and token. This scenario is slightly different (in particular
 
        since the message sender is also the message recipient), but
 
        sufficiently similar to use an HMAC. Benefits compared to a plain
 
        SHA1 hash includes resistance against a length extension attack.
 

	
 
        The HMAC key consists of the following values (known only to the
 
        server and authorized users):
 

	
 
        * per-application secret (the `app_instance_uuid` setting), without
 
          which an attacker cannot counterfeit tokens
 
        * hashed user password, invalidating the token upon password change
 

	
 
        The HMAC message consists of the following values (potentially known
 
        to an attacker):
 

	
 
        * session ID (the anti-CSRF token), requiring an attacker to have
 
          access to the browser session in which the token was created
 
        * numeric user ID, limiting the token to a specific user (yet allowing
 
          users to be renamed)
 
        * user email address
 
        * time of token issue (a Unix timestamp, to enable token expiration)
 

	
 
        The key and message values are separated by NUL characters, which are
 
        guaranteed not to occur in any of the values.
 
        """
 
        app_secret = config.get('app_instance_uuid')
 
        return hmac.new(
 
            '\0'.join([app_secret, user.password]).encode('utf-8'),
 
            msg='\0'.join([session_id, str(user.user_id), user.email, str(timestamp)]).encode('utf-8'),
 
            digestmod=hashlib.sha1,
 
        ).hexdigest()
 

	
 
    def send_reset_password_email(self, data):
 
        """
 
        Sends email with a password reset token and link to the password
 
        reset confirmation page with all information (including the token)
 
        pre-filled. Also returns URL of that page, only without the token,
 
        allowing users to copy-paste or manually enter the token from the
 
        email.
 
        """
 
        from kallithea.lib.celerylib import tasks
 
        from kallithea.model import notification
 
        from kallithea.model import async_tasks, notification
 

	
 
        user_email = data['email']
 
        user = db.User.get_by_email(user_email)
 
        timestamp = int(time.time())
 
        if user is not None:
 
            if self.can_change_password(user):
 
                log.debug('password reset user %s found', user)
 
                token = self.get_reset_password_token(user,
 
                                                      timestamp,
 
                                                      webutils.session_csrf_secret_token())
 
                # URL must be fully qualified; but since the token is locked to
 
                # the current browser session, we must provide a URL with the
 
                # current scheme and hostname, rather than the canonical_url.
 
                link = webutils.url('reset_password_confirmation', qualified=True,
 
                             email=user_email,
 
                             timestamp=timestamp,
 
                             token=token)
 
            else:
 
                log.debug('password reset user %s found but was managed', user)
 
                token = link = None
 
            reg_type = notification.EmailNotificationModel.TYPE_PASSWORD_RESET
 
            body = notification.EmailNotificationModel().get_email_tmpl(
 
                reg_type, 'txt',
 
                user=user.short_contact,
 
                reset_token=token,
 
                reset_url=link)
 
            html_body = notification.EmailNotificationModel().get_email_tmpl(
 
                reg_type, 'html',
 
                user=user.short_contact,
 
                reset_token=token,
 
                reset_url=link)
 
            log.debug('sending email')
 
            tasks.send_email([user_email], _("Password reset link"), body, html_body)
 
            async_tasks.send_email([user_email], _("Password reset link"), body, html_body)
 
            log.info('send new password mail to %s', user_email)
 
        else:
 
            log.debug("password reset email %s not found", user_email)
 

	
 
        return webutils.url('reset_password_confirmation',
 
                     email=user_email,
 
                     timestamp=timestamp)
 

	
 
    def verify_reset_password_token(self, email, timestamp, token):
 
        user = db.User.get_by_email(email)
 
        if user is None:
 
            log.debug("user with email %s not found", email)
 
            return False
 

	
 
        token_age = int(time.time()) - int(timestamp)
 

	
 
        if token_age < 0:
 
            log.debug('timestamp is from the future')
 
            return False
 

	
 
        if token_age > UserModel.password_reset_token_lifetime:
 
            log.debug('password reset token expired')
 
            return False
 

	
 
        expected_token = self.get_reset_password_token(user,
 
                                                       timestamp,
 
                                                       webutils.session_csrf_secret_token())
 
        log.debug('computed password reset token: %s', expected_token)
 
        log.debug('received password reset token: %s', token)
 
        return expected_token == token
 

	
 
    def reset_password(self, user_email, new_passwd):
 
        from kallithea.lib.celerylib import tasks
 
        from kallithea.model import async_tasks
 
        user = db.User.get_by_email(user_email)
 
        if user is not None:
 
            if not self.can_change_password(user):
 
                raise Exception('trying to change password for external user')
 
            user.password = get_crypt_password(new_passwd)
 
            meta.Session().commit()
 
            log.info('change password for %s', user_email)
 
        if new_passwd is None:
 
            raise Exception('unable to set new password')
 

	
 
        tasks.send_email([user_email],
 
        async_tasks.send_email([user_email],
 
                 _('Password reset notification'),
 
                 _('The password to your account %s has been changed using password reset form.') % (user.username,))
 
        log.info('send password reset mail to %s', user_email)
 

	
 
        return True
 

	
 
    def has_perm(self, user, perm):
 
        perm = db.Permission.guess_instance(perm)
 
        user = db.User.guess_instance(user)
 

	
 
        return db.UserToPerm.query().filter(db.UserToPerm.user == user) \
 
            .filter(db.UserToPerm.permission == perm).scalar() is not None
 

	
 
    def grant_perm(self, user, perm):
 
        """
 
        Grant user global permissions
 

	
 
        :param user:
 
        :param perm:
 
        """
 
        user = db.User.guess_instance(user)
 
        perm = db.Permission.guess_instance(perm)
 
        # if this permission is already granted skip it
 
        _perm = db.UserToPerm.query() \
 
            .filter(db.UserToPerm.user == user) \
 
            .filter(db.UserToPerm.permission == perm) \
 
            .scalar()
 
        if _perm:
 
            return
 
        new = db.UserToPerm()
 
        new.user = user
 
        new.permission = perm
 
        meta.Session().add(new)
 
        return new
 

	
 
    def revoke_perm(self, user, perm):
 
        """
 
        Revoke users global permissions
 

	
 
        :param user:
 
        :param perm:
 
        """
 
        user = db.User.guess_instance(user)
 
        perm = db.Permission.guess_instance(perm)
 

	
 
        db.UserToPerm.query().filter(
 
            db.UserToPerm.user == user,
 
            db.UserToPerm.permission == perm,
kallithea/tests/functional/test_login.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
import re
 
import time
 
import urllib.parse
 

	
 
import mock
 
from tg.util.webtest import test_context
 

	
 
import kallithea.lib.celerylib.tasks
 
import kallithea.model.async_tasks
 
from kallithea.lib import webutils
 
from kallithea.lib.utils2 import check_password, generate_api_key
 
from kallithea.model import db, meta, validators
 
from kallithea.model.api_key import ApiKeyModel
 
from kallithea.model.user import UserModel
 
from kallithea.tests import base
 
from kallithea.tests.fixture import Fixture
 

	
 

	
 
fixture = Fixture()
 

	
 

	
 
class TestLoginController(base.TestController):
 

	
 
    def test_index(self):
 
        response = self.app.get(base.url(controller='login', action='index'))
 
        assert response.status == '200 OK'
 
        # Test response...
 

	
 
    def test_login_admin_ok(self):
 
        response = self.app.post(base.url(controller='login', action='index'),
 
                                 {'username': base.TEST_USER_ADMIN_LOGIN,
 
                                  'password': base.TEST_USER_ADMIN_PASS,
 
                                  '_session_csrf_secret_token': self.session_csrf_secret_token()})
 
        assert response.status == '302 Found'
 
        self.assert_authenticated_user(response, base.TEST_USER_ADMIN_LOGIN)
 

	
 
        response = response.follow()
 
        response.mustcontain('/%s' % base.HG_REPO)
 

	
 
    def test_login_regular_ok(self):
 
        response = self.app.post(base.url(controller='login', action='index'),
 
                                 {'username': base.TEST_USER_REGULAR_LOGIN,
 
                                  'password': base.TEST_USER_REGULAR_PASS,
 
                                  '_session_csrf_secret_token': self.session_csrf_secret_token()})
 

	
 
        assert response.status == '302 Found'
 
        self.assert_authenticated_user(response, base.TEST_USER_REGULAR_LOGIN)
 

	
 
        response = response.follow()
 
        response.mustcontain('/%s' % base.HG_REPO)
 

	
 
    def test_login_regular_email_ok(self):
 
        response = self.app.post(base.url(controller='login', action='index'),
 
                                 {'username': base.TEST_USER_REGULAR_EMAIL,
 
                                  'password': base.TEST_USER_REGULAR_PASS,
 
                                  '_session_csrf_secret_token': self.session_csrf_secret_token()})
 

	
 
@@ -365,97 +365,97 @@ class TestLoginController(base.TestContr
 
        assert ret.email == email
 
        assert ret.name == name
 
        assert ret.lastname == lastname
 
        assert ret.api_key is not None
 
        assert ret.admin == False
 

	
 
    #==========================================================================
 
    # PASSWORD RESET
 
    #==========================================================================
 

	
 
    def test_forgot_password_wrong_mail(self):
 
        bad_email = 'username%wrongmail.org'
 
        response = self.app.post(
 
                        base.url(controller='login', action='password_reset'),
 
                            {'email': bad_email,
 
                             '_session_csrf_secret_token': self.session_csrf_secret_token()})
 

	
 
        response.mustcontain('An email address must contain a single @')
 

	
 
    def test_forgot_password(self):
 
        response = self.app.get(base.url(controller='login',
 
                                    action='password_reset'))
 
        assert response.status == '200 OK'
 

	
 
        username = 'test_password_reset_1'
 
        password = 'qweqwe'
 
        email = 'username@example.com'
 
        name = 'passwd'
 
        lastname = 'reset'
 
        timestamp = int(time.time())
 

	
 
        new = db.User()
 
        new.username = username
 
        new.password = password
 
        new.email = email
 
        new.name = name
 
        new.lastname = lastname
 
        new.api_key = generate_api_key()
 
        meta.Session().add(new)
 
        meta.Session().commit()
 

	
 
        token = UserModel().get_reset_password_token(
 
            db.User.get_by_username(username), timestamp, self.session_csrf_secret_token())
 

	
 
        collected = []
 
        def mock_send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
            collected.append((recipients, subject, body, html_body))
 

	
 
        with mock.patch.object(kallithea.lib.celerylib.tasks, 'send_email', mock_send_email), \
 
        with mock.patch.object(kallithea.model.async_tasks, 'send_email', mock_send_email), \
 
                mock.patch.object(time, 'time', lambda: timestamp):
 
            response = self.app.post(base.url(controller='login',
 
                                         action='password_reset'),
 
                                     {'email': email,
 
                                      '_session_csrf_secret_token': self.session_csrf_secret_token()})
 

	
 
        self.checkSessionFlash(response, 'A password reset confirmation code has been sent')
 

	
 
        ((recipients, subject, body, html_body),) = collected
 
        assert recipients == ['username@example.com']
 
        assert subject == 'Password reset link'
 
        assert '\n%s\n' % token in body
 
        (confirmation_url,) = (line for line in body.splitlines() if line.startswith('http://'))
 
        assert ' href="%s"' % confirmation_url.replace('&', '&amp;').replace('@', '%40') in html_body
 

	
 
        d = urllib.parse.parse_qs(urllib.parse.urlparse(confirmation_url).query)
 
        assert d['token'] == [token]
 
        assert d['timestamp'] == [str(timestamp)]
 
        assert d['email'] == [email]
 

	
 
        response = response.follow()
 

	
 
        # BAD TOKEN
 

	
 
        bad_token = "bad"
 

	
 
        response = self.app.post(base.url(controller='login',
 
                                     action='password_reset_confirmation'),
 
                                 {'email': email,
 
                                  'timestamp': timestamp,
 
                                  'password': "p@ssw0rd",
 
                                  'password_confirm': "p@ssw0rd",
 
                                  'token': bad_token,
 
                                  '_session_csrf_secret_token': self.session_csrf_secret_token(),
 
                                 })
 
        assert response.status == '200 OK'
 
        response.mustcontain('Invalid password reset token')
 

	
 
        # GOOD TOKEN
 

	
 
        response = self.app.get(confirmation_url)
 
        assert response.status == '200 OK'
 
        response.mustcontain("You are about to set a new password for the email address %s" % email)
 
        response.mustcontain('<form action="%s" method="post">' % base.url(controller='login', action='password_reset_confirmation'))
 
        response.mustcontain('value="%s"' % self.session_csrf_secret_token())
 
        response.mustcontain('value="%s"' % token)
 
        response.mustcontain('value="%s"' % timestamp)
 
        response.mustcontain('value="username@example.com"')
kallithea/tests/models/test_notifications.py
Show inline comments
 
import os
 
import re
 

	
 
import mock
 
from tg.util.webtest import test_context
 

	
 
import kallithea.lib.celerylib
 
import kallithea.lib.celerylib.tasks
 
import kallithea.lib.helpers as h
 
import kallithea.model.async_tasks
 
from kallithea.model import db, meta
 
from kallithea.model.notification import EmailNotificationModel, NotificationModel
 
from kallithea.model.user import UserModel
 
from kallithea.tests import base
 

	
 

	
 
class TestNotifications(base.TestController):
 

	
 
    def setup_method(self, method):
 
        meta.Session.remove()
 
        u1 = UserModel().create_or_update(username='u1',
 
                                        password='qweqwe',
 
                                        email='u1@example.com',
 
                                        firstname='u1', lastname='u1')
 
        meta.Session().commit()
 
        self.u1 = u1.user_id
 

	
 
        u2 = UserModel().create_or_update(username='u2',
 
                                        password='qweqwe',
 
                                        email='u2@example.com',
 
                                        firstname='u2', lastname='u3')
 
        meta.Session().commit()
 
        self.u2 = u2.user_id
 

	
 
        u3 = UserModel().create_or_update(username='u3',
 
                                        password='qweqwe',
 
                                        email='u3@example.com',
 
                                        firstname='u3', lastname='u3')
 
        meta.Session().commit()
 
        self.u3 = u3.user_id
 

	
 
    def test_create_notification(self):
 
        with test_context(self.app):
 
            usrs = [self.u1, self.u2]
 

	
 
            def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
                assert recipients == ['u2@example.com']
 
                assert subject == 'Test Message'
 
                assert body == "hi there"
 
                assert '>hi there<' in html_body
 
                assert from_name == 'u1 u1'
 
            with mock.patch.object(kallithea.lib.celerylib.tasks, 'send_email', send_email):
 
            with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email):
 
                NotificationModel().create(created_by=self.u1,
 
                                                   body='hi there',
 
                                                   recipients=usrs)
 

	
 
    @mock.patch.object(h, 'canonical_url', (lambda arg, **kwargs: 'http://%s/?%s' % (arg, '&'.join('%s=%s' % (k, v) for (k, v) in sorted(kwargs.items())))))
 
    def test_dump_html_mails(self):
 
        # Exercise all notification types and dump them to one big html file
 
        l = []
 

	
 
        def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
 
            l.append('<hr/>\n')
 
            l.append('<h1>%s</h1>\n' % desc) # desc is from outer scope
 
            l.append('<pre>\n')
 
            l.append('From: %s <name@example.com>\n' % from_name)
 
            l.append('To: %s\n' % ' '.join(recipients))
 
            l.append('Subject: %s\n' % subject)
 
            l.append('</pre>\n')
 
            l.append('<hr/>\n')
 
            l.append('<pre>%s</pre>\n' % body)
 
            l.append('<hr/>\n')
 
            l.append(html_body)
 
            l.append('<hr/>\n')
 

	
 
        with test_context(self.app):
 
            with mock.patch.object(kallithea.lib.celerylib.tasks, 'send_email', send_email):
 
            with mock.patch.object(kallithea.model.async_tasks, 'send_email', send_email):
 
                pr_kwargs = dict(
 
                    pr_nice_id='#7',
 
                    pr_title='The Title',
 
                    pr_title_short='The Title',
 
                    pr_url='http://pr.org/7',
 
                    pr_target_repo='http://mainline.com/repo',
 
                    pr_target_branch='trunk',
 
                    pr_source_repo='https://dev.org/repo',
 
                    pr_source_branch='devbranch',
 
                    pr_owner=db.User.get(self.u2),
 
                    pr_owner_username='u2'
 
                    )
 

	
 
                for type_, body, kwargs in [
 
                        (NotificationModel.TYPE_CHANGESET_COMMENT,
 
                         'This is the new \'comment\'.\n\n - and here it ends indented.',
 
                         dict(
 
                            short_id='cafe1234',
 
                            raw_id='cafe1234c0ffeecafe',
 
                            branch='brunch',
 
                            cs_comment_user='Opinionated User (jsmith)',
 
                            cs_comment_url='http://comment.org',
 
                            is_mention=[False, True],
 
                            message='This changeset did something clever which is hard to explain',
 
                            message_short='This changeset did something cl...',
 
                            status_change=[None, 'Approved'],
 
                            cs_target_repo='http://example.com/repo_target',
 
                            cs_url='http://changeset.com',
 
                            cs_author_username=db.User.get(self.u2).username,
 
                            cs_author=db.User.get(self.u2))),
 
                        (NotificationModel.TYPE_MESSAGE,
 
                         'This is the \'body\' of the "test" message\n - nothing interesting here except indentation.',
 
                         dict()),
 
                        #(NotificationModel.TYPE_MENTION, '$body', None), # not used
 
                        (NotificationModel.TYPE_REGISTRATION,
 
                         'Registration body',
 
                         dict(
 
                            new_username='newbie',
 
                            registered_user_url='http://newbie.org',
 
                            new_email='new@email.com',
 
                            new_full_name='New Full Name')),
 
                        (NotificationModel.TYPE_PULL_REQUEST,
 
                         'This PR is \'awesome\' because it does <stuff>\n - please approve indented!',
 
                         dict(
 
                            pr_user_created='Requesting User (root)', # pr_owner should perhaps be used for @mention in description ...
 
                            is_mention=[False, True],
 
                            pr_revisions=[('123abc'*7, "Introduce one and two\n\nand that's it"), ('567fed'*7, 'Make one plus two equal tree')],
 
                            org_repo_name='repo_org',
 
                            **pr_kwargs)),
 
                        (NotificationModel.TYPE_PULL_REQUEST_COMMENT,
 
                         'Me too!\n\n - and indented on second line',
 
                         dict(
 
                            closing_pr=[False, True],
 
                            is_mention=[False, True],
 
                            pr_comment_user='Opinionated User (jsmith)',
 
                            pr_comment_url='http://pr.org/comment',
 
                            status_change=[None, 'Under Review'],
 
                            **pr_kwargs)),
 
                        ]:
 
                    kwargs['repo_name'] = 'repo/name'
 
                    params = [(type_, type_, body, kwargs)]
 
                    for param_name in ['is_mention', 'status_change', 'closing_pr']: # TODO: inline/general
 
                        if not isinstance(kwargs.get(param_name), list):
 
                            continue
 
                        new_params = []
 
                        for v in kwargs[param_name]:
 
                            for desc, type_, body, kwargs in params:
 
                                kwargs = dict(kwargs)
 
                                kwargs[param_name] = v
 
                                new_params.append(('%s, %s=%r' % (desc, param_name, v), type_, body, kwargs))
 
                        params = new_params
 

	
 
                    for desc, type_, body, kwargs in params:
 
                        # desc is used as "global" variable
 
                        NotificationModel().create(created_by=self.u1,
 
                                                           body=body, email_kwargs=kwargs,
 
                                                           recipients=[self.u2], type_=type_)
 

	
 
                # Email type TYPE_PASSWORD_RESET has no corresponding notification type - test it directly:
 
                desc = 'TYPE_PASSWORD_RESET'
 
                kwargs = dict(user='John Doe', reset_token='decbf64715098db5b0bd23eab44bd792670ab746', reset_url='http://reset.com/decbf64715098db5b0bd23eab44bd792670ab746')
 
                kallithea.lib.celerylib.tasks.send_email(['john@doe.com'],
 
                kallithea.model.async_tasks.send_email(['john@doe.com'],
 
                    "Password reset link",
 
                    EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'txt', **kwargs),
 
                    EmailNotificationModel().get_email_tmpl(EmailNotificationModel.TYPE_PASSWORD_RESET, 'html', **kwargs),
 
                    from_name=db.User.get(self.u1).full_name_or_username)
 

	
 
        out = '<!doctype html>\n<html lang="en">\n<head><title>Notifications</title><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"></head>\n<body>\n%s\n</body>\n</html>\n' % \
 
            re.sub(r'<(/?(?:!doctype|html|head|title|meta|body)\b[^>]*)>', r'<!--\1-->', ''.join(l))
 

	
 
        outfn = os.path.join(os.path.dirname(__file__), 'test_dump_html_mails.out.html')
 
        reffn = os.path.join(os.path.dirname(__file__), 'test_dump_html_mails.ref.html')
 
        with open(outfn, 'w') as f:
 
            f.write(out)
 
        with open(reffn) as f:
 
            ref = f.read()
 
        assert ref == out # copy test_dump_html_mails.out.html to test_dump_html_mails.ref.html to update expectations
 
        os.unlink(outfn)
kallithea/tests/other/test_mail.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
import mock
 

	
 
import kallithea
 
from kallithea.model import db
 
from kallithea.tests import base
 

	
 

	
 
class smtplib_mock(object):
 

	
 
    @classmethod
 
    def SMTP(cls, server, port):
 
        return smtplib_mock()
 

	
 
    def ehlo(self):
 
        pass
 

	
 
    def quit(self):
 
        pass
 

	
 
    def sendmail(self, sender, dest, msg):
 
        smtplib_mock.lastsender = sender
 
        smtplib_mock.lastdest = set(dest)
 
        smtplib_mock.lastmsg = msg
 

	
 

	
 
@mock.patch('kallithea.lib.celerylib.tasks.smtplib', smtplib_mock)
 
@mock.patch('kallithea.model.async_tasks.smtplib', smtplib_mock)
 
class TestMail(base.TestController):
 

	
 
    def test_send_mail_trivial(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = ['rcpt1', 'rcpt2']
 
        envelope_from = 'noreply@mailserver.org'
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 

	
 
    def test_send_mail_no_recipients(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = []
 
        envelope_from = 'noreply@mailserver.org'
 
        email_to = 'admin@mailserver.org'
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
            'email_to': email_to,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL, email_to])
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 

	
 
    def test_send_mail_no_recipients_multiple_email_to(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = []
 
        envelope_from = 'noreply@mailserver.org'
 
        email_to = 'admin@mailserver.org,admin2@example.com'
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
            'email_to': email_to,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL] + email_to.split(','))
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 

	
 
    def test_send_mail_no_recipients_no_email_to(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = []
 
        envelope_from = 'noreply@mailserver.org'
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body)
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body)
 

	
 
        assert smtplib_mock.lastdest == set([base.TEST_USER_ADMIN_EMAIL])
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert 'From: %s' % envelope_from in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 

	
 
    def test_send_mail_with_author(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = ['rcpt1', 'rcpt2']
 
        envelope_from = 'noreply@mailserver.org'
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 
        author = db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN)
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert 'From: "Kallithea Admin (no-reply)" <%s>' % envelope_from in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 

	
 
    def test_send_mail_with_author_full_mail_from(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = ['ræcpt1', 'receptor2 <rcpt2@example.com>', 'tæst@example.com', 'Tæst <test@example.com>']
 
        envelope_addr = 'noreply@mailserver.org'
 
        envelope_from = 'Söme Næme <%s>' % envelope_addr
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 
        author = db.User.get_by_username(base.TEST_USER_REGULAR_LOGIN)
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body, from_name=author.full_name_or_username)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert 'From: "Kallithea Admin (no-reply)" <%s>' % envelope_addr in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 

	
 
    def test_send_mail_extra_headers(self):
 
        mailserver = 'smtp.mailserver.org'
 
        recipients = ['rcpt1', 'rcpt2']
 
        envelope_from = 'noreply@mailserver.org'
 
        subject = 'subject'
 
        body = 'body'
 
        html_body = 'html_body'
 
        author = db.User(name='foo', lastname='(fubar) "baz"')
 
        headers = {'extra': 'yes'}
 

	
 
        config_mock = {
 
            'smtp_server': mailserver,
 
            'app_email_from': envelope_from,
 
        }
 
        with mock.patch('kallithea.lib.celerylib.tasks.config', config_mock):
 
            kallithea.lib.celerylib.tasks.send_email(recipients, subject, body, html_body,
 
        with mock.patch('kallithea.model.async_tasks.config', config_mock):
 
            kallithea.model.async_tasks.send_email(recipients, subject, body, html_body,
 
                                                     from_name=author.full_name_or_username, headers=headers)
 

	
 
        assert smtplib_mock.lastdest == set(recipients)
 
        assert smtplib_mock.lastsender == envelope_from
 
        assert r'From: "foo (fubar) \"baz\" (no-reply)" <%s>' % envelope_from in smtplib_mock.lastmsg
 
        assert 'Subject: %s' % subject in smtplib_mock.lastmsg
 
        assert body in smtplib_mock.lastmsg
 
        assert html_body in smtplib_mock.lastmsg
 
        assert 'extra: yes' in smtplib_mock.lastmsg
 
        # verify that headers dict hasn't mutated by send_email
 
        assert headers == {'extra': 'yes'}
scripts/deps.py
Show inline comments
 
@@ -86,127 +86,127 @@ subprocess
 
sys
 
tarfile
 
tempfile
 
textwrap
 
tgext
 
threading
 
time
 
traceback
 
traitlets
 
types
 
urllib
 
urlobject
 
uuid
 
warnings
 
webhelpers2
 
webob
 
webtest
 
whoosh
 
win32traceutil
 
zipfile
 
'''.split())
 

	
 
top_modules = set('''
 
kallithea.alembic
 
kallithea.bin
 
kallithea.config
 
kallithea.controllers
 
kallithea.templates.py
 
scripts
 
'''.split())
 

	
 
bottom_external_modules = set('''
 
tg
 
mercurial
 
sqlalchemy
 
alembic
 
formencode
 
pygments
 
dulwich
 
beaker
 
psycopg2
 
docs
 
setup
 
conftest
 
'''.split())
 

	
 
normal_modules = set('''
 
kallithea
 
kallithea.lib.celerylib.tasks
 
kallithea.lib
 
kallithea.lib.auth
 
kallithea.lib.auth_modules
 
kallithea.lib.base
 
kallithea.lib.celerylib
 
kallithea.lib.db_manage
 
kallithea.lib.helpers
 
kallithea.lib.hooks
 
kallithea.lib.indexers
 
kallithea.lib.utils
 
kallithea.lib.utils2
 
kallithea.lib.vcs
 
kallithea.lib.webutils
 
kallithea.model
 
kallithea.model.async_tasks
 
kallithea.model.scm
 
kallithea.templates.py
 
'''.split())
 

	
 
shown_modules = normal_modules | top_modules
 

	
 
# break the chains somehow - this is a cleanup TODO list
 
known_violations = [
 
('kallithea.lib.auth_modules', 'kallithea.lib.auth'),  # needs base&facade
 
('kallithea.lib.utils', 'kallithea.model'),  # clean up utils
 
('kallithea.lib.utils', 'kallithea.model.db'),
 
('kallithea.lib.utils', 'kallithea.model.scm'),
 
('kallithea.lib.celerylib.tasks', 'kallithea.lib.helpers'),
 
('kallithea.lib.celerylib.tasks', 'kallithea.lib.hooks'),
 
('kallithea.lib.celerylib.tasks', 'kallithea.lib.indexers'),
 
('kallithea.lib.celerylib.tasks', 'kallithea.model'),
 
('kallithea.model.async_tasks', 'kallithea.lib.helpers'),
 
('kallithea.model.async_tasks', 'kallithea.lib.hooks'),
 
('kallithea.model.async_tasks', 'kallithea.lib.indexers'),
 
('kallithea.model.async_tasks', 'kallithea.model'),
 
('kallithea.model', 'kallithea.lib.auth'),  # auth.HasXXX
 
('kallithea.model', 'kallithea.lib.auth_modules'),  # validators
 
('kallithea.model', 'kallithea.lib.helpers'),
 
('kallithea.model', 'kallithea.lib.hooks'),  # clean up hooks
 
('kallithea.model', 'kallithea.model.scm'),
 
('kallithea.model.scm', 'kallithea.lib.hooks'),
 
]
 

	
 
extra_edges = [
 
('kallithea.config', 'kallithea.controllers'),  # through TG
 
('kallithea.lib.auth', 'kallithea.lib.auth_modules'),  # custom loader
 
]
 

	
 

	
 
def normalize(s):
 
    """Given a string with dot path, return the string it should be shown as."""
 
    parts = s.replace('.__init__', '').split('.')
 
    short_2 = '.'.join(parts[:2])
 
    short_3 = '.'.join(parts[:3])
 
    short_4 = '.'.join(parts[:4])
 
    if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']:
 
        return 'scripts'
 
    if short_3 == 'kallithea.model.meta':
 
        return 'kallithea.model.db'
 
    if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']:
 
        return 'kallithea.lib.vcs.ssh'
 
    if short_4 in shown_modules:
 
        return short_4
 
    if short_3 in shown_modules:
 
        return short_3
 
    if short_2 in shown_modules:
 
        return short_2
 
    if short_2 == 'kallithea.tests':
 
        return None
 
    if parts[0] in ignored_modules:
 
        return None
 
    assert parts[0] in bottom_external_modules, parts
 
    return parts[0]
 

	
 

	
 
def main(filenames):
 
    if not filenames or filenames[0].startswith('-'):
 
        print('''\
 
Usage:
 
    hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
 
    dot -Tsvg deps.dot > deps.svg
 
        ''')
 
        raise SystemExit(1)
0 comments (0 inline, 0 general)