If scm param is given also compare if given scm is the same as expected

    from scm parameter

    :param repo_name:

    :param base_path:

    :param scm:

    :return True: if given path is a valid repository

    """

    # TODO: paranoid security checks?

    full_path = os.path.join(safe_str(base_path), safe_str(repo_name))

    try:

        scm_ = get_scm(full_path)

        if scm:

            return scm_[0] == scm

        return True

    except VCSError:

        return False

def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):

    """

    Returns True if given path is a repository group False otherwise

    :param repo_name:

    :param base_path:

    """

    full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))

    # check if it's not a repo

    if is_valid_repo(repo_group_name, base_path):

        return False

    try:

        # we need to check bare git repos at higher level

        # since we might match branches/hooks/info/objects or possible

        # other things inside bare git repo

        get_scm(os.path.dirname(full_path))

        return False

    except VCSError:

        pass

    # check if it's a valid path

    if skip_path_check or os.path.isdir(full_path):

        return True

    return False

# propagated from mercurial documentation

ui_sections = ['alias', 'auth',

                'decode/encode', 'defaults',

                'diff', 'email',

                'extensions', 'format',

                'merge-patterns', 'merge-tools',

                'hooks', 'http_proxy',

                'smtp', 'patch',

                'paths', 'profiling',

                'server', 'trusted',

                'ui', 'web', ]

def make_ui(repo_path=None):

    """

    Create an Mercurial 'ui' object based on database Ui settings, possibly

    augmenting with content from a hgrc file.

    """

    baseui = ui.ui()

    # clean the baseui object

    baseui._ocfg = config.config()

    baseui._ucfg = config.config()

    baseui._tcfg = config.config()

    sa = meta.Session()

    for ui_ in sa.query(Ui).all():

        if ui_.ui_active:

            ui_val = '' if ui_.ui_value is None else safe_str(ui_.ui_value)

            log.debug('config from db: [%s] %s=%r', ui_.ui_section,

                      ui_.ui_key, ui_val)

            baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),

                             ui_val)

    # force set push_ssl requirement to False, Kallithea handles that

    baseui.setconfig('web', 'push_ssl', False)

    baseui.setconfig('web', 'allow_push', '*')

    # prevent interactive questions for ssh password / passphrase

    ssh = baseui.config('ui', 'ssh', default='ssh')

    baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)

    # push / pull hooks

    baseui.setconfig('hooks', 'changegroup.kallithea_log_push_action', 'python:kallithea.lib.hooks.log_push_action')

    baseui.setconfig('hooks', 'outgoing.kallithea_log_pull_action', 'python:kallithea.lib.hooks.log_pull_action')

    if repo_path is not None:

        hgrc_path = os.path.join(repo_path, '.hg', 'hgrc')

        if os.path.isfile(hgrc_path):

            log.debug('reading hgrc from %s', hgrc_path)

            cfg = config.config()

            cfg.read(hgrc_path)

            for section in ui_sections:

                for k, v in cfg.items(section):

                    log.debug('config from file: [%s] %s=%s', section, k, v)

                    baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))

        else:

            log.debug('hgrc file is not present at %s, skipping...', hgrc_path)

    return baseui

def set_app_settings(config):

    """

    Updates app config with new settings from database

    :param config:

    """

    hgsettings = Setting.get_app_settings()

    for k, v in hgsettings.items():

        config[k] = v

def set_vcs_config(config):

    """

    Patch VCS config with some Kallithea specific stuff

    :param config: kallithea.CONFIG

    """

    settings.BACKENDS = {

        'hg': 'kallithea.lib.vcs.backends.hg.MercurialRepository',

        'git': 'kallithea.lib.vcs.backends.git.GitRepository',

    }

    settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')

    settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()

    settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',

                                                        'utf-8'), sep=',')

def set_indexer_config(config):

    """

    Update Whoosh index mapping

    :param config: kallithea.CONFIG

    """

    log.debug('adding extra into INDEX_EXTENSIONS')

    kallithea.config.conf.INDEX_EXTENSIONS.extend(re.split(r'\s+', config.get('index.extensions', '')))

    log.debug('adding extra into INDEX_FILENAMES')

    kallithea.config.conf.INDEX_FILENAMES.extend(re.split(r'\s+', config.get('index.filenames', '')))

def map_groups(path):

    """

    Given a full path to a repository, create all nested groups that this

    repo is inside. This function creates parent-child relationships between

    groups and creates default perms for all new groups.

    :param paths: full path to repository

    """

    from kallithea.model.repo_group import RepoGroupModel

    sa = meta.Session()

    groups = path.split(Repository.url_sep())

    parent = None

    group = None

    # last element is repo in nested groups structure

    groups = groups[:-1]

    rgm = RepoGroupModel()

    owner = User.get_first_admin()

    for lvl, group_name in enumerate(groups):

        group_name = u'/'.join(groups[:lvl] + [group_name])

        group = RepoGroup.get_by_group_name(group_name)

        desc = '%s group' % group_name

        # skip folders that are now removed repos

        if REMOVED_REPO_PAT.match(group_name):

            break

        if group is None:

            log.debug('creating group level: %s group_name: %s',

                      lvl, group_name)

            group = RepoGroup(group_name, parent)

            group.group_description = desc

            group.owner = owner

            sa.add(group)

            rgm._create_default_perms(group)

            sa.flush()

        parent = group

    return group

                   install_git_hooks=False, user=None, overwrite_git_hooks=False):

    """

    are created, if remove_obsolete is True it also check for db entries

    :param remove_obsolete: check for obsolete entries in database

    :param install_git_hooks: if this is True, also check and install git hook

        for a repo if missing

    :param overwrite_git_hooks: if this is True, overwrite any existing git hooks

        that may be encountered (even if user-deployed)

    """

    from kallithea.model.repo import RepoModel

    from kallithea.model.scm import ScmModel

    sa = meta.Session()

    repo_model = RepoModel()

    if user is None:

        user = User.get_first_admin()

    added = []

    # creation defaults

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_downloads = defs.get('repo_enable_downloads')

    private = defs.get('repo_private')

        group = map_groups(name)

        unicode_name = safe_unicode(name)

        db_repo = repo_model.get_by_repo_name(unicode_name)

        # found repo that is on filesystem not in Kallithea database

        if not db_repo:

            log.info('repository %s not found, creating now', name)

            added.append(name)

            desc = (repo.description

                    if repo.description != 'unknown'

                    else '%s repository' % name)

            new_repo = repo_model._create_repo(

                repo_name=name,

                repo_type=repo.alias,

                description=desc,

                repo_group=getattr(group, 'group_id', None),

                owner=user,

                enable_downloads=enable_downloads,

                enable_statistics=enable_statistics,

                private=private,

                state=Repository.STATE_CREATED

            )

            sa.commit()

            # we added that repo just now, and make sure it has githook

            # installed, and updated server info

            if new_repo.repo_type == 'git':

                git_repo = new_repo.scm_instance

                ScmModel().install_git_hooks(git_repo)

                # update repository server-info

                log.debug('Running update server info')

                git_repo._update_server_info()

            new_repo.update_changeset_cache()

        elif install_git_hooks:

            if db_repo.repo_type == 'git':

                ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks)

    removed = []

    # remove from database those repositories that are not in the filesystem

    for repo in sa.query(Repository).all():

            if remove_obsolete:

                log.debug("Removing non-existing repository found in db `%s`",

                          repo.repo_name)

                try:

                    RepoModel().delete(repo, forks='detach', fs_remove=False)

                    sa.commit()

                except Exception:

                    #don't hold further removals on error

                    log.error(traceback.format_exc())

                    sa.rollback()

            removed.append(repo.repo_name)

    return added, removed

def load_rcextensions(root_path):

    path = os.path.join(root_path, 'rcextensions', '__init__.py')

    if os.path.isfile(path):

        rcext = create_module('rc', path)

        EXT = kallithea.EXTENSIONS = rcext

        log.debug('Found rcextensions now loading %s...', rcext)

        # Additional mappings that are not present in the pygments lexers

        kallithea.config.conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))

        # OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)

        if getattr(EXT, 'INDEX_EXTENSIONS', []):

            log.debug('settings custom INDEX_EXTENSIONS')

            kallithea.config.conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])

        # ADDITIONAL MAPPINGS

        log.debug('adding extra into INDEX_EXTENSIONS')

        kallithea.config.conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))

        # auto check if the module is not missing any data, set to default if is

        # this will help autoupdate new feature of rcext module

        #from kallithea.config import rcextensions

        #for k in dir(rcextensions):

        #    if not k.startswith('_') and not hasattr(EXT, k):

        #        setattr(EXT, k, getattr(rcextensions, k))

#==============================================================================

# MISC

#==============================================================================

git_req_ver = StrictVersion('1.7.4')

def check_git_version():

    """

    Checks what version of git is installed on the system, and raise a system exit

    if it's too old for Kallithea to work properly.

    """

    if 'git' not in kallithea.BACKENDS:

        return None

    if not settings.GIT_EXECUTABLE_PATH:

        log.warning('No git executable configured - check "git_path" in the ini file.')

        return None

    try:

        stdout, stderr = GitRepository._run_git_command(['--version'])

    except RepositoryError as e:

        # message will already have been logged as error

        log.warning('No working git executable found - check "git_path" in the ini file.')

        return None

    if stderr:

        log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, stderr)

    if not stdout:

        log.warning('No working git executable found - check "git_path" in the ini file.')

        return None

    output = stdout.strip()

    m = re.search(r"\d+.\d+.\d+", output)

    if m:

        ver = StrictVersion(m.group(0))

        log.debug('Git executable: "%s", version %s (parsed from: "%s")',

                  settings.GIT_EXECUTABLE_PATH, ver, output)

        if ver < git_req_ver:

            log.error('Kallithea detected %s version %s, which is too old '

                      'for the system to function properly. '

                      'Please upgrade to version %s or later. '

                      'If you strictly need Mercurial repositories, you can '

                      'clear the "git_path" setting in the ini file.',

                      settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)

            log.error("Terminating ...")

            sys.exit(1)

    else:

        ver = StrictVersion('0.0.0')

        log.warning('Error finding version number in "%s --version" stdout:\n%s',

                    settings.GIT_EXECUTABLE_PATH, output)

    return ver

#===============================================================================

# CACHE RELATED METHODS

#===============================================================================

# set cache regions for beaker so celery can utilise it

def setup_cache_regions(settings):

    # Create dict with just beaker cache configs with prefix stripped

    cache_settings = {'regions': None}

    prefix = 'beaker.cache.'

    for key in settings:

        if key.startswith(prefix):

            name = key[len(prefix):]

            cache_settings[name] = settings[key]

    # Find all regions, apply defaults, and apply to beaker

    if cache_settings['regions']:

        for region in cache_settings['regions'].split(','):

            region = region.strip()

            prefix = region + '.'

            region_settings = {}

            for key in cache_settings:

                if key.startswith(prefix):

                    name = key[len(prefix):]

                    region_settings[name] = cache_settings[key]

            region_settings.setdefault('expire',

                                       cache_settings.get('expire', '60'))

            region_settings.setdefault('lock_dir',

                                       cache_settings.get('lock_dir'))

            region_settings.setdefault('data_dir',

                                       cache_settings.get('data_dir'))

            region_settings.setdefault('type',

                                       cache_settings.get('type', 'memory'))

            beaker.cache.cache_regions[region] = region_settings

def conditional_cache(region, prefix, condition, func):

    """

    Conditional caching function use like::

        def _c(arg):

            #heavy computation function

            return data

        # depending from condition the compute is wrapped in cache or not

        compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)

        return compute(arg)

    :param region: name of cache region

    :param prefix: cache region prefix

    :param condition: condition for cache to be triggered, and return data cached

    :param func: wrapped heavy function to compute

    """

    wrapped = func

    if condition:

        log.debug('conditional_cache: True, wrapping call of '

                  'func: %s into %s region cache' % (region, func))

        wrapped = _cache_decorate((prefix,), None, None, region)(func)

    return wrapped

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.model.scm

~~~~~~~~~~~~~~~~~~~

Scm model for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 9, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import os

import posixpath

import re

import sys

import traceback

import pkg_resources

from tg.i18n import ugettext as _

import kallithea

from kallithea import BACKENDS

from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel, HasUserGroupPermissionLevel

from kallithea.lib.exceptions import IMCCommitError, NonRelativePathError

from kallithea.lib.hooks import process_pushed_raw_ids

from kallithea.lib.utils import action_logger, get_filesystem_repos, make_ui

from kallithea.lib.utils2 import safe_str, safe_unicode, set_hook_environment

from kallithea.lib.vcs import get_backend

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.exceptions import RepositoryError

from kallithea.lib.vcs.nodes import FileNode

from kallithea.lib.vcs.utils.lazy import LazyProperty

from kallithea.model.db import PullRequest, RepoGroup, Repository, Session, Ui, User, UserFollowing, UserLog

log = logging.getLogger(__name__)

class UserTemp(object):

    def __init__(self, user_id):

        self.user_id = user_id

    def __repr__(self):

        return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)

class RepoTemp(object):

    def __init__(self, repo_id):

        self.repo_id = repo_id

    def __repr__(self):

        return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)

class _PermCheckIterator(object):

    def __init__(self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None):

        """

        Creates iterator from given list of objects, additionally

        checking permission for them from perm_set var

        :param obj_list: list of db objects

        :param obj_attr: attribute of object to pass into perm_checker

        :param perm_set: list of permissions to check

        :param perm_checker: callable to check permissions against

        """

        self.obj_list = obj_list

        self.obj_attr = obj_attr

        self.perm_set = perm_set

        self.perm_checker = perm_checker

        self.extra_kwargs = extra_kwargs or {}

    def __len__(self):

        return len(self.obj_list)

    def __repr__(self):

        return '<%s (%s)>' % (self.__class__.__name__, self.__len__())

    def __iter__(self):

        for db_obj in self.obj_list:

            # check permission at this level

            name = getattr(db_obj, self.obj_attr, None)

            if not self.perm_checker(*self.perm_set)(

                    name, self.__class__.__name__, **self.extra_kwargs):

                continue

            yield db_obj

class RepoList(_PermCheckIterator):

    def __init__(self, db_repo_list, perm_level, extra_kwargs=None):

        super(RepoList, self).__init__(obj_list=db_repo_list,

                    obj_attr='repo_name', perm_set=[perm_level],

                    perm_checker=HasRepoPermissionLevel,

                    extra_kwargs=extra_kwargs)

class RepoGroupList(_PermCheckIterator):

    def __init__(self, db_repo_group_list, perm_level, extra_kwargs=None):

        super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,

                    obj_attr='group_name', perm_set=[perm_level],

                    perm_checker=HasRepoGroupPermissionLevel,

                    extra_kwargs=extra_kwargs)

class UserGroupList(_PermCheckIterator):

    def __init__(self, db_user_group_list, perm_level, extra_kwargs=None):

        super(UserGroupList, self).__init__(obj_list=db_user_group_list,

                    obj_attr='users_group_name', perm_set=[perm_level],

                    perm_checker=HasUserGroupPermissionLevel,

                    extra_kwargs=extra_kwargs)

class ScmModel(object):

    """

    Generic Scm Model

    """

    def __get_repo(self, instance):

        cls = Repository

        if isinstance(instance, cls):

            return instance

        elif isinstance(instance, int):

            return cls.get(instance)

        elif isinstance(instance, basestring):

            if instance.isdigit():

                return cls.get(int(instance))

            return cls.get_by_repo_name(instance)

        elif instance is not None:

            raise Exception('given object must be int, basestr or Instance'

                            ' of %s got %s' % (type(cls), type(instance)))

    @LazyProperty

    def repos_path(self):

        """

        Gets the repositories root path from database

        """

        q = Ui.query().filter(Ui.ui_key == '/').one()

        return q.ui_value

    def repo_scan(self, repos_path=None):

        """

        Listing of repositories in given path. This path should not be a

        :param repos_path: path to directory containing repositories

        """

        if repos_path is None:

            repos_path = self.repos_path

        log.info('scanning for repositories in %s', repos_path)

        baseui = make_ui()

        repos = {}

        for name, path in get_filesystem_repos(repos_path):

            # name need to be decomposed and put back together using the /

            # since this is internal storage separator for kallithea

            name = Repository.normalize_repo_name(name)

            try:

                if name in repos:

                    raise RepositoryError('Duplicate repository name %s '

                                          'found in %s' % (name, path))

                else:

                    klass = get_backend(path[0])

                    if path[0] == 'hg' and path[0] in BACKENDS:

                        repos[name] = klass(safe_str(path[1]), baseui=baseui)

                    if path[0] == 'git' and path[0] in BACKENDS:

                        repos[name] = klass(path[1])

            except OSError:

                continue

        log.debug('found %s paths with repositories', len(repos))

        return repos

    def get_repos(self, repos):

        """Return the repos the user has access to"""

        return RepoList(repos, perm_level='read')

    def get_repo_groups(self, groups=None):

        """Return the repo groups the user has access to

        If no groups are specified, use top level groups.

        """

        if groups is None:

            groups = RepoGroup.query() \

                .filter(RepoGroup.parent_group_id == None).all()

        return RepoGroupList(groups, perm_level='read')

    def mark_for_invalidation(self, repo_name):

        """

        Mark caches of this repo invalid in the database.

        :param repo_name: the repo for which caches should be marked invalid

        """

        log.debug("Marking %s as invalidated and update cache", repo_name)

        repo = Repository.get_by_repo_name(repo_name)

        if repo is not None:

            repo.set_invalidate()

            repo.update_changeset_cache()

    def toggle_following_repo(self, follow_repo_id, user_id):

        f = UserFollowing.query() \

            .filter(UserFollowing.follows_repository_id == follow_repo_id) \

            .filter(UserFollowing.user_id == user_id).scalar()

        if f is not None:

            try:

                Session().delete(f)

                action_logger(UserTemp(user_id),

                              'stopped_following_repo',

                              RepoTemp(follow_repo_id))

                return

            except Exception:

                log.error(traceback.format_exc())

                raise

        try:

            f = UserFollowing()

            f.user_id = user_id

            f.follows_repository_id = follow_repo_id

            Session().add(f)

            action_logger(UserTemp(user_id),

                          'started_following_repo',

                          RepoTemp(follow_repo_id))

        except Exception:

            log.error(traceback.format_exc())

            raise

    def toggle_following_user(self, follow_user_id, user_id):

        f = UserFollowing.query() \

            .filter(UserFollowing.follows_user_id == follow_user_id) \

            .filter(UserFollowing.user_id == user_id).scalar()

        if f is not None:

            try:

                Session().delete(f)

                return

            except Exception:

                log.error(traceback.format_exc())

                raise

        try:

            f = UserFollowing()

            f.user_id = user_id

            f.follows_user_id = follow_user_id

            Session().add(f)

        except Exception:

            log.error(traceback.format_exc())

            raise

    def is_following_repo(self, repo_name, user_id):

        r = Repository.query() \

            .filter(Repository.repo_name == repo_name).scalar()

        f = UserFollowing.query() \

            .filter(UserFollowing.follows_repository == r) \

            .filter(UserFollowing.user_id == user_id).scalar()

        return f is not None

    def is_following_user(self, username, user_id):

        u = User.get_by_username(username)

        f = UserFollowing.query() \

            .filter(UserFollowing.follows_user == u) \

            .filter(UserFollowing.user_id == user_id).scalar()

        return f is not None

    def get_followers(self, repo):

        repo = Repository.guess_instance(repo)

        return UserFollowing.query() \

                .filter(UserFollowing.follows_repository == repo).count()

    def get_forks(self, repo):

        repo = Repository.guess_instance(repo)

        return Repository.query() \

                .filter(Repository.fork == repo).count()

    def get_pull_requests(self, repo):

        repo = Repository.guess_instance(repo)

        return PullRequest.query() \

                .filter(PullRequest.other_repo == repo) \

                .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()

    def mark_as_fork(self, repo, fork, user):

        repo = self.__get_repo(repo)

        fork = self.__get_repo(fork)

        if fork and repo.repo_id == fork.repo_id:

            raise Exception("Cannot set repository as fork of itself")

        if fork and repo.repo_type != fork.repo_type:

            raise RepositoryError("Cannot set repository as fork of repository with other type")

        repo.fork = fork

        return repo

    def _handle_push(self, repo, username, ip_addr, action, repo_name, revisions):

        """

        Handle that the repository has changed.

        Adds an action log entry with the new revisions, and the head revision

        cache and in-memory caches are invalidated/updated.

        :param username: username who pushes

        :param action: push/push_local/push_remote

        :param repo_name: name of repo

        :param revisions: list of revisions that we pushed

        """

        set_hook_environment(username, ip_addr, repo_name, repo_alias=repo.alias, action=action)

        process_pushed_raw_ids(revisions) # also calls mark_for_invalidation

    def _get_IMC_module(self, scm_type):

        """

        Returns InMemoryCommit class based on scm_type

        :param scm_type:

        """

        if scm_type == 'hg':

            from kallithea.lib.vcs.backends.hg import MercurialInMemoryChangeset

            return MercurialInMemoryChangeset

        if scm_type == 'git':

            from kallithea.lib.vcs.backends.git import GitInMemoryChangeset

            return GitInMemoryChangeset

        raise Exception('Invalid scm_type, must be one of hg,git got %s'

                        % (scm_type,))

    def pull_changes(self, repo, username, ip_addr, clone_uri=None):