:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import datetime

import os

import re

import shutil

import click

import kallithea.bin.kallithea_cli_base as cli_base

from kallithea.lib.utils import REMOVED_REPO_PAT, repo2db_mapper

from kallithea.model.db import Repository, Ui

from kallithea.model.meta import Session

from kallithea.model.scm import ScmModel

@cli_base.register_command(config_file_initialize_app=True)

@click.option('--remove-missing', is_flag=True,

        help='Remove missing repositories from the Kallithea database.')

def repo_scan(remove_missing):

    """Scan filesystem for repositories.

    Search the configured repository root for new repositories and add them

    In normal operation, Kallithea will keep caches up-to-date

    automatically. However, if repositories are externally modified, e.g. by

    a direct push via the filesystem rather than via a Kallithea URL,

    Kallithea is not aware of it. In this case, you should manually run this

    command to update the repository cache.

    If no repositories are specified, the caches of all repositories are

    updated.

    """

    if not repositories:

        repo_list = Repository.query().all()

    else:

        repo_list = list(Repository.query()

                        .filter(Repository.repo_name.in_(repo_names)))

    for repo in repo_list:

        # update latest revision metadata in database

        repo.update_changeset_cache()

        # invalidate in-memory VCS object cache... will be repopulated on

        # first access

        repo.set_invalidate()

    Session().commit()

        if c.gist.is_expired:

            log.error('Gist expired at %s',

                      time_to_datetime(c.gist.gist_expires))

            raise HTTPNotFound()

        try:

            c.file_changeset, c.files = GistModel().get_gist_files(gist_id,

                                                            revision=revision)

        except VCSError:

            log.error(traceback.format_exc())

            raise HTTPNotFound()

        if format == 'raw':

            response.content_type = 'text/plain'

            return content

        return render('admin/gists/show.html')

    @LoginRequired()

    def edit(self, gist_id, format='html'):

        c.gist = Gist.get_or_404(gist_id)

        if c.gist.is_expired:

            log.error('Gist expired at %s',

                      time_to_datetime(c.gist.gist_expires))

            raise HTTPNotFound()

import formencode

from formencode import htmlfill

from tg import app_globals, request

from tg import tmpl_context as c

from tg.i18n import ugettext as _

from tg.i18n import ungettext

from webob.exc import HTTPForbidden, HTTPFound, HTTPInternalServerError, HTTPNotFound

from kallithea.config.routing import url

from kallithea.lib import helpers as h

from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoGroupPermissionLevelDecorator, LoginRequired

from kallithea.lib.base import BaseController, render

from kallithea.model.db import RepoGroup, Repository

from kallithea.model.forms import RepoGroupForm, RepoGroupPermsForm

from kallithea.model.meta import Session

from kallithea.model.repo import RepoModel

from kallithea.model.repo_group import RepoGroupModel

from kallithea.model.scm import AvailableRepoGroupChoices, RepoGroupList

log = logging.getLogger(__name__)

class RepoGroupsController(BaseController):

        repo_group_name = lambda repo_group_name, children_groups: (

            template.get_def("repo_group_name")

            .render_unicode(repo_group_name, children_groups, _=_, h=h, c=c)

        )

        repo_group_actions = lambda repo_group_id, repo_group_name, gr_count: (

            template.get_def("repo_group_actions")

            .render_unicode(repo_group_id, repo_group_name, gr_count, _=_, h=h, c=c,

                    ungettext=ungettext)

        )

        for repo_gr in group_iter:

            repo_count = repo_gr.repositories.count()

            repo_groups_data.append({

                "raw_name": repo_gr.group_name,

                "group_name": repo_group_name(repo_gr.group_name, children_groups),

                "desc": h.escape(repo_gr.group_description),

                "repos": repo_count,

                "owner": h.person(repo_gr.owner),

                "action": repo_group_actions(repo_gr.group_id, repo_gr.group_name,

                                             repo_count)

            })

        c.data = {

        action = 'user_commented_revision:%s' % revision

    action_logger(request.authuser, action, c.db_repo, request.ip_addr)

    if pull_request and close_pr:

        PullRequestModel().close_pull_request(pull_request_id)

        action_logger(request.authuser,

                      'user_closed_pull_request:%s' % pull_request_id,

                      c.db_repo, request.ip_addr)

    Session().commit()

    data = {

    }

    if comment is not None:

        c.comment = comment

        data.update(comment.get_dict())

        data.update({'rendered_text':

                     render('changeset/changeset_comment_block.html')})

    return data

def delete_cs_pr_comment(repo_name, comment_id):

    """Delete a comment from a changeset or pull request"""

    co = ChangesetComment.get_or_404(comment_id)

        _url = h.canonical_url('changeset_home', repo_name=c.db_repo.repo_name,

                   revision=cs.raw_id)

        desc_msg.append('changeset: <a href="%s">%s</a>' % (_url, cs.raw_id[:8]))

        desc_msg.append('<pre>')

        desc_msg.append(h.urlify_text(cs.message))

        desc_msg.append('\n')

        desc_msg.extend(changes)

        if str2bool(CONFIG.get('rss_include_diff', False)):

            desc_msg.append('\n\n')

            desc_msg.append(safe_unicode(raw_diff))

        desc_msg.append('</pre>')

    def _feed(self, repo_name, feeder):

        """Produce a simple feed"""

        @cache_region('long_term', '_get_feed_from_cache')

        def _get_feed_from_cache(*_cache_keys):  # parameters are not really used - only as caching key

            header = dict(

                title=_('%s %s feed') % (c.site_name, repo_name),

                link=h.canonical_url('summary_home', repo_name=repo_name),

                description=_('Changes on %s repository') % repo_name,

            )

import ipaddr

from decorator import decorator

from sqlalchemy.orm import joinedload

from sqlalchemy.orm.exc import ObjectDeletedError

from tg import request

from tg.i18n import ugettext as _

from webob.exc import HTTPForbidden, HTTPFound

from kallithea import __platform__, is_unix, is_windows

from kallithea.config.routing import url

from kallithea.lib.caching_query import FromCache

from kallithea.lib.utils import conditional_cache, get_repo_group_slug, get_repo_slug, get_user_group_slug

from kallithea.lib.vcs.utils.lazy import LazyProperty

from kallithea.model.db import (

    Permission, RepoGroup, Repository, User, UserApiKeys, UserGroup, UserGroupMember, UserGroupRepoGroupToPerm, UserGroupRepoToPerm, UserGroupToPerm, UserGroupUserGroupToPerm, UserIpMap, UserToPerm)

from kallithea.model.meta import Session

from kallithea.model.user import UserModel

log = logging.getLogger(__name__)

class PasswordGenerator(object):

    """

        return request.authuser.has_user_group_permission_level(user_group_name, self.required_perm, purpose)

#==============================================================================

# SPECIAL VERSION TO HANDLE MIDDLEWARE AUTH

#==============================================================================

class HasPermissionAnyMiddleware(object):

    def __init__(self, *perms):

        self.required_perms = set(perms)

    def __call__(self, authuser, repo_name, purpose=None):

        try:

            ok = authuser.permissions['repositories'][repo_name] in self.required_perms

        except KeyError:

            ok = False

        log.debug('Middleware check %s for %s for repo %s (%s): %s', authuser.username, self.required_perms, repo_name, purpose, ok)

        return ok

def check_ip_access(source_ip, allowed_ips=None):

    """

    Checks if source_ip is a subnet of any of allowed_ips.

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Created on Nov 17, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

from kallithea.lib import auth_modules

from kallithea.lib.compat import hybrid_property

from kallithea.model.db import Setting

log = logging.getLogger(__name__)

class KallitheaAuthPlugin(auth_modules.KallitheaExternalAuthPlugin):

    def __init__(self):

        pass

    @hybrid_property

    def name(self):

        # if cannot fetch username, it's a no-go for this plugin to proceed

        if not username:

            return None

        # old attrs fetched from Kallithea database

        admin = getattr(userobj, 'admin', False)

        email = environ.get(settings.get('email_header'), getattr(userobj, 'email', ''))

        firstname = environ.get(settings.get('firstname_header'), getattr(userobj, 'firstname', ''))

        lastname = environ.get(settings.get('lastname_header'), getattr(userobj, 'lastname', ''))

        user_data = {

            'username': username,

            'groups': [],

            'email': email or '',

            'admin': admin or False,

            'extern_name': username,

        }

        log.info('user `%s` authenticated correctly', user_data['username'])

        return user_data

    def get_managed_fields(self):

        fields = ['username', 'password']

        if(Setting.get_by_name('auth_container_email_header').app_settings_value):

:created_on: Created on Nov 17, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

from kallithea.lib import auth_modules

from kallithea.lib.compat import hybrid_property

from kallithea.lib.exceptions import LdapConnectionError, LdapImportError, LdapPasswordError, LdapUsernameError

log = logging.getLogger(__name__)

try:

    import ldap

    import ldap.filter

except ImportError:

    # means that python-ldap is not installed

    ldap = None

            log.debug('Got ldap DN response %s', user_dn)

            get_ldap_attr = lambda k: ldap_attrs.get(settings.get(k), [''])[0]

            # old attrs fetched from Kallithea database

            admin = getattr(userobj, 'admin', False)

            email = getattr(userobj, 'email', '')

            firstname = getattr(userobj, 'firstname', '')

            lastname = getattr(userobj, 'lastname', '')

            user_data = {

                'username': username,

                'groups': [],

                'email': get_ldap_attr('attr_email') or email,

                'admin': admin,

                'extern_name': user_dn,

            }

            log.info('user %s authenticated correctly', user_data['username'])

            return user_data

        except LdapUsernameError:

            log.info('Error authenticating %s with LDAP: User not found', username)

        except LdapPasswordError:

            log.info('Error authenticating %s with LDAP: Password error', username)

    c.repository_forks: number of forks

    c.repository_following: weather the current user is following the current repo

    """

    def _before(self, *args, **kwargs):

        super(BaseRepoController, self)._before(*args, **kwargs)

        if c.repo_name:  # extracted from routes

            _dbr = Repository.get_by_repo_name(c.repo_name)

            if not _dbr:

                return

            log.debug('Found repository in database %s with state `%s`',

            route = getattr(request.environ.get('routes.route'), 'name', '')

            # allow to delete repos that are somehow damages in filesystem

            if route in ['delete_repo']:

                return

            if _dbr.repo_state in [Repository.STATE_PENDING]:

                if route in ['repo_creating_home']:

                    return

                check_url = url('repo_creating_home', repo_name=c.repo_name)

                raise webob.exc.HTTPFound(location=check_url)

    it's safe to use in urls

    :param raw_id:

    :param path:

    """

    return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12])

class _FilesBreadCrumbs(object):

    def __call__(self, repo_name, rev, paths):

        url_l = [link_to(repo_name, url('files_home',

                                        repo_name=repo_name,

                                        revision=rev, f_path=''),

                         class_='ypjax-link')]

        paths_l = paths.split('/')

        for cnt, p in enumerate(paths_l):

            if p != '':

                url_l.append(link_to(p,

                                     url('files_home',

                                         repo_name=repo_name,

                                         revision=rev,

                                         f_path='/'.join(paths_l[:cnt + 1])

def changed_tooltip(nodes):

    """

    Generates a html string for changed nodes in changeset page.

    It limits the output to 30 entries

    :param nodes: LazyNodesGenerator

    """

    if nodes:

        pref = ': <br/> '

        suf = ''

        if len(nodes) > 30:

            suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)

                                             for x in nodes[:30]]) + suf)

    else:

        return ': ' + _('No files')

def fancy_file_stats(stats):

    """

    Displays a fancy two colored bar for number of added/deleted

    lines of code on file

    :param stats: two element list of added/deleted lines of code

    """

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import time

import mercurial.scmutil

from kallithea.lib import helpers as h

from kallithea.lib.exceptions import UserCreationError

from kallithea.lib.utils import action_logger, make_ui

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.model.db import Repository, User

def _get_scm_size(alias, root_path):

    if not alias.startswith('.'):

        alias += '.'

    size_scm, size_root = 0, 0

    for path, dirs, files in os.walk(safe_str(root_path)):

        if path.find(alias) != -1:

            for f in files:

    connect to the database.

    """

    import paste.deploy

    import kallithea.config.middleware

    extras = get_hook_environment()

    path_to_ini_file = extras['config']

    kallithea.CONFIG = paste.deploy.appconfig('config:' + path_to_ini_file)

    #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging

    kallithea.config.middleware.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf)

    # fix if it's not a bare repo

    if repo_path.endswith(os.sep + '.git'):

        repo_path = repo_path[:-5]

    repo = Repository.get_by_full_path(repo_path)

    if not repo:

        raise OSError('Repository %s not found in database'

                      % (safe_str(repo_path)))

    baseui = make_ui()

    return baseui, repo

        self.index_location = index_location

        if not index_location:

            raise Exception('You have to provide index location')

        self.repo_location = repo_location

        if not repo_location:

            raise Exception('You have to provide repositories location')

        self.repo_paths = ScmModel().repo_scan(self.repo_location)

        # filter repo list

        if repo_list:

            self.filtered_repo_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_list:

                    self.filtered_repo_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_paths

        # filter update repo list

        self.filtered_repo_update_paths = {}

        if repo_update_list:

            self.filtered_repo_update_paths = {}

            for repo_name, repo in self.repo_paths.items():

            log.info('Cannot run incremental index since it does not '

                     'yet exist - running full build')

        elif not exists_in(self.index_location, IDX_NAME):

            log.info('Running full index build, as the file content '

                     'index does not exist')

        elif not exists_in(self.index_location, CHGSET_IDX_NAME):

            log.info('Running full index build, as the changeset '

                     'index does not exist')

        else:

            self.initial = False

    def _get_index_revision(self, repo):

        landing_rev = 'tip'

        if db_repo:

            _rev_type, _rev = db_repo.landing_rev

            landing_rev = _rev

        return landing_rev

    def _get_index_changeset(self, repo, index_rev=None):

        if not index_rev:

            index_rev = self._get_index_revision(repo)

        cs = repo.get_changeset(index_rev)

        return cs

                u_content = u''

            else:

                log.debug('    >> %s', path)

                u_content = safe_unicode(bytes_content)

                indexed_w_content += 1

        else:

            log.debug('    >> %s - not indexable', path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        writer.add_document(

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

    def index_changesets(self, writer, repo_name, repo, start_rev=None):

        """

        Add all changeset in the vcs repo starting at start_rev

        to the index writer

        :param writer: the whoosh index writer to add to

            start_rev = repo[0].raw_id

        log.debug('Indexing changesets in %s, starting at rev %s',

                  repo_name, start_rev)

        indexed = 0

        cs_iter = repo.get_changesets(start=start_rev)

        total = len(cs_iter)

        for cs in cs_iter:

            indexed += 1

            log.debug('    >> %s %s/%s', cs, indexed, total)

            writer.add_document(

                date=cs._timestamp,

                author=cs.author,

                message=cs.message,

                last=cs.last,

            )

        return indexed

    def index_files(self, file_idx_writer, repo_name, repo):

        """

        Index files for given repo_name

        :param file_idx_writer: the whoosh index writer to add to

        :param repo_name: name of the repository we're indexing

        :param repo: instance of vcs repo

        """

            # Assume we have a function that gathers the filenames of the

            # documents to be indexed

            ri_cnt_total = 0  # indexed

            riwc_cnt_total = 0  # indexed with content

            for repo_name, repo in sorted(self.repo_paths.items()):

                log.debug('Updating file index for repo %s', repo_name)

                # skip indexing if there aren't any revisions

                if len(repo) < 1:

                    continue

                ri_cnt = 0   # indexed

                riwc_cnt = 0  # indexed with content

                for path in self.get_paths(repo):

                    if path in to_index or path not in indexed_paths:

                        # This is either a file that's changed, or a new file

                        # that wasn't indexed before. So index it!

                        i, iwc = self.add_doc(writer, path, repo, repo_name)

                        writer_is_dirty = True

                        ri_cnt += i

                        ri_cnt_total += 1

                        riwc_cnt += iwc

                        riwc_cnt_total += iwc

                log.debug('added %s files %s with content for repo %s',

                             ri_cnt + riwc_cnt, riwc_cnt, repo.path

                )

            log.debug('indexed %s files in total and %s with content',

This file was forked by the Kallithea project in July 2014.

"""

import logging

import os

import socket

import traceback

from webob import Request, Response, exc

import kallithea

from kallithea.lib.vcs import subprocessio

log = logging.getLogger(__name__)

class FileWrapper(object):

    def __init__(self, fd, content_length):

        self.fd = fd

        self.content_length = content_length

        self.remain = content_length

            raise OSError('%s missing git signature' % content_path)

        self.content_path = content_path

        self.valid_accepts = ['application/x-%s-result' %

                              c for c in self.commands]

        self.repo_name = repo_name

    def _get_fixedpath(self, path):

        """

        Small fix for repo_path

        :param path:

        """

        assert path.startswith('/' + self.repo_name + '/')

        return path[len(self.repo_name) + 2:].strip('/')

    def inforefs(self, req, environ):

        """

        WSGI Response producer for HTTP GET Git Smart

        HTTP /info/refs request.

        """

        git_command = req.GET.get('service')

        if git_command not in self.commands:

            log.debug('command %s not allowed', git_command)

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import re

from kallithea.lib.base import BaseVCSController, get_path_info

from kallithea.lib.hooks import log_pull_action

from kallithea.lib.middleware.pygrack import make_wsgi_app

from kallithea.lib.utils import make_ui

from kallithea.model.db import Repository

log = logging.getLogger(__name__)

GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)$')

cmd_mapping = {

    'git-receive-pack': 'push',

    'git-upload-pack': 'pull',

    scm_alias = 'git'

    @classmethod

    def parse_request(cls, environ):

        path_info = get_path_info(environ)

        m = GIT_PROTO_PAT.match(path_info)

        if m is None:

            return None

        class parsed_request(object):

            # See https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols#_the_smart_protocol

            cmd = m.group(2)

            query_string = environ['QUERY_STRING']

            if cmd == 'info/refs' and query_string.startswith('service='):

                service = query_string.split('=', 1)[1]

                action = cmd_mapping.get(service)

            else:

                service = None

                action = cmd_mapping.get(cmd)

        return parsed_request

"""

import logging

import os

import urllib.parse

import mercurial.hgweb

from kallithea.lib.base import BaseVCSController, get_path_info

from kallithea.lib.utils import make_ui

log = logging.getLogger(__name__)

def get_header_hgarg(environ):

    """Decode the special Mercurial encoding of big requests over multiple headers.

    >>> get_header_hgarg({})

    ''

    >>> get_header_hgarg({'HTTP_X_HGARG_0': ' ', 'HTTP_X_HGARG_1': 'a','HTTP_X_HGARG_2': '','HTTP_X_HGARG_3': 'b+c %20'})

    'ab+c %20'

    """

    scm_alias = 'hg'

    @classmethod

    def parse_request(cls, environ):

        http_accept = environ.get('HTTP_ACCEPT', '')

        if not http_accept.startswith('application/mercurial'):

            return None

        path_info = get_path_info(environ)