message=message,

                    nodes=nodes,

                    parent_cs=c.cs,

                    author=author,

                )

                webutils.flash(_('Successfully committed to %s') % node_path,

                        category='success')

            except NonRelativePathError as e:

                webutils.flash(_('Location must be relative path and must not '

                          'contain .. in path'), category='warning')

                raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,

                                    revision='tip'))

            except (NodeError, NodeAlreadyExistsError) as e:

                webutils.flash(_(e), category='error')

            except Exception:

                log.error(traceback.format_exc())

                webutils.flash(_('Error occurred during commit'), category='error')

            raise HTTPFound(location=url('changeset_home',

                                repo_name=c.repo_name, revision='tip'))

        return base.render('files/files_add.html')

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    def archivefile(self, repo_name, fname):

        fileformat = None

        revision = None

        ext = None

        subrepos = request.GET.get('subrepos') == 'true'

        for a_type, ext_data in settings.ARCHIVE_SPECS.items():

            archive_spec = fname.split(ext_data[1])

            if len(archive_spec) == 2 and archive_spec[1] == '':

                fileformat = a_type or ext_data[1]

                revision = archive_spec[0]

                ext = ext_data[1]

        try:

            dbrepo = RepoModel().get_by_repo_name(repo_name)

            if not dbrepo.enable_downloads:

                return _('Downloads disabled') # TODO: do something else?

            if c.db_repo_scm_instance.alias == 'hg':

                # patch and reset hooks section of UI config to not run any

                # hooks on fetching archives with subrepos

                for k, v in c.db_repo_scm_instance._repo.ui.configitems('hooks'):

                    c.db_repo_scm_instance._repo.ui.setconfig('hooks', k, None)

            cs = c.db_repo_scm_instance.get_changeset(revision)

            content_type = settings.ARCHIVE_SPECS[fileformat][0]

        except ChangesetDoesNotExistError:

            return _('Unknown revision %s') % revision

        except EmptyRepositoryError:

            return _('Empty repository')

        except (ImproperArchiveTypeError, KeyError):

            return _('Unknown archive type')

        rev_name = cs.raw_id[:12]

        archive_name = '%s-%s%s' % (repo_name.replace('/', '_'), rev_name, ext)

        archive_path = None

        cached_archive_path = None

        archive_cache_dir = kallithea.CONFIG.get('archive_cache_dir')

        if archive_cache_dir and not subrepos: # TODO: subrepo caching?

            if not os.path.isdir(archive_cache_dir):

                os.makedirs(archive_cache_dir)

            cached_archive_path = os.path.join(archive_cache_dir, archive_name)

            if os.path.isfile(cached_archive_path):

                log.debug('Found cached archive in %s', cached_archive_path)

                archive_path = cached_archive_path

            else:

                log.debug('Archive %s is not yet cached', archive_name)

        if archive_path is None:

            # generate new archive

            fd, archive_path = tempfile.mkstemp()

            log.debug('Creating new temp archive in %s', archive_path)

            with os.fdopen(fd, 'wb') as stream:

                cs.fill_archive(stream=stream, kind=fileformat, subrepos=subrepos)

                # stream (and thus fd) has been closed by cs.fill_archive

            if cached_archive_path is not None:

                # we generated the archive - move it to cache

                log.debug('Storing new archive in %s', cached_archive_path)

                shutil.move(archive_path, cached_archive_path)

                archive_path = cached_archive_path

        def get_chunked_archive(archive_path):

            stream = open(archive_path, 'rb')

            while True:

                data = stream.read(16 * 1024)

                if not data:

                    break

                yield data

            stream.close()

            if archive_path != cached_archive_path:

                log.debug('Destroying temp archive %s', archive_path)

                os.remove(archive_path)

        userlog.action_logger(user=request.authuser,

                      action='user_downloaded_archive:%s' % (archive_name),

                      repo=repo_name, ipaddr=request.ip_addr, commit=True)

        response.content_disposition = str('attachment; filename=%s' % (archive_name))

        response.content_type = str(content_type)

        return get_chunked_archive(archive_path)

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    def diff(self, repo_name, f_path):

        ignore_whitespace_diff = h.get_ignore_whitespace_diff(request.GET)

        diff_context_size = h.get_diff_context_size(request.GET)

        diff2 = request.GET.get('diff2', '')

        diff1 = request.GET.get('diff1', '') or diff2

        c.action = request.GET.get('diff')

        c.no_changes = diff1 == diff2

        c.f_path = f_path

        c.big_diff = False

        fulldiff = request.GET.get('fulldiff')

        c.changes = OrderedDict()

        c.changes[diff2] = []

        # special case if we want a show rev only, it's impl here

        # to reduce JS and callbacks

        if request.GET.get('show_rev'):

            if asbool(request.GET.get('annotate', 'False')):

                _url = url('files_annotate_home', repo_name=c.repo_name,

                           revision=diff1, f_path=c.f_path)

            else:

                _url = url('files_home', repo_name=c.repo_name,

                           revision=diff1, f_path=c.f_path)

            raise HTTPFound(location=_url)

        try:

            if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:

                c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)

                try:

                    node1 = c.changeset_1.get_node(f_path)

                    if node1.is_dir():

                        raise NodeError('%s path is a %s not a file'

                                        % (node1, type(node1)))

                except NodeDoesNotExistError:

                    c.changeset_1 = EmptyChangeset(cs=diff1,

                                                   revision=c.changeset_1.revision,

                                                   repo=c.db_repo_scm_instance)

                    node1 = FileNode(f_path, '', changeset=c.changeset_1)

            else:

                c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)

                node1 = FileNode(f_path, '', changeset=c.changeset_1)

            if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:

                c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)

                try:

                    node2 = c.changeset_2.get_node(f_path)

                    if node2.is_dir():

                        raise NodeError('%s path is a %s not a file'

                                        % (node2, type(node2)))

                except NodeDoesNotExistError:

                    c.changeset_2 = EmptyChangeset(cs=diff2,

                                                   revision=c.changeset_2.revision,

                                                   repo=c.db_repo_scm_instance)

                    node2 = FileNode(f_path, '', changeset=c.changeset_2)

            else:

                c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)

                node2 = FileNode(f_path, '', changeset=c.changeset_2)

        except (RepositoryError, NodeError):

            log.error(traceback.format_exc())

            raise HTTPFound(location=url('files_home', repo_name=c.repo_name,

                                f_path=f_path))

        if c.action == 'download':

            raw_diff = diffs.get_gitdiff(node1, node2,

                                      ignore_whitespace=ignore_whitespace_diff,

                                      context=diff_context_size)

            diff_name = '%s_vs_%s.diff' % (diff1, diff2)

            response.content_type = 'text/plain'

            response.content_disposition = (

                'attachment; filename=%s' % diff_name

            )

            return raw_diff

        elif c.action == 'raw':

            raw_diff = diffs.get_gitdiff(node1, node2,

                                      ignore_whitespace=ignore_whitespace_diff,

                                      context=diff_context_size)

            response.content_type = 'text/plain'

            return raw_diff

        else:

            fid = h.FID(diff2, node2.path)

            diff_limit = None if fulldiff else self.cut_off_limit

                                         filenode_new=node2,

                                         diff_limit=diff_limit,

                                         ignore_whitespace=ignore_whitespace_diff,

                                         line_context=diff_context_size)

            c.file_diff_data = [(fid, fid, op, a_path, node2.path, diff, st)]

            return base.render('files/file_diff.html')

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    def diff_2way(self, repo_name, f_path):

        diff1 = request.GET.get('diff1', '')

        diff2 = request.GET.get('diff2', '')

        try:

            if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:

                c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)

                try:

                    node1 = c.changeset_1.get_node(f_path)

                    if node1.is_dir():

                        raise NodeError('%s path is a %s not a file'

                                        % (node1, type(node1)))

                except NodeDoesNotExistError:

                    c.changeset_1 = EmptyChangeset(cs=diff1,

                                                   revision=c.changeset_1.revision,

                                                   repo=c.db_repo_scm_instance)

                    node1 = FileNode(f_path, '', changeset=c.changeset_1)

            else:

                c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)

                node1 = FileNode(f_path, '', changeset=c.changeset_1)

            if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:

                c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)

                try:

                    node2 = c.changeset_2.get_node(f_path)

                    if node2.is_dir():

                        raise NodeError('%s path is a %s not a file'

                                        % (node2, type(node2)))

                except NodeDoesNotExistError:

                    c.changeset_2 = EmptyChangeset(cs=diff2,

                                                   revision=c.changeset_2.revision,

                                                   repo=c.db_repo_scm_instance)

                    node2 = FileNode(f_path, '', changeset=c.changeset_2)

            else:

                c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)

                node2 = FileNode(f_path, '', changeset=c.changeset_2)

        except ChangesetDoesNotExistError as e:

            msg = _('Such revision does not exist for this repository')

            webutils.flash(msg, category='error')

            raise HTTPNotFound()

        c.node1 = node1

        c.node2 = node2

        c.cs1 = c.changeset_1

        c.cs2 = c.changeset_2

        return base.render('files/diff_2way.html')

    def _get_node_history(self, cs, f_path, changesets=None):

        """

        get changesets history for given node

        :param cs: changeset to calculate history

        :param f_path: path for node to calculate history for

        :param changesets: if passed don't calculate history and take

            changesets defined in this list

        """

        # calculate history based on tip

        tip_cs = c.db_repo_scm_instance.get_changeset()

        if changesets is None:

            try:

                changesets = tip_cs.get_file_history(f_path)

            except (NodeDoesNotExistError, ChangesetError):

                # this node is not present at tip !

                changesets = cs.get_file_history(f_path)

        hist_l = []

        changesets_group = ([], _("Changesets"))

        branches_group = ([], _("Branches"))

        tags_group = ([], _("Tags"))

        for chs in changesets:

            # TODO: loop over chs.branches ... but that will not give all the bogus None branches for Git ...

            _branch = chs.branch

            n_desc = '%s (%s)' % (h.show_id(chs), _branch)

            changesets_group[0].append((chs.raw_id, n_desc,))

        hist_l.append(changesets_group)

        for name, chs in c.db_repo_scm_instance.branches.items():

            branches_group[0].append((chs, name),)

        hist_l.append(branches_group)

        for name, chs in c.db_repo_scm_instance.tags.items():

            tags_group[0].append((chs, name),)

        hist_l.append(tags_group)

        return hist_l, changesets

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    @base.jsonify

    def nodelist(self, repo_name, revision, f_path):

        if request.environ.get('HTTP_X_PARTIAL_XHR'):

            cs = self.__get_cs(revision)

            _d, _f = ScmModel().get_nodes(repo_name, cs.raw_id, f_path,

                                          flat=False)

            return {'nodes': _d + _f}

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.diffs

~~~~~~~~~~~~~~~~~~~

Set of diffing helpers, previously part of vcs

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Dec 4, 2011

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import difflib

import logging

import re

from tg.i18n import ugettext as _

from kallithea.lib import webutils

from kallithea.lib.utils2 import safe_str

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.exceptions import VCSError

from kallithea.lib.vcs.nodes import FileNode, SubModuleNode

log = logging.getLogger(__name__)

def _safe_id(idstring):

    r"""Make a string safe for including in an id attribute.

    The HTML spec says that id attributes 'must begin with

    a letter ([A-Za-z]) and may be followed by any number

    of letters, digits ([0-9]), hyphens ("-"), underscores

    ("_"), colons (":"), and periods (".")'. These regexps

    are slightly over-zealous, in that they remove colons

    and periods unnecessarily.

    Whitespace is transformed into underscores, and then

    anything which is not a hyphen or a character that

    matches \w (alphanumerics and underscore) is removed.

    """

    # Transform all whitespace to underscore

    idstring = re.sub(r'\s', "_", idstring)

    # Remove everything that is not a hyphen or a member of \w

    idstring = re.sub(r'(?!-)\W', "", idstring).lower()

    return idstring

def as_html(table_class='code-difftable', line_class='line',

            old_lineno_class='lineno old', new_lineno_class='lineno new',

            no_lineno_class='lineno',

            code_class='code', parsed_lines=None):

    """

    Return given diff as html table with customized css classes

    """

    _html_empty = True

    _html = []

    _html.append('''<table class="%(table_class)s">\n''' % {

        'table_class': table_class

    })

    for file_info in parsed_lines:

        count_no_lineno = 0  # counter to allow comments on lines without new/old line numbers

        for chunk in file_info['chunks']:

            _html_empty = False

            for change in chunk:

                _html.append('''<tr class="%(lc)s %(action)s">\n''' % {

                    'lc': line_class,

                    'action': change['action']

                })

                if change['old_lineno'] or change['new_lineno']:

                    ###########################################################

                    # OLD LINE NUMBER

                    ###########################################################

                    anchor_old = "%(filename)s_o%(oldline_no)s" % {

                        'filename': _safe_id(file_info['filename']),

                        'oldline_no': change['old_lineno']

                    }

                    anchor_old_id = ''

                    if change['old_lineno']:

                        anchor_old_id = 'id="%s"' % anchor_old

                    _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {

                        'a_id': anchor_old_id,

                        'olc': old_lineno_class,

                    })

                    _html.append('''<a href="%(url)s" data-pseudo-content="%(label)s"></a>''' % {

                        'label': change['old_lineno'],

                        'url': '#%s' % anchor_old,

                    })

                    _html.append('''</td>\n''')

                    ###########################################################

                    # NEW LINE NUMBER

                    ###########################################################

                    anchor_new = "%(filename)s_n%(newline_no)s" % {

                        'filename': _safe_id(file_info['filename']),

                        'newline_no': change['new_lineno']

                    }

                    anchor_new_id = ''

                    if change['new_lineno']:

                        anchor_new_id = 'id="%s"' % anchor_new

                    _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {

                        'a_id': anchor_new_id,

                        'nlc': new_lineno_class

                    })

                    _html.append('''<a href="%(url)s" data-pseudo-content="%(label)s"></a>''' % {

                        'label': change['new_lineno'],

                        'url': '#%s' % anchor_new,

                    })

                    _html.append('''</td>\n''')

                else:

                    ###########################################################

                    # NO LINE NUMBER

                    ###########################################################

                    anchor = "%(filename)s_%(count_no_lineno)s" % {

                        'filename': _safe_id(file_info['filename']),

                        'count_no_lineno': count_no_lineno,

                    }

                    count_no_lineno += 1

                    _html.append('''\t<td id="%(anchor)s" class="%(olc)s" colspan="2">''' % {

                        'anchor': anchor,

                        'olc': no_lineno_class,

                    })

                    _html.append('''</td>\n''')

                ###########################################################

                # CODE

                ###########################################################

                _html.append('''\t<td class="%(cc)s">''' % {

                    'cc': code_class,

                })

                _html.append('''\n\t\t<div class="add-bubble"><div>&nbsp;</div></div><pre>%(code)s</pre>\n''' % {

                    'code': change['line']

                })

                _html.append('''\t</td>''')

                _html.append('''\n</tr>\n''')

    _html.append('''</table>''')

    if _html_empty:

        return None

    return ''.join(_html)

def wrap_to_table(html):

    """Given a string with html, return it wrapped in a table, similar to what

    as_html returns."""

    return '''\

              <table class="code-difftable">

                <tr class="line">

                <td class="lineno new"></td>

                <td class="code"><pre>%s</pre></td>

                </tr>

              </table>''' % html

                ignore_whitespace=True, line_context=3):

    """

    Checks for diff_limit and presents a message if the diff is too big.

    """

    if filenode_old is None:

        filenode_old = FileNode(filenode_new.path, '', EmptyChangeset())

    op = None

    a_path = filenode_old.path # default, might be overriden by actual rename in diff

    if filenode_old.is_binary or filenode_new.is_binary:

        html_diff = wrap_to_table(_('Binary file'))

        stats = (0, 0)

    elif diff_limit != -1 and (

            diff_limit is None or

            (filenode_old.size < diff_limit and filenode_new.size < diff_limit)):

        raw_diff = get_gitdiff(filenode_old, filenode_new,

                                ignore_whitespace=ignore_whitespace,

                                context=line_context)

        diff_processor = DiffProcessor(raw_diff)

        if diff_processor.parsed: # there should be exactly one element, for the specified file

            f = diff_processor.parsed[0]

            op = f['operation']

            a_path = f['old_filename']

        html_diff = as_html(parsed_lines=diff_processor.parsed)

        stats = diff_processor.stat()

    else:

        html_diff = wrap_to_table(_('Changeset was too big and was cut off, use '

                               'diff menu to display this diff'))

        stats = (0, 0)

    if not html_diff:

        submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]

        if submodules:

            html_diff = wrap_to_table(webutils.escape('Submodule %r' % submodules[0]))

        else:

            html_diff = wrap_to_table(_('No changes detected'))

    cs1 = filenode_old.changeset.raw_id

    cs2 = filenode_new.changeset.raw_id

    return cs1, cs2, a_path, html_diff, stats, op

def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):

    """

    Returns git style diff between given ``filenode_old`` and ``filenode_new``.

    """

    # make sure we pass in default context

    context = context or 3

    submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]

    if submodules:

        return b''

    for filenode in (filenode_old, filenode_new):

        if not isinstance(filenode, FileNode):

            raise VCSError("Given object should be FileNode object, not %s"

                % filenode.__class__)

    repo = filenode_new.changeset.repository

    old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)

    new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)

    vcs_gitdiff = get_diff(repo, old_raw_id, new_raw_id, filenode_new.path,

                           ignore_whitespace, context)

    return vcs_gitdiff

def get_diff(scm_instance, rev1, rev2, path=None, ignore_whitespace=False, context=3):

    """

    A thin wrapper around vcs lib get_diff.

    """

    try:

        return scm_instance.get_diff(rev1, rev2, path=path,

                                     ignore_whitespace=ignore_whitespace, context=context)

    except MemoryError:

        webutils.flash('MemoryError: Diff is too big', category='error')

        return b''

NEW_FILENODE = 1

DEL_FILENODE = 2

MOD_FILENODE = 3

RENAMED_FILENODE = 4

COPIED_FILENODE = 5

CHMOD_FILENODE = 6

BIN_FILENODE = 7

class DiffProcessor(object):

    """

    Give it a unified or git diff and it returns a list of the files that were

    mentioned in the diff together with a dict of meta information that

    can be used to render it in a HTML template.

    """

    _diff_git_re = re.compile(b'^diff --git', re.MULTILINE)

    def __init__(self, diff, vcs='hg', diff_limit=None, inline_diff=True):

        """

        :param diff:   a text in diff format

        :param vcs: type of version control hg or git

        :param diff_limit: define the size of diff that is considered "big"

            based on that parameter cut off will be triggered, set to None

            to show full diff

        """

        if not isinstance(diff, bytes):

            raise Exception('Diff must be bytes - got %s' % type(diff))

        self._diff = memoryview(diff)

        self.adds = 0

        self.removes = 0

        self.diff_limit = diff_limit

        self.limited_diff = False

        self.vcs = vcs

        self.parsed = self._parse_gitdiff(inline_diff=inline_diff)

    def _parse_gitdiff(self, inline_diff):

        """Parse self._diff and return a list of dicts with meta info and chunks for each file.

        Might set limited_diff.

        Optionally, do an extra pass and to extra markup of one-liner changes.

        """

        _files = [] # list of dicts with meta info and chunks

        starts = [m.start() for m in self._diff_git_re.finditer(self._diff)]

        starts.append(len(self._diff))

        for start, end in zip(starts, starts[1:]):

            if self.diff_limit and end > self.diff_limit:

                self.limited_diff = True

                continue

            head, diff_lines = _get_header(self.vcs, self._diff[start:end])

            op = None

            stats = {

                'added': 0,

                'deleted': 0,

                'binary': False,

                'ops': {},

            }

            if head['deleted_file_mode']:

                op = 'removed'

                stats['binary'] = True

                stats['ops'][DEL_FILENODE] = 'deleted file'

            elif head['new_file_mode']:

                op = 'added'

                stats['binary'] = True

                stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']

            else:  # modify operation, can be cp, rename, chmod

                # CHMOD

                if head['new_mode'] and head['old_mode']:

                    op = 'modified'

                    stats['binary'] = True

                    stats['ops'][CHMOD_FILENODE] = ('modified file chmod %s => %s'

                                        % (head['old_mode'], head['new_mode']))

                # RENAME

                if (head['rename_from'] and head['rename_to']

                      and head['rename_from'] != head['rename_to']):

                    op = 'renamed'

                    stats['binary'] = True

                    stats['ops'][RENAMED_FILENODE] = ('file renamed from %s to %s'

                                    % (head['rename_from'], head['rename_to']))

                # COPY

                if head.get('copy_from') and head.get('copy_to'):

                    op = 'modified'

                    stats['binary'] = True

                    stats['ops'][COPIED_FILENODE] = ('file copied from %s to %s'

                                        % (head['copy_from'], head['copy_to']))

                # FALL BACK: detect missed old style add or remove

                if op is None:

                    if not head['a_file'] and head['b_file']:

                        op = 'added'

                        stats['binary'] = True

                        stats['ops'][NEW_FILENODE] = 'new file'

                    elif head['a_file'] and not head['b_file']:

                        op = 'removed'

                        stats['binary'] = True

                        stats['ops'][DEL_FILENODE] = 'deleted file'

                # it's not ADD not DELETE

                if op is None:

                    op = 'modified'

                    stats['binary'] = True

                    stats['ops'][MOD_FILENODE] = 'modified file'

            # a real non-binary diff

            if head['a_file'] or head['b_file']:

                chunks, added, deleted = _parse_lines(diff_lines)