context=line_context)

        diff_processor = DiffProcessor(raw_diff, html=True)

        if diff_processor.parsed: # there should be exactly one element, for the specified file

            f = diff_processor.parsed[0]

            op = f['operation']

            a_path = f['old_filename']

        html_diff = as_html(parsed_lines=diff_processor.parsed)

        stats = diff_processor.stat()

    else:

        html_diff = wrap_to_table(_('Changeset was too big and was cut off, use '

                               'diff menu to display this diff'))

        stats = (0, 0)

    if not html_diff:

        submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]

        if submodules:

            html_diff = wrap_to_table(webutils.escape('Submodule %r' % submodules[0]))

        else:

            html_diff = wrap_to_table(_('No changes detected'))

    cs1 = filenode_old.changeset.raw_id

    cs2 = filenode_new.changeset.raw_id

    return cs1, cs2, a_path, html_diff, stats, op

def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):

    """

    Returns git style diff between given ``filenode_old`` and ``filenode_new``.

    """

    # make sure we pass in default context

    context = context or 3

    submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]

    if submodules:

        return b''

    for filenode in (filenode_old, filenode_new):

        if not isinstance(filenode, FileNode):

            raise VCSError("Given object should be FileNode object, not %s"

                % filenode.__class__)

    repo = filenode_new.changeset.repository

    old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)

    new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)

    vcs_gitdiff = get_diff(repo, old_raw_id, new_raw_id, filenode_new.path,

                           ignore_whitespace, context)

    return vcs_gitdiff

def get_diff(scm_instance, rev1, rev2, path=None, ignore_whitespace=False, context=3):

    """

    A thin wrapper around vcs lib get_diff.

    """

    try:

        return scm_instance.get_diff(rev1, rev2, path=path,

                                     ignore_whitespace=ignore_whitespace, context=context)

    except MemoryError:

        webutils.flash('MemoryError: Diff is too big', category='error')

        return b''

NEW_FILENODE = 1

DEL_FILENODE = 2

MOD_FILENODE = 3

RENAMED_FILENODE = 4

COPIED_FILENODE = 5

CHMOD_FILENODE = 6

BIN_FILENODE = 7

class DiffProcessor(object):

    """

    Give it a unified or git diff and it returns a list of the files that were

    mentioned in the diff together with a dict of meta information that

    can be used to render it in a HTML template or as text.

    """

    _diff_git_re = re.compile(b'^diff --git', re.MULTILINE)

    def __init__(self, diff, vcs='hg', diff_limit=None, html=True):

        """

        :param diff:   a text in diff format

        :param vcs: type of version control hg or git

        :param diff_limit: define the size of diff that is considered "big"

            based on that parameter cut off will be triggered, set to None

            to show full diff

        """

        if not isinstance(diff, bytes):

            raise Exception('Diff must be bytes - got %s' % type(diff))

        self._diff = memoryview(diff)

        self.adds = 0

        self.removes = 0

        self.diff_limit = diff_limit

        self.limited_diff = False

        self.vcs = vcs

        self.parsed = self._parse_gitdiff(html=html)

    def _parse_gitdiff(self, html):

        """Parse self._diff and return a list of dicts with meta info and chunks for each file.

        Might set limited_diff.

        Optionally, do an extra pass and to extra markup of one-liner changes.

        """

        _files = [] # list of dicts with meta info and chunks

        starts = [m.start() for m in self._diff_git_re.finditer(self._diff)]

        starts.append(len(self._diff))

        for start, end in zip(starts, starts[1:]):

            if self.diff_limit and end > self.diff_limit:

                self.limited_diff = True

                continue

            head, diff_lines = _get_header(self.vcs, self._diff[start:end])

            op = None

            stats = {

                'added': 0,

                'deleted': 0,

                'binary': False,

                'ops': {},

            }

            if head['deleted_file_mode']:

                op = 'removed'

                stats['binary'] = True

                stats['ops'][DEL_FILENODE] = 'deleted file'

            elif head['new_file_mode']:

                op = 'added'

                stats['binary'] = True

                stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']

            else:  # modify operation, can be cp, rename, chmod

                # CHMOD

                if head['new_mode'] and head['old_mode']:

                    op = 'modified'

                    stats['binary'] = True

                    stats['ops'][CHMOD_FILENODE] = ('modified file chmod %s => %s'

                                        % (head['old_mode'], head['new_mode']))

                # RENAME

                if (head['rename_from'] and head['rename_to']

                      and head['rename_from'] != head['rename_to']):

                    op = 'renamed'

                    stats['binary'] = True

                    stats['ops'][RENAMED_FILENODE] = ('file renamed from %s to %s'

                                    % (head['rename_from'], head['rename_to']))

                # COPY

                if head.get('copy_from') and head.get('copy_to'):

                    op = 'modified'

                    stats['binary'] = True

                    stats['ops'][COPIED_FILENODE] = ('file copied from %s to %s'

                                        % (head['copy_from'], head['copy_to']))

                # FALL BACK: detect missed old style add or remove

                if op is None:

                    if not head['a_file'] and head['b_file']:

                        op = 'added'

                        stats['binary'] = True

                        stats['ops'][NEW_FILENODE] = 'new file'

                    elif head['a_file'] and not head['b_file']:

                        op = 'removed'

                        stats['binary'] = True

                        stats['ops'][DEL_FILENODE] = 'deleted file'

                # it's not ADD not DELETE

                if op is None:

                    op = 'modified'

                    stats['binary'] = True

                    stats['ops'][MOD_FILENODE] = 'modified file'

            # a real non-binary diff

            if head['a_file'] or head['b_file']:

                chunks, added, deleted = _parse_lines(diff_lines)

                stats['binary'] = False

                stats['added'] = added

                stats['deleted'] = deleted

                # explicit mark that it's a modified file

                if op == 'modified':

                    stats['ops'][MOD_FILENODE] = 'modified file'

            else:  # Git binary patch (or empty diff)

                # Git binary patch

                if head['bin_patch']:

                    stats['ops'][BIN_FILENODE] = 'binary diff not shown'

                chunks = []

            if op == 'removed' and chunks:

                # a way of seeing deleted content could perhaps be nice - but

                # not with the current UI

                chunks = []

            _files.append({

                'old_filename':     head['a_path'],

                'filename':         head['b_path'],

                'old_revision':     head['a_blob_id'],

                'new_revision':     head['b_blob_id'],

                'chunks':           chunks,

                'operation':        op,

                'stats':            stats,

            })

        if not html:

            return _files

        for diff_data in _files:

            for chunk in diff_data['chunks']:

                for change in chunk:

                    change['line'] = _escaper(change['line'])

                # highlight inline changes when one del is followed by one add

                lineiter = iter(chunk)

                try:

                    peekline = next(lineiter)

                    while True:

                        # find a first del line

                        while peekline['action'] != 'del':

                            peekline = next(lineiter)

                        delline = peekline

                        peekline = next(lineiter)

                        # if not followed by add, eat all following del lines

                        if peekline['action'] != 'add':

                            while peekline['action'] == 'del':

                                peekline = next(lineiter)

                            continue

                        # found an add - make sure it is the only one

                        addline = peekline

                        try:

                            peekline = next(lineiter)

                        except StopIteration:

                            # add was last line - ok

                            _highlight_inline_diff(delline, addline)

                            raise

                        if peekline['action'] != 'add':

                            # there was only one add line - ok

                            _highlight_inline_diff(delline, addline)

                except StopIteration:

                    pass

        return _files

    def stat(self):

        """

        Returns tuple of added, and removed lines for this instance

        """

        return self.adds, self.removes

_escape_re = re.compile(r'(&)|(<)|(>)|(\t)($)?|(\r)|( $)')

def _escaper(diff_line):

    r"""

    Do HTML escaping/markup of a single diff line (excluding first +/- column)

    >>> _escaper('foobar')

    'foobar'

    >>> _escaper('@foo & bar')

    '@foo & bar'

    >>> _escaper('foo < bar')

    'foo < bar'

    >>> _escaper('foo > bar')

    'foo > bar'

    >>> _escaper('<foo>')

    '<foo>'

    >>> _escaper('foo\tbar')

    'foo<u>\t</u>bar'

    >>> _escaper('foo\rbar\r')

    'foo<u class="cr"></u>bar<u class="cr"></u>'

    >>> _escaper('foo\t')

    'foo<u>\t</u><i></i>'

    >>> _escaper('foo ')

    'foo <i></i>'

    >>> _escaper('foo  ')

    'foo  <i></i>'

    >>> _escaper('')

    ''

    >>> _escaper(' ')

    ' <i></i>'

    >>> _escaper('\t')

    '<u>\t</u><i></i>'

    >>> _escaper('\t  ')

    '<u>\t</u>  <i></i>'

    >>> _escaper('  \t')

    '  <u>\t</u><i></i>'

    >>> _escaper('\t\t  ')

    '<u>\t</u><u>\t</u>  <i></i>'

    >>> _escaper('  \t\t')

    '  <u>\t</u><u>\t</u><i></i>'

    >>> _escaper('foo&bar<baz>  ')

    'foo&amp;bar&lt;baz&gt;  <i></i>'

    """

    def substitute(m):

        groups = m.groups()

        if groups[0]:

            return '&'

        if groups[1]:

            return '<'

        if groups[2]:

            return '>'

        if groups[3]:

            if groups[4] is not None:  # end of line

                return '<u>\t</u><i></i>'

            return '<u>\t</u>'

        if groups[5]:

            return '<u class="cr"></u>'

        if groups[6]:

            return ' <i></i>'

        assert False

    return _escape_re.sub(substitute, diff_line)

_git_header_re = re.compile(br"""

    ^diff[ ]--git[ ](?P<a_path_quote>"?)a/(?P<a_path>.+?)(?P=a_path_quote)[ ](?P<b_path_quote>"?)b/(?P<b_path>.+?)(?P=a_path_quote)\n

    (?:^old[ ]mode[ ](?P<old_mode>\d+)\n

       ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?

    (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%\n

       ^rename[ ]from[ ](?P<rename_from>.+)\n

       ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?

    (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?

    (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?

    (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)

        \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?

    (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?

    (?:^---[ ](?P<a_file_quote>"?)(a/(?P<a_file>.+?)(?P=a_file_quote)|/dev/null)\t?(?:\n|$))?

    (?:^\+\+\+[ ](?P<b_file_quote>"?)(b/(?P<b_file>.+?)(?P=b_file_quote)|/dev/null)\t?(?:\n|$))?

""", re.VERBOSE | re.MULTILINE)

_hg_header_re = re.compile(br"""

    ^diff[ ]--git[ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n

    (?:^old[ ]mode[ ](?P<old_mode>\d+)\n

       ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?

    (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%(?:\n|$))?

    (?:^rename[ ]from[ ](?P<rename_from>.+)\n

       ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?

    (?:^copy[ ]from[ ](?P<copy_from>.+)\n

       ^copy[ ]to[ ](?P<copy_to>.+)(?:\n|$))?

    (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?

    (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?

    (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)

        \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?

    (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?

    (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?

    (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?

""", re.VERBOSE | re.MULTILINE)

_header_next_check = re.compile(br'''(?!@)(?!literal )(?!delta )''')

_git_bs_escape_re = re.compile(r'\\(?:([^0-9])|([0-9]{3}))')

_git_bs_escape_dict = {'\\': '\\', '"': '"', 'r': '\r', 'n': '\n', 't': '\t'}

def _git_bs_unescape_m(m):

    c = m.group(1)

    if c is not None:

        return _git_bs_escape_dict.get(c) or ('\\' + c)

    return chr(int(m.group(2), 8))

def _get_header(vcs, diff_chunk):

    """

    Parses a Git diff for a single file (header and chunks) and returns a tuple with:

    1. A dict with meta info:

        a_path, b_path, similarity_index, rename_from, rename_to,

        old_mode, new_mode, new_file_mode, deleted_file_mode,

        a_blob_id, b_blob_id, b_mode, a_file, b_file

    2. An iterator yielding lines with simple HTML markup.

    """

    match = None

    if vcs == 'git':

        match = _git_header_re.match(diff_chunk)

    elif vcs == 'hg':

        match = _hg_header_re.match(diff_chunk)

    if match is None:

         {'added': 0,

          'deleted': 0,

          'binary': True,

          'ops': {NEW_FILENODE: 'new file 100644',

                  BIN_FILENODE: 'binary diff not shown'}}),

        ('img/baseline-20px.png', 'removed',

         {'added': 0,

          'deleted': 0,

          'binary': True,

          'ops': {DEL_FILENODE: 'deleted file',

                  BIN_FILENODE: 'binary diff not shown'}}),

        ('index.html', 'modified',

         {'added': 3,

          'deleted': 2,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('js/global.js', 'removed',

         {'added': 0,

          'deleted': 75,

          'binary': False,

          'ops': {DEL_FILENODE: 'deleted file'}}),

        ('js/jquery/hashgrid.js', 'added',

         {'added': 340,

          'deleted': 0,

          'binary': False,

          'ops': {NEW_FILENODE: 'new file 100755'}}),

        ('less/docs.less', 'modified',

         {'added': 34,

          'deleted': 0,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('less/scaffolding.less', 'modified',

         {'added': 1,

          'deleted': 3,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('readme.markdown', 'modified',

         {'added': 1,

          'deleted': 10,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

    ],

    'diff_with_diff_data.diff': [

        ('vcs/backends/base.py', 'modified',

         {'added': 18,

          'deleted': 2,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('vcs/backends/git/repository.py', 'modified',

         {'added': 46,

          'deleted': 15,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('vcs/backends/hg.py', 'modified',

         {'added': 22,

          'deleted': 3,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('vcs/tests/test_git.py', 'modified',

         {'added': 5,

          'deleted': 5,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

        ('vcs/tests/test_repository.py', 'modified',

         {'added': 174,

          'deleted': 2,

          'binary': False,

          'ops': {MOD_FILENODE: 'modified file'}}),

    ],

    'git_diff_modify_binary_file.diff': [

        ('file.name', 'modified',

         {'added': 0,

          'deleted': 0,

          'binary': True,

          'ops': {MOD_FILENODE: 'modified file',

                  BIN_FILENODE: 'binary diff not shown'}})

    ],

    'hg_diff_copy_file.diff': [

        ('file2', 'modified',

         {'added': 0,

          'deleted': 0,

          'binary': True,

          'ops': {COPIED_FILENODE: 'file copied from file1 to file2'}}),

    ],

    'hg_diff_copy_and_modify_file.diff': [

        ('file3', 'modified',

         {'added': 1,

          'deleted': 0,

          'binary': False,

          'ops': {COPIED_FILENODE: 'file copied from file2 to file3',

                  MOD_FILENODE: 'modified file'}}),

    ],

    'hg_diff_copy_and_chmod_file.diff': [

        ('file4', 'modified',

         {'added': 0,

          'deleted': 0,

          'binary': True,

          'ops': {COPIED_FILENODE: 'file copied from file3 to file4',

                  CHMOD_FILENODE: 'modified file chmod 100644 => 100755'}}),

    ],

    'hg_diff_copy_chmod_and_edit_file.diff': [

        ('file5', 'modified',

         {'added': 2,

          'deleted': 1,

          'binary': False,

          'ops': {COPIED_FILENODE: 'file copied from file4 to file5',

                  CHMOD_FILENODE: 'modified file chmod 100755 => 100644',

                  MOD_FILENODE: 'modified file'}}),

    ],

    'hg_diff_rename_space_cr.diff': [

        ('oh yes', 'renamed',

         {'added': 3,

          'deleted': 2,

          'binary': False,

          'ops': {RENAMED_FILENODE: 'file renamed from oh no to oh yes'}}),

    ],

    'git_diff_quoting.diff': [

        ('"foo"',

         'added',

         {'added': 1,

          'binary': False,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ("'foo'",

         'added',

         {'added': 1,

          'binary': False,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ("'foo'" '"foo"',

         'added',

         {'added': 1,

          'binary': False,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ('a\r\nb',  # Note: will be parsed correctly, but other parts of Kallithea can't handle it

         'added',

         {'added': 1,

          'binary': False,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ('foo\rfoo',  # Note: will be parsed correctly, but other parts of Kallithea can't handle it

         'added',

        {'added': 0,

         'binary': True,

         'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ('foo bar',

         'added',

         {'added': 1,

          'binary': False,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ('test',

         'added',

         {'added': 1,

          'binary': False,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ('esc\033foo',  # Note: will be parsed and handled correctly, but without good UI

         'added',

         {'added': 0,

          'binary': True,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

        ('tab\tfoo',  # Note: will be parsed and handled correctly, but without good UI

         'added',

         {'added': 0,

          'binary': True,

          'deleted': 0,

          'ops': {1: 'new file 100644'}}),

    ],

}

class TestDiffLib(base.TestController):

    @base.parametrize('diff_fixture', DIFF_FIXTURES)

    def test_diff(self, diff_fixture):

        raw_diff = fixture.load_resource(diff_fixture, strip=False)

        vcs = 'hg'

        if diff_fixture.startswith('git_'):

            vcs = 'git'

        diff_processor = DiffProcessor(raw_diff, vcs=vcs)

        data = [(x['filename'], x['operation'], x['stats']) for x in diff_processor.parsed]

        expected_data = DIFF_FIXTURES[diff_fixture]

        assert expected_data == data

    def test_diff_markup(self):

        raw_diff = fixture.load_resource('markuptest.diff', strip=False)

        diff_processor = DiffProcessor(raw_diff)

        chunks = diff_processor.parsed[0]['chunks']

        #from pprint import pprint; pprint(chunks[1])

        l = ['\n']

            l.append('%(action)-7s %(new_lineno)3s %(old_lineno)3s %(line)r\n' % d)

        s = ''.join(l)

        assert s == r'''

context         '@@ -51,8 +51,15 @@'

unmod    51  51 '<u>\t</u>begin();'

unmod    52  52 '<u>\t</u><i></i>'

add      53     '<u>\t</u>int foo;<u class="cr"></u>'

add      54     '<u>\t</u>int bar; <u class="cr"></u>'

add      55     '<u>\t</u>int baz;<u>\t</u><u class="cr"></u>'

add      56     '<u>\t</u>int space; <i></i>'

add      57     '<u>\t</u>int tab;<u>\t</u><i></i>'

add      58     '<u>\t</u><i></i>'

unmod    59  53 ' <i></i>'

del          54 '<u>\t</u>#define MAX_STEPS (48)'

add      60     '<u>\t</u><u class="cr"></u>'

add      61     '<u>\t</u>#define MAX_STEPS (64)<u class="cr"></u>'

unmod    62  55 ''

del          56 '<u>\t</u>#define MIN_STEPS (<del>48</del>)'

add      63     '<u>\t</u>#define MIN_STEPS (<ins>42</ins>)'

unmod    64  57 ''

del          58 '<u>\t</u>#define <del>MORE_STEPS</del><u>\t</u><del>+</del>(<del>48</del>)<del><u>\t</u></del><del><i></i></del>'

add      65     '<u>\t</u>#define <ins>LESS_STEPS</ins><u>\t</u>(<ins>42</ins>)<ins> <i></i></ins>'

'''