# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.controllers.summary

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Summary controller for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 18, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import calendar

import itertools

import logging

import traceback

from datetime import date, timedelta

from time import mktime

from beaker.cache import cache_region

from tg import request

from tg import tmpl_context as c

from tg.i18n import ugettext as _

from webob.exc import HTTPBadRequest

import kallithea.lib.helpers as h

from kallithea.lib import ext_json

from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired

from kallithea.lib.base import BaseRepoController, jsonify, render

from kallithea.lib.celerylib.tasks import get_commits_stats

from kallithea.lib.markup_renderer import MarkupRenderer

from kallithea.lib.page import Page

from kallithea.lib.utils2 import safe_int, safe_str

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, NodeDoesNotExistError

from kallithea.lib.vcs.nodes import FileNode

from kallithea.model.db import Statistics

log = logging.getLogger(__name__)

README_FILES = [''.join([x[0][0], x[1][0]]) for x in

                    sorted(list(itertools.product(ALL_READMES, ALL_EXTS)),

                           key=lambda y:y[0][1] + y[1][1])]

class SummaryController(BaseRepoController):

    def __get_readme_data(self, db_repo):

        repo_name = db_repo.repo_name

        log.debug('Looking for README file')

        @cache_region('long_term_file', '_get_readme_from_cache')

        def _get_readme_from_cache(*_cache_keys):  # parameters are not really used - only as caching key

            readme_data = None

            readme_file = None

            try:

                # gets the landing revision! or tip if fails

                cs = db_repo.get_landing_changeset()

                if isinstance(cs, EmptyChangeset):

                    raise EmptyRepositoryError()

                renderer = MarkupRenderer()

                for f in README_FILES:

                    try:

                        readme = cs.get_node(f)

                        if not isinstance(readme, FileNode):

                            continue

                        readme_file = f

                        log.debug('Found README file `%s` rendering...',

                                  readme_file)

                        readme_data = renderer.render(safe_str(readme.content),

                                                      filename=f)

                        break

                    except NodeDoesNotExistError:

                        continue

            except ChangesetError:

                log.error(traceback.format_exc())

                pass

            except EmptyRepositoryError:

                pass

            return readme_data, readme_file

        kind = 'README'

        return _get_readme_from_cache(repo_name, kind, c.db_repo.changeset_cache.get('raw_id'))

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    def index(self, repo_name):

        p = safe_int(request.GET.get('page'), 1)

        size = safe_int(request.GET.get('size'), 10)

        try:

            collection = c.db_repo_scm_instance.get_changesets(reverse=True)

        except EmptyRepositoryError as e:

            h.flash(e, category='warning')

            collection = []

        c.cs_pagination = Page(collection, page=p, items_per_page=size)

        page_revisions = [x.raw_id for x in list(c.cs_pagination)]

        c.cs_comments = c.db_repo.get_comments(page_revisions)

        c.cs_statuses = c.db_repo.statuses(page_revisions)

        c.ssh_repo_url = None

        if request.authuser.is_default_user:

            username = None

        else:

            username = request.authuser.username

            if c.ssh_enabled:

                c.ssh_repo_url = c.db_repo.clone_url(clone_uri_tmpl=c.clone_ssh_tmpl)

        c.clone_repo_url = c.db_repo.clone_url(clone_uri_tmpl=c.clone_uri_tmpl, with_id=False, username=username)

        c.clone_repo_url_id = c.db_repo.clone_url(clone_uri_tmpl=c.clone_uri_tmpl, with_id=True, username=username)

        if c.db_repo.enable_statistics:

            c.show_stats = True

        else:

            c.show_stats = False

        stats = Statistics.query() \

            .filter(Statistics.repository == c.db_repo) \

            .scalar()

        c.stats_percentage = 0

        if stats and stats.languages:

            lang_stats_d = ext_json.loads(stats.languages)

            lang_stats = [(x, {"count": y,

                               "desc": LANGUAGES_EXTENSIONS_MAP.get(x, '?')})

                          for x, y in lang_stats_d.items()]

            lang_stats.sort(key=lambda k: (-k[1]['count'], k[0]))

            c.trending_languages = lang_stats[:10]

        else:

            c.trending_languages = []

        c.enable_downloads = c.db_repo.enable_downloads

        c.readme_data, c.readme_file = \

            self.__get_readme_data(c.db_repo)

        return render('summary/summary.html')

    @LoginRequired()

    @HasRepoPermissionLevelDecorator('read')

    @jsonify

    def repo_size(self, repo_name):

        if request.is_xhr:

            return c.db_repo._repo_size()

        else:

            raise HTTPBadRequest()

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    def statistics(self, repo_name):

        if c.db_repo.enable_statistics:

            c.show_stats = True

            c.no_data_msg = _('No data ready yet')

        else:

            c.show_stats = False

            c.no_data_msg = _('Statistics are disabled for this repository')

        td = date.today() + timedelta(days=1)

        td_1m = td - timedelta(days=calendar.monthrange(td.year, td.month)[1])

        td_1y = td - timedelta(days=365)

        ts_min_m = mktime(td_1m.timetuple())

        ts_min_y = mktime(td_1y.timetuple())

        ts_max_y = mktime(td.timetuple())

        c.ts_min = ts_min_m

        c.ts_max = ts_max_y

        stats = Statistics.query() \

            .filter(Statistics.repository == c.db_repo) \

            .scalar()

        c.stats_percentage = 0

        if stats and stats.languages:

            c.commit_data = ext_json.loads(stats.commit_activity)

            c.overview_data = ext_json.loads(stats.commit_activity_combined)

            lang_stats_d = ext_json.loads(stats.languages)

            lang_stats = [(x, {"count": y,

                               "desc": LANGUAGES_EXTENSIONS_MAP.get(x, '?')})

                          for x, y in lang_stats_d.items()]

            lang_stats.sort(key=lambda k: (-k[1]['count'], k[0]))

            c.trending_languages = lang_stats[:10]

            last_rev = stats.stat_on_revision + 1

            c.repo_last_rev = c.db_repo_scm_instance.count() \

                if c.db_repo_scm_instance.revisions else 0

            if last_rev == 0 or c.repo_last_rev == 0:

                pass

            else:

                c.stats_percentage = '%.2f' % ((float((last_rev)) /

                                                c.repo_last_rev) * 100)

        else:

            c.commit_data = {}

            c.overview_data = ([[ts_min_y, 0], [ts_max_y, 10]])

            c.trending_languages = []

        recurse_limit = 500  # don't recurse more than 500 times when parsing

        get_commits_stats(c.db_repo.repo_name, ts_min_y, ts_max_y, recurse_limit)

        return render('summary/statistics.html')

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.celerylib.tasks

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Kallithea task modules, containing all task that suppose to be run

by celery daemon

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Oct 6, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import email.utils

import os

import traceback

from collections import OrderedDict

from operator import itemgetter

from time import mktime

import celery.utils.log

from tg import config

import kallithea

from kallithea.lib.helpers import person

from kallithea.lib.hooks import log_create_repository

from kallithea.lib.indexers.daemon import WhooshIndexingDaemon

from kallithea.lib.rcmail.smtp_mailer import SmtpMailer

from kallithea.lib.utils import action_logger

from kallithea.lib.utils2 import asbool, ascii_bytes

from kallithea.lib.vcs.utils import author_email

from kallithea.model.db import RepoGroup, Repository, Setting, Statistics, User

from kallithea.model.repo import RepoModel

__all__ = ['whoosh_index', 'get_commits_stats', 'send_email']

log = celery.utils.log.get_task_logger(__name__)

@celerylib.task

@celerylib.locked_task

@celerylib.dbsession

def whoosh_index(repo_location, full_index):

    celerylib.get_session() # initialize database connection

    index_location = config['index_dir']

    WhooshIndexingDaemon(index_location=index_location,

                         repo_location=repo_location) \

                         .run(full_index=full_index)

# for js data compatibility cleans the key for person from '

def akc(k):

    return person(k).replace('"', '')

@celerylib.task

@celerylib.dbsession

def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):

    DBS = celerylib.get_session()

    lockkey = celerylib.__get_lockkey('get_commits_stats', repo_name, ts_min_y,

                            ts_max_y)

    lockkey_path = config.get('cache_dir') or config['app_conf']['cache_dir']  # Backward compatibility for TurboGears < 2.4

    log.info('running task with lockkey %s', lockkey)

    try:

        lock = celerylib.DaemonLock(os.path.join(lockkey_path, lockkey))

        co_day_auth_aggr = {}

        commits_by_day_aggregate = {}

        repo = Repository.get_by_repo_name(repo_name)

        if repo is None:

            return True

        repo = repo.scm_instance

        repo_size = repo.count()

        # return if repo have no revisions

        if repo_size < 1:

            lock.release()

            return True

        skip_date_limit = True

        parse_limit = int(config.get('commit_parse_limit'))

        last_rev = None

        last_cs = None

        timegetter = itemgetter('time')

        dbrepo = DBS.query(Repository) \

            .filter(Repository.repo_name == repo_name).scalar()

        cur_stats = DBS.query(Statistics) \

            .filter(Statistics.repository == dbrepo).scalar()

        if cur_stats is not None:

            last_rev = cur_stats.stat_on_revision

        if last_rev == repo.get_changeset().revision and repo_size > 1:

            # pass silently without any work if we're not on first revision or

            # current state of parsing revision(from db marker) is the

            # last revision

            lock.release()

            return True

        if cur_stats:

            commits_by_day_aggregate = OrderedDict(ext_json.loads(

                                        cur_stats.commit_activity_combined))

            co_day_auth_aggr = ext_json.loads(cur_stats.commit_activity)

        log.debug('starting parsing %s', parse_limit)

        last_rev = last_rev + 1 if last_rev and last_rev >= 0 else 0

        log.debug('Getting revisions from %s to %s',

             last_rev, last_rev + parse_limit

        )

        for cs in repo[last_rev:last_rev + parse_limit]:

            log.debug('parsing %s', cs)

            last_cs = cs  # remember last parsed changeset

            tt = cs.date.timetuple()

            k = mktime(tt[:3] + (0, 0, 0, 0, 0, 0))

            if akc(cs.author) in co_day_auth_aggr:

                try:

                    l = [timegetter(x) for x in

                         co_day_auth_aggr[akc(cs.author)]['data']]

                    time_pos = l.index(k)

                except ValueError:

                    time_pos = None

                if time_pos is not None and time_pos >= 0:

                    datadict = \

                        co_day_auth_aggr[akc(cs.author)]['data'][time_pos]

                    datadict["commits"] += 1

                    datadict["added"] += len(cs.added)

                    datadict["changed"] += len(cs.changed)

                    datadict["removed"] += len(cs.removed)

                else:

                    if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                        datadict = {"time": k,

                                    "commits": 1,

                                    "added": len(cs.added),

                                    "changed": len(cs.changed),

                                    "removed": len(cs.removed),

                                   }

                        co_day_auth_aggr[akc(cs.author)]['data'] \

                            .append(datadict)

            else:

                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                    co_day_auth_aggr[akc(cs.author)] = {

                                        "label": akc(cs.author),

                                        "data": [{"time": k,

                                                 "commits": 1,

                                                 "added": len(cs.added),

                                                 "changed": len(cs.changed),

                                                 "removed": len(cs.removed),

                                                 }],

                                        "schema": ["commits"],

                                        }

            # gather all data by day

            if k in commits_by_day_aggregate:

                commits_by_day_aggregate[k] += 1

            else:

                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),

                               key=itemgetter(0))

        if not co_day_auth_aggr:

            co_day_auth_aggr[akc(repo.contact)] = {

                "label": akc(repo.contact),

                "data": [0, 1],

                "schema": ["commits"],

            }

        stats = cur_stats if cur_stats else Statistics()

        stats.commit_activity = ascii_bytes(ext_json.dumps(co_day_auth_aggr))

        stats.commit_activity_combined = ascii_bytes(ext_json.dumps(overview_data))

        log.debug('last revision %s', last_rev)

        leftovers = len(repo.revisions[last_rev:])

        log.debug('revisions to parse %s', leftovers)

        if last_rev == 0 or leftovers < parse_limit:

            log.debug('getting code trending stats')

            stats.languages = ascii_bytes(ext_json.dumps(__get_codes_stats(repo_name)))

        try:

            stats.repository = dbrepo

            stats.stat_on_revision = last_cs.revision if last_cs else 0

            DBS.add(stats)

            DBS.commit()

        except:

            log.error(traceback.format_exc())

            DBS.rollback()

            lock.release()

            return False

        # final release

        lock.release()

        # execute another task if celery is enabled

        if len(repo.revisions) > 1 and kallithea.CELERY_APP and recurse_limit > 0:

            get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit - 1)

        elif recurse_limit <= 0:

            log.debug('Not recursing - limit has been reached')

        else:

            log.debug('Not recursing')

    except celerylib.LockHeld:

        log.info('Task with key %s already running', lockkey)

        return 'Task with key %s already running' % lockkey

@celerylib.task

@celerylib.dbsession

def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):

    """

    Sends an email with defined parameters from the .ini files.

    :param recipients: list of recipients, if this is None, the defined email

        address from field 'email_to' and all admins is used instead

    :param subject: subject of the mail

    :param body: body of the mail

    :param html_body: html version of body

    :param headers: dictionary of prepopulated e-mail headers

    :param from_name: full name to be used as sender of this mail - often a

    .full_name_or_username value

    """

    assert isinstance(recipients, list), recipients

    if headers is None:

        headers = {}

    else:

        # do not modify the original headers object passed by the caller

        headers = headers.copy()

    email_config = config

    email_prefix = email_config.get('email_prefix', '')

    if email_prefix:

        subject = "%s %s" % (email_prefix, subject)

    if not recipients:

        # if recipients are not defined we send to email_config + all admins

        recipients = [u.email for u in User.query()

                      .filter(User.admin == True).all()]

        if email_config.get('email_to') is not None:

            recipients += email_config.get('email_to').split(',')

        # If there are still no recipients, there are no admins and no address

        # configured in email_to, so return.

        if not recipients:

            log.error("No recipients specified and no fallback available.")

            return False

        log.warning("No recipients specified for '%s' - sending to admins %s", subject, ' '.join(recipients))

    # SMTP sender

    envelope_from = email_config.get('app_email_from', 'Kallithea')

    # 'From' header

    if from_name is not None:

        # set From header based on from_name but with a generic e-mail address

        # In case app_email_from is in "Some Name <e-mail>" format, we first

        # extract the e-mail address.

        envelope_addr = author_email(envelope_from)

        headers['From'] = '"%s" <%s>' % (

            email.utils.quote('%s (no-reply)' % from_name),

            envelope_addr)

    user = email_config.get('smtp_username')

    passwd = email_config.get('smtp_password')

    mail_server = email_config.get('smtp_server')

    mail_port = email_config.get('smtp_port')

    tls = asbool(email_config.get('smtp_use_tls'))

    ssl = asbool(email_config.get('smtp_use_ssl'))

    debug = asbool(email_config.get('debug'))

    smtp_auth = email_config.get('smtp_auth')

    logmsg = ("Mail details:\n"

              "recipients: %s\n"

              "headers: %s\n"

              "subject: %s\n"

              "body:\n%s\n"

              "html:\n%s\n"

              % (' '.join(recipients), headers, subject, body, html_body))

    if mail_server:

        log.debug("Sending e-mail. " + logmsg)

    else:

        log.error("SMTP mail server not configured - cannot send e-mail.")

        log.warning(logmsg)

        return False

    try:

        m = SmtpMailer(envelope_from, user, passwd, mail_server, smtp_auth,

                       mail_port, ssl, tls, debug=debug)

        m.send(recipients, subject, body, html_body, headers=headers)

    except:

        log.error('Mail sending failed')

        log.error(traceback.format_exc())

        return False

    return True

@celerylib.task

@celerylib.dbsession

def create_repo(form_data, cur_user):

    DBS = celerylib.get_session()

    cur_user = User.guess_instance(cur_user)

    owner = cur_user

    repo_name = form_data['repo_name']

    repo_name_full = form_data['repo_name_full']

    repo_type = form_data['repo_type']

    description = form_data['repo_description']

    private = form_data['repo_private']

    clone_uri = form_data.get('clone_uri')

    repo_group = form_data['repo_group']

    landing_rev = form_data['repo_landing_rev']

    copy_fork_permissions = form_data.get('copy_permissions')

    copy_group_permissions = form_data.get('repo_copy_permissions')

    fork_of = form_data.get('fork_parent_id')

    state = form_data.get('repo_state', Repository.STATE_PENDING)

    # repo creation defaults, private and repo_type are filled in form

    defs = Setting.get_default_repo_settings(strip_prefix=True)

    enable_statistics = defs.get('repo_enable_statistics')

    enable_downloads = defs.get('repo_enable_downloads')

    try:

        repo = RepoModel()._create_repo(

            repo_name=repo_name_full,

            repo_type=repo_type,

            description=description,

            owner=owner,

            private=private,

            clone_uri=clone_uri,

            repo_group=repo_group,

            landing_rev=landing_rev,

            fork_of=fork_of,

            copy_fork_permissions=copy_fork_permissions,

            copy_group_permissions=copy_group_permissions,

            enable_statistics=enable_statistics,

            enable_downloads=enable_downloads,

            state=state

        )

        action_logger(cur_user, 'user_created_repo',

                      form_data['repo_name_full'], '')

        DBS.commit()

        # now create this repo on Filesystem

        RepoModel()._create_filesystem_repo(

            repo_name=repo_name,

            repo_type=repo_type,

            repo_group=RepoGroup.guess_instance(repo_group),

            clone_uri=clone_uri,

        )

        repo = Repository.get_by_repo_name(repo_name_full)

        log_create_repository(repo.get_dict(), created_by=owner.username)

        # update repo changeset caches initially

        repo.update_changeset_cache()

        # set new created state

        repo.set_state(Repository.STATE_CREATED)

        DBS.commit()

    except Exception as e:

        log.warning('Exception %s occurred when forking repository, '

                    'doing cleanup...' % e)

        # rollback things manually !

        repo = Repository.get_by_repo_name(repo_name_full)

        if repo:

            Repository.delete(repo.repo_id)

            DBS.commit()

            RepoModel()._delete_filesystem_repo(repo)

        raise

    return True

@celerylib.task

@celerylib.dbsession

def create_repo_fork(form_data, cur_user):

    """

    Creates a fork of repository using interval VCS methods

    :param form_data:

    :param cur_user:

    """

    DBS = celerylib.get_session()

    base_path = kallithea.CONFIG['base_path']

    cur_user = User.guess_instance(cur_user)

    repo_name = form_data['repo_name']  # fork in this case

    repo_name_full = form_data['repo_name_full']

    repo_type = form_data['repo_type']

    owner = cur_user

    private = form_data['private']

    clone_uri = form_data.get('clone_uri')

    repo_group = form_data['repo_group']

    landing_rev = form_data['landing_rev']

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

~~~~~~~~~~~~~~~~~~~~~

Various config settings for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Mar 7, 2012

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

from kallithea.lib import pygmentsutils

# language map is also used by whoosh indexer, which for those specified

# extensions will index it's content

LANGUAGES_EXTENSIONS_MAP = pygmentsutils.get_extension_descriptions()

# Whoosh index targets

# Extensions we want to index content of using whoosh

INDEX_EXTENSIONS = list(LANGUAGES_EXTENSIONS_MAP)

# Filenames we want to index content of using whoosh

INDEX_FILENAMES = pygmentsutils.get_index_filenames()

# list of readme files to search in file tree and display in summary

# attached weights defines the search  order lower is first

ALL_READMES = [

    ('readme', 0), ('README', 0), ('Readme', 0),

    ('doc/readme', 1), ('doc/README', 1), ('doc/Readme', 1),

    ('Docs/readme', 2), ('Docs/README', 2), ('Docs/Readme', 2),

    ('DOCS/readme', 2), ('DOCS/README', 2), ('DOCS/Readme', 2),

    ('docs/readme', 2), ('docs/README', 2), ('docs/Readme', 2),

]

# extension together with weights to search lower is first

RST_EXTS = [

    ('', 0), ('.rst', 1), ('.rest', 1),

    ('.RST', 2), ('.REST', 2),

    ('.txt', 3), ('.TXT', 3)

]

MARKDOWN_EXTS = [

    ('.md', 1), ('.MD', 1),

    ('.mkdn', 2), ('.MKDN', 2),

    ('.mdown', 3), ('.MDOWN', 3),

    ('.markdown', 4), ('.MARKDOWN', 4)

]

PLAIN_EXTS = [('.text', 2), ('.TEXT', 2)]

ALL_EXTS = MARKDOWN_EXTS + RST_EXTS + PLAIN_EXTS

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.indexers.daemon

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

A daemon will read from task table and run tasks

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Jan 26, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import os

import sys

import traceback

from os.path import dirname

from shutil import rmtree

from time import mktime

from whoosh.index import create_in, exists_in, open_dir

from whoosh.qparser import QueryParser

from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA

from kallithea.lib.utils2 import safe_str

from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, NodeDoesNotExistError, RepositoryError

from kallithea.model.db import Repository

from kallithea.model.scm import ScmModel

# Add location of top level folder to sys.path

project_path = dirname(dirname(dirname(dirname(os.path.realpath(__file__)))))

sys.path.append(project_path)

log = logging.getLogger('whoosh_indexer')

class WhooshIndexingDaemon(object):

    """

    Daemon for atomic indexing jobs

    """

    def __init__(self, indexname=IDX_NAME, index_location=None,

                 repo_location=None, repo_list=None,

                 repo_update_list=None):

        self.indexname = indexname

        self.index_location = index_location

        if not index_location:

            raise Exception('You have to provide index location')

        self.repo_location = repo_location

        if not repo_location:

            raise Exception('You have to provide repositories location')

        self.repo_paths = ScmModel().repo_scan(self.repo_location)

        # filter repo list

        if repo_list:

            repo_list = set(repo_list)

            self.filtered_repo_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_list:

                    self.filtered_repo_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_paths

        # filter update repo list

        self.filtered_repo_update_paths = {}

        if repo_update_list:

            self.filtered_repo_update_paths = {}

            for repo_name, repo in self.repo_paths.items():

                if repo_name in repo_update_list:

                    self.filtered_repo_update_paths[repo_name] = repo

            self.repo_paths = self.filtered_repo_update_paths

        self.initial = True

        if not os.path.isdir(self.index_location):

            os.makedirs(self.index_location)

            log.info('Cannot run incremental index since it does not '

                     'yet exist - running full build')

        elif not exists_in(self.index_location, IDX_NAME):

            log.info('Running full index build, as the file content '

                     'index does not exist')

        elif not exists_in(self.index_location, CHGSET_IDX_NAME):

            log.info('Running full index build, as the changeset '

                     'index does not exist')

        else:

            self.initial = False

    def _get_index_revision(self, repo):

        db_repo = Repository.get_by_repo_name(repo.name)

        landing_rev = 'tip'

        if db_repo:

            _rev_type, _rev = db_repo.landing_rev

            landing_rev = _rev

        return landing_rev

    def _get_index_changeset(self, repo, index_rev=None):

        if not index_rev:

            index_rev = self._get_index_revision(repo)

        cs = repo.get_changeset(index_rev)

        return cs

    def get_paths(self, repo):

        """

        recursive walk in root dir and return a set of all path in that dir

        based on repository walk function

        """

        index_paths_ = set()

        try:

            cs = self._get_index_changeset(repo)

            for _topnode, _dirs, files in cs.walk('/'):

                for f in files:

                    index_paths_.add(os.path.join(repo.path, f.path))

        except RepositoryError:

            log.debug(traceback.format_exc())

            pass

        return index_paths_

    def get_node(self, repo, path, index_rev=None):

        """

        gets a filenode based on given full path.

        :param repo: scm repo instance

        :param path: full path including root location

        :return: FileNode

        """

        # FIXME: paths should be normalized ... or even better: don't include repo.path