node = self.get_node(repo, path, index_rev)

        except (ChangesetError, NodeDoesNotExistError):

            log.debug("    >> %s - not found in %s %s", path, repo, index_rev)

            return 0, 0

        indexed = indexed_w_content = 0

        if self.is_indexable_node(node):

            bytes_content = node.content

            if b'\0' in bytes_content:

                log.warning('    >> %s - no text content', path)

                u_content = ''

            else:

                log.debug('    >> %s', path)

                u_content = safe_str(bytes_content)

                indexed_w_content += 1

        else:

            log.debug('    >> %s - not indexable', path)

            # just index file name without it's content

            u_content = ''

            indexed += 1

        writer.add_document(

            fileid=path,

            repository_rawname=repo_name,

            repository=repo_name,

            path=path,

            content=u_content,

            modtime=self.get_node_mtime(node),

            extension=node.extension

        )

        return indexed, indexed_w_content

    def index_changesets(self, writer, repo_name, repo, start_rev=None):

        """

        Add all changeset in the vcs repo starting at start_rev

        to the index writer

        :param writer: the whoosh index writer to add to

        :param repo_name: name of the repository from whence the

          changeset originates including the repository group

        :param repo: the vcs repository instance to index changesets for,

          the presumption is the repo has changesets to index

        :param start_rev=None: the full sha id to start indexing from

          if start_rev is None then index from the first changeset in

          the repo

        """

        if start_rev is None:

            start_rev = repo[0].raw_id

        log.debug('Indexing changesets in %s, starting at rev %s',

                  repo_name, start_rev)

        indexed = 0

        cs_iter = repo.get_changesets(start=start_rev)

        total = len(cs_iter)

        for cs in cs_iter:

            indexed += 1

            log.debug('    >> %s %s/%s', cs, indexed, total)

            writer.add_document(

                raw_id=cs.raw_id,

                date=cs._timestamp,

                repository_rawname=repo_name,

                repository=repo_name,

                author=cs.author,

                message=cs.message,

                last=cs.last,

                added=' '.join(node.path for node in cs.added).lower(),

                removed=' '.join(node.path for node in cs.removed).lower(),

                changed=' '.join(node.path for node in cs.changed).lower(),

                parents=' '.join(cs.raw_id for cs in cs.parents),

            )

        return indexed

    def index_files(self, file_idx_writer, repo_name, repo):

        """

        Index files for given repo_name

        :param file_idx_writer: the whoosh index writer to add to

        :param repo_name: name of the repository we're indexing

        :param repo: instance of vcs repo

        """

        i_cnt = iwc_cnt = 0

        log.debug('Building file index for %s @revision:%s', repo_name,

    @LazyProperty

    def last_change(self):

        """

        Returns last change made on this repository as datetime object

        """

        return date_fromtimestamp(self._get_mtime(), makedate()[1])

    def _get_mtime(self):

        try:

            return time.mktime(self.get_changeset().date.timetuple())

        except RepositoryError:

            idx_loc = '' if self.bare else '.git'

            # fallback to filesystem

            in_path = os.path.join(self.path, idx_loc, "index")

            he_path = os.path.join(self.path, idx_loc, "HEAD")

            if os.path.exists(in_path):

                return os.stat(in_path).st_mtime

            else:

                return os.stat(he_path).st_mtime

    @LazyProperty

    def description(self):

        return safe_str(self._repo.get_description() or b'unknown')

    @property

    def branches(self):

        if not self.revisions:

            return {}

        _branches = [(safe_str(key), ascii_str(sha))

                     for key, (sha, type_) in self._parsed_refs.items() if type_ == b'H']

        return OrderedDict(sorted(_branches, key=(lambda ctx: ctx[0]), reverse=False))

    @LazyProperty

    def closed_branches(self):

        return {}

    @LazyProperty

    def tags(self):

        return self._get_tags()

    def _get_tags(self):

        if not self.revisions:

            return {}

        _tags = [(safe_str(key), ascii_str(sha))

                 for key, (sha, type_) in self._parsed_refs.items() if type_ == b'T']

        return OrderedDict(sorted(_tags, key=(lambda ctx: ctx[0]), reverse=True))

    def tag(self, name, user, revision=None, message=None, date=None,

                mercurial.commands.clone(self.baseui, safe_bytes(url), safe_bytes(self.path), **opts)

                # Don't try to create if we've already cloned repo

                create = False

            return mercurial.localrepo.instance(self.baseui, safe_bytes(self.path), create=create)

        except (mercurial.error.Abort, mercurial.error.RepoError) as err:

            if create:

                msg = "Cannot create repository at %s. Original error was %s" \

                    % (self.name, err)

            else:

                msg = "Not valid repository at %s. Original error was %s" \

                    % (self.name, err)

            raise RepositoryError(msg)

    @LazyProperty

    def in_memory_changeset(self):

        return inmemory.MercurialInMemoryChangeset(self)

    @LazyProperty

    def description(self):

        _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)

        return safe_str(_desc or b'unknown')

    @LazyProperty

    def last_change(self):

        """

        Returns last change made on this repository as datetime object

        """

        return date_fromtimestamp(self._get_mtime(), makedate()[1])

    def _get_mtime(self):

        try:

            return time.mktime(self.get_changeset().date.timetuple())

        except RepositoryError:

            # fallback to filesystem

            cl_path = os.path.join(self.path, '.hg', "00changelog.i")

            st_path = os.path.join(self.path, '.hg', "store")

            if os.path.exists(cl_path):

                return os.stat(cl_path).st_mtime

            else:

                return os.stat(st_path).st_mtime

    def _get_revision(self, revision):

        """

        Given any revision identifier, returns a 40 char string with revision hash.

        :param revision: str or int or None

        """

                            .append(datadict)

            else:

                if k >= ts_min_y and k <= ts_max_y or skip_date_limit:

                    co_day_auth_aggr[akc(cs.author)] = {

                                        "label": akc(cs.author),

                                        "data": [{"time": k,

                                                 "commits": 1,

                                                 "added": len(cs.added),

                                                 "changed": len(cs.changed),

                                                 "removed": len(cs.removed),

                                                 }],

                                        "schema": ["commits"],

                                        }

            # gather all data by day

            if k in commits_by_day_aggregate:

                commits_by_day_aggregate[k] += 1

            else:

                commits_by_day_aggregate[k] = 1

        overview_data = sorted(commits_by_day_aggregate.items(),

                               key=itemgetter(0))

        stats = cur_stats if cur_stats else db.Statistics()

        stats.commit_activity = ascii_bytes(ext_json.dumps(co_day_auth_aggr))

        stats.commit_activity_combined = ascii_bytes(ext_json.dumps(overview_data))

        log.debug('last revision %s', last_rev)

        leftovers = len(scm_repo.revisions[last_rev:])

        log.debug('revisions to parse %s', leftovers)

        if last_rev == 0 or leftovers < parse_limit:

            log.debug('getting code trending stats')

            stats.languages = ascii_bytes(ext_json.dumps(__get_codes_stats(repo_name)))

        try:

            stats.repository = dbrepo

            stats.stat_on_revision = last_cs.revision if last_cs else 0

            DBS.add(stats)

            DBS.commit()

        except:

            log.error(traceback.format_exc())

            DBS.rollback()

            lock.release()

            return False

        # final release