kallithea Changeset - aee4c451566f

Changeset - aee4c451566f

Parent rev.

Child rev.

[Not reviewed]

default

0 5 0

Mads Kiilerich (mads) - 5 years ago 2020-11-01 06:18:48
mads@kiilerich.com

vcs: cleanup around get_backend and get_repo

5 files changed with 12 insertions and 28 deletions:

kallithea/lib/vcs/backends/__init__.py

kallithea/lib/vcs/backends/base.py

kallithea/lib/vcs/backends/git/repository.py

kallithea/model/db.py

kallithea/model/scm.py

0 comments (0 inline, 0 general)

kallithea/lib/vcs/backends/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.backends
     ~~~~~~~~~~~~
     Main package for scm backends
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import os
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import VCSError
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.lib.vcs.utils.imports import import_class
 from kallithea.lib.vcs.utils.paths import abspath
 def get_repo(path=None, alias=None, create=False):
+def get_repo(path=None, alias=None, create=False, baseui=None):
     """
     Returns ``Repository`` object of type linked with given ``alias`` at
     the specified ``path``. If ``alias`` is not given it will try to guess it
     using get_scm method
     """
     if create:
         if not (path or alias):
             raise TypeError("If create is specified, we need path and scm type")
         return get_backend(alias)(path, create=True)
     if path is None:
         path = abspath(os.path.curdir)
     try:
         scm, path = get_scm(path, search_up=True)
         path = abspath(path)
         alias = scm
     except VCSError:
         raise VCSError("No scm found at %s" % path)
     if alias is None:
         alias = get_scm(path)[0]
     backend = get_backend(alias)
     repo = backend(path, create=create)
+    repo = backend(path, create=create, baseui=baseui)
     return repo
 def get_backend(alias):
     """
     Returns ``Repository`` class identified by the given alias or raises
     VCSError if alias is not recognized or backend class cannot be imported.
     """
     if alias not in settings.BACKENDS:
         raise VCSError("Given alias '%s' is not recognized! Allowed aliases:\n"
             "%s" % (alias, '", "'.join(settings.BACKENDS)))
     backend_path = settings.BACKENDS[alias]
     klass = import_class(backend_path)
     return klass
 def get_supported_backends():
     """
     Returns list of aliases of supported backends.
     """
     return settings.BACKENDS.keys()

kallithea/lib/vcs/backends/base.py

➞

Show inline comments

@@ @@ -837,231 +837,231 @@ class BaseInMemoryChangeset(object): @@
         for node in filenodes:
             if node.path in (n.path for n in self.removed):
                 raise NodeAlreadyRemovedError("Node at %s is already marked "
                     "as removed" % node.path)
         try:
             self.repository.get_changeset()
         except EmptyRepositoryError:
             raise EmptyRepositoryError("Nothing to change - try to *add* new "
                 "nodes rather than changing them")
         for node in filenodes:
             if node.path in (n.path for n in self.changed):
                 raise NodeAlreadyChangedError("Node at '%s' is already "
                     "marked as changed" % node.path)
             self.changed.append(node)
     def remove(self, *filenodes):
         """
         Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be
         *removed* in next commit.
         :raises ``NodeAlreadyRemovedError``: if node has been already marked to
           be *removed*
         :raises ``NodeAlreadyChangedError``: if node has been already marked to
           be *changed*
         """
         for node in filenodes:
             if node.path in (n.path for n in self.removed):
                 raise NodeAlreadyRemovedError("Node is already marked to "
                     "for removal at %s" % node.path)
             if node.path in (n.path for n in self.changed):
                 raise NodeAlreadyChangedError("Node is already marked to "
                     "be changed at %s" % node.path)
             # We only mark node as *removed* - real removal is done by
             # commit method
             self.removed.append(node)
     def reset(self):
         """
         Resets this instance to initial state (cleans ``added``, ``changed``
         and ``removed`` lists).
         """
         self.added = []
         self.changed = []
         self.removed = []
         self.parents = []
     def get_ipaths(self):
         """
         Returns generator of paths from nodes marked as added, changed or
         removed.
         """
         for node in itertools.chain(self.added, self.changed, self.removed):
             yield node.path
     def get_paths(self):
         """
         Returns list of paths from nodes marked as added, changed or removed.
         """
         return list(self.get_ipaths())
     def check_integrity(self, parents=None):
         """
         Checks in-memory changeset's integrity. Also, sets parents if not
         already set.
         :raises CommitError: if any error occurs (i.e.
           ``NodeDoesNotExistError``).
         """
         if not self.parents:
             parents = parents or []
             if len(parents) == 0:
                 try:
                     parents = [self.repository.get_changeset(), None]
                 except EmptyRepositoryError:
                     parents = [None, None]
             elif len(parents) == 1:
                 parents += [None]
             self.parents = parents
         # Local parents, only if not None
         parents = [p for p in self.parents if p]
         # Check nodes marked as added
         for p in parents:
             for node in self.added:
                 try:
                     p.get_node(node.path)
                 except NodeDoesNotExistError:
                     pass
                 else:
                     raise NodeAlreadyExistsError("Node at %s already exists "
                         "at %s" % (node.path, p))
         # Check nodes marked as changed
         missing = set(node.path for node in self.changed)
         not_changed = set(node.path for node in self.changed)
         if self.changed and not parents:
             raise NodeDoesNotExistError(self.changed[0].path)
         for p in parents:
             for node in self.changed:
                 try:
                     old = p.get_node(node.path)
                     missing.remove(node.path)
                     # if content actually changed, remove node from unchanged
                     if old.content != node.content:
                         not_changed.remove(node.path)
                 except NodeDoesNotExistError:
                     pass
         if self.changed and missing:
             raise NodeDoesNotExistError("Node at %s is missing "
                 "(parents: %s)" % (node.path, parents))
         if self.changed and not_changed:
             raise NodeNotChangedError("Node at %s wasn't actually changed "
                 "since parents' changesets: %s" % (not_changed.pop(),
                     parents)
+            )
         # Check nodes marked as removed
         if self.removed and not parents:
             raise NodeDoesNotExistError("Cannot remove node at %s as there "
                 "were no parents specified" % self.removed[0].path)
         really_removed = set()
         for p in parents:
             for node in self.removed:
                 try:
                     p.get_node(node.path)
                     really_removed.add(node.path)
                 except ChangesetError:
                     pass
         not_removed = list(set(node.path for node in self.removed) - really_removed)
         if not_removed:
             raise NodeDoesNotExistError("Cannot remove node at %s from "
                 "following parents: %s" % (not_removed[0], parents))
     def commit(self, message, author, parents=None, branch=None, date=None,
             **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
         .. note::
             While overriding this method each backend's should call
             ``self.check_integrity(parents)`` in the first place.
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <joe.doe@example.com>"
         :param parents: single parent or sequence of parents from which commit
           would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
         :raises ``CommitError``: if any error occurs while committing
         """
         raise NotImplementedError
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset. It's possible to pass hash when creating
     an EmptyChangeset
     """
     def __init__(self, cs='0' * 40, repo=None, requested_revision=None,
                  alias=None, revision=-1, message='', author='', date=None):
         self._empty_cs = cs
         self.revision = revision
         self.message = message
         self.author = author
         self.date = date or datetime.datetime.fromtimestamp(0)
         self.repository = repo
         self.requested_revision = requested_revision
         self.alias = alias
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset, useful for web
         representation.
         """
         return self._empty_cs
     @LazyProperty
     def branch(self):
         return get_backend(self.alias).DEFAULT_BRANCH_NAME
     @LazyProperty
     def branches(self):
-        return [get_backend(self.alias).DEFAULT_BRANCH_NAME]
+        return [self.branch]
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
     def get_file_changeset(self, path):
         return self
     def get_file_content(self, path):
         return b''
     def get_file_size(self, path):
         return 0
 class CollectionGenerator(object):
     def __init__(self, repo, revs):
         self.repo = repo
         self.revs = revs
     def __len__(self):
         return len(self.revs)
     def __iter__(self):
         for rev in self.revs:
             yield self.repo.get_changeset(rev)
     def __getitem__(self, what):
         """Return either a single element by index, or a sliced collection."""
         if isinstance(what, slice):
             return CollectionGenerator(self.repo, self.revs[what])
         else:
             # single item
             return self.repo.get_changeset(self.revs[what])
     def __repr__(self):
         return '<CollectionGenerator[len:%s]>' % (len(self))

kallithea/lib/vcs/backends/git/repository.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.backends.git.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Git repository implementation.
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import errno
 import logging
 import os
 import re
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 import mercurial.util  # import url as hg_url
 from dulwich.client import SubprocessGitClient
 from dulwich.config import ConfigFile
 from dulwich.objects import Tag
 from dulwich.repo import NotGitRepository, Repo
 from dulwich.server import update_server_info
 from kallithea.lib.vcs import subprocessio
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError,
                                           TagDoesNotExistError)
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, makedate, safe_bytes, safe_str
 from kallithea.lib.vcs.utils.helpers import get_urllib_request_handlers
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath, get_user_home
 from . import changeset, inmemory, workdir
 SHA_PATTERN = re.compile(r'^([0-9a-fA-F]{12}|[0-9a-fA-F]{40})$')
 log = logging.getLogger(__name__)
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = 'master'
     scm = 'git'
     def __init__(self, repo_path, create=False, src_url=None,
                  update_after_clone=False, bare=False):
                  update_after_clone=False, bare=False, baseui=None):
         baseui  # unused
         self.path = abspath(repo_path)
         self.repo = self._get_repo(create, src_url, update_after_clone, bare)
         self.bare = self.repo.bare
     @property
     def _config_files(self):
         return [
             self.bare and abspath(self.path, 'config')
                       or abspath(self.path, '.git', 'config'),
              abspath(get_user_home(), '.gitconfig'),
+         ]
     @property
     def _repo(self):
         return self.repo
     @property
     def head(self):
         try:
             return self._repo.head()
         except KeyError:
             return None
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         try:
             self.revisions[0]
         except (KeyError, IndexError):
             return True
         return False
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
     @classmethod
     def _run_git_command(cls, cmd, cwd=None):
         """
         Runs given ``cmd`` as git command and returns output bytes in a tuple
         (stdout, stderr) ... or raise RepositoryError.
         :param cmd: git command to be executed
         :param cwd: passed directly to subprocess
         """
         # need to clean fix GIT_DIR !
         gitenv = dict(os.environ)
         gitenv.pop('GIT_DIR', None)
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         assert isinstance(cmd, list), cmd
         cmd = [settings.GIT_EXECUTABLE_PATH, '-c', 'core.quotepath=false'] + cmd
         try:
             p = subprocessio.SubprocessIOChunker(cmd, cwd=cwd, env=gitenv, shell=False)
         except (EnvironmentError, OSError) as err:
             # output from the failing process is in str(EnvironmentError)
             msg = ("Couldn't run git command %s.\n"
                    "Subprocess failed with '%s': %s\n" %
                    (cmd, type(err).__name__, err)
             ).strip()
             log.error(msg)
             raise RepositoryError(msg)
         try:
             stdout = b''.join(p.output)
             stderr = b''.join(p.error)
         finally:
             p.close()
         # TODO: introduce option to make commands fail if they have any stderr output?
         if stderr:
             log.debug('stderr from %s:\n%s', cmd, stderr)
         else:
             log.debug('stderr from %s: None', cmd)
         return stdout, stderr
     def run_git_command(self, cmd):
         """
         Runs given ``cmd`` as git command with cwd set to current repo.
         Returns stdout as unicode str ... or raise RepositoryError.
         """
         cwd = None
         if os.path.isdir(self.path):
             cwd = self.path
         stdout, _stderr = self._run_git_command(cmd, cwd=cwd)
         return safe_str(stdout)
     @classmethod
     def _check_url(cls, url):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that git will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
         if url.startswith('git://'):
             return True
         if '+' in url[:url.find('://')]:
             url = url[url.find('+') + 1:]
         url_obj = mercurial.util.url(safe_bytes(url))
         test_uri, handlers = get_urllib_request_handlers(url_obj)
         if not test_uri.endswith(b'info/refs'):
             test_uri = test_uri.rstrip(b'/') + b'/info/refs'
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
         req = urllib.request.Request(
             "%s?%s" % (
                 safe_str(test_uri),
                 urllib.parse.urlencode({"service": 'git-upload-pack'})
             ))
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
             raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
         # now detect if it's proper git repo
         gitdata = resp.read()
         if b'service=git-upload-pack' not in gitdata:
             raise urllib.error.URLError(
                 "url [%s] does not look like an git" % cleaned_uri)
         return True
     def _get_repo(self, create, src_url=None, update_after_clone=False,
                   bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError("Location already exist")
         if src_url and not create:
             raise RepositoryError("Create should be set to True if src_url is "
                                   "given (clone operation creates repository)")
         try:
             if create and src_url:
                 GitRepository._check_url(src_url)
                 self.clone(src_url, update_after_clone, bare)
                 return Repo(self.path)
             elif create:
                 os.makedirs(self.path)
                 if bare:
                     return Repo.init_bare(self.path)
                 else:
                     return Repo.init(self.path)
             else:
                 return Repo(self.path)
         except (NotGitRepository, OSError) as err:
             raise RepositoryError(err)
     def _get_all_revisions(self):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
         try:
             self._repo.head()
         except KeyError:
             return []
         rev_filter = settings.GIT_REV_FILTER
         cmd = ['rev-list', rev_filter, '--reverse', '--date-order']
         try:
             so = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return so.splitlines()
     def _get_all_revisions2(self):
         # alternate implementation using dulwich
         includes = [ascii_str(sha) for key, (sha, type_) in self._parsed_refs.items()
                     if type_ != b'T']

kallithea/model/db.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.db
 ~~~~~~~~~~~~~~~~~~
 Database Models for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 08, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import base64
 import collections
 import datetime
 import functools
 import hashlib
 import logging
 import os
 import time
 import traceback
 import ipaddr
 import sqlalchemy
 import urlobject
 from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, String, Unicode, UnicodeText, UniqueConstraint
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import class_mapper, joinedload, relationship, validates
 from tg.i18n import lazy_ugettext as _
 from webob.exc import HTTPNotFound
 import kallithea
 from kallithea.lib import ext_json, ssh, webutils
 from kallithea.lib.exceptions import DefaultUserException
 from kallithea.lib.utils2 import asbool, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, urlreadable
-from kallithea.lib.vcs import get_backend, get_repo
 from kallithea.lib.vcs import get_repo
 from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
 from kallithea.lib.vcs.utils import author_email, author_name
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.model import meta
 log = logging.getLogger(__name__)
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 class BaseDbModel(object):
     """
     Base Model for all classes
     """
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         # Note: not a normal dict - iterator gives "users.firstname", but keys gives "firstname"
         return class_mapper(cls).c.keys()
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         # also use __json__() if present to get additional fields
         _json_attr = getattr(self, '__json__', None)
         if _json_attr:
             # update with attributes from __json__
             if callable(_json_attr):
                 _json_attr = _json_attr()
             for k, val in _json_attr.items():
                 d[k] = val
         return d
     def get_appstruct(self):
         """return list with keys and values tuples corresponding
         to this model data """
         return [
             (k, getattr(self, k))
             for k in self._get_keys()
+        ]
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
     @classmethod
     def query(cls):
         return meta.Session().query(cls)
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
     @classmethod
     def guess_instance(cls, value, callback=None):
         """Haphazardly attempt to convert `value` to a `cls` instance.
         If `value` is None or already a `cls` instance, return it. If `value`
         is a number (or looks like one if you squint just right), assume it's
         a database primary key and let SQLAlchemy sort things out. Otherwise,
         fall back to resolving it using `callback` (if specified); this could
         e.g. be a function that looks up instances by name (though that won't
         work if the name begins with a digit). Otherwise, raise Exception.
         """
         if value is None:
             return None
         if isinstance(value, cls):
             return value
         if isinstance(value, int):
             return cls.get(value)
         if isinstance(value, str) and value.isdigit():
             return cls.get(int(value))
         if callback is not None:
             return callback(value)
         raise Exception(
             'given object must be int, long or Instance of %s '
             'got %s, no callback provided' % (cls, type(value))
+        )
     @classmethod
     def get_or_404(cls, id_):
         try:
             id_ = int(id_)
         except (TypeError, ValueError):
             raise HTTPNotFound
         res = cls.query().get(id_)
         if res is None:
             raise HTTPNotFound
         return res
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         meta.Session().delete(obj)
     def __repr__(self):
         return '<DB:%s>' % (self.__class__.__name__)
 _table_args_default_dict = {'extend_existing': True,
                             'mysql_engine': 'InnoDB',
                             'sqlite_autoincrement': True,
+                           }
 class Setting(meta.Base, BaseDbModel):
     __tablename__ = 'settings'
     __table_args__ = (
         _table_args_default_dict,
+    )
     SETTINGS_TYPES = {
         'str': safe_bytes,
         'int': safe_int,
         'unicode': safe_str,
         'bool': asbool,
         'list': functools.partial(aslist, sep=',')
+    }
     app_settings_id = Column(Integer(), primary_key=True)
     app_settings_name = Column(String(255), nullable=False, unique=True)
     _app_settings_value = Column("app_settings_value", Unicode(4096), nullable=False)
     _app_settings_type = Column("app_settings_type", String(255), nullable=True) # FIXME: not nullable?
     def __init__(self, key='', val='', type='unicode'):
         self.app_settings_name = key
         self.app_settings_value = val
         self.app_settings_type = type
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert isinstance(val, str)
         return val
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         _type = self.app_settings_type
         converter = self.SETTINGS_TYPES.get(_type) or self.SETTINGS_TYPES['unicode']
         return converter(v)
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use str in app_settings_value
         """
         self._app_settings_value = safe_str(val)
     @hybrid_property
     def app_settings_type(self):
         return self._app_settings_type
     @app_settings_type.setter
     def app_settings_type(self, val):
         if val not in self.SETTINGS_TYPES:
             raise Exception('type must be one of %s got %s'
                             % (list(self.SETTINGS_TYPES), val))
         self._app_settings_type = val
     def __repr__(self):
         return "<%s %s.%s=%r>" % (
             self.__class__.__name__,
             self.app_settings_name, self.app_settings_type, self.app_settings_value
+        )
     @classmethod
     def get_by_name(cls, key):
         return cls.query() \
             .filter(cls.app_settings_name == key).scalar()
     @classmethod
     def get_by_name_or_create(cls, key, val='', type='unicode'):
         res = cls.get_by_name(key)
         if res is None:
             res = cls(key, val, type)
         return res
     @classmethod
     def create_or_update(cls, key, val=None, type=None):
@@ @@ -1142,394 +1141,386 @@ class Repository(meta.Base, BaseDbModel) @@
             description=repo.description,
             landing_rev=repo.landing_rev,
             owner=repo.owner.username,
             fork_of=repo.fork.repo_name if repo.fork else None,
             enable_statistics=repo.enable_statistics,
             enable_downloads=repo.enable_downloads,
             last_changeset=repo.changeset_cache,
+        )
         if with_revision_names:
             scm_repo = repo.scm_instance_no_cache()
             data.update(dict(
                 tags=scm_repo.tags,
                 branches=scm_repo.branches,
                 bookmarks=scm_repo.bookmarks,
             ))
         if with_pullrequests:
             data['pull_requests'] = repo.pull_requests_other
         settings = Setting.get_app_settings()
         repository_fields = asbool(settings.get('repository_fields'))
         if repository_fields:
             for f in self.extra_fields:
                 data[f.field_key_prefixed] = f.field_value
         return data
     @property
     def last_db_change(self):
         return self.updated_on
     @property
     def clone_uri_hidden(self):
         clone_uri = self.clone_uri
         if clone_uri:
             url_obj = urlobject.URLObject(self.clone_uri)
             if url_obj.password:
                 clone_uri = url_obj.with_password('*****')
         return clone_uri
     def clone_url(self, clone_uri_tmpl, with_id=False, username=None):
         if '{repo}' not in clone_uri_tmpl and '_{repoid}' not in clone_uri_tmpl:
             log.error("Configured clone_uri_tmpl %r has no '{repo}' or '_{repoid}' and cannot toggle to use repo id URLs", clone_uri_tmpl)
         elif with_id:
             clone_uri_tmpl = clone_uri_tmpl.replace('{repo}', '_{repoid}')
         else:
             clone_uri_tmpl = clone_uri_tmpl.replace('_{repoid}', '{repo}')
         prefix_url = webutils.canonical_url('home')
         return get_clone_url(clone_uri_tmpl=clone_uri_tmpl,
                              prefix_url=prefix_url,
                              repo_name=self.repo_name,
                              repo_id=self.repo_id,
                              username=username)
     def set_state(self, state):
         self.repo_state = state
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
     def get_changeset(self, rev=None):
         return get_changeset_safe(self.scm_instance, rev)
     def get_landing_changeset(self):
         """
         Returns landing changeset, or if that doesn't exist returns the tip
         """
         _rev_type, _rev = self.landing_rev
         cs = self.get_changeset(_rev)
         if isinstance(cs, EmptyChangeset):
             return self.get_changeset()
         return cs
     def update_changeset_cache(self, cs_cache=None):
         """
         Update cache of last changeset for repository, keys should be::
             short_id
             raw_id
             revision
             message
             date
             author
         :param cs_cache:
         """
         if cs_cache is None:
             cs_cache = EmptyChangeset()
             # use no-cache version here
             scm_repo = self.scm_instance_no_cache()
             if scm_repo:
                 cs_cache = scm_repo.get_changeset()
         if isinstance(cs_cache, BaseChangeset):
             cs_cache = cs_cache.__json__()
         if (not self.changeset_cache or cs_cache['raw_id'] != self.changeset_cache['raw_id']):
             _default = datetime.datetime.fromtimestamp(0)
             last_change = cs_cache.get('date') or _default
             log.debug('updated repo %s with new cs cache %s',
                       self.repo_name, cs_cache)
             self.updated_on = last_change
             self.changeset_cache = cs_cache
             meta.Session().commit()
         else:
             log.debug('changeset_cache for %s already up to date with %s',
                       self.repo_name, cs_cache['raw_id'])
     @property
     def tip(self):
         return self.get_changeset('tip')
     @property
     def author(self):
         return self.tip.author
     @property
     def last_change(self):
         return self.scm_instance.last_change
     def get_comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query() \
             .filter(ChangesetComment.repo == self)
         if revisions is not None:
             if not revisions:
                 return {} # don't use sql 'in' on empty set
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = collections.defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
     def statuses(self, revisions):
         """
         Returns statuses for this repository.
         PRs without any votes do _not_ show up as unreviewed.
         :param revisions: list of revisions to get statuses for
         """
         if not revisions:
             return {}
         statuses = ChangesetStatus.query() \
             .filter(ChangesetStatus.repo == self) \
             .filter(ChangesetStatus.version == 0) \
             .filter(ChangesetStatus.revision.in_(revisions))
         grouped = {}
         for stat in statuses.all():
             pr_id = pr_nice_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_nice_id = PullRequest.make_nice_id(pr_id)
                 pr_repo = stat.pull_request.other_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo, pr_nice_id,
                                       stat.author]
         return grouped
     def _repo_size(self):
         log.debug('calculating repository size...')
         return webutils.format_byte_size(self.scm_instance.size)
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
     def set_invalidate(self):
         """
         Flush SA session caches of instances of on disk repo.
         """
         try:
             del self._scm_instance
         except AttributeError:
             pass
     _scm_instance = None  # caching inside lifetime of SA session
     @property
     def scm_instance(self):
         if self._scm_instance is None:
             return self.scm_instance_no_cache()  # will populate self._scm_instance
         return self._scm_instance
     def scm_instance_no_cache(self):
         repo_full_path = self.repo_full_path
         alias = get_scm(repo_full_path)[0]
         log.debug('Creating instance of %s repository from %s',
                   alias, self.repo_full_path)
         backend = get_backend(alias)
         if alias == 'hg':
             self._scm_instance = backend(repo_full_path, create=False, baseui=self._ui)
         else:
             self._scm_instance = backend(repo_full_path, create=False)
         log.debug('Creating instance of repository at %s', repo_full_path)
         self._scm_instance = get_repo(repo_full_path, baseui=self._ui)
         return self._scm_instance
     def __json__(self):
         return dict(
             repo_id=self.repo_id,
             repo_name=self.repo_name,
             landing_rev=self.landing_rev,
+        )
 class RepoGroup(meta.Base, BaseDbModel):
     __tablename__ = 'groups'
     __table_args__ = (
         _table_args_default_dict,
+    )
     SEP = ' &raquo; '
     group_id = Column(Integer(), primary_key=True)
     group_name = Column(Unicode(255), nullable=False, unique=True) # full path
     parent_group_id = Column('group_parent_id', Integer(), ForeignKey('groups.group_id'), nullable=True)
     group_description = Column(Unicode(10000), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     parent_group = relationship('RepoGroup', remote_side=group_id)
     owner = relationship('User')
     @classmethod
     def query(cls, sorted=False):
         """Add RepoGroup-specific helpers for common query constructs.
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(RepoGroup, cls).query()
         if sorted:
             q = q.order_by(sqlalchemy.func.lower(RepoGroup.group_name))
         return q
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
     def __repr__(self):
         return "<%s %s: %s>" % (self.__class__.__name__,
                                 self.group_id, self.group_name)
     @classmethod
     def _generate_choice(cls, repo_group):
         """Return tuple with group_id and name as html literal"""
         if repo_group is None:
             return (-1, '-- %s --' % _('top level'))
         return repo_group.group_id, webutils.literal(cls.SEP.join(repo_group.full_path_splitted))
     @classmethod
     def groups_choices(cls, groups):
         """Return tuples with group_id and name as html literal."""
         return sorted((cls._generate_choice(g) for g in groups),
                       key=lambda c: c[1].split(cls.SEP))
     @classmethod
     def guess_instance(cls, value):
         return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
     @classmethod
     def get_by_group_name(cls, group_name, case_insensitive=False):
         group_name = group_name.rstrip('/')
         if case_insensitive:
             gr = cls.query() \
                 .filter(sqlalchemy.func.lower(cls.group_name) == sqlalchemy.func.lower(group_name))
         else:
             gr = cls.query() \
                 .filter(cls.group_name == group_name)
         return gr.scalar()
     @property
     def parents(self):
         groups = []
         group = self.parent_group
         while group is not None:
             groups.append(group)
             group = group.parent_group
             assert group not in groups, group # avoid recursion on bad db content
         groups.reverse()
         return groups
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
     @property
     def name(self):
         return self.group_name.split(kallithea.URL_SEP)[-1]
     @property
     def full_path(self):
         return self.group_name
     @property
     def full_path_splitted(self):
         return self.group_name.split(kallithea.URL_SEP)
     @property
     def repositories(self):
         return Repository.query(sorted=True).filter_by(group=self)
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
         return cnt + children_count(self)
     def _recursive_objects(self, include_repos=True):
         all_ = []
         def _get_members(root_gr):
             if include_repos:
                 for r in root_gr.repositories:
                     all_.append(r)
             childs = root_gr.children.all()
             if childs:
                 for gr in childs:
                     all_.append(gr)
                     _get_members(gr)
         _get_members(self)
         return [self] + all_
     def recursive_groups_and_repos(self):
         """
         Recursive return all groups, with repositories in those groups
         """
         return self._recursive_objects()
     def recursive_groups(self):
         """
         Returns all children groups for this group including children of children
         """
         return self._recursive_objects(include_repos=False)
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return kallithea.URL_SEP.join(path_prefix + [group_name])
     def get_api_data(self):
         """
         Common function for generating api data
         """
         group = self
         data = dict(
             group_id=group.group_id,
             group_name=group.group_name,
             group_description=group.group_description,
             parent_group=group.parent_group.group_name if group.parent_group else None,
             repositories=[x.repo_name for x in group.repositories],
             owner=group.owner.username
+        )
         return data
 class Permission(meta.Base, BaseDbModel):
     __tablename__ = 'permissions'
     __table_args__ = (
         Index('p_perm_name_idx', 'permission_name'),
         _table_args_default_dict,
+    )
     PERMS = (
         ('hg.admin', _('Kallithea Administrator')),
         ('repository.none', _('Default user has no access to new repositories')),
         ('repository.read', _('Default user has read access to new repositories')),
         ('repository.write', _('Default user has write access to new repositories')),
         ('repository.admin', _('Default user has admin access to new repositories')),

kallithea/model/scm.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.scm
 ~~~~~~~~~~~~~~~~~~~
 Scm model for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 9, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import posixpath
 import re
 import sys
 import traceback
 import pkg_resources
 from tg.i18n import ugettext as _
 import kallithea
 from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel, HasUserGroupPermissionLevel
 from kallithea.lib.exceptions import IMCCommitError, NonRelativePathError
 from kallithea.lib.hooks import process_pushed_raw_ids
 from kallithea.lib.utils import action_logger, get_filesystem_repos, make_ui
 from kallithea.lib.utils2 import safe_bytes, set_hook_environment
-from kallithea.lib.vcs import get_backend
+from kallithea.lib.vcs import get_repo
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
+from kallithea.lib.vcs.exceptions import RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.model import db, meta
 log = logging.getLogger(__name__)
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 class _PermCheckIterator(object):
     def __init__(self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
     def __len__(self):
         return len(self.obj_list)
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not self.perm_checker(*self.perm_set)(
                     name, self.__class__.__name__, **self.extra_kwargs):
                 continue
             yield db_obj
 class RepoList(_PermCheckIterator):
     def __init__(self, db_repo_list, perm_level, extra_kwargs=None):
         super(RepoList, self).__init__(obj_list=db_repo_list,
                     obj_attr='repo_name', perm_set=[perm_level],
                     perm_checker=HasRepoPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class RepoGroupList(_PermCheckIterator):
     def __init__(self, db_repo_group_list, perm_level, extra_kwargs=None):
         super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
                     obj_attr='group_name', perm_set=[perm_level],
                     perm_checker=HasRepoGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class UserGroupList(_PermCheckIterator):
     def __init__(self, db_user_group_list, perm_level, extra_kwargs=None):
         super(UserGroupList, self).__init__(obj_list=db_user_group_list,
                     obj_attr='users_group_name', perm_set=[perm_level],
                     perm_checker=HasUserGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class ScmModel(object):
     """
     Generic Scm Model
     """
     def __get_repo(self, instance):
         cls = db.Repository
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, int):
             return cls.get(instance)
         elif isinstance(instance, str):
             if instance.isdigit():
                 return cls.get(int(instance))
             return cls.get_by_repo_name(instance)
         raise Exception('given object must be int, basestr or Instance'
                         ' of %s got %s' % (type(cls), type(instance)))
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         q = db.Ui.query().filter(db.Ui.ui_key == '/').one()
         return q.ui_value
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects mapping to
         vcs instances.
         :param repos_path: path to directory containing repositories
         """
         if repos_path is None:
             repos_path = self.repos_path
         log.info('scanning for repositories in %s', repos_path)
         baseui = make_ui()
         repos = {}
         for name, path in get_filesystem_repos(repos_path):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for kallithea
             name = db.Repository.normalize_repo_name(name)
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
                     klass = get_backend(path[0])
                     if path[0] == 'hg' and path[0] in kallithea.BACKENDS:
                         repos[name] = klass(path[1], baseui=baseui)
                     if path[0] == 'git' and path[0] in kallithea.BACKENDS:
                         repos[name] = klass(path[1])
             except OSError:
                     repos[name] = get_repo(path[1], baseui=baseui)
             except (OSError, VCSError):
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
     def get_repos(self, repos):
         """Return the repos the user has access to"""
         return RepoList(repos, perm_level='read')
     def get_repo_groups(self, groups=None):
         """Return the repo groups the user has access to
         If no groups are specified, use top level groups.
         """
         if groups is None:
             groups = db.RepoGroup.query() \
                 .filter(db.RepoGroup.parent_group_id == None).all()
         return RepoGroupList(groups, perm_level='read')
     def mark_for_invalidation(self, repo_name):
         """
         Mark caches of this repo invalid in the database.
         :param repo_name: the repo for which caches should be marked invalid
         """
         log.debug("Marking %s as invalidated and update cache", repo_name)
         repo = db.Repository.get_by_repo_name(repo_name)
         if repo is not None:
             repo.set_invalidate()
             repo.update_changeset_cache()
     def toggle_following_repo(self, follow_repo_id, user_id):
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_repository_id == follow_repo_id) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 meta.Session().delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = db.UserFollowing()
             f.user_id = user_id
             f.follows_repository_id = follow_repo_id
             meta.Session().add(f)
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
     def toggle_following_user(self, follow_user_id, user_id):
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_user_id == follow_user_id) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 meta.Session().delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = db.UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             meta.Session().add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def is_following_repo(self, repo_name, user_id):
         r = db.Repository.query() \
             .filter(db.Repository.repo_name == repo_name).scalar()
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_repository == r) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
         return f is not None
     def is_following_user(self, username, user_id):
         u = db.User.get_by_username(username)
         f = db.UserFollowing.query() \
             .filter(db.UserFollowing.follows_user == u) \
             .filter(db.UserFollowing.user_id == user_id).scalar()
         return f is not None
     def get_followers(self, repo):
         repo = db.Repository.guess_instance(repo)
         return db.UserFollowing.query() \
                 .filter(db.UserFollowing.follows_repository == repo).count()
     def get_forks(self, repo):
         repo = db.Repository.guess_instance(repo)
         return db.Repository.query() \
                 .filter(db.Repository.fork == repo).count()
     def get_pull_requests(self, repo):
         repo = db.Repository.guess_instance(repo)
         return db.PullRequest.query() \
                 .filter(db.PullRequest.other_repo == repo) \
                 .filter(db.PullRequest.status != db.PullRequest.STATUS_CLOSED).count()
     def mark_as_fork(self, repo, fork, user):
         repo = self.__get_repo(repo)
         fork = self.__get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError("Cannot set repository as fork of repository with other type")
         repo.fork = fork
         return repo
     def _handle_push(self, repo, username, ip_addr, action, repo_name, revisions):
         """
         Handle that the repository has changed.
         Adds an action log entry with the new revisions, and the head revision
         cache and in-memory caches are invalidated/updated.
         :param username: username who pushes
         :param action: push/push_local/push_remote
         :param repo_name: name of repo
         :param revisions: list of revisions that we pushed
         """
         set_hook_environment(username, ip_addr, repo_name, repo_alias=repo.alias, action=action)
         process_pushed_raw_ids(revisions) # also calls mark_for_invalidation
     def pull_changes(self, repo, username, ip_addr, clone_uri=None):
         """
         Pull from "clone URL" or fork origin.
         """
         dbrepo = self.__get_repo(repo)
         if clone_uri is None:
             clone_uri = dbrepo.clone_uri or dbrepo.fork and dbrepo.fork.repo_full_path
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
         repo = dbrepo.scm_instance
         repo_name = dbrepo.repo_name
         try:
             if repo.alias == 'git':
                 repo.fetch(clone_uri)
                 # git doesn't really have something like post-fetch action
                 # we fake that now.
                 # TODO: extract fetched revisions ... somehow ...
                 self._handle_push(repo,
                                   username=username,
                                   ip_addr=ip_addr,
                                   action='push_remote',
                                   repo_name=repo_name,
                                   revisions=[])
             else:
                 set_hook_environment(username, ip_addr, dbrepo.repo_name,
                                            repo.alias, action='push_remote')
                 repo.pull(clone_uri)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def commit_change(self, repo, repo_name, cs, user, ip_addr, author, message,
                       content, f_path):
         """
         Commit a change to a single file
         :param repo: a db_repo.scm_instance
         """
         user = db.User.guess_instance(user)
         imc = repo.in_memory_changeset
         imc.change(FileNode(f_path, content, mode=cs.get_file_mode(f_path)))
         try:
             tip = imc.commit(message=message, author=author,
                              parents=[cs], branch=cs.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             # clear caches - we also want a fresh object if commit fails
             self.mark_for_invalidation(repo_name)

0 comments (0 inline, 0 general)