@@ -427,50 +427,52 @@ def repo2db_mapper(initial_repo_list, re
if not db_repo:
log.info('repository %s not found, creating now' % name)
added.append(name)
desc = (repo.description
if repo.description != 'unknown'
else '%s repository' % name)
new_repo = rm.create_repo(
repo_name=name,
repo_type=repo.alias,
description=desc,
repos_group=getattr(group, 'group_id', None),
owner=user,
just_db=True
)
# we added that repo just now, and make sure it has githook
# installed
if new_repo.repo_type == 'git':
ScmModel().install_git_hook(new_repo.scm_instance)
elif install_git_hook:
if db_repo.repo_type == 'git':
ScmModel().install_git_hook(db_repo.scm_instance)
# during starting install all cache keys for all repositories in the
# system, this will register all repos and multiple instances
key, _prefix, _org_key = CacheInvalidation._get_key(name)
log.debug("Creating a cache key for %s instance_id:`%s`" % (name, _prefix))
CacheInvalidation._get_or_create_key(key, _prefix, _org_key, commit=False)
CacheInvalidation.invalidate(name)
log.debug("Creating a cache key for %s instance_id=>`%s`"
% (name, _prefix or '-'))
sa.commit()
removed = []
if remove_obsolete:
# remove from database those repositories that are not in the filesystem
for repo in sa.query(Repository).all():
if repo.repo_name not in initial_repo_list.keys():
log.debug("Removing non-existing repository found in db `%s`" %
repo.repo_name)
try:
sa.delete(repo)
removed.append(repo.repo_name)
except:
#don't hold further removals on error
log.error(traceback.format_exc())
sa.rollback()
return added, removed
# set cache regions for beaker so celery can utilise it
def add_cache(settings):
cache_settings = {'regions': None}
for key in settings.keys():
@@ -916,49 +916,49 @@ class Repository(Base, BaseModel):
pr_repo = pr.other_repo.repo_name
grouped[rev] = [stat, status_lbl, pr_id, pr_repo]
for stat in statuses.all():
pr_id = pr_repo = None
if stat.pull_request:
pr_id = stat.pull_request.pull_request_id
pr_repo = stat.pull_request.other_repo.repo_name
grouped[stat.revision] = [str(stat.status), stat.status_lbl,
pr_id, pr_repo]
return grouped
#==========================================================================
# SCM CACHE INSTANCE
@property
def invalidate(self):
return CacheInvalidation.invalidate(self.repo_name)
def set_invalidate(self):
"""
set a cache for invalidation for this instance
CacheInvalidation.set_invalidate(self.repo_name)
CacheInvalidation.set_invalidate(repo_name=self.repo_name)
@LazyProperty
def scm_instance(self):
return self.scm_instance_cached()
return self.__get_instance()
def scm_instance_cached(self, cache_map=None):
@cache_region('long_term')
def _c(repo_name):
rn = self.repo_name
log.debug('Getting cached instance of repo')
if cache_map:
# get using prefilled cache_map
invalidate_repo = cache_map[self.repo_name]
if invalidate_repo:
invalidate_repo = (None if invalidate_repo.cache_active
else invalidate_repo)
else:
# get from invalidate
invalidate_repo = self.invalidate
if invalidate_repo is not None:
@@ -1417,108 +1417,118 @@ class CacheInvalidation(Base, BaseModel)
def __unicode__(self):
return u"<%s('%s:%s')>" % (self.__class__.__name__,
self.cache_id, self.cache_key)
def prefix(self):
_split = self.cache_key.split(self.cache_args, 1)
if _split and len(_split) == 2:
return _split[0]
return ''
@classmethod
def clear_cache(cls):
cls.query().delete()
def _get_key(cls, key):
Wrapper for generating a key, together with a prefix
:param key:
import rhodecode
prefix = ''
org_key = key
iid = rhodecode.CONFIG.get('instance_id')
if iid:
prefix = iid
#remove specific suffixes like _README or _RSS
key = remove_suffix(key, '_README')
key = remove_suffix(key, '_RSS')
key = remove_suffix(key, '_ATOM')
return "%s%s" % (prefix, key), prefix, key
return "%s%s" % (prefix, key), prefix, org_key
def get_by_key(cls, key):
return cls.query().filter(cls.cache_key == key).scalar()
def _get_or_create_key(cls, key, prefix, org_key, commit=True):
def get_by_repo_name(cls, repo_name):
return cls.query().filter(cls.cache_args == repo_name).all()
def _get_or_create_key(cls, key, repo_name, commit=True):
inv_obj = Session().query(cls).filter(cls.cache_key == key).scalar()
if not inv_obj:
inv_obj = CacheInvalidation(key, org_key)
inv_obj = CacheInvalidation(key, repo_name)
Session().add(inv_obj)
if commit:
Session().commit()
except Exception:
Session().rollback()
return inv_obj
def invalidate(cls, key):
Returns Invalidation object if this given key should be invalidated
None otherwise. `cache_active = False` means that this cache
state is not valid and needs to be invalidated
repo_name = key
repo_name = remove_suffix(repo_name, '_README')
repo_name = remove_suffix(repo_name, '_RSS')
repo_name = remove_suffix(repo_name, '_ATOM')
# adds instance prefix
key, _prefix, _org_key = cls._get_key(key)
inv = cls._get_or_create_key(key, _prefix, _org_key)
inv = cls._get_or_create_key(key, repo_name)
if inv and inv.cache_active is False:
return inv
def set_invalidate(cls, key):
def set_invalidate(cls, key=None, repo_name=None):
Mark this Cache key for invalidation
Mark this Cache key for invalidation, either by key or whole
cache sets based on repo_name
if key:
inv_objs = Session().query(cls).filter(cls.cache_key == key).all()
elif repo_name:
inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
inv_objs = Session().query(cls).filter(cls.cache_args == _org_key).all()
log.debug('marking %s key[s] %s for invalidation' % (len(inv_objs),
_org_key))
log.debug('marking %s key[s] for invalidation based on key=%s,repo_name=%s'
% (len(inv_objs), key, repo_name))
for inv_obj in inv_objs:
if inv_obj:
inv_obj.cache_active = False
print inv_obj
def set_valid(cls, key):
Mark this cache key as active and currently cached
inv_obj = cls.get_by_key(key)
inv_obj.cache_active = True
def get_cache_map(cls):
class cachemapdict(dict):
def __init__(self, *args, **kwargs):
@@ -270,49 +270,49 @@ class ScmModel(BaseModel):
repos_path=self.repos_path,
order_by=sort_key)
repo_iter = CachedRepoList(all_repos,
return repo_iter
def get_repos_groups(self, all_groups=None):
if all_groups is None:
all_groups = RepoGroup.query()\
.filter(RepoGroup.group_parent_id == None).all()
group_iter = GroupList(all_groups)
return group_iter
def mark_for_invalidation(self, repo_name):
Puts cache invalidation task into db for
further global cache invalidation
:param repo_name: this repo that should invalidation take place
CacheInvalidation.set_invalidate(repo_name)
CacheInvalidation.set_invalidate(repo_name=repo_name)
def toggle_following_repo(self, follow_repo_id, user_id):
f = self.sa.query(UserFollowing)\
.filter(UserFollowing.follows_repo_id == follow_repo_id)\
.filter(UserFollowing.user_id == user_id).scalar()
if f is not None:
self.sa.delete(f)
action_logger(UserTemp(user_id),
'stopped_following_repo',
RepoTemp(follow_repo_id))
return
raise
f = UserFollowing()
f.user_id = user_id
f.follows_repo_id = follow_repo_id
self.sa.add(f)
@@ -1606,48 +1606,49 @@ input.disabled {
div.form div.fields div.field div.button {
margin: 0;
padding: 0 0 0 8px;
}
#content div.box table.noborder {
border: 1px solid transparent;
#content div.box table {
width: 100%;
border-collapse: separate;
padding: 0;
border: 1px solid #eee;
-webkit-border-radius: 4px;
-moz-border-radius: 4px;
border-radius: 4px;
#content div.box table th {
background: #eee;
border-bottom: 1px solid #ddd;
padding: 5px 0px 5px 5px;
text-align: left;
#content div.box table th.left {
#content div.box table th.right {
text-align: right;
#content div.box table th.center {
text-align: center;
#content div.box table th.selected {
vertical-align: middle;
#content div.box table td {
background: #fff;
border-bottom: 1px solid #cdcdcd;
padding: 5px;
@@ -175,53 +175,62 @@
${h.submit('remote_pull_%s' % c.repo_info.repo_name,_('Pull changes from remote location'),class_="ui-btn",onclick="return confirm('"+_('Confirm to pull changes from remote side')+"');")}
<div class="field" style="border:none">
<ul>
<li><a href="${c.repo_info.clone_uri}">${c.repo_info.clone_uri}</a></li>
</ul>
</div>
${h.end_form()}
%endif
<h3>${_('Cache')}</h3>
${h.form(url('repo_cache', repo_name=c.repo_info.repo_name),method='delete')}
<div class="form">
<div class="fields">
${h.submit('reset_cache_%s' % c.repo_info.repo_name,_('Invalidate repository cache'),class_="ui-btn",onclick="return confirm('"+_('Confirm to invalidate repository cache')+"');")}
<div class="field" style="border:none;color:#888">
<li>${_('Manually invalidate cache for this repository. On first access repository will be cached again')}
</li>
<div class="field" style="border:none;">
${_('List of cached values')}
<table>
<tr>
<th>${_('Prefix')}</th>
<th>${_('Key')}</th>
<th>${_('Active')}</th>
</tr>
%for cache in c.repo_info.cache_keys:
<li>INSTANCE ID:${cache.prefix or '-'} ${cache.cache_args} CACHED: ${h.bool2icon(cache.cache_active)}</li>
<td>${cache.prefix or '-'}</td>
<td>${cache.cache_key}</td>
<td>${h.bool2icon(cache.cache_active)}</td>
%endfor
</table>
<h3>${_('Public journal')}</h3>
${h.form(url('repo_public_journal', repo_name=c.repo_info.repo_name),method='put')}
${h.hidden('auth_token',str(h.get_token()))}
<div class="field">
%if c.in_public_journal:
${h.submit('set_public_%s' % c.repo_info.repo_name,_('Remove from public journal'),class_="ui-btn")}
%else:
${h.submit('set_public_%s' % c.repo_info.repo_name,_('Add to public journal'),class_="ui-btn")}
<li>${_('All actions made on this repository will be accessible to everyone in public journal')}
Status change: