@@ -73,37 +73,37 @@ def run_task(task, *args, **kwargs):
log.error(traceback.format_exc())
except KeyError, e:
log.debug('Unable to connect to celeryd. Sync execution')
except Exception, e:
log.debug('executing task %s in sync mode', task)
return ResultWrapper(task(*args, **kwargs))
def __get_lockkey(func, *fargs, **fkwargs):
params = list(fargs)
params.extend(['%s-%s' % ar for ar in fkwargs.items()])
func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)
lockkey = 'task_%s.lock' % \
md5(func_name + '-' + '-'.join(map(str, params))).hexdigest()
return lockkey
def locked_task(func):
def __wrapper(func, *fargs, **fkwargs):
lockkey = __get_lockkey(func, *fargs, **fkwargs)
lockkey_path = dn(dn(dn(os.path.abspath(__file__))))
lockkey_path = config['here']
log.info('running task with lockkey %s', lockkey)
try:
l = DaemonLock(jn(lockkey_path, lockkey))
l = DaemonLock(file_=jn(lockkey_path, lockkey))
ret = func(*fargs, **fkwargs)
l.release()
return ret
except LockHeld:
log.info('LockHeld')
return 'Task with key %s already running' % lockkey
return decorator(__wrapper, func)
@@ -76,52 +76,53 @@ def get_repos_path():
q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
return q.ui_value
@task(ignore_result=True)
@locked_task
def whoosh_index(repo_location, full_index):
#log = whoosh_index.get_logger()
from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
index_location = config['index_dir']
WhooshIndexingDaemon(index_location=index_location,
repo_location=repo_location, sa=get_session())\
.run(full_index=full_index)
def get_commits_stats(repo_name, ts_min_y, ts_max_y):
log = get_commits_stats.get_logger()
except:
log = logging.getLogger(__name__)
lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y,
ts_max_y)
lockkey_path = dn(dn(dn(dn(os.path.abspath(__file__)))))
lock = l = DaemonLock(jn(lockkey_path, lockkey))
lock = l = DaemonLock(file_=jn(lockkey_path, lockkey))
#for js data compatibilty cleans the key for person from '
akc = lambda k: person(k).replace('"', "")
co_day_auth_aggr = {}
commits_by_day_aggregate = {}
repos_path = get_repos_path()
repo = get_repo(safe_str(os.path.join(repos_path, repo_name)))
repo_size = len(repo.revisions)
#return if repo have no revisions
if repo_size < 1:
lock.release()
return True
skip_date_limit = True
parse_limit = int(config['app_conf'].get('commit_parse_limit'))
last_rev = 0
last_cs = None
timegetter = itemgetter('time')
sa = get_session()
dbrepo = sa.query(Repository)\
.filter(Repository.repo_name == repo_name).scalar()
@@ -80,49 +80,49 @@ class MakeIndex(BasePasterCommand):
summary = "Creates index for full text search given configuration file"
group_name = "RhodeCode"
takes_config_file = -1
parser = Command.standard_parser(verbose=True)
def command(self):
from pylons import config
add_cache(config)
engine = engine_from_config(config, 'sqlalchemy.db1.')
init_model(engine)
repo_location = self.options.repo_location \
if self.options.repo_location else RepoModel().repos_path
repo_list = map(strip, self.options.repo_list.split(',')) \
if self.options.repo_list else None
#======================================================================
# WHOOSH DAEMON
from rhodecode.lib.pidlock import LockHeld, DaemonLock
l = DaemonLock(file=jn(dn(dn(index_location)), 'make_index.lock'))
l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock'))
repo_location=repo_location,
repo_list=repo_list)\
.run(full_index=self.options.full_index)
sys.exit(1)
def update_parser(self):
self.parser.add_option('--repo-location',
action='store',
dest='repo_location',
help="Specifies repositories location to index OPTIONAL",
)
self.parser.add_option('--index-only',
dest='repo_list',
help="Specifies a comma separated list of repositores "
"to build index on OPTIONAL",
self.parser.add_option('-f',
action='store_true',
dest='full_index',
help="Specifies that index should be made full i.e"
@@ -8,59 +8,59 @@ from multiprocessing.util import Finaliz
from rhodecode import __platform__, PLATFORM_WIN
if __platform__ in PLATFORM_WIN:
import ctypes
def kill(pid, sig):
"""kill function for Win32"""
kernel32 = ctypes.windll.kernel32
handle = kernel32.OpenProcess(1, 0, pid)
return (0 != kernel32.TerminateProcess(handle, 0))
else:
kill = os.kill
class LockHeld(Exception):
pass
class DaemonLock(object):
"""daemon locking
USAGE:
l = DaemonLock(desc='test lock')
l = DaemonLock(file_='/path/tolockfile',desc='test lock')
main()
"""
def __init__(self, file=None, callbackfn=None,
def __init__(self, file_=None, callbackfn=None,
desc='daemon lock', debug=False):
self.pidfile = file if file else os.path.join(
self.pidfile = file_ if file_ else os.path.join(
os.path.dirname(__file__),
'running.lock')
self.callbackfn = callbackfn
self.desc = desc
self.debug = debug
self.held = False
#run the lock automatically !
self.lock()
self._finalize = Finalize(self, DaemonLock._on_finalize,
args=(self, debug), exitpriority=10)
@staticmethod
def _on_finalize(lock, debug):
if lock.held:
if debug:
print 'leck held finilazing and running lock.release()'
def lock(self):
locking function, if lock is present it
will raise LockHeld exception
lockname = '%s' % (os.getpid())
@@ -465,49 +465,49 @@ def get_current_revision():
return None
#==============================================================================
# TEST FUNCTIONS AND CREATORS
def create_test_index(repo_location, config, full_index):
Makes default test index
:param config: test config
:param full_index:
from rhodecode.lib.pidlock import DaemonLock, LockHeld
repo_location = repo_location
index_location = os.path.join(config['app_conf']['index_dir'])
if not os.path.exists(index_location):
os.makedirs(index_location)
l = DaemonLock(file=jn(dn(index_location), 'make_index.lock'))
l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))
repo_location=repo_location)\
def create_test_env(repos_test_path, config):
"""Makes a fresh database and
install test repository into tmp dir
from rhodecode.lib.db_manage import DbManage
from rhodecode.tests import HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, \
HG_FORK, GIT_FORK, TESTS_TMP_PATH
import tarfile
import shutil
from os.path import abspath
# PART ONE create db
dbconf = config['sqlalchemy.db1.url']
log.debug('making test db %s', dbconf)
# create test dir if it doesn't exist
Status change: