@@ -17,12 +17,14 @@ news
- email map, allowing users to have multiple email addresses mapped into
their accounts
- changed setup-app into setup-rhodecode and added default options to it.
- new git repos are created as bare now by default
- #464 added links to groups in permission box
- #465 mentions autocomplete inside comments boxes
- #469 added --update-only option to whoosh to re-index only given list
of repos in index
fixes
+++++
- improved translations
- fixes issue #455 Creating an archive generates an exception on Windows
@@ -90,23 +90,26 @@ class MakeIndex(BasePasterCommand):
init_model(engine)
index_location = config['index_dir']
repo_location = self.options.repo_location \
if self.options.repo_location else RepoModel().repos_path
repo_list = map(strip, self.options.repo_list.split(',')) \
if self.options.repo_list else None
repo_update_list = map(strip, self.options.repo_update_list.split(',')) \
if self.options.repo_update_list else None
load_rcextensions(config['here'])
#======================================================================
# WHOOSH DAEMON
from rhodecode.lib.pidlock import LockHeld, DaemonLock
from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
try:
l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock'))
WhooshIndexingDaemon(index_location=index_location,
repo_location=repo_location,
repo_list=repo_list,)\
repo_list=repo_list,
repo_update_list=repo_update_list)\
.run(full_index=self.options.full_index)
l.release()
except LockHeld:
sys.exit(1)
def update_parser(self):
@@ -116,13 +119,20 @@ class MakeIndex(BasePasterCommand):
help="Specifies repositories location to index OPTIONAL",
)
self.parser.add_option('--index-only',
action='store',
dest='repo_list',
help="Specifies a comma separated list of repositores "
"to build index on OPTIONAL",
"to build index on. If not given all repositories "
"are scanned for indexing. OPTIONAL",
self.parser.add_option('--update-only',
dest='repo_update_list',
"to re-build index on. OPTIONAL",
self.parser.add_option('-f',
action='store_true',
dest='full_index',
help="Specifies that index should be made full i.e"
" destroy old and build from scratch",
@@ -50,36 +50,47 @@ from whoosh.index import create_in, open
log = logging.getLogger('whoosh_indexer')
class WhooshIndexingDaemon(object):
"""
Daemon for atomic jobs
Daemon for atomic indexing jobs
def __init__(self, indexname=IDX_NAME, index_location=None,
repo_location=None, sa=None, repo_list=None):
repo_location=None, sa=None, repo_list=None,
repo_update_list=None):
self.indexname = indexname
self.index_location = index_location
if not index_location:
raise Exception('You have to provide index location')
self.repo_location = repo_location
if not repo_location:
raise Exception('You have to provide repositories location')
self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
#filter repo list
if repo_list:
filtered_repo_paths = {}
self.filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
filtered_repo_paths[repo_name] = repo
self.filtered_repo_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_paths
self.repo_paths = filtered_repo_paths
#filter update repo list
self.filtered_repo_update_paths = {}
if repo_update_list:
if repo_name in repo_update_list:
self.filtered_repo_update_paths[repo_name] = repo
self.repo_paths = self.filtered_repo_update_paths
self.initial = False
if not os.path.isdir(self.index_location):
os.makedirs(self.index_location)
log.info('Cannot run incremental index since it does not'
' yet exist running full build')
@@ -169,14 +180,14 @@ class WhooshIndexingDaemon(object):
log.debug('>> COMMITING CHANGES <<')
writer.commit(merge=True)
log.debug('>>> FINISHED BUILDING INDEX <<<')
def update_index(self):
log.debug(('STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths))
log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))
idx = open_dir(self.index_location, indexname=self.indexname)
# The set of all paths in the index
indexed_paths = set()
# The set of all paths we need to re-index
to_index = set()
@@ -184,33 +195,35 @@ class WhooshIndexingDaemon(object):
reader = idx.reader()
writer = idx.writer()
# Loop over the stored fields in the index
for fields in reader.all_stored_fields():
indexed_path = fields['path']
indexed_repo_path = fields['repository']
indexed_paths.add(indexed_path)
repo = self.repo_paths[fields['repository']]
if not indexed_repo_path in self.filtered_repo_update_paths:
continue
repo = self.repo_paths[indexed_repo_path]
node = self.get_node(repo, indexed_path)
except (ChangesetError, NodeDoesNotExistError):
# This file was deleted since it was indexed
log.debug('removing from index %s' % indexed_path)
writer.delete_by_term('path', indexed_path)
else:
# Check if this file was changed since it was indexed
indexed_time = fields['modtime']
mtime = self.get_node_mtime(node)
if mtime > indexed_time:
# The file has changed, delete it and add it to the list of
# files to reindex
log.debug('adding to reindex list %s' % indexed_path)
to_index.add(indexed_path)
# Loop over the files in the filesystem
# Assume we have a function that gathers the filenames of the
# documents to be indexed
ri_cnt = riwc_cnt = 0
Status change: