kallithea Changeset - 4a18e6bf6b87

Changeset - 4a18e6bf6b87

Parent rev.

Child rev.

[Not reviewed]

default

0 2 0

Mads Kiilerich (mads) - 5 years ago 2020-12-18 22:03:10
mads@kiilerich.com

Grafted from: a888884b3b86

model: simplify how get_commits_stats task group on author

Avoid using the caching h.person . We want to get rid of the model
dependency on helpers.

The stats are persisted, and any temporary incorrectness in the long term
cached h.person will thus remain forever. It is thus arguably better to avoid
using it in this place.

get_commits_stats is also a long running task, so speed is not *that* critical.
And generally, processing commits in order will have a lot of the same
committers, so a local cache will have a good hit rate.

(Alternatively, h.person could perhaps be in user model ... but that's not how
it is now.)

2 files changed with 21 insertions and 4 deletions:

kallithea/model/async_tasks.py

scripts/deps.py

0 comments (0 inline, 0 general)

kallithea/model/async_tasks.py

➞

Show inline comments

@@ @@ -37,17 +37,16 @@ from operator import itemgetter @@
 from time import mktime
 import celery.utils.log
 from tg import config
 import kallithea
 import kallithea.lib.helpers as h
 from kallithea.lib import celerylib, conf, ext_json, hooks
 from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
 from kallithea.lib.utils2 import asbool, ascii_bytes
 from kallithea.lib.vcs.utils import author_email
+from kallithea.lib.vcs.utils import author_email, author_name
 from kallithea.model import db, repo, userlog
 __all__ = ['whoosh_index', 'get_commits_stats', 'send_email']
@@ @@ -63,12 +62,25 @@ def whoosh_index(repo_location, full_ind @@
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location) \
                          .run(full_index=full_index)
 def _author_username(author):
     """Return the username of the user identified by the email part of the 'author' string,
     default to the name or email.
     Kind of similar to h.person() ."""
     email = author_email(author)
     if email:
         user = db.User.get_by_email(email)
         if user is not None:
             return user.username
     # Still nothing?  Just pass back the author name if any, else the email
     return author_name(author) or email
 @celerylib.task
 @celerylib.dbsession
 def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
     DBS = celerylib.get_session()
     lockkey = celerylib.__get_lockkey('get_commits_stats', repo_name, ts_min_y,
                             ts_max_y)
@@ @@ -121,19 +133,25 @@ def get_commits_stats(repo_name, ts_min_ @@
         log.debug('starting parsing %s', parse_limit)
         last_rev = last_rev + 1 if last_rev and last_rev >= 0 else 0
         log.debug('Getting revisions from %s to %s',
              last_rev, last_rev + parse_limit
+        )
         usernames_cache = {}
         for cs in scm_repo[last_rev:last_rev + parse_limit]:
             log.debug('parsing %s', cs)
             last_cs = cs  # remember last parsed changeset
             tt = cs.date.timetuple()
             k = mktime(tt[:3] + (0, 0, 0, 0, 0, 0))
             username = h.person(cs.author)
             # get username from author - similar to what h.person does
             username = usernames_cache.get(cs.author)
             if username is None:
                 username = _author_username(cs.author)
                 usernames_cache[cs.author] = username
             if username in co_day_auth_aggr:
                 try:
                     l = [timegetter(x) for x in
                          co_day_auth_aggr[username]['data']]
                     time_pos = l.index(k)
                 except ValueError:

scripts/deps.py

➞

Show inline comments

@@ @@ -155,13 +155,12 @@ shown_modules = normal_modules | top_mod @@
 # break the chains somehow - this is a cleanup TODO list
 known_violations = [
 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'),  # needs base&facade
 ('kallithea.lib.utils', 'kallithea.model'),  # clean up utils
 ('kallithea.lib.utils', 'kallithea.model.db'),
 ('kallithea.lib.utils', 'kallithea.model.scm'),
 ('kallithea.model.async_tasks', 'kallithea.lib.helpers'),
 ('kallithea.model.async_tasks', 'kallithea.lib.hooks'),
 ('kallithea.model.async_tasks', 'kallithea.lib.indexers'),
 ('kallithea.model.async_tasks', 'kallithea.model'),
 ('kallithea.model', 'kallithea.lib.auth'),  # auth.HasXXX
 ('kallithea.model', 'kallithea.lib.auth_modules'),  # validators
 ('kallithea.model', 'kallithea.lib.hooks'),  # clean up hooks

0 comments (0 inline, 0 general)