kallithea Changeset - 116151b6bfb2

Changeset - 116151b6bfb2

Parent rev.

Child rev.

[Not reviewed]

default

0 8 0

Mads Kiilerich (mads) - 5 years ago 2020-12-30 00:14:57
mads@kiilerich.com

Grafted from: 84df601909c0

celery: drop tracking of task_id - we use ignore_result=True and will never get anything back

There is thus no need for configuration of celery.result_backend .

The alternative would be to fix it. That could give better error reporting from
failing repo creations, but would require quite a bit of additional changes
before it actually works reliably.

8 files changed with 3 insertions and 36 deletions:

development.ini

kallithea/controllers/admin/repos.py

kallithea/controllers/api/api.py

kallithea/controllers/forks.py

kallithea/lib/celerylib/__init__.py

kallithea/templates/admin/repos/repo_creating.html

kallithea/templates/ini/template.ini.mako

kallithea/tests/api/api_base.py

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -250,28 +250,24 @@ ssh_enabled = false @@
 ####################################
 ##         CELERY CONFIG          ##
 ####################################
 ## Note: Celery doesn't support Windows.
 use_celery = false
 ## Celery config settings from https://docs.celeryproject.org/en/4.4.0/userguide/configuration.html prefixed with 'celery.'.
 ## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
 celery.broker_url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 celery.result_backend = db+sqlite:///celery-results.db
 #celery.amqp.task.result.expires = 18000
 celery.worker_concurrency = 2
 celery.worker_max_tasks_per_child = 100
 ####################################
 ##          BEAKER CACHE          ##
 ####################################
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.regions = long_term,long_term_file

kallithea/controllers/admin/repos.py

➞

Show inline comments

@@ @@ -108,35 +108,34 @@ class ReposController(base.BaseRepoContr @@
             return htmlfill.render(
                 base.render('admin/repos/repo_add.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 force_defaults=False,
                 encoding="UTF-8")
         try:
             # create is done sometimes async on celery, db transaction
             # management is handled there.
             task = RepoModel().create(form_result, request.authuser.user_id)
             task_id = task.task_id
         except Exception:
             log.error(traceback.format_exc())
             msg = (_('Error creating repository %s')
                    % form_result.get('repo_name'))
             webutils.flash(msg, category='error')
             raise HTTPFound(location=url('home'))
         raise HTTPFound(location=webutils.url('repo_creating_home',
                               repo_name=form_result['repo_name_full'],
-                              task_id=task_id))
                               ))
     @NotAnonymous()
     def create_repository(self):
         self.__load_defaults()
         if not c.repo_groups:
             raise HTTPForbidden
         parent_group = request.GET.get('parent_group')
         ## apply the defaults from defaults page
         defaults = db.Setting.get_default_repo_settings(strip_prefix=True)
         if parent_group:
             prg = db.RepoGroup.get(parent_group)
@@ @@ -149,41 +148,32 @@ class ReposController(base.BaseRepoContr @@
         return htmlfill.render(
             base.render('admin/repos/repo_add.html'),
             defaults=defaults,
             errors={},
             prefix_error=False,
             encoding="UTF-8",
             force_defaults=False)
     @LoginRequired()
     def repo_creating(self, repo_name):
         c.repo = repo_name
         c.task_id = request.GET.get('task_id')
         if not c.repo:
             raise HTTPNotFound()
         return base.render('admin/repos/repo_creating.html')
     @LoginRequired()
     @base.jsonify
     def repo_check(self, repo_name):
         c.repo = repo_name
         task_id = request.GET.get('task_id')
         if task_id and task_id not in ['None']:
             if kallithea.CELERY_APP:
                 task_result = kallithea.CELERY_APP.AsyncResult(task_id)
                 if task_result.failed():
                     raise HTTPInternalServerError(task_result.traceback)
         repo = db.Repository.get_by_repo_name(repo_name)
         if repo and repo.repo_state == db.Repository.STATE_CREATED:
             if repo.clone_uri:
                 webutils.flash(_('Created repository %s from %s')
                         % (repo.repo_name, repo.clone_uri_hidden), category='success')
             else:
                 repo_url = webutils.link_to(repo.repo_name,
                                      webutils.url('summary_home',
                                            repo_name=repo.repo_name))
                 fork = repo.fork
                 if fork is not None:
                     fork_name = fork.repo_name

kallithea/controllers/api/api.py

➞

Show inline comments

@@ @@ -1257,31 +1257,29 @@ class ApiController(JSONRPCController): @@
                 repo_type=repo_type,
                 repo_description=description,
                 repo_private=private,
                 clone_uri=clone_uri,
                 repo_group=repo_group,
                 repo_landing_rev=landing_rev,
                 enable_statistics=enable_statistics,
                 enable_downloads=enable_downloads,
                 repo_copy_permissions=copy_permissions,
+            )
             task = RepoModel().create(form_data=data, cur_user=owner.username)
             task_id = task.task_id
             # no commit, it's done in RepoModel, or async via celery
             return dict(
                 msg="Created new repository `%s`" % (repo_name,),
                 success=True,  # cannot return the repo data here since fork
                                # can be done async
                 task=task_id
+            )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to create repository `%s`' % (repo_name,))
     # permission check inside
     def update_repo(self, repoid, name=None,
                     owner=None,
                     group=None,
                     description=None, private=None,
                     clone_uri=None, landing_rev=None,
@@ @@ -1432,31 +1430,29 @@ class ApiController(JSONRPCController): @@
                 repo_name_full=fork_name,
                 repo_group=repo_group,
                 repo_type=repo.repo_type,
                 description=description,
                 private=private,
                 copy_permissions=copy_permissions,
                 landing_rev=landing_rev,
                 update_after_clone=False,
                 fork_parent_id=repo.repo_id,
+            )
             task = RepoModel().create_fork(form_data, cur_user=owner.username)
             # no commit, it's done in RepoModel, or async via celery
             task_id = task.task_id
             return dict(
                 msg='Created fork of `%s` as `%s`' % (repo.repo_name,
                                                       fork_name),
                 success=True,  # cannot return the repo data here since fork
                                # can be done async
                 task=task_id
+            )
         except Exception:
             log.error(traceback.format_exc())
             raise JSONRPCError(
                 'failed to fork repository `%s` as `%s`' % (repo_name,
                                                             fork_name)
+            )
     # permission check inside
     def delete_repo(self, repoid, forks=''):
         """
         Deletes a repository. This command can be executed only using api_key belonging

kallithea/controllers/forks.py

➞

Show inline comments

@@ @@ -134,40 +134,38 @@ class ForksController(base.BaseRepoContr @@
             force_defaults=False)
     @LoginRequired()
     @HasPermissionAnyDecorator('hg.admin', 'hg.fork.repository')
     @HasRepoPermissionLevelDecorator('read')
     def fork_create(self, repo_name):
         self.__load_defaults()
         c.repo_info = db.Repository.get_by_repo_name(repo_name)
         _form = RepoForkForm(old_data={'repo_type': c.repo_info.repo_type},
                              repo_groups=c.repo_groups,
                              landing_revs=c.landing_revs_choices)()
         form_result = {}
         task_id = None
         try:
             form_result = _form.to_python(dict(request.POST))
             # an approximation that is better than nothing
             if not db.Ui.get_by_key('hooks', db.Ui.HOOK_UPDATE).ui_active:
                 form_result['update_after_clone'] = False
             # create fork is done sometimes async on celery, db transaction
             # management is handled there.
             task = RepoModel().create_fork(form_result, request.authuser.user_id)
             task_id = task.task_id
         except formencode.Invalid as errors:
             return htmlfill.render(
                 base.render('forks/fork.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
         except Exception:
             log.error(traceback.format_exc())
             webutils.flash(_('An error occurred during repository forking %s') %
                     repo_name, category='error')
         raise HTTPFound(location=webutils.url('repo_creating_home',
                               repo_name=form_result['repo_name_full'],
-                              task_id=task_id))
                               ))

kallithea/lib/celerylib/__init__.py

➞

Show inline comments

@@ @@ -44,26 +44,24 @@ log = logging.getLogger(__name__) @@
 class FakeTask(object):
     """Fake a sync result to make it look like a finished task"""
     def __init__(self, result):
         self.result = result
     def failed(self):
         return False
     traceback = None # if failed
     task_id = None
 def task(f_org):
     """Wrapper of celery.task.task, running async if CELERY_APP
     """
     if kallithea.CELERY_APP:
         def f_async(*args, **kwargs):
             log.info('executing %s task', f_org.__name__)
             try:
                 f_org(*args, **kwargs)
             finally:
                 meta.Session.remove()  # prevent reuse of auto created db sessions

kallithea/templates/admin/repos/repo_creating.html

➞

Show inline comments

@@ @@ -37,25 +37,25 @@ @@
         <div id="progress_error" style="display: none;">
             <div class="alert alert-danger">
                 ${_("We're sorry but error occurred during this operation. Please check your Kallithea server logs, or contact administrator.")}
             </div>
         </div>
     </div>
 </div>
 <script>
 'use strict';
 (function worker() {
   $.ajax({
-    url: ${h.js(h.url('repo_check_home', repo_name=c.repo_name, repo=c.repo, task_id=c.task_id))},
     url: ${h.js(h.url('repo_check_home', repo_name=c.repo_name, repo=c.repo))},
     success: function(data) {
       if(data.result === true){
           //redirect to created fork if our ajax loop tells us to do so.
           window.location = ${h.js(h.url('summary_home', repo_name = c.repo))};
+      }
     },
     complete: function(resp) {
       if (resp.status == 200){
           // Schedule the next request when the current one's complete
           setTimeout(worker, 1000);
+      }
       else{

kallithea/templates/ini/template.ini.mako

➞

Show inline comments

@@ @@ -325,28 +325,24 @@ ssh_locale = ${ssh_locale} @@
 <%text>##</%text>##################################
 <%text>##</%text>         CELERY CONFIG          ##
 <%text>##</%text>##################################
 <%text>##</%text> Note: Celery doesn't support Windows.
 use_celery = false
 <%text>##</%text> Celery config settings from https://docs.celeryproject.org/en/4.4.0/userguide/configuration.html prefixed with 'celery.'.
 <%text>##</%text> Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
 celery.broker_url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
 celery.result_backend = db+sqlite:///celery-results.db
 #celery.amqp.task.result.expires = 18000
 celery.worker_concurrency = 2
 celery.worker_max_tasks_per_child = 100
 <%text>##</%text>##################################
 <%text>##</%text>          BEAKER CACHE          ##
 <%text>##</%text>##################################
 beaker.cache.data_dir = %(here)s/data/cache/data
 beaker.cache.lock_dir = %(here)s/data/cache/lock
 beaker.cache.regions = long_term,long_term_file

kallithea/tests/api/api_base.py

➞

Show inline comments

@@ @@ -780,25 +780,24 @@ class _BaseTestApi(object): @@
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,
+        )
         response = api_call(self, params)
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
+        }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
     @base.parametrize('repo_name', [
         '',
         '.',
         '..',
         ':',
         '/',
         '<test>',
@@ @@ -851,25 +850,24 @@ class _BaseTestApi(object): @@
         # create group before creating repo
         rg = fixture.create_repo_group(repo_group_name)
         meta.Session().commit()
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
                                   repo_type=self.REPO_TYPE,)
         response = api_call(self, params)
         expected = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
+        }
         self._compare_ok(id_, expected, given=response.body)
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         fixture.destroy_repo(repo_name)
         fixture.destroy_repo_group(repo_group_name)
     def test_api_create_repo_in_repo_group_without_permission(self):
         repo_group_basename = 'api-repo-repo'
         repo_group_name = '%s/%s' % (TEST_REPO_GROUP, repo_group_basename)
         repo_name = '%s/api-repo' % repo_group_name
@@ @@ -884,25 +882,24 @@ class _BaseTestApi(object): @@
         meta.Session().commit()
         id_, params = _build_data(self.apikey_regular, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
+        )
         response = api_call(self, params)
         # Current result when API access control is different from Web:
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
+        }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
         # Expected and arguably more correct result:
         #expected = 'failed to create repository `%s`' % repo_name
         #self._compare_error(id_, expected, given=response.body)
         fixture.destroy_repo_group(repo_group_name)
     def test_api_create_repo_unknown_owner(self):
@@ @@ -922,45 +919,43 @@ class _BaseTestApi(object): @@
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
+        )
         response = api_call(self, params)
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
+        }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
     def test_api_create_repo_by_non_admin(self):
         repo_name = 'api-repo'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey_regular, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
+        )
         response = api_call(self, params)
         repo = RepoModel().get_by_repo_name(repo_name)
         assert repo is not None
         ret = {
             'msg': 'Created new repository `%s`' % repo_name,
             'success': True,
             'task': None,
+        }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(repo_name)
     def test_api_create_repo_by_non_admin_specify_owner(self):
         repo_name = 'api-repo'
         owner = 'i-dont-exist'
         id_, params = _build_data(self.apikey_regular, 'create_repo',
                                   repo_name=repo_name,
                                   repo_type=self.REPO_TYPE,
                                   owner=owner)
@@ @@ -1251,46 +1246,44 @@ class _BaseTestApi(object): @@
         fork_name = 'api-repo-fork'
         id_, params = _build_data(self.apikey, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
+        )
         response = api_call(self, params)
         ret = {
             'msg': 'Created fork of `%s` as `%s`' % (self.REPO,
                                                      fork_name),
             'success': True,
             'task': None,
+        }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(fork_name)
     @base.parametrize('fork_name', [
         'api-repo-fork',
         '%s/api-repo-fork' % TEST_REPO_GROUP,
     ])
     def test_api_fork_repo_non_admin(self, fork_name):
         id_, params = _build_data(self.apikey_regular, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
+        )
         response = api_call(self, params)
         ret = {
             'msg': 'Created fork of `%s` as `%s`' % (self.REPO,
                                                      fork_name),
             'success': True,
             'task': None,
+        }
         expected = ret
         self._compare_ok(id_, expected, given=response.body)
         fixture.destroy_repo(fork_name)
     def test_api_fork_repo_non_admin_specify_owner(self):
         fork_name = 'api-repo-fork'
         id_, params = _build_data(self.apikey_regular, 'fork_repo',
                                   repoid=self.REPO,
                                   fork_name=fork_name,
                                   owner=base.TEST_USER_ADMIN_LOGIN,
+        )

0 comments (0 inline, 0 general)