# -*- coding: utf-8 -*-
"""
rhodecode.controllers.files
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Files controller for RhodeCode
:created_on: Apr 21, 2010
:author: marcink
:copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
:license: GPLv3, see COPYING for more details.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import with_statement
import os
import logging
import traceback
import tempfile
import shutil
from pylons import request, response, tmpl_context as c, url
from pylons.i18n.translation import _
from pylons.controllers.util import redirect
from rhodecode.lib.utils import jsonify
from rhodecode.lib import diffs
from rhodecode.lib import helpers as h
from rhodecode.lib.compat import OrderedDict
from rhodecode.lib.utils2 import convert_line_endings, detect_mode, safe_str,\
str2bool
from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
from rhodecode.lib.base import BaseRepoController, render
from rhodecode.lib.vcs.backends.base import EmptyChangeset
from rhodecode.lib.vcs.conf import settings
from rhodecode.lib.vcs.exceptions import RepositoryError, \
ChangesetDoesNotExistError, EmptyRepositoryError, \
ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,\
NodeDoesNotExistError, ChangesetError, NodeError
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.model.repo import RepoModel
from rhodecode.model.scm import ScmModel
from rhodecode.model.db import Repository
from rhodecode.controllers.changeset import anchor_url, _ignorews_url,\
_context_url, get_line_ctx, get_ignore_ws
from webob.exc import HTTPNotFound
log = logging.getLogger(__name__)
class FilesController(BaseRepoController):
def __before__(self):
super(FilesController, self).__before__()
c.cut_off_limit = self.cut_off_limit
def __get_cs_or_redirect(self, rev, repo_name, redirect_after=True):
Safe way to get changeset if error occur it redirects to tip with
proper message
:param rev: revision to fetch
:param repo_name: repo name to redirect after
try:
return c.rhodecode_repo.get_changeset(rev)
except EmptyRepositoryError, e:
if not redirect_after:
return None
url_ = url('files_add_home',
repo_name=c.repo_name,
revision=0, f_path='')
add_new = h.link_to(_('click here to add new file'), url_)
h.flash(h.literal(_('There are no files yet %s') % add_new),
category='warning')
redirect(h.url('summary_home', repo_name=repo_name))
except RepositoryError, e: # including ChangesetDoesNotExistError
h.flash(str(e), category='error')
redirect(h.url('files_home', repo_name=repo_name, revision='tip'))
raise HTTPNotFound()
def __get_filenode_or_redirect(self, repo_name, cs, path):
Returns file_node, if error occurs or given path is directory,
it'll redirect to top level path
:param repo_name: repo_name
:param cs: given changeset
:param path: path to lookup
file_node = cs.get_node(path)
if file_node.is_dir():
raise RepositoryError('given path is a directory')
except RepositoryError, e:
redirect(h.url('files_home', repo_name=repo_name,
revision=cs.raw_id))
return file_node
@LoginRequired()
@HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
'repository.admin')
def index(self, repo_name, revision, f_path, annotate=False):
# redirect to given revision from form if given
post_revision = request.POST.get('at_rev', None)
if post_revision:
cs = self.__get_cs_or_redirect(post_revision, repo_name)
redirect(url('files_home', repo_name=c.repo_name,
revision=cs.raw_id, f_path=f_path))
c.changeset = self.__get_cs_or_redirect(revision, repo_name)
c.branch = request.GET.get('branch', None)
c.f_path = f_path
c.annotate = annotate
cur_rev = c.changeset.revision
# prev link
prev_rev = c.rhodecode_repo.get_changeset(cur_rev).prev(c.branch)
c.url_prev = url('files_home', repo_name=c.repo_name,
revision=prev_rev.raw_id, f_path=f_path)
if c.branch:
c.url_prev += '?branch=%s' % c.branch
except (ChangesetDoesNotExistError, VCSError):
c.url_prev = '#'
# next link
next_rev = c.rhodecode_repo.get_changeset(cur_rev).next(c.branch)
c.url_next = url('files_home', repo_name=c.repo_name,
revision=next_rev.raw_id, f_path=f_path)
c.url_next += '?branch=%s' % c.branch
c.url_next = '#'
# files or dirs
c.file = c.changeset.get_node(f_path)
if c.file.is_file():
c.load_full_history = False
file_last_cs = c.file.last_changeset
c.file_changeset = (c.changeset
if c.changeset.revision < file_last_cs.revision
else file_last_cs)
#determine if we're on branch head
_branches = c.rhodecode_repo.branches
c.on_branch_head = revision in _branches.keys() + _branches.values()
_hist = []
c.file_history = []
if c.load_full_history:
c.file_history, _hist = self._get_node_history(c.changeset, f_path)
c.authors = []
for a in set([x.author for x in _hist]):
c.authors.append((h.email(a), h.person(a)))
else:
c.authors = c.file_history = []
h.flash(str(e), category='warning')
revision='tip'))
if request.environ.get('HTTP_X_PARTIAL_XHR'):
return render('files/files_ypjax.html')
return render('files/files.html')
def history(self, repo_name, revision, f_path, annotate=False):
return render('files/files_history_box.html')
def rawfile(self, repo_name, revision, f_path):
cs = self.__get_cs_or_redirect(revision, repo_name)
file_node = self.__get_filenode_or_redirect(repo_name, cs, f_path)
response.content_disposition = 'attachment; filename=%s' % \
safe_str(f_path.split(Repository.url_sep())[-1])
response.content_type = file_node.mimetype
return file_node.content
def raw(self, repo_name, revision, f_path):
raw_mimetype_mapping = {
# map original mimetype to a mimetype used for "show as raw"
# you can also provide a content-disposition to override the
# default "attachment" disposition.
# orig_type: (new_type, new_dispo)
# show images inline:
'image/x-icon': ('image/x-icon', 'inline'),
'image/png': ('image/png', 'inline'),
'image/gif': ('image/gif', 'inline'),
'image/jpeg': ('image/jpeg', 'inline'),
'image/svg+xml': ('image/svg+xml', 'inline'),
}
mimetype = file_node.mimetype
mimetype, dispo = raw_mimetype_mapping[mimetype]
except KeyError:
# we don't know anything special about this, handle it safely
if file_node.is_binary:
# do same as download raw for binary files
mimetype, dispo = 'application/octet-stream', 'attachment'
# do not just use the original mimetype, but force text/plain,
# otherwise it would serve text/html and that might be unsafe.
# Note: underlying vcs library fakes text/plain mimetype if the
# mimetype can not be determined and it thinks it is not
# binary.This might lead to erroneous text display in some
# cases, but helps in other cases, like with text files
# without extension.
mimetype, dispo = 'text/plain', 'inline'
if dispo == 'attachment':
dispo = 'attachment; filename=%s' % \
safe_str(f_path.split(os.sep)[-1])
response.content_disposition = dispo
response.content_type = mimetype
@HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
def edit(self, repo_name, revision, f_path):
repo = c.rhodecode_db_repo
if repo.enable_locking and repo.locked[0]:
h.flash(_('This repository is has been locked by %s on %s')
% (h.person_by_id(repo.locked[0]),
h.fmt_date(h.time_to_datetime(repo.locked[1]))),
'warning')
return redirect(h.url('files_home',
repo_name=repo_name, revision='tip'))
# check if revision is a branch identifier- basically we cannot
# create multiple heads via file editing
_branches = repo.scm_instance.branches
# check if revision is a branch name or branch hash
if revision not in _branches.keys() + _branches.values():
h.flash(_('You can only edit files with revision '
'being a valid branch '), category='warning')
repo_name=repo_name, revision='tip',
f_path=f_path))
r_post = request.POST
c.cs = self.__get_cs_or_redirect(revision, repo_name)
c.file = self.__get_filenode_or_redirect(repo_name, c.cs, f_path)
if c.file.is_binary:
return redirect(url('files_home', repo_name=c.repo_name,
revision=c.cs.raw_id, f_path=f_path))
c.default_message = _('Edited file %s via RhodeCode') % (f_path)
if r_post:
old_content = c.file.content
sl = old_content.splitlines(1)
first_line = sl[0] if sl else ''
# modes: 0 - Unix, 1 - Mac, 2 - DOS
mode = detect_mode(first_line, 0)
content = convert_line_endings(r_post.get('content'), mode)
message = r_post.get('message') or c.default_message
author = self.rhodecode_user.full_contact
if content == old_content:
h.flash(_('No changes'),
h.flash(_('No changes'), category='warning')
return redirect(url('changeset_home', repo_name=c.repo_name,
self.scm_model.commit_change(repo=c.rhodecode_repo,
repo_name=repo_name, cs=c.cs,
user=self.rhodecode_user.user_id,
author=author, message=message,
content=content, f_path=f_path)
h.flash(_('Successfully committed to %s') % f_path,
category='success')
except Exception:
log.error(traceback.format_exc())
h.flash(_('Error occurred during commit'), category='error')
return redirect(url('changeset_home',
repo_name=c.repo_name, revision='tip'))
return render('files/files_edit.html')
def add(self, repo_name, revision, f_path):
repo = Repository.get_by_repo_name(repo_name)
c.cs = self.__get_cs_or_redirect(revision, repo_name,
redirect_after=False)
if c.cs is None:
c.cs = EmptyChangeset(alias=c.rhodecode_repo.alias)
c.default_message = (_('Added file via RhodeCode'))
unix_mode = 0
content = convert_line_endings(r_post.get('content'), unix_mode)
filename = r_post.get('filename')
location = r_post.get('location')
file_obj = r_post.get('upload_file', None)
if file_obj is not None and hasattr(file_obj, 'filename'):
filename = file_obj.filename
content = file_obj.file
if not content:
h.flash(_('No content'), category='warning')
if not filename:
h.flash(_('No filename'), category='warning')
if location.startswith('/') or location.startswith('.') or '../' in location:
h.flash(_('location must be relative path and must not '
'contain .. in path'), category='warning')
if location:
location = os.path.normpath(location)
filename = os.path.basename(filename)
node_path = os.path.join(location, filename)
self.scm_model.create_node(repo=c.rhodecode_repo,
content=content, f_path=node_path)
h.flash(_('Successfully committed to %s') % node_path,
except (NodeError, NodeAlreadyExistsError), e:
h.flash(_(e), category='error')
return render('files/files_add.html')
def archivefile(self, repo_name, fname):
fileformat = None
revision = None
ext = None
subrepos = request.GET.get('subrepos') == 'true'
for a_type, ext_data in settings.ARCHIVE_SPECS.items():
archive_spec = fname.split(ext_data[1])
if len(archive_spec) == 2 and archive_spec[1] == '':
fileformat = a_type or ext_data[1]
revision = archive_spec[0]
ext = ext_data[1]
dbrepo = RepoModel().get_by_repo_name(repo_name)
if dbrepo.enable_downloads is False:
return _('downloads disabled')
if c.rhodecode_repo.alias == 'hg':
# patch and reset hooks section of UI config to not run any
# hooks on fetching archives with subrepos
for k, v in c.rhodecode_repo._repo.ui.configitems('hooks'):
c.rhodecode_repo._repo.ui.setconfig('hooks', k, None)
cs = c.rhodecode_repo.get_changeset(revision)
content_type = settings.ARCHIVE_SPECS[fileformat][0]
except ChangesetDoesNotExistError:
return _('Unknown revision %s') % revision
except EmptyRepositoryError:
return _('Empty repository')
except (ImproperArchiveTypeError, KeyError):
return _('Unknown archive type')
# archive cache
from rhodecode import CONFIG
rev_name = cs.raw_id[:12]
archive_name = '%s-%s%s' % (safe_str(repo_name.replace('/', '_')),
safe_str(rev_name), ext)
use_cached_archive = False # defines if we use cached version of archive
archive_cache_enabled = CONFIG.get('archive_cache_dir')
if not subrepos and archive_cache_enabled:
#check if we it's ok to write
if not os.path.isdir(CONFIG['archive_cache_dir']):
os.makedirs(CONFIG['archive_cache_dir'])
cached_archive_path = os.path.join(CONFIG['archive_cache_dir'], archive_name)
if os.path.isfile(cached_archive_path):
log.debug('Found cached archive in %s' % cached_archive_path)
fd, archive = None, cached_archive_path
use_cached_archive = True
log.debug('Archive %s is not yet cached' % (archive_name))
if not use_cached_archive:
#generate new archive
fd, archive = tempfile.mkstemp()
t = open(archive, 'wb')
log.debug('Creating new temp archive in %s' % archive)
cs.fill_archive(stream=t, kind=fileformat, subrepos=subrepos)
if archive_cache_enabled:
#if we generated the archive and use cache rename that
log.debug('Storing new archive in %s' % cached_archive_path)
shutil.move(archive, cached_archive_path)
archive = cached_archive_path
finally:
t.close()
def get_chunked_archive(archive):
stream = open(archive, 'rb')
while True:
data = stream.read(16 * 1024)
if not data:
stream.close()
if fd: # fd means we used temporary file
os.close(fd)
if not archive_cache_enabled:
log.debug('Destroing temp archive %s' % archive)
os.remove(archive)
break
yield data
response.content_disposition = str('attachment; filename=%s' % (archive_name))
response.content_type = str(content_type)
return get_chunked_archive(archive)
def diff(self, repo_name, f_path):
ignore_whitespace = request.GET.get('ignorews') == '1'
line_context = request.GET.get('context', 3)
diff1 = request.GET.get('diff1', '')
diff2 = request.GET.get('diff2', '')
c.action = request.GET.get('diff')
c.no_changes = diff1 == diff2
c.big_diff = False
c.anchor_url = anchor_url
@@ -100,256 +100,278 @@ removed extra unicode conversion in diff
extra_environ={'HTTP_X_PARTIAL_XHR': '1'},)
#test or history
response.mustcontain("""<optgroup label="Changesets">
<option selected="selected" value="8911406ad776fdd3d0b9932a2e89677e57405a48">r167:8911406ad776 (default)</option>
<option value="aa957ed78c35a1541f508d2ec90e501b0a9e3167">r165:aa957ed78c35 (default)</option>
<option value="48e11b73e94c0db33e736eaeea692f990cb0b5f1">r140:48e11b73e94c (default)</option>
<option value="adf3cbf483298563b968a6c673cd5bde5f7d5eea">r126:adf3cbf48329 (default)</option>
<option value="6249fd0fb2cfb1411e764129f598e2cf0de79a6f">r113:6249fd0fb2cf (git)</option>
<option value="75feb4c33e81186c87eac740cee2447330288412">r109:75feb4c33e81 (default)</option>
<option value="9a4dc232ecdc763ef2e98ae2238cfcbba4f6ad8d">r108:9a4dc232ecdc (default)</option>
<option value="595cce4efa21fda2f2e4eeb4fe5f2a6befe6fa2d">r107:595cce4efa21 (default)</option>
<option value="4a8bd421fbc2dfbfb70d85a3fe064075ab2c49da">r104:4a8bd421fbc2 (default)</option>
<option value="57be63fc8f85e65a0106a53187f7316f8c487ffa">r102:57be63fc8f85 (default)</option>
<option value="5530bd87f7e2e124a64d07cb2654c997682128be">r101:5530bd87f7e2 (git)</option>
<option value="e516008b1c93f142263dc4b7961787cbad654ce1">r99:e516008b1c93 (default)</option>
<option value="41f43fc74b8b285984554532eb105ac3be5c434f">r93:41f43fc74b8b (default)</option>
<option value="cc66b61b8455b264a7a8a2d8ddc80fcfc58c221e">r92:cc66b61b8455 (default)</option>
<option value="73ab5b616b3271b0518682fb4988ce421de8099f">r91:73ab5b616b32 (default)</option>
<option value="e0da75f308c0f18f98e9ce6257626009fdda2b39">r82:e0da75f308c0 (default)</option>
<option value="fb2e41e0f0810be4d7103bc2a4c7be16ee3ec611">r81:fb2e41e0f081 (default)</option>
<option value="602ae2f5e7ade70b3b66a58cdd9e3e613dc8a028">r76:602ae2f5e7ad (default)</option>
<option value="a066b25d5df7016b45a41b7e2a78c33b57adc235">r73:a066b25d5df7 (default)</option>
<option value="637a933c905958ce5151f154147c25c1c7b68832">r61:637a933c9059 (web)</option>
<option value="0c21004effeb8ce2d2d5b4a8baf6afa8394b6fbc">r60:0c21004effeb (web)</option>
<option value="a1f39c56d3f1d52d5fb5920370a2a2716cd9a444">r59:a1f39c56d3f1 (web)</option>
<option value="97d32df05c715a3bbf936bf3cc4e32fb77fe1a7f">r58:97d32df05c71 (web)</option>
<option value="08eaf14517718dccea4b67755a93368341aca919">r57:08eaf1451771 (web)</option>
<option value="22f71ad265265a53238359c883aa976e725aa07d">r56:22f71ad26526 (web)</option>
<option value="97501f02b7b4330924b647755663a2d90a5e638d">r49:97501f02b7b4 (web)</option>
<option value="86ede6754f2b27309452bb11f997386ae01d0e5a">r47:86ede6754f2b (web)</option>
<option value="014c40c0203c423dc19ecf94644f7cac9d4cdce0">r45:014c40c0203c (web)</option>
<option value="ee87846a61c12153b51543bf860e1026c6d3dcba">r30:ee87846a61c1 (default)</option>
<option value="9bb326a04ae5d98d437dece54be04f830cf1edd9">r26:9bb326a04ae5 (default)</option>
<option value="536c1a19428381cfea92ac44985304f6a8049569">r24:536c1a194283 (default)</option>
<option value="dc5d2c0661b61928834a785d3e64a3f80d3aad9c">r8:dc5d2c0661b6 (default)</option>
<option value="3803844fdbd3b711175fc3da9bdacfcd6d29a6fb">r7:3803844fdbd3 (default)</option>
</optgroup>
<optgroup label="Branches">
<option value="27cd5cce30c96924232dffcd24178a07ffeb5dfc">default</option>
<option value="97e8b885c04894463c51898e14387d80c30ed1ee">git</option>
<option value="2e6a2bf9356ca56df08807f4ad86d480da72a8f4">web</option>
<optgroup label="Tags">
<option value="27cd5cce30c96924232dffcd24178a07ffeb5dfc">tip</option>
<option value="fd4bdb5e9b2a29b4393a4ac6caef48c17ee1a200">0.1.4</option>
<option value="17544fbfcd33ffb439e2b728b5d526b1ef30bfcf">0.1.3</option>
<option value="a7e60bff65d57ac3a1a1ce3b12a70f8a9e8a7720">0.1.2</option>
<option value="eb3a60fc964309c1a318b8dfe26aa2d1586c85ae">0.1.1</option>
""")
def test_file_annotation(self):
self.log_user()
response = self.app.get(url(controller='files', action='index',
repo_name=HG_REPO,
revision='27cd5cce30c96924232dffcd24178a07ffeb5dfc',
f_path='vcs/nodes.py',
annotate=True))
response.mustcontain("""<span style="text-transform: uppercase;"><a href="#">branch: default</a></span>""")
def test_file_annotation_history(self):
response = self.app.get(url(controller='files', action='history',
annotate=True),
extra_environ={'HTTP_X_PARTIAL_XHR': '1'})
</optgroup>""")
def test_file_annotation_git(self):
repo_name=GIT_REPO,
revision='master',
def test_archival(self):
_set_downloads(HG_REPO, set_to=True)
for arch_ext, info in ARCHIVE_SPECS.items():
short = '27cd5cce30c9%s' % arch_ext
fname = '27cd5cce30c96924232dffcd24178a07ffeb5dfc%s' % arch_ext
filename = '%s-%s' % (HG_REPO, short)
response = self.app.get(url(controller='files',
action='archivefile',
fname=fname))
self.assertEqual(response.status, '200 OK')
heads = [
('Pragma', 'no-cache'),
('Cache-Control', 'no-cache'),
('Content-Disposition', 'attachment; filename=%s' % filename),
('Content-Type', '%s; charset=utf-8' % info[0]),
]
self.assertEqual(response.response._headers.items(), heads)
def test_archival_wrong_ext(self):
for arch_ext in ['tar', 'rar', 'x', '..ax', '.zipz']:
response.mustcontain('Unknown archive type')
def test_archival_wrong_revision(self):
for rev in ['00x000000', 'tar', 'wrong', '@##$@$42413232', '232dffcd']:
fname = '%s.zip' % rev
response.mustcontain('Unknown revision')
#==========================================================================
# RAW FILE
def test_raw_file_ok(self):
response = self.app.get(url(controller='files', action='rawfile',
f_path='vcs/nodes.py'))
self.assertEqual(response.content_disposition, "attachment; filename=nodes.py")
self.assertEqual(response.content_type, "text/x-python")
def test_raw_file_wrong_cs(self):
rev = u'ERRORce30c96924232dffcd24178a07ffeb5dfc'
f_path = 'vcs/nodes.py'
revision=rev,
f_path=f_path), status=404)
msg = """Revision %s does not exist for this repository""" % (rev)
self.checkSessionFlash(response, msg)
self.assertEqual('http://localhost/%s/files/tip/' % HG_REPO, response.headers['location'])
response.mustcontain(msg)
def test_raw_file_wrong_f_path(self):
rev = '27cd5cce30c96924232dffcd24178a07ffeb5dfc'
f_path = 'vcs/ERRORnodes.py'
msg = "There is no file nor directory at the given path: '%s' at revision %s" % (f_path, rev[:12])
msg = "There is no file nor directory at the given path: '%s' at revision %s" % (f_path, rev[:12])
# RAW RESPONSE - PLAIN
def test_raw_ok(self):
response = self.app.get(url(controller='files', action='raw',
self.assertEqual(response.content_type, "text/plain")
def test_raw_wrong_cs(self):
rev = u'ERRORcce30c96924232dffcd24178a07ffeb5dfc'
def test_raw_wrong_f_path(self):
def test_ajaxed_files_list(self):
response = self.app.get(
url('files_nodelist_home', repo_name=HG_REPO,f_path='/',revision=rev),
url('files_nodelist_home', repo_name=HG_REPO, f_path='/',
revision=rev),
extra_environ={'HTTP_X_PARTIAL_XHR': '1'},
)
response.mustcontain("vcs/web/simplevcs/views/repository.py")
def test_add_file_view_hg(self):
response = self.app.get(url('files_add_home',
revision='tip', f_path='/'))
def test_add_file_view_git(self):
def test_edit_file_view_hg(self):
response = self.app.get(url('files_edit_home',
revision='tip', f_path='vcs/nodes.py'))
def test_edit_file_view_git(self):
Status change: