Files @ 3c503044e9f1
Branch filter:

Location: kallithea/scripts/docs-headings.py - annotation

mads
mysql: bump sqlalchemy.url MariaDB/MySQL charset to to 'utf8mb4' to get full UTF-8 support

The change in 210e76d69b62 only changed character_set_database, as shown by output after:

--- a/kallithea/model/base.py
+++ b/kallithea/model/base.py
@@ -46,3 +46,8 @@ def init_model(engine):
engine_str = obfuscate_url_pw(str(engine.url))
log.info("initializing db for %s", engine_str)
meta.Base.metadata.bind = engine
+
+ meta.Session.configure(bind=engine)
+ for a, b in meta.Session().execute('''show variables''').fetchall():
+ if 'character_set_' in a:
+ print(a, repr(b))

Before, with charset=utf8, the utf8mb3 charset was used all the way through the stack:

[kallithea.model.base] initializing db for mysql://kallithea-test:XXXXX@localhost/kallithea-test?charset=utf8
character_set_client 'utf8'
character_set_connection 'utf8'
character_set_database 'utf8mb4'
character_set_filesystem 'binary'
character_set_results 'utf8'
character_set_server 'latin1'
character_set_system 'utf8'

With explicit charset=utf8mb4:

[kallithea.model.base] initializing db for mysql://kallithea-test:XXXXX@localhost/kallithea-test?charset=utf8mb4
character_set_client 'utf8mb4'
character_set_connection 'utf8mb4'
character_set_database 'utf8mb4'
character_set_filesystem 'binary'
character_set_results 'utf8mb4'
character_set_server 'latin1'
character_set_system 'utf8'
#!/usr/bin/env python3

"""
Consistent formatting of rst section titles
"""

import re
import subprocess


spaces = [
    (0, 1), # we assume this is a over-and-underlined header
    (2, 1),
    (1, 1),
    (1, 0),
    (1, 0),
    ]

# http://sphinx-doc.org/rest.html :
#   for the Python documentation, this convention is used which you may follow:
#   # with overline, for parts
#   * with overline, for chapters
#   =, for sections
#   -, for subsections
#   ^, for subsubsections
#   ", for paragraphs
pystyles = ['#', '*', '=', '-', '^', '"']

# match on a header line underlined with one of the valid characters
headermatch = re.compile(r'''\n*(.+)\n([][!"#$%&'()*+,./:;<=>?@\\^_`{|}~-])\2{2,}\n+''', flags=re.MULTILINE)


def main():
    filenames = subprocess.check_output(['hg', 'loc', 'set:**.rst+kallithea/i18n/how_to']).splitlines()
    for fn in filenames:
        fn = fn.decode()
        print('processing %s' % fn)
        s = open(fn).read()

        # find levels and their styles
        lastpos = 0
        styles = []
        for markup in headermatch.findall(s):
            style = markup[1]
            if style in styles:
                stylepos = styles.index(style)
                if stylepos > lastpos + 1:
                    print('bad style %r with level %s - was at %s' % (style, stylepos, lastpos))
            else:
                stylepos = len(styles)
                if stylepos > lastpos + 1:
                    print('bad new style %r - expected %r' % (style, styles[lastpos + 1]))
                else:
                    styles.append(style)
            lastpos = stylepos

        # remove superfluous spacing (may however be restored by header spacing)
        s = re.sub(r'''(\n\n)\n*''', r'\1', s, flags=re.MULTILINE)

        if styles:
            newstyles = pystyles[pystyles.index(styles[0]):]

            def subf(m):
                title, style = m.groups()
                level = styles.index(style)
                before, after = spaces[level]
                newstyle = newstyles[level]
                return '\n' * (before + 1) + title + '\n' + newstyle * len(title) + '\n' * (after + 1)
            s = headermatch.sub(subf, s)

        # remove superfluous spacing when headers are adjacent
        s = re.sub(r'''(\n.+\n([][!"#$%&'()*+,./:;<=>?@\\^_`{|}~-])\2{2,}\n\n\n)\n*''', r'\1', s, flags=re.MULTILINE)
        # fix trailing space and spacing before link sections
        s = s.strip() + '\n'
        s = re.sub(r'''\n+((?:\.\. _[^\n]*\n)+)$''', r'\n\n\n\1', s)

        open(fn, 'w').write(s)

    print(subprocess.check_output(['hg', 'diff'] + filenames))

if __name__ == '__main__':
    main()