Changeset - 3c503044e9f1
[Not reviewed]
default
0 3 0
Mads Kiilerich (mads) - 6 years ago 2020-07-01 14:58:56
mads@kiilerich.com
mysql: bump sqlalchemy.url MariaDB/MySQL charset to to 'utf8mb4' to get full UTF-8 support

The change in 210e76d69b62 only changed character_set_database, as shown by output after:

--- a/kallithea/model/base.py
+++ b/kallithea/model/base.py
@@ -46,3 +46,8 @@ def init_model(engine):
engine_str = obfuscate_url_pw(str(engine.url))
log.info("initializing db for %s", engine_str)
meta.Base.metadata.bind = engine
+
+ meta.Session.configure(bind=engine)
+ for a, b in meta.Session().execute('''show variables''').fetchall():
+ if 'character_set_' in a:
+ print(a, repr(b))

Before, with charset=utf8, the utf8mb3 charset was used all the way through the stack:

[kallithea.model.base] initializing db for mysql://kallithea-test:XXXXX@localhost/kallithea-test?charset=utf8
character_set_client 'utf8'
character_set_connection 'utf8'
character_set_database 'utf8mb4'
character_set_filesystem 'binary'
character_set_results 'utf8'
character_set_server 'latin1'
character_set_system 'utf8'

With explicit charset=utf8mb4:

[kallithea.model.base] initializing db for mysql://kallithea-test:XXXXX@localhost/kallithea-test?charset=utf8mb4
character_set_client 'utf8mb4'
character_set_connection 'utf8mb4'
character_set_database 'utf8mb4'
character_set_filesystem 'binary'
character_set_results 'utf8mb4'
character_set_server 'latin1'
character_set_system 'utf8'
3 files changed with 4 insertions and 4 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -316,97 +316,97 @@ session.secret = development-not-secret
 

	
 
## Database storage of session data.
 
#session.type = ext:database
 
#session.sa.url = postgresql://postgres:qwe@localhost/kallithea
 
#session.table_name = db_session
 

	
 
####################################
 
##        ERROR HANDLING          ##
 
####################################
 

	
 
## Show a nice error page for application HTTP errors and exceptions (default true)
 
#errorpage.enabled = true
 

	
 
## Enable Backlash client-side interactive debugger (default false)
 
## WARNING: *THIS MUST BE false IN PRODUCTION ENVIRONMENTS!!!*
 
## This debug mode will allow all visitors to execute malicious code.
 
#debug = false
 
debug = true
 

	
 
## Enable Backlash server-side error reporting (unless debug mode handles it client-side) (default true)
 
#trace_errors.enable = true
 
## Errors will be reported by mail if trace_errors.error_email is set.
 

	
 
## Propagate email settings to ErrorReporter of TurboGears2
 
## You do not normally need to change these lines
 
get trace_errors.smtp_server = smtp_server
 
get trace_errors.smtp_port = smtp_port
 
get trace_errors.from_address = error_email_from
 
get trace_errors.error_email = email_to
 
get trace_errors.smtp_username = smtp_username
 
get trace_errors.smtp_password = smtp_password
 
get trace_errors.smtp_use_tls = smtp_use_tls
 

	
 

	
 
##################################
 
##        LOGVIEW CONFIG        ##
 
##################################
 

	
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
#########################
 
##      DB CONFIG      ##
 
#########################
 

	
 
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
#sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4
 
## Note: the mysql:// prefix should also be used for MariaDB
 

	
 
sqlalchemy.pool_recycle = 3600
 

	
 
################################
 
##   ALEMBIC CONFIGURATION    ##
 
################################
 

	
 
[alembic]
 
script_location = kallithea:alembic
 

	
 
################################
 
##   LOGGING CONFIGURATION    ##
 
################################
 

	
 
[loggers]
 
keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash
 

	
 
[handlers]
 
keys = console, console_color, console_color_sql, null
 

	
 
[formatters]
 
keys = generic, color_formatter, color_formatter_sql
 

	
 
#############
 
## LOGGERS ##
 
#############
 

	
 
[logger_root]
 
level = NOTSET
 
#handlers = console
 
## For coloring based on log level:
 
handlers = console_color
 

	
 
[logger_routes]
 
#level = WARN
 
level = DEBUG
 
handlers =
 
qualname = routes.middleware
 
## "level = DEBUG" logs the route matched and routing variables.
 

	
 
[logger_beaker]
 
#level = WARN
 
level = DEBUG
 
handlers =
 
qualname = beaker.container
 

	
 
[logger_templates]
docs/contributing.rst
Show inline comments
 
@@ -56,97 +56,97 @@ latest upstream changes::
 

	
 
        hg pull
 
        hg update
 

	
 
Review the :ref:`contributing-guidelines` and :ref:`coding-guidelines`.
 

	
 
If you are new to Mercurial, refer to Mercurial `Quick Start`_ and `Beginners
 
Guide`_ on the Mercurial wiki.
 

	
 
Now, make some changes and test them (see :ref:`contributing-tests`). Don't
 
forget to add new tests to cover new functionality or bug fixes.
 

	
 
For documentation changes, run ``make html`` from the ``docs`` directory to
 
generate the HTML result, then review them in your browser.
 

	
 
Before submitting any changes, run the cleanup script::
 

	
 
        ./scripts/run-all-cleanup
 

	
 
When you are completely ready, you can send your changes to the community for
 
review and inclusion. Most commonly used methods are sending patches to the
 
mailing list (via ``hg email``) or by creating a pull request on Bitbucket_.
 

	
 
.. _contributing-tests:
 

	
 

	
 
Running tests
 
-------------
 

	
 
After finishing your changes make sure all tests pass cleanly. Run the testsuite
 
by invoking ``py.test`` from the project root::
 

	
 
    py.test
 

	
 
Note that on unix systems, the temporary directory (``/tmp`` or where
 
``$TMPDIR`` points) must allow executable files; Git hooks must be executable,
 
and the test suite creates repositories in the temporary directory. Linux
 
systems with /tmp mounted noexec will thus fail.
 

	
 
Tests can be run on PostgreSQL like::
 

	
 
    sudo -u postgres createuser 'kallithea-test' --pwprompt  # password password
 
    sudo -u postgres createdb 'kallithea-test' --owner 'kallithea-test'
 
    REUSE_TEST_DB='postgresql://kallithea-test:password@localhost/kallithea-test' py.test
 

	
 
Tests can be run on MariaDB/MySQL like::
 

	
 
    echo "GRANT ALL PRIVILEGES ON \`kallithea-test\`.* TO 'kallithea-test'@'localhost' IDENTIFIED BY 'password'" | sudo -u mysql mysql
 
    TEST_DB='mysql://kallithea-test:password@localhost/kallithea-test?charset=utf8' py.test
 
    TEST_DB='mysql://kallithea-test:password@localhost/kallithea-test?charset=utf8mb4' py.test
 

	
 
You can also use ``tox`` to run the tests with all supported Python versions.
 

	
 
When running tests, Kallithea generates a `test.ini` based on template values
 
in `kallithea/tests/conftest.py` and populates the SQLite database specified
 
there.
 

	
 
It is possible to avoid recreating the full test database on each invocation of
 
the tests, thus eliminating the initial delay. To achieve this, run the tests as::
 

	
 
    gearbox serve -c /tmp/kallithea-test-XXX/test.ini --pid-file=test.pid --daemon
 
    KALLITHEA_WHOOSH_TEST_DISABLE=1 KALLITHEA_NO_TMP_PATH=1 py.test
 
    kill -9 $(cat test.pid)
 

	
 
In these commands, the following variables are used::
 

	
 
    KALLITHEA_WHOOSH_TEST_DISABLE=1 - skip whoosh index building and tests
 
    KALLITHEA_NO_TMP_PATH=1 - disable new temp path for tests, used mostly for testing_vcs_operations
 

	
 
You can run individual tests by specifying their path as argument to py.test.
 
py.test also has many more options, see `py.test -h`. Some useful options
 
are::
 

	
 
    -k EXPRESSION         only run tests which match the given substring
 
                          expression. An expression is a python evaluable
 
                          expression where all names are substring-matched
 
                          against test names and their parent classes. Example:
 
    -x, --exitfirst       exit instantly on first error or failed test.
 
    --lf                  rerun only the tests that failed at the last run (or
 
                          all if none failed)
 
    --ff                  run all tests but run the last failures first. This
 
                          may re-order tests and thus lead to repeated fixture
 
                          setup/teardown
 
    --pdb                 start the interactive Python debugger on errors.
 
    -s, --capture=no      don't capture stdout (any stdout output will be
 
                          printed immediately)
 

	
 
Performance tests
 
^^^^^^^^^^^^^^^^^
 

	
 
A number of performance tests are present in the test suite, but they are
 
not run in a standard test run. These tests are useful to
 
evaluate the impact of certain code changes with respect to performance.
 

	
 
To run these tests::
 

	
 
    env TEST_PERFORMANCE=1 py.test kallithea/tests/performance
 

	
kallithea/lib/paster_commands/template.ini.mako
Show inline comments
 
@@ -418,99 +418,99 @@ get trace_errors.error_email = email_to
 
get trace_errors.smtp_username = smtp_username
 
get trace_errors.smtp_password = smtp_password
 
get trace_errors.smtp_use_tls = smtp_use_tls
 

	
 
%if error_aggregation_service == 'sentry':
 
<%text>##</%text>##############
 
<%text>##</%text>  [sentry]  ##
 
<%text>##</%text>##############
 

	
 
<%text>##</%text> sentry is a alternative open source error aggregator
 
<%text>##</%text> you must install python packages `sentry` and `raven` to enable
 

	
 
sentry.dsn = YOUR_DNS
 
sentry.servers =
 
sentry.name =
 
sentry.key =
 
sentry.public_key =
 
sentry.secret_key =
 
sentry.project =
 
sentry.site =
 
sentry.include_paths =
 
sentry.exclude_paths =
 

	
 
%endif
 

	
 
<%text>##</%text>################################
 
<%text>##</%text>        LOGVIEW CONFIG        ##
 
<%text>##</%text>################################
 

	
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
<%text>##</%text>#######################
 
<%text>##</%text>      DB CONFIG      ##
 
<%text>##</%text>#######################
 

	
 
%if database_engine == 'sqlite':
 
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
%else:
 
#sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
%endif
 
%if database_engine == 'postgres':
 
sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea
 
%else:
 
#sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea
 
%endif
 
%if database_engine == 'mysql':
 
sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8
 
sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4
 
%else:
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4
 
%endif
 
<%text>##</%text> Note: the mysql:// prefix should also be used for MariaDB
 

	
 
sqlalchemy.pool_recycle = 3600
 

	
 
<%text>##</%text>##############################
 
<%text>##</%text>   ALEMBIC CONFIGURATION    ##
 
<%text>##</%text>##############################
 

	
 
[alembic]
 
script_location = kallithea:alembic
 

	
 
<%text>##</%text>##############################
 
<%text>##</%text>   LOGGING CONFIGURATION    ##
 
<%text>##</%text>##############################
 

	
 
[loggers]
 
keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash
 

	
 
[handlers]
 
keys = console, console_color, console_color_sql, null
 

	
 
[formatters]
 
keys = generic, color_formatter, color_formatter_sql
 

	
 
<%text>##</%text>###########
 
<%text>##</%text> LOGGERS ##
 
<%text>##</%text>###########
 

	
 
[logger_root]
 
level = NOTSET
 
handlers = console
 
<%text>##</%text> For coloring based on log level:
 
#handlers = console_color
 

	
 
[logger_routes]
 
level = WARN
 
handlers =
 
qualname = routes.middleware
 
<%text>##</%text> "level = DEBUG" logs the route matched and routing variables.
 

	
 
[logger_beaker]
 
level = WARN
 
handlers =
 
qualname = beaker.container
 

	
 
[logger_templates]
 
level = WARN
0 comments (0 inline, 0 general)