Changeset - 3c503044e9f1
Parent rev.
Child rev.
[Not reviewed]
default
0 3 0
Mads Kiilerich (mads) - 6 years ago 2020-07-01 14:58:56
mads@kiilerich.com
mysql: bump sqlalchemy.url MariaDB/MySQL charset to to 'utf8mb4' to get full UTF-8 support

The change in 210e76d69b62 only changed character_set_database, as shown by output after:

--- a/kallithea/model/base.py
+++ b/kallithea/model/base.py
@@ -46,3 +46,8 @@ def init_model(engine):
engine_str = obfuscate_url_pw(str(engine.url))
log.info("initializing db for %s", engine_str)
meta.Base.metadata.bind = engine
+
+ meta.Session.configure(bind=engine)
+ for a, b in meta.Session().execute('''show variables''').fetchall():
+ if 'character_set_' in a:
+ print(a, repr(b))

Before, with charset=utf8, the utf8mb3 charset was used all the way through the stack:

[kallithea.model.base] initializing db for mysql://kallithea-test:XXXXX@localhost/kallithea-test?charset=utf8
character_set_client 'utf8'
character_set_connection 'utf8'
character_set_database 'utf8mb4'
character_set_filesystem 'binary'
character_set_results 'utf8'
character_set_server 'latin1'
character_set_system 'utf8'

With explicit charset=utf8mb4:

[kallithea.model.base] initializing db for mysql://kallithea-test:XXXXX@localhost/kallithea-test?charset=utf8mb4
character_set_client 'utf8mb4'
character_set_connection 'utf8mb4'
character_set_database 'utf8mb4'
character_set_filesystem 'binary'
character_set_results 'utf8mb4'
character_set_server 'latin1'
character_set_system 'utf8'
3 files changed with 4 insertions and 4 deletions:
development.ini
1
1
docs/contributing.rst
1
1
kallithea/lib/paster_commands/template.ini.mako
2
2
0 comments (0 inline, 0 general)
development.ini
➞
Show inline comments
 
@@ -340,49 +340,49 @@ debug = true
 
## You do not normally need to change these lines
 
get trace_errors.smtp_server = smtp_server
 
get trace_errors.smtp_port = smtp_port
 
get trace_errors.from_address = error_email_from
 
get trace_errors.error_email = email_to
 
get trace_errors.smtp_username = smtp_username
 
get trace_errors.smtp_password = smtp_password
 
get trace_errors.smtp_use_tls = smtp_use_tls
 

			




	
	
	
		
 

			




	
	
	
		
 
##################################

			




	
	
	
		
 
##        LOGVIEW CONFIG        ##

			




	
	
	
		
 
##################################

			




	
	
	
		
 

			




	
	
	
		
 
logview.sqlalchemy = #faa

			




	
	
	
		
 
logview.pylons.templating = #bfb

			




	
	
	
		
 
logview.pylons.util = #eee

			




	
	
	
		
 

			




	
	
	
		
 
#########################

			




	
	
	
		
 
##      DB CONFIG      ##

			




	
	
	
		
 
#########################

			




	
	
	
		
 

			




	
	
	
		
 
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60

			




	
	
	
		
 
#sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea

			




	
	
	
		
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8

			




	
	
	
		
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4

			




	
	
	
		
 
## Note: the mysql:// prefix should also be used for MariaDB

			




	
	
	
		
 

			




	
	
	
		
 
sqlalchemy.pool_recycle = 3600

			




	
	
	
		
 

			




	
	
	
		
 
################################

			




	
	
	
		
 
##   ALEMBIC CONFIGURATION    ##

			




	
	
	
		
 
################################

			




	
	
	
		
 

			




	
	
	
		
 
[alembic]

			




	
	
	
		
 
script_location = kallithea:alembic

			




	
	
	
		
 

			




	
	
	
		
 
################################

			




	
	
	
		
 
##   LOGGING CONFIGURATION    ##

			




	
	
	
		
 
################################

			




	
	
	
		
 

			




	
	
	
		
 
[loggers]

			




	
	
	
		
 
keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash

			




	
	
	
		
 

			




	
	
	
		
 
[handlers]

			




	
	
	
		
 
keys = console, console_color, console_color_sql, null

			




	
	
	
		
 

			




	
	
	
		
 
[formatters]

			




	
	
	
		
 
keys = generic, color_formatter, color_formatter_sql

			




	
	
	
		
 

			




        

    


    
    

    

        

                

                    docs/contributing.rst
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -80,49 +80,49 @@ mailing list (via ``hg email``) or by cr

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
Running tests

			




	
	
	
		
 
-------------

			




	
	
	
		
 

			




	
	
	
		
 
After finishing your changes make sure all tests pass cleanly. Run the testsuite

			




	
	
	
		
 
by invoking ``py.test`` from the project root::

			




	
	
	
		
 

			




	
	
	
		
 
    py.test

			




	
	
	
		
 

			




	
	
	
		
 
Note that on unix systems, the temporary directory (``/tmp`` or where

			




	
	
	
		
 
``$TMPDIR`` points) must allow executable files; Git hooks must be executable,

			




	
	
	
		
 
and the test suite creates repositories in the temporary directory. Linux

			




	
	
	
		
 
systems with /tmp mounted noexec will thus fail.

			




	
	
	
		
 

			




	
	
	
		
 
Tests can be run on PostgreSQL like::

			




	
	
	
		
 

			




	
	
	
		
 
    sudo -u postgres createuser 'kallithea-test' --pwprompt  # password password

			




	
	
	
		
 
    sudo -u postgres createdb 'kallithea-test' --owner 'kallithea-test'

			




	
	
	
		
 
    REUSE_TEST_DB='postgresql://kallithea-test:password@localhost/kallithea-test' py.test

			




	
	
	
		
 

			




	
	
	
		
 
Tests can be run on MariaDB/MySQL like::

			




	
	
	
		
 

			




	
	
	
		
 
    echo "GRANT ALL PRIVILEGES ON \`kallithea-test\`.* TO 'kallithea-test'@'localhost' IDENTIFIED BY 'password'" | sudo -u mysql mysql

			




	
	
	
		
 
    TEST_DB='mysql://kallithea-test:password@localhost/kallithea-test?charset=utf8' py.test

			




	
	
	
		
 
    TEST_DB='mysql://kallithea-test:password@localhost/kallithea-test?charset=utf8mb4' py.test

			




	
	
	
		
 

			




	
	
	
		
 
You can also use ``tox`` to run the tests with all supported Python versions.

			




	
	
	
		
 

			




	
	
	
		
 
When running tests, Kallithea generates a `test.ini` based on template values

			




	
	
	
		
 
in `kallithea/tests/conftest.py` and populates the SQLite database specified

			




	
	
	
		
 
there.

			




	
	
	
		
 

			




	
	
	
		
 
It is possible to avoid recreating the full test database on each invocation of

			




	
	
	
		
 
the tests, thus eliminating the initial delay. To achieve this, run the tests as::

			




	
	
	
		
 

			




	
	
	
		
 
    gearbox serve -c /tmp/kallithea-test-XXX/test.ini --pid-file=test.pid --daemon

			




	
	
	
		
 
    KALLITHEA_WHOOSH_TEST_DISABLE=1 KALLITHEA_NO_TMP_PATH=1 py.test

			




	
	
	
		
 
    kill -9 $(cat test.pid)

			




	
	
	
		
 

			




	
	
	
		
 
In these commands, the following variables are used::

			




	
	
	
		
 

			




	
	
	
		
 
    KALLITHEA_WHOOSH_TEST_DISABLE=1 - skip whoosh index building and tests

			




	
	
	
		
 
    KALLITHEA_NO_TMP_PATH=1 - disable new temp path for tests, used mostly for testing_vcs_operations

			




	
	
	
		
 

			




	
	
	
		
 
You can run individual tests by specifying their path as argument to py.test.

			




	
	
	
		
 
py.test also has many more options, see `py.test -h`. Some useful options

			




	
	
	
		
 
are::

			




	
	
	
		
 

			




	
	
	
		
 
    -k EXPRESSION         only run tests which match the given substring

			




        

    


    
    

    

        

                

                    kallithea/lib/paster_commands/template.ini.mako
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
		
 
@@ -442,51 +442,51 @@ sentry.exclude_paths =

			




	
	
	
		
 

			




	
	
	
		
 
<%text>##</%text>################################

			




	
	
	
		
 
<%text>##</%text>        LOGVIEW CONFIG        ##

			




	
	
	
		
 
<%text>##</%text>################################

			




	
	
	
		
 

			




	
	
	
		
 
logview.sqlalchemy = #faa

			




	
	
	
		
 
logview.pylons.templating = #bfb

			




	
	
	
		
 
logview.pylons.util = #eee

			




	
	
	
		
 

			




	
	
	
		
 
<%text>##</%text>#######################

			




	
	
	
		
 
<%text>##</%text>      DB CONFIG      ##

			




	
	
	
		
 
<%text>##</%text>#######################

			




	
	
	
		
 

			




	
	
	
		
 
%if database_engine == 'sqlite':

			




	
	
	
		
 
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60

			




	
	
	
		
 
%else:

			




	
	
	
		
 
#sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60

			




	
	
	
		
 
%endif

			




	
	
	
		
 
%if database_engine == 'postgres':

			




	
	
	
		
 
sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea

			




	
	
	
		
 
%else:

			




	
	
	
		
 
#sqlalchemy.url = postgresql://kallithea:password@localhost/kallithea

			




	
	
	
		
 
%endif

			




	
	
	
		
 
%if database_engine == 'mysql':

			




	
	
	
		
 
sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8

			




	
	
	
		
 
sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4

			




	
	
	
		
 
%else:

			




	
	
	
		
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8

			




	
	
	
		
 
#sqlalchemy.url = mysql://kallithea:password@localhost/kallithea?charset=utf8mb4

			




	
	
	
		
 
%endif

			




	
	
	
		
 
<%text>##</%text> Note: the mysql:// prefix should also be used for MariaDB

			




	
	
	
		
 

			




	
	
	
		
 
sqlalchemy.pool_recycle = 3600

			




	
	
	
		
 

			




	
	
	
		
 
<%text>##</%text>##############################

			




	
	
	
		
 
<%text>##</%text>   ALEMBIC CONFIGURATION    ##

			




	
	
	
		
 
<%text>##</%text>##############################

			




	
	
	
		
 

			




	
	
	
		
 
[alembic]

			




	
	
	
		
 
script_location = kallithea:alembic

			




	
	
	
		
 

			




	
	
	
		
 
<%text>##</%text>##############################

			




	
	
	
		
 
<%text>##</%text>   LOGGING CONFIGURATION    ##

			




	
	
	
		
 
<%text>##</%text>##############################

			




	
	
	
		
 

			




	
	
	
		
 
[loggers]

			




	
	
	
		
 
keys = root, routes, kallithea, sqlalchemy, tg, gearbox, beaker, templates, whoosh_indexer, werkzeug, backlash

			




	
	
	
		
 

			




	
	
	
		
 
[handlers]

			




	
	
	
		
 
keys = console, console_color, console_color_sql, null

			




	
	
	
		
 

			




	
	
	
		
 
[formatters]

			




	
	
	
		
 
keys = generic, color_formatter, color_formatter_sql

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        Server instance: clio-28956
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2025 by various authors & licensed under GPLv3.
            – Support