I am trying to use LocalExecutor instead of the default SequentialExecutor which forces to use a different database then SQLlite.
I wanted to try MySQL, however I am seeing issues with MySQL version 5.6, 5.7? Not sure if it is related to version compatibility.
Would love to see any documentation related to Airflow versions and compatible MySQL versions.
Update:
Here is the Ooops error I am seeing in th UI when click on any of the DAG related buttons while using MySQL backend:
Traceback (most recent call last):
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask/app.py", line 1982, in wsgi_app
response = self.full_dispatch_request()
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask/app.py", line 1614, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask/app.py", line 1517, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask/app.py", line 1612, in full_dispatch_request
rv = self.dispatch_request()
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask/app.py", line 1598, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask_admin/base.py", line 69, in inner
return self._run_view(f, *args, **kwargs)
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask_admin/base.py", line 368, in _run_view
return fn(self, *args, **kwargs)
File "/home/ec2-user/.local/lib/python2.7/site-packages/flask_login.py", line 755, in decorated_view
return func(*args, **kwargs)
File "/home/ec2-user/.local/lib/python2.7/site-packages/airflow/www/utils.py", line 364, in view_func
return f(*args, **kwargs)
File "/home/ec2-user/.local/lib/python2.7/site-packages/airflow/www/utils.py", line 268, in wrapper
session.commit()
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 927, in commit
self.transaction.commit()
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 467, in commit
self._prepare_impl()
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 447, in _prepare_impl
self.session.flush()
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2209, in flush
self._flush(objects)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2329, in _flush
transaction.rollback(_capture_exception=True)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2293, in _flush
flush_context.execute()
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
rec.execute(self)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 548, in execute
uow
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
mapper, table, insert)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 835, in _emit_insert_statements
execute(statement, params)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
return meth(self, multiparams, params)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
compiled_sql, distilled_params
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
context)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
exc_info
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
context)
File "/home/ec2-user/.local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 470, in do_execute
cursor.execute(statement, parameters)
File "/usr/local/lib64/python2.7/site-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib64/python2.7/site-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib64/python2.7/site-packages/pymysql/connections.py", line 516, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib64/python2.7/site-packages/pymysql/connections.py", line 727, in _read_query_result
result.read()
File "/usr/local/lib64/python2.7/site-packages/pymysql/connections.py", line 1066, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib64/python2.7/site-packages/pymysql/connections.py", line 683, in _read_packet
packet.check_error()
File "/usr/local/lib64/python2.7/site-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib64/python2.7/site-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
ProgrammingError: (pymysql.err.ProgrammingError) (1064, u"You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '[(\\'dag_id\\', u\\'DataPipeline\\')]'')' at line 1") [SQL: u'INSERT INTO log (dttm, dag_id, task_id, event, execution_date, owner, extra) VALUES (%(dttm)s, %(dag_id)s, %(task_id)s, %(event)s, %(execution_date)s, %(owner)s, %(extra)s)'] [parameters: {'task_id': None, 'extra': "[('dag_id', u'DataPipeline')]", 'execution_date': None, 'event': 'tree', 'owner': 'anonymous', 'dttm': datetime.datetime(2018, 10, 6, 5, 28, 31, 180060, tzinfo=<Timezone [UTC]>), 'dag_id': u'DataPipeline'}]
AIRFLOW-1405 reads:
MySQL supports fractional seconds only from version
5.6.4on.You can not use airflow >= 1.8 with earlier versions.
5.7 should work, while field last_scheduler_run has the expected size.
I was using Python27 all these time, once I update my linux to Python36 all started working. Now I am using Airflow1.10 with Mysql5.6.4
One more thing to keep in mind, browser cookies should be cleared once you update your python version from python27 to python36 for this to work in the UI or else you will see the same error.
Hope this helps someone!
来源：https://stackoverflow.com/questions/52675779/which-version-of-mysql-is-compatible-with-airflow-version-1-10