Add fluentd configuration for MySQL error.log
- changed default logging to log file
- parametrized log file path
- added fluentd configuration
- introduced fluentd multi_format parser plugin
Currently error logs sent to syslog are represented
with severity ERROR, which is misleading for Notice
and Warning messages during service startup/shutdown.
Change-Id: I55106775fa54b412b50ecc10c12930445da15d03
Related-bug: PROD-21904 (PROD:21904)
diff --git a/README.rst b/README.rst
index 112a2db..b6be8e5 100644
--- a/README.rst
+++ b/README.rst
@@ -152,7 +152,9 @@
- ``galera_xtrabackup_parallel``
Default is 4
- ``galera_error_log_enabled``
- Default is ``false``
+ Default is ``true``
+- ``galera_error_log_path``
+ Default is ``/var/log/mysql/error.log``
Usage:
@@ -166,6 +168,7 @@
galera_wsrep_slave_threads: 8
galera_xtrabackup_parallel: 2
galera_error_log_enabled: true
+ galera_error_log_path: /var/log/mysql/error.log
Usage
=====
diff --git a/galera/files/my.cnf b/galera/files/my.cnf
index cb9687e..ede65c1 100644
--- a/galera/files/my.cnf
+++ b/galera/files/my.cnf
@@ -18,7 +18,9 @@
{% endif %}
[mysqld_safe]
+{%- if not service.get('error_log_enabled', False) %}
syslog
+{%- endif %}
[mysqld]
datadir=/var/lib/mysql
@@ -32,9 +34,7 @@
character-set-server=utf8
default-storage-engine=innodb
{%- if service.get('error_log_enabled', False) %}
-log_error = /var/log/mysql/error.log
-{%- else %}
-#log_error = /var/log/mysql/error.log
+log_error = {{ service.get('error_log_path', '/var/log/mysql/error.log') }}
{%- endif %}
skip-external-locking
skip-name-resolve
diff --git a/galera/meta/fluentd.yml b/galera/meta/fluentd.yml
new file mode 100644
index 0000000..09cf83b
--- /dev/null
+++ b/galera/meta/fluentd.yml
@@ -0,0 +1,55 @@
+{%- if pillar.get('fluentd', {}).get('agent', {}).get('enabled', False) %}
+agent:
+ config:
+ label:
+ mysql:
+ input:
+ tail_mysql:
+ type: tail
+ tag: mysql.log
+ path: {{ service.get('error_log_path', '/var/log/mysql/error.log') }}
+ pos_file: {{ pillar.fluentd.agent.dir.positiondb }}/mysql.pos
+ parser:
+ type: multi_format
+ patterns:
+ # mysql format I: https://regex101.com/r/xmjZ8H/1
+ - type: regexp
+ time_key: Timestamp
+ time_format: '%F %H:%M:%S'
+ keep_time_key: false
+ format: '/^(?<Timestamp>.*) (?<Pid>\d+) \[(?<Severity>.*)\] (?<Payload>.*)$/'
+ # mysql format II: https://regex101.com/r/ucqhB3/1
+ - type: regexp
+ time_key: Timestamp
+ time_format: '%H:%M:%S'
+ keep_time_key: false
+ format: '/^(?<Pid>\d+) (?<Timestamp>.*) mysqld\_safe (?<Payload>.*)$/'
+ filter:
+ wsrep_severity:
+ type: record_transformer
+ tag: mysql.log
+ enable_ruby: true
+ remove_keys: priority
+ record:
+ - name: Severity
+ value: '${ {"Note"=>5,"Warning"=>4,"ERROR"=>3}.fetch(record["Severity"], 6) }'
+ mysql_record:
+ require:
+ - wsrep_severity
+ type: record_transformer
+ tag: mysql.log
+ enable_ruby: true
+ remove_keys: priority
+ record:
+ - name: service
+ value: mysql.service
+ - name: source
+ value: error_log
+ - name: severity_label
+ value: '${ {"TRACE"=>8,"DEBUG"=>7,"INFO"=>6,"NOTICE"=>5,"WARNING"=>4,"ERROR"=>3,"CRITICAL"=>2,"ALERT"=>1,"EMERGENCY"=>0}.key(record["Severity"].to_i) }'
+ match:
+ push_to_default:
+ tag: mysql.**
+ type: relabel
+ label: default_output
+{%- endif %}
diff --git a/metadata/service/master/cluster.yml b/metadata/service/master/cluster.yml
index d48ffcb..af0bf1c 100644
--- a/metadata/service/master/cluster.yml
+++ b/metadata/service/master/cluster.yml
@@ -14,7 +14,8 @@
galera_innodb_write_io_threads: 8
galera_wsrep_slave_threads: 8
galera_xtrabackup_parallel: 4
- galera_error_log_enabled: false
+ galera_error_log_enabled: true
+ galera_error_log_path: /var/log/mysql/error.log
galera:
master:
enabled: true
@@ -40,6 +41,7 @@
wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
error_log_enabled: ${_param:galera_error_log_enabled}
+ error_log_path: ${_param:galera_error_log_path}
mysql:
server:
users:
diff --git a/metadata/service/master/container.yml b/metadata/service/master/container.yml
index 2850f77..f4957a6 100644
--- a/metadata/service/master/container.yml
+++ b/metadata/service/master/container.yml
@@ -9,7 +9,8 @@
galera_innodb_write_io_threads: 8
galera_wsrep_slave_threads: 8
galera_xtrabackup_parallel: 4
- galera_error_log_enabled: false
+ galera_error_log_enabled: true
+ galera_error_log_path: /var/log/mysql/error.log
kubernetes:
control:
configmap:
@@ -42,10 +43,11 @@
wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
error_log_enabled: ${_param:galera_error_log_enabled}
+ error_log_path: ${_param:galera_error_log_path}
mysql:
server:
users:
- name: haproxy
host: localhost
- name: haproxy
- host: '%'
\ No newline at end of file
+ host: '%'
diff --git a/metadata/service/master/single.yml b/metadata/service/master/single.yml
index 9e6061b..bed28d6 100644
--- a/metadata/service/master/single.yml
+++ b/metadata/service/master/single.yml
@@ -14,7 +14,8 @@
galera_innodb_write_io_threads: 8
galera_wsrep_slave_threads: 8
galera_xtrabackup_parallel: 4
- galera_error_log_enabled: false
+ galera_error_log_enabled: true
+ galera_error_log_path: /var/log/mysql/error.log
galera:
master:
enabled: true
@@ -36,6 +37,7 @@
wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
error_log_enabled: ${_param:galera_error_log_enabled}
+ error_log_path: ${_param:galera_error_log_path}
mysql:
server:
users:
diff --git a/metadata/service/slave/cluster.yml b/metadata/service/slave/cluster.yml
index f1430a5..49eb6bb 100644
--- a/metadata/service/slave/cluster.yml
+++ b/metadata/service/slave/cluster.yml
@@ -14,7 +14,8 @@
galera_innodb_write_io_threads: 8
galera_wsrep_slave_threads: 8
galera_xtrabackup_parallel: 4
- galera_error_log_enabled: false
+ galera_error_log_enabled: true
+ galera_error_log_path: /var/log/mysql/error.log
galera:
slave:
enabled: true
@@ -40,6 +41,7 @@
wsrep_slave_threads: ${_param:galera_wsrep_slave_threads}
xtrabackup_parallel: ${_param:galera_xtrabackup_parallel}
error_log_enabled: ${_param:galera_error_log_enabled}
+ error_log_path: ${_param:galera_error_log_path}
mysql:
server:
users:
diff --git a/metadata/service/support.yml b/metadata/service/support.yml
index fb5051f..4fb4539 100644
--- a/metadata/service/support.yml
+++ b/metadata/service/support.yml
@@ -5,6 +5,8 @@
enabled: true
heka:
enabled: true
+ fluentd:
+ enabled: true
sensu:
enabled: false
sphinx: