cocluto/mysql2sqlite.py

import re

def mysql_to_sqlite( mysql_sql_code, truncate_hex_strings = False ):
	"""
	converts a mysql-compatible sql code into a sqlite-ompatible sql code
	
	note: the original code was found on internet, then tweaked
	"""
	content = mysql_sql_code

	# unused commands
	COMMAND_RE = re.compile(r'^(SET).*?;\n$', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	content = COMMAND_RE.sub('', content)

	# sqlite doesn't like COMMENT= , remove it properly before the table constraint filter because the table constraint filter is not clever enough to cope with ; inside comment strings
	# ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='A list of URL aliases for Drupal paths; a user may visit...';
	COMMENTS_EQUAL_RE = re.compile(r'\s+COMMENT=\'[^\']*\'', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	# content = re.sub(r'^-- Tab[.]', 'toto', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL)
	content = COMMENTS_EQUAL_RE.sub('', content)

	# table constraints
	TCONS_RE = re.compile(r'\)(\s*(CHARSET|DEFAULT|ENGINE)(=.*?)?\s*)+;', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	content = TCONS_RE.sub(');', content)

	# remove comments
	# `nid` int(10) UNSIGNED NOT NULL DEFAULT '0' COMMENT 'The node.nid this record affects.',
	COMMENTS_RE = re.compile(r'\s+COMMENT\s+\'[^\']*\'', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	# content = re.sub(r'^-- Tab[.]', 'toto', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL)
	content = COMMENTS_RE.sub('', content)

	# sqlite doesn't like ' being escaped as \', use '' instead
	content = re.sub(r'\\\'', '\'\'', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )

	if truncate_hex_strings:
		# sqlite doesn't like too big hex strings 0x613a343a7b733a383a
		content = re.sub(r'0x[0-9a-f]+', '0xdeadbeef', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )

	# sqlite doesn't understand 
	# `format` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
	content = re.sub(r'\s+CHARACTER SET\s+[^\s]+', '', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )
	
	# sqlite doesn't know the utf8_bin :
	# `format` varchar(100) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL
	#no such collation sequence: utf8_bin
	content = re.sub(r'\s+COLLATE\s+utf8_bin\s+', ' ', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )

	# sqlite doesn't like 'unsigned' as in `ip_address_3` tinyint(3) unsigned NOT NULL default '27',
	content = re.sub(r' unsigned ', ' ', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )

	# sqlite doesn't like 'enum' as in `type` enum('normal','light_out_management') NOT NULL default 'normal',,
	content = re.sub(r' enum\([^\)]*\) ', ' varchar(255) ', content)

	# sqlite doesn't support much of alter table (https://www.sqlite.org/lang_altertable.html). The following is not supported :
	# ALTER TABLE `blocked_ips`
	# ADD PRIMARY KEY (`iid`),
	# ADD KEY `blocked_ip` (`ip`);
	content = re.sub(r'alter table [^;]*;', '', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )


	# COMMIT;
	# sqlite3.OperationalError: cannot commit - no transaction is active
	content = re.sub(r'commit\s*;', '', content, flags=re.IGNORECASE | re.MULTILINE | re.DOTALL )

	# insert multiple values
	# INSERTVALS_RE = re.compile(r'^(INSERT INTO.*?VALUES)\s*\((.*)\*;', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	INSERTVALS_RE = re.compile(r'^(INSERT INTO.*?VALUES)\s*([^;]*);', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	#INSERTVALS_RE = re.compile(r'^(INSERT INTO.*?VALUES)\s*((\[^\)](\)));$', re.IGNORECASE | re.MULTILINE | re.DOTALL)
	INSERTVALS_SPLIT_RE = re.compile(r'\)\s*,\s*\(', re.IGNORECASE | re.MULTILINE | re.DOTALL)


	def insertvals_replacer(match):
		insert, values = match.groups()
		# print("insert=%s"%insert)
		# print("values=%s"%values)
		values = re.sub('^\s*\(' ,'', values)
		values = re.sub('\)\s*$' ,'', values)
		replacement = ''
		for vals in INSERTVALS_SPLIT_RE.split(values):
			#print("vals=%s"%vals)
			replacement = '%s\n%s (%s);' % (replacement, insert, vals)
		return replacement

	content = INSERTVALS_RE.sub(insertvals_replacer, content)
	return content