#!/usr/bin/env python3

VERSION = '0.7.2'
'''
mhegepgsnoop.py - Freeview DVB-T MHEG-5 to XMLTV EPG converter
Version 0.7.0 JSW modified version converted to Python 3
Copyright (C) 2011  David Moore <dmoo1790@ihug.co.nz>
Contributors: Bruce Wilson <acaferacer@gmail.com>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
extra_help='''
Several python modules (sqlite3, zlib, difflib, etc.) are required. There is no error-checking in this
version to confirm whether these modules are available on your system.

This script started as a Python port of the mheg2xmltv.sh bash script originally created by SolorVox
and updated by myself. It takes DVB-T MHEG-5 EPG data collected by directly reading data from a DVB demux device,
or by using dvbsnoop to read data, and creates an XML file suitable for importing into the MythTV EPG. The
default is to read from the DVB demux device.

It uses fuzzy matching of the channel name extracted from the MHEG data to the callsign, name and xmltvid
fields in the MythTV database channels table to link the extracted EPG data to MythTV xmltv IDs. This
means that at least one of callsign, name or xmltvid for each MythTV channel must resemble the MHEG channel
name for each broadcast channel. Any channels that don't match are dropped, i.e., the EPG data is not
written to the output xml file. Use the -v option to see a list of the broadcast channel names in the verbose output.

This script does NOT:

	- Require access to the internet. (So you don't get any more data than what is broadcast via MHEG-5.)
	- Require configuration. Optional settings are command line arguments.
	- Update MythTV EPG. You need to run mythfilldatabase to import the xml file generated by this script.

Example usage to collect EPG data from default adapter and write to tvguide.xml:
	mhegepgsnoop.py -o tvguide.xml

Example usage to collect EPG data using dvbsnoop from default adapter and write to tvguide.xml:
	mhegepgsnoop.py -o tvguide.xml -s

Example usage to collect EPG data from specified demux device and write to tvguide.xml:
	mhegepgsnoop.py -o tvguide.xml -d "/dev/dvb/adapter1/demux0"

Example usage to collect EPG data using dvbsnoop from adapter 1 and write to tvguide.xml with verbose output:
	mhegepgsnoop.py -o tvguide.xml -e "-adapter 1" -vs

Example usage to collect EPG data from default adapter and write to tvguide.xml and specify MySQL user & password:
	mhegepgsnoop.py -o tvguide.xml -m "-u myuser -pmypassword"

Example usage to collect EPG data from specified demux device, write to tvguide.xml, connect to mythconverg using
Python bindings, verbose output:
	mhegepgsnoop.py -o tvguide.xml -d "/dev/dvb/adapter1/demux0" -vp

Example usage to collect EPG data from specified demux device, write to tvguide.xml, use a channel map file, verbose output:
	mhegepgsnoop.py -o tvguide.xml -d "/dev/dvb/adapter1/demux0" -v -f chanmap.txt

Example usage to collect EPG data using from default adapter, write to tvguide.xml, tune to channel number 3:
	mhegepgsnoop.py -o tvguide.xml -t 3

Example usage to collect EPG data from default adapter, write to tvguide.xml, and strip some text from titles:
	mhegepgsnoop.py -o tvguide.xml -c "All New |Movie: "
	mhegepgsnoop.py -o tvguide.xml -c
'''
'''
REVISION HISTORY

JSW =  Stephen Worthington <stephen@jsw.gen.nz>

0.1	First release
0.2	Some code clean up. Added messages.
0.3	Added bindings for Linux DVB API
0.3.1	Clean up.
0.3.2	Fixed problem where channel matching over-wrote best match with later, worse match.
	Bruce Wilson fixed bug in date calculation for months less than 10.
0.3.3	Add option for channel map file instead of mysql lookup
0.3.4	Add option to generate program start/stop times in UTC + timezone offset format for users
	with "auto" timezone setting for xmltv
0.3.5	Changed/added time options to generate (a) times in <local time> +<offset> format or
	(b) times in UTC format for users with "auto" timezone setting for xmltv
0.3.6	Fix bug introduced in midnight corrections due to using non-local time
0.4	Add option to use MythTV Python bindings for database access
	Add error trapping
	Added help text and more examples
0.4.1	Added xml declaration and doctype headers
	Changed to cElementTree instead of ElementTree. Supposed to be faster and use less memory.
0.5	Added tuning option for DVB-T tuners
0.5.1	Tidy up
	Add hierarchy parameter to tuning
	Add version number to header
	Delete channels not listed in channel map file
0.5.2	Fixed bug in match_channels. Trapped list index errors in tune.
0.5.3	Add -c option to clean "All New" from the front of titles.
0.6	Added argparse because it's easier to handle variable numbers of arguments compared to optparse
	Put default options in a class to simplify using argparse or optparse
	Refactored various options into the global options list and removed unneccessary option tests by adding defaults
	Added regex stripping of unwanted text from titles. Bit of a hack getting it working for Python with/without argparse.
	Removed many global variables
	Fixed apparently long standing bug in dvbsnoop code
0.6.1	Fixed bug in code handling optparse and argparse due to clean_titles2 not being set in some cases.
	Fixed handling of some tuning parameters with default values of 'a'.
	Added '-T' option to tune by chanid (which is unique) instead of channum.
0.6.1 JSW Change selects used to get channels from MythTV to only get DVB-T channels.
0.6.1 JSW Increase the buffer size used to read from the demux device to prevent buffer overflows.
0.6.2 JSW Fixed extra + bug in line 734.
0.7.0 JSW Conversion to Python 3.
    Added -b option.
	Replaced build_modules code to speed it up (runs in 1/10th the time).
	Removed optparse (not needed with Python 3).
	Prevented hang on tuner open error - it will now timeout and die.
0.7.1 JSW Fixed a problem with -b option where the database was being accessed when only the map file was being used.
0.7.2 JSW Add --hdhr option, Mike Brady's changes to allow using dvbsnoop option with a capture file created using HDHomerun tuners.

TO DO

Confirm correct function at year end and at daylight savings changes.
Tidy up code using DVB API.
Do we need to set flags in dmx_sct_filter_params?

'''
####### Start code pasted from linuxdvb module ##########################
"""
Python bindings for Linux DVB API v5.1
see headers in linux/dvb/ for reference
"""
import ctypes

# the following 41 lines are copied verbatim from the python v4l2 binding

_IOC_NRBITS = 8
_IOC_TYPEBITS = 8
_IOC_SIZEBITS = 14
_IOC_DIRBITS = 2

_IOC_NRSHIFT = 0
_IOC_TYPESHIFT = _IOC_NRSHIFT + _IOC_NRBITS
_IOC_SIZESHIFT = _IOC_TYPESHIFT + _IOC_TYPEBITS
_IOC_DIRSHIFT = _IOC_SIZESHIFT + _IOC_SIZEBITS

_IOC_NONE = 0
_IOC_WRITE = 1
_IOC_READ  = 2


def _IOC(dir_, type_, nr, size):
	return (
		ctypes.c_int32(dir_ << _IOC_DIRSHIFT).value |
		ctypes.c_int32(ord(type_) << _IOC_TYPESHIFT).value |
		ctypes.c_int32(nr << _IOC_NRSHIFT).value |
		ctypes.c_int32(size << _IOC_SIZESHIFT).value)


def _IOC_TYPECHECK(t):
	return ctypes.sizeof(t)


def _IO(type_, nr):
	return _IOC(_IOC_NONE, type_, nr, 0)


def _IOW(type_, nr, size):
	return _IOC(_IOC_WRITE, type_, nr, _IOC_TYPECHECK(size))


def _IOR(type_, nr, size):
	return _IOC(_IOC_READ, type_, nr, _IOC_TYPECHECK(size))


def _IOWR(type_, nr, size):
	return _IOC(_IOC_READ | _IOC_WRITE, type_, nr, _IOC_TYPECHECK(size))

# end code cribbed from v4l2 binding

def _binrange(start, stop):
	'''returns a list of ints from start to stop in increments of one binary lshift'''
	out = list()
	out.append(start)
	if start == 0:
		start = 1
		out.append(start)
	while start < stop:
		start = start << 1
		out.append(start)
	return out

#
# frontend 
#

fe_type = list([
	'FE_QPSK',
	'FE_QAM',
	'FE_OFDM',
	'FE_ATSC'
])
for i, name in enumerate(fe_type):
	exec(name + '=' + str(i))


fe_caps = dict(list(zip(_binrange(0, 0x800000) + _binrange(0x10000000, 0x80000000), (
	'FE_IS_STUPID',
	'FE_CAN_INVERSION_AUTO',
	'FE_CAN_FEC_1_2',
	'FE_CAN_FEC_2_3',
	'FE_CAN_FEC_3_4',
	'FE_CAN_FEC_4_5',
	'FE_CAN_FEC_5_6',
	'FE_CAN_FEC_6_7',
	'FE_CAN_FEC_7_8',
	'FE_CAN_FEC_8_9',
	'FE_CAN_FEC_AUTO',
	'FE_CAN_QPSK',
	'FE_CAN_QAM_16',
	'FE_CAN_QAM_32',
	'FE_CAN_QAM_64',
	'FE_CAN_QAM_128',
	'FE_CAN_QAM_256',
	'FE_CAN_QAM_AUTO',
	'FE_CAN_TRANSMISSION_MODE_AUTO',
	'FE_CAN_BANDWIDTH_AUTO',
	'FE_CAN_GUARD_INTERVAL_AUTO',
	'FE_CAN_HIERARCHY_AUTO',
	'FE_CAN_8VSB',
	'FE_CAN_16VSB',
	'FE_HAS_EXTENDED_CAPS',
	'FE_CAN_2G_MODULATION',
	'FE_NEEDS_BENDING',
	'FE_CAN_RECOVER',
	'FE_CAN_MUTE_TS'
))))
for val, name in list(fe_caps.items()):
	exec(name + '=' + str(val))


class dvb_frontend_info(ctypes.Structure):
	_fields_ = [
		('name', ctypes.c_char * 128),
		('type', ctypes.c_uint),
		('frequency_min', ctypes.c_uint32),
		('frequency_max', ctypes.c_uint32),
		('frequency_stepsize', ctypes.c_uint32),
		('frequency_tolerance', ctypes.c_uint32),
		('symbol_rate_min', ctypes.c_uint32),
		('symbol_rate_max', ctypes.c_uint32),
		('symbol_rate_tolerance', ctypes.c_uint32),
		('notifier_delay', ctypes.c_uint32),
		('caps', ctypes.c_uint32)
	]


class dvb_diseqc_master_cmd(ctypes.Structure):
	_fields_ = [
		('msg', ctypes.c_uint8 * 6),
		('msg_len', ctypes.c_uint8)
	]


class dvb_diseqc_slave_reply(ctypes.Structure):
	_fields_ = [
		('msg', ctypes.c_uint8 * 4),
		('msg_len', ctypes.c_uint8),
		('timeout', ctypes.c_int)
	]


fe_sec_voltage = list([
	'SEC_VOLTAGE_13',
	'SEC_VOLTAGE_18',
	'SEC_VOLTAGE_OFF'
])
for i, name in enumerate(fe_sec_voltage):
	exec(name + '=' + str(i))


fe_sec_tone_mode = list([
	'SEC_TONE_ON',
	'SEC_TONE_OFF'
])
for i, name in enumerate(fe_sec_tone_mode):
	exec(name + '=' + str(i))


fe_sec_mini_cmd = list([
	'SEC_MINI_A',
	'SEC_MINI_B'
])
for i, name in enumerate(fe_sec_mini_cmd):
	exec(name + '=' + str(i))


fe_status = dict(list(zip(_binrange(0x01, 0x40), (
	'FE_HAS_SIGNAL',
	'FE_HAS_CARRIER',
	'FE_HAS_VITERBI',
	'FE_HAS_SYNC',
	'FE_HAS_LOCK',
	'FE_TIMEDOUT',
	'FE_REINIT'
))))
for val, name in list(fe_status.items()):
	exec(name + '=' + str(val))


fe_spectral_inversion = list([
	'INVERSION_OFF',
	'INVERSION_ON',
	'INVERSION_AUTO'
])
for i, name in enumerate(fe_spectral_inversion):
	exec(name + '=' + str(i))


fe_code_rate = list([
	'FEC_NONE',
	'FEC_1_2',
	'FEC_2_3',
	'FEC_3_4',
	'FEC_4_5',
	'FEC_5_6',
	'FEC_6_7',
	'FEC_7_8',
	'FEC_8_9',
	'FEC_AUTO',
	'FEC_3_5',
	'FEC_9_10'
])
for i, name in enumerate(fe_code_rate):
	exec(name + '=' + str(i))


fe_modulation = list([
	'QPSK',
	'QAM_16',
	'QAM_32',
	'QAM_64',
	'QAM_128',
	'QAM_256',
	'QAM_AUTO',
	'VSB_8',
	'VSB_16',
	'PSK_8',
	'APSK_16',
	'APSK_32',
	'DQPSK'
])
for i, name in enumerate(fe_modulation):
	exec(name + '=' + str(i))


fe_transmit_mode = list([
	'TRANSMISSION_MODE_2K',
	'TRANSMISSION_MODE_8K',
	'TRANSMISSION_MODE_AUTO',
	'TRANSMISSION_MODE_4K'
])
for i, name in enumerate(fe_transmit_mode):
	exec(name + '=' + str(i))


fe_bandwidth = list([
	'BANDWIDTH_8_MHZ',
	'BANDWIDTH_7_MHZ',
	'BANDWIDTH_6_MHZ',
	'BANDWIDTH_AUTO'
])
for i, name in enumerate(fe_bandwidth):
	exec(name + '=' + str(i))


fe_guard_interval = list([
	'GUARD_INTERVAL_1_32',
	'GUARD_INTERVAL_1_16',
	'GUARD_INTERVAL_1_8',
	'GUARD_INTERVAL_1_4',
	'GUARD_INTERVAL_AUTO'
])
for i, name in enumerate(fe_guard_interval):
	exec(name + '=' + str(i))


fe_hierarchy = list([
	'HIERARCHY_NONE',
	'HIERARCHY_1',
	'HIERARCHY_2',
	'HIERARCHY_4',
	'HIERARCHY_AUTO'
])
for i, name in enumerate(fe_hierarchy):
	exec(name + '=' + str(i))


class dvb_qpsk_parameters(ctypes.Structure):
	_fields_ = [
		('symbol_rate', ctypes.c_uint32),
		('fec_inner', ctypes.c_uint)
	]


class dvb_qam_parameters(ctypes.Structure):
	_fields_ = [
		('symbol_rate', ctypes.c_uint32),
		('fec_inner', ctypes.c_uint),
		('modulation', ctypes.c_uint)
	]


class dvb_vsb_parameters(ctypes.Structure):
	_fields_ = [
		('modulation', ctypes.c_uint)
	]


class dvb_ofdm_parameters(ctypes.Structure):
	_fields_ = [
		('bandwidth', ctypes.c_uint),
		('code_rate_HP', ctypes.c_uint),
		('code_rate_LP', ctypes.c_uint),
		('constellation', ctypes.c_uint),
		('transmission_mode', ctypes.c_uint),
		('guard_interval', ctypes.c_uint),
		('hierarchy_information', ctypes.c_uint)
	]


class dvb_frontend_parameters(ctypes.Structure):
	class _u(ctypes.Union):
		_fields_ = [
			('qpsk', dvb_qpsk_parameters),
			('qam', dvb_qam_parameters),
			('ofdm', dvb_ofdm_parameters),
			('vsb', dvb_vsb_parameters)
		]

	_fields_ = [
		('frequency', ctypes.c_uint32),
		('inversion', ctypes.c_uint),
		('u', _u)
	]


class dvb_frontend_event(ctypes.Structure):
	_fields_ = [
		('status', ctypes.c_uint),
		('parameters', dvb_frontend_parameters)
	]


s2api_commands = list([
	'DTV_UNDEFINED',
	'DTV_TUNE',
	'DTV_CLEAR',
	'DTV_FREQUENCY',
	'DTV_MODULATION',
	'DTV_BANDWIDTH_HZ',
	'DTV_INVERSION',
	'DTV_DISEQC_MASTER',
	'DTV_SYMBOL_RATE',
	'DTV_INNER_FEC',
	'DTV_VOLTAGE',
	'DTV_TONE',
	'DTV_PILOT',
	'DTV_ROLLOFF',
	'DTV_DISEQC_SLAVE_REPLY',
	'DTV_FE_CAPABILITY_COUNT',
	'DTV_FE_CAPABILITY',
	'DTV_DELIVERY_SYSTEM',
	'DTV_ISDBT_PARTIAL_RECEPTION',
	'DTV_ISDBT_SOUND_BROADCASTING',
	'DTV_ISDBT_SB_SUBCHANNEL_ID',
	'DTV_ISDBT_SB_SEGMENT_IDX',
	'DTV_ISDBT_SB_SEGMENT_COUNT',
	'DTV_ISDBT_LAYERA_FEC',
	'DTV_ISDBT_LAYERA_MODULATION',
	'DTV_ISDBT_LAYERA_SEGMENT_COUNT',
	'DTV_ISDBT_LAYERA_TIME_INTERLEAVING',
	'DTV_ISDBT_LAYERB_FEC',
	'DTV_ISDBT_LAYERB_MODULATION',
	'DTV_ISDBT_LAYERB_SEGMENT_COUNT',
	'DTV_ISDBT_LAYERB_TIME_INTERLEAVING',
	'DTV_ISDBT_LAYERC_FEC',
	'DTV_ISDBT_LAYERC_MODULATION',
	'DTV_ISDBT_LAYERC_SEGMENT_COUNT',
	'DTV_ISDBT_LAYERC_TIME_INTERLEAVING',
	'DTV_API_VERSION',
	'DTV_CODE_RATE_HP',
	'DTV_CODE_RATE_LP',
	'DTV_GUARD_INTERVAL',
	'DTV_TRANSMISSION_MODE',
	'DTV_HIERARCHY',
	'DTV_ISDBT_LAYER_ENABLED',
	'DTV_ISDBS_TS_ID'
])
for i, name in enumerate(s2api_commands):
	exec(name + '=' + str(i))
DTV_MAX_COMMAND = DTV_ISDBS_TS_ID


fe_pilot = list([
	'PILOT_ON',
	'PILOT_OFF',
	'PILOT_AUTO'
])
for i, name in enumerate(fe_pilot):
	exec(name + '=' + str(i))


fe_rolloff = list([
	'ROLLOFF_35',
	'ROLLOFF_20',
	'ROLLOFF_25',
	'ROLLOFF_AUTO'
])
for i, name in enumerate(fe_rolloff):
	exec(name + '=' + str(i))


fe_delivery_system = list([
	'SYS_UNDEFINED',
	'SYS_DVBC_ANNEX_AC',
	'SYS_DVBC_ANNEX_B',
	'SYS_DVBT',
	'SYS_DSS',
	'SYS_DVBS',
	'SYS_DVBS2',
	'SYS_DVBH',
	'SYS_ISDBT',
	'SYS_ISDBS',
	'SYS_ISDBC',
	'SYS_ATSC',
	'SYS_ATSCMH',
	'SYS_DMBTH',
	'SYS_CMMB',
	'SYS_DAB',
	'SYS_DCII_C_QPSK',
	'SYS_DCII_I_QPSK',
	'SYS_DCII_Q_QPSK',
	'SYS_DCII_C_OQPSK'
])
for i, name in enumerate(fe_delivery_system):
	exec(name + '=' + str(i))


class dtv_cmds_h(ctypes.Structure):
	_fields_ = [
		('name', ctypes.c_char_p),
		('cmd', ctypes.c_uint32),
		('set', ctypes.c_uint32),
		('buffer', ctypes.c_uint32),
		('reserved', ctypes.c_uint32)
	]


class dtv_property(ctypes.Structure):
	class _u(ctypes.Union):
		class _s(ctypes.Structure):
			_fields_ = [
				('data', ctypes.c_uint8 * 32),
				('len', ctypes.c_uint32),
				('reserved1', ctypes.c_uint32 * 3),
				('reserved2', ctypes.c_void_p)
			]


		_fields_ = [
			('data', ctypes.c_uint32),
			('buffer', _s)
		]


	_fields_ = [
		('cmd', ctypes.c_uint32),
		('reserved', ctypes.c_uint32 * 3),
		('u', _u),
		('result', ctypes.c_int)
	]

	_pack_ = True


class dtv_properties(ctypes.Structure):
	_fields_ = [
		('num', ctypes.c_uint32),
		('props', ctypes.POINTER(dtv_property))
	]


FE_SET_PROPERTY = _IOW('o', 82, dtv_properties)
FE_GET_PROPERTY = _IOR('o', 83, dtv_properties)

DTV_IOCTL_MAX_MSGS = 64
FE_TUNE_MODE_ONESHOT = 0x01

FE_GET_INFO = _IOR('o', 61, dvb_frontend_info)

FE_DISEQC_RESET_OVERLOAD = _IO('o', 62)
FE_DISEQC_SEND_MASTER_CMD = _IOW('o', 63, dvb_diseqc_master_cmd)
FE_DISEQC_RECV_SLAVE_REPLY = _IOR('o', 64, dvb_diseqc_slave_reply)
FE_DISEQC_SEND_BURST = _IO('o', 65)

FE_SET_TONE = _IO('o', 66)
FE_SET_VOLTAGE = _IO('o', 67)
FE_ENABLE_HIGH_LNB_VOLTAGE = _IO('o', 68)

FE_READ_STATUS = _IOR('o', 69, ctypes.c_uint)
FE_READ_BER = _IOR('o', 70, ctypes.c_uint32)
FE_READ_SIGNAL_STRENGTH = _IOR('o', 71, ctypes.c_uint16)
FE_READ_SNR = _IOR('o', 72, ctypes.c_uint16)
FE_READ_UNCORRECTED_BLOCKS = _IOR('o', 73, ctypes.c_uint32)

FE_SET_FRONTEND = _IOW('o', 76, dvb_frontend_parameters)
FE_GET_FRONTEND = _IOR('o', 77, dvb_frontend_parameters)
FE_SET_FRONTEND_TUNE_MODE = _IO('o', 81)
FE_GET_EVENT = _IOR('o', 78, dvb_frontend_event)

FE_DISHNETWORK_SEND_LEGACY_CMD = _IO('o', 80)

#
# demux
#

DMX_FILTER_SIZE = 16

class dmx_filter(ctypes.Structure):
	_fields_ = [
		('filter', ctypes.c_uint8 * DMX_FILTER_SIZE),
		('mask', ctypes.c_uint8 * DMX_FILTER_SIZE),
		('mode', ctypes.c_uint8 * DMX_FILTER_SIZE)
	]

class dmx_sct_filter_params(ctypes.Structure):
	_fields_ = [
		('pid', ctypes.c_uint16),
		('filter', dmx_filter),
		('timeout', ctypes.c_uint32),
		('flags', ctypes.c_uint32)
	]

DMX_CHECK_CRC = 0x01
DMX_ONESHOT = 0x02
DMX_IMMEDIATE_START = 0x04
DMX_KERNEL_CLIENT = 0x8000

DMX_START = _IO('o', 41)
DMX_STOP = _IO('o', 42)
DMX_SET_BUFFER_SIZE = _IO('o', 45)
DMX_SET_FILTER = _IOW('o', 43, dmx_sct_filter_params)

####### End code pasted from linuxdvb module ##########################

try:
	from xml.etree import cElementTree as ET
except ImportError:
	from xml.etree import ElementTree as ET
import difflib, fileinput, os, re, select, sqlite3, struct, sys, time, unicodedata, zlib
from subprocess import Popen, PIPE, STDOUT
import fcntl
import traceback
from array import array
import argparse

def main():
	global c, c2, unsquashed_modules, chanlist, channels, MythDB, options

	lines = []
	unsquashed_modules = []
	chanlist = []
	channels = []
	raw_tuning_info = []
	
	icons = {'hd.png':'HDTV', 'dolby.png':'dolby', 'ear.png':'teletext'}
	ratings = {'ao.png':'AO', 'g.png':'G', 'pgr.png':'PGR'}

	class defaults:
		demux_device = '/dev/dvb/adapter0/demux0'
		extra_args = ''
		mysql_args = "-u root"
		output_file = "/tmp/xmltv.xml" # Output filename
		map_file = False # Channel map filename. Set to FALSE if -f option is not used.
		both = False
		tune_ch = False
		tune_chanid = False
		use_dvbsnoop = False
		verbosity = False
		timezone = False
		UTC = False
		clean_titles = False
		py_bind = False
	defs = defaults()

	options = argparse_parse(defs)

	verbose("Options selected = " + str(options))

	# Test for dvbsnoop
	if options.use_dvbsnoop and which("dvbsnoop") == None:
		print("You need to install dvbsnoop")
		sys.exit(1)

	# Set up database in memory to simplify handling shows crossing midnight
	conn = sqlite3.connect(":memory:")
	conn.text_factory = sqlite3.OptimizedUnicode	# need this to handle inserting unicode strings
	conn.row_factory = sqlite3.Row
	c = conn.cursor()
	c2 = conn.cursor()
	c.execute('''create table programs(start, stop, channel, title, desc,
			episode_id, dolby_flag, teletext_flag, hd_flag, rating, start_time, local_start, local_stop)''')

	if options.py_bind:
		try:
			from MythTV import MythDB
		except Exception as inst:
			print('\n' + str(inst))
			print('Unable to load MythTV Python module. Exiting.')
			sys.exit(1)

	(chaninfo_map, chaninfo_db) = get_chan_info(options.map_file, options.py_bind, options.both)

	if options.tune_ch or options.tune_chanid:
		verbose('\nGetting tuning info for channel ' + (('ID ' + str(options.tune_chanid)) if options.tune_chanid else str(options.tune_ch)) + '\n')
		tuning_fields = ['frequency','inversion','bandwidth','hp_code_rate','lp_code_rate','constellation','transmission_mode','guard_interval','hierarchy']
		sql_select = 'select '
		for t in tuning_fields:
			sql_select += t + ','
		sql_select = sql_select.rstrip(',') + ' from channel join dtv_multiplex on channel.mplexid=dtv_multiplex.mplexid where (dtv_multiplex.mod_sys=\'UNDEFINED\' or left(dtv_multiplex.mod_sys, 5)=\'DVB-T\') and '
		sql_select += ('chanid=' + str(options.tune_chanid)) if options.tune_chanid else ('channum=' + str(options.tune_ch))
		tuning_info = get_tune_info(options.py_bind, sql_select, options.tune_ch, tuning_fields)
		d = options.demux_device.rstrip('/')
		frontend = d[0:d.rfind('/') + 1] + 'frontend' + d[len(d) - 1]
		verbose('\nTuning ' + frontend + ' to channel ' + (('ID ' + str(options.tune_chanid)) if options.tune_chanid else str(options.tune_ch)) + '\n')
		try:
			try:
				fefd = open(frontend, 'rb+')
			except:
				verbose('Unable to open ' + frontend + ' for tuning. Possibly in use. Continuing without tuning.\n')
				options.tune_ch = False
			if options.tune_ch:
				tune(frontend, fefd, tuning_info)
		except Exception as inst:
			print(inst)
			traceback.print_exc()
			print('Unable to tune', frontend, '. Exiting')
			sys.exit(1)

	datablocks = []
	module_numbers = []
	if options.use_dvbsnoop:
		verbose("\nUsing dvbsnoop to collect data\n")
		the_pid = find_pid()
		if the_pid == -1:
			verbose("PID not found\n")
			sys.exit(1)
		else:
			the_pid = find_pid2(the_pid)
		download(the_pid, datablocks, module_numbers)
	else:
		verbose("\nOpening read from device " + options.demux_device + " to collect data\n")
		try:
			dmxfd = open(options.demux_device, 'rb', 2*1024*1024)
		except IOError as e:
			(errno, strerr) = e.args
			verbose("Could not open demux device " + options.demux_device + ". I/O error " + str(errno) + ": " + strerr + "\n")
			sys.exit(1)
		except:
			verbose("Unexpected error: " + str(sys.exc_info()[0]))
			sys.exit(1)
		retval = fcntl.ioctl(dmxfd, DMX_SET_BUFFER_SIZE, 20*4096)
		if retval != 0:
			verbose('Unable to set DMX_SET_BUFFER_SIZE, returned value = ' + str(retval) + ', aborting')
			sys.exit(1)
		demux_filter = dmx_sct_filter_params()
		the_pid = find_pid3(dmxfd, demux_filter)
		if the_pid != -1:
			download2(the_pid, datablocks, module_numbers, dmxfd, demux_filter)
		else:
			verbose("Could not find DSM-CC carousel PID.\n")
			sys.exit(1)
		dmxfd.close()

	if options.tune_ch:
		fefd.close()

	build_modules(datablocks, module_numbers)
	lines = get_MHEG_data()
	verbose("Extracting EPG info\n\n--- Channel Names ---\n")
	parse_MHEG_data(lines, icons, ratings)
	if options.map_file:
		verbose("\nMatching MHEG channels to MythTV channels using map file\n")
		map_channels(chaninfo_map)
	if not options.map_file or options.both:
		verbose("\nFuzzy matching MHEG channels to MythTV channels\n")
		match_channels(chaninfo_db)
	delete_unmapped_channels()
	verbose("Fixing start/stop times for shows crossing midnight\n")
	fix_midnight()
	verbose("Building XML file: " + options.output_file + "\n")
	build_xml()
	prettyup_xml()
	if options.clean_titles:
		if isinstance(options.clean_titles, bool): # A hack to handle both argparse and optparse
			options.clean_titles = 'All New '
		do_clean_titles(options.clean_titles)
	sys.exit(0)

def argparse_parse(defs):
	parser = argparse.ArgumentParser(epilog=extra_help, 
									formatter_class=argparse.RawDescriptionHelpFormatter, 
									description='Convert DVB-T MHEG EPG data to xmltv for import into MythTV EPG')
	parser.add_argument('-d', dest='demux_device', action='store', default=defs.demux_device, 
					help='Specify DVB demux device, e.g., "/dev/dvb/adapter1/demux0"')
	parser.add_argument('-e', dest='extra_args', action='store', default=defs.extra_args, 
					help='Extra dvbsnoop arguments, e.g., "-adapter 1"')
	parser.add_argument('-m', dest='mysql_args', action='store', default=defs.mysql_args, 
					help='MySQL arguments, e.g., "-u myuser -pmypassword"' + ' (default "' + defs.mysql_args + '")')
	parser.add_argument('-o', dest='output_file', action='store', default=defs.output_file, 
					help='Output filename (default '+defs.output_file+')')
	parser.add_argument('-v', dest='verbosity', action='store_true', default=defs.verbosity,
					help='Enable verbose output. Disable or redirect stdout & stderr to file if you get broken pipe error 32.')
	parser.add_argument('-s', dest='use_dvbsnoop', action='store_true', default=defs.use_dvbsnoop, 
					help="Use dvbsnoop instead of direct DVB API access (default FALSE)")
	parser.add_argument('-f', dest='map_file', action='store', default=defs.map_file, 
					help='Channel map filename (tab separated, default ' + str(defs.map_file) + ')')
	parser.add_argument('-b', dest='both', action='store_true', default=defs.both, 
					help='Match channels with both fuzzy matching and then the map file (default False)')
	parser.add_argument('-z', dest='timezone', action='store_true', default=defs.timezone, 
					help='Generate local + offset program start/stop times (default local time)')
	parser.add_argument('-u', dest='UTC', action='store_true', default=defs.UTC, 
					help='Generate UTC start/stop times (default local time)')
	parser.add_argument('-p', dest='py_bind', action='store_true', default=defs.py_bind,
					help='Use MythTV Python bindings for database access (default FALSE)')
	parser.add_argument('-t', dest='tune_ch', action='store', type=int, default=defs.tune_ch, 
					help='Tune to a specified channel number. DVB-T only. Will not tune the adapter if it appears to be locked by another app (e.g., MythTV) but will continue to try to download the EPG.')
	parser.add_argument('-T', dest='tune_chanid', action='store', type=int, default=defs.tune_chanid, 
					help='Tune to a specified channel ID. DVB-T only. Will not tune the adapter if it appears to be locked by another app (e.g., MythTV) but will continue to try to download the EPG.')
	parser.add_argument('-c', dest='clean_titles', nargs='?', const='All New ', default=defs.clean_titles,
					help='''Clean up silly things prepended to titles. Default (just -c with no string specified) removes "All New ".
					Customize the text to be removed by specifying a matching regular expression string like this: 
					"Remove this|And this" or this: "Remove this".''')
	parser.add_argument('--hdhr', action='store_true', default=False, 
					help='Use dvbsnoop to read a .ts file captured externally from an HDHomerun tuner.  Implies -s and removes -n 1 option from the dvbsnoop command line.  Requires -e.')
	args = parser.parse_args()
	if args.hdhr:
		args.n1 = ''
		args.use_dvbsnoop = True
		if args.extra_args == defs.extra_args:
			print('Error: --hdhr option requires -e option.')
			sys.exit(1)
	else:
		args.n1 = '-n 1 '
	return args

def get_tune_info(py_bind, sql_select, tune_ch, tuning_fields):
	if py_bind:
		try:
			db = MythDB()
			dbconn = db.cursor()
			dbconn.execute(sql_select)
			raw_tuning_info = dbconn.fetchone()
			dbconn.close()
			if raw_tuning_info == None or len(raw_tuning_info) != len(tuning_fields):
				print('Unable to get tuning info for channel ' + str(tune_ch) + '.  Exiting.')
				sys.exit(1)
			tuning_info = dict(list(zip(tuning_fields,raw_tuning_info)))
		except Exception as inst:
			print(inst)
			print('Unable to get tuning info for channel ' + str(tune_ch) + '.  Exiting.')
			sys.exit(1) 
	else:
		try:
			f = os.popen('mysql -ss ' + options.mysql_args + ' -e "' + sql_select + '" mythconverg')
			raw_tuning_info = f.read().rstrip('\n').split('\t')
			f.close()
			if len(raw_tuning_info) != len(tuning_fields):
				print('Unable to get tuning info for channel ' + str(tune_ch) + '.  Exiting.')
				sys.exit(1)
			tuning_info = dict(list(zip(tuning_fields,raw_tuning_info)))
		except Exception as inst:
			print(inst)
			print('Unable to get tuning info for channel ' + str(tune_ch) + '.  Exiting.')
			sys.exit(1)
	tuning_info['frequency'] = str(tuning_info['frequency'])
	for ch in tuning_info:
		verbose(ch + ': ' + str(tuning_info[ch]) + '\n')
	return tuning_info

def get_chan_info(map_file, py_bind, both):
	chaninfo_map = []
	chaninfo_db = []
	if map_file:
		# Get channel info from map file
		verbose("\nGetting channel xmltv ids from map file " + map_file + "\n")
		f = open(map_file)
		chaninfo1 = f.readlines()
		f.close()
		for ch in chaninfo1:
			verbose(ch)
			chaninfo_map.append(ch.strip("\n").split("\t"))
	if py_bind:
		# Get database channel info using Python bindings
		verbose("\nGetting channel info from MythTV database using Python bindings\n")
		try:
			db = MythDB()
			dbconn = db.cursor()
			dbconn.execute("desc channel deleted")
			deleted = dbconn.fetchall()
			if deleted == ():
				deleted = ''
			else:
				deleted =  ' and c.deleted is NULL'
			dbconn.execute("select callsign, name, xmltvid from channel c, dtv_multiplex d where c.mplexid is not NULL and c.mplexid = d.mplexid and (d.mod_sys='UNDEFINED' or d.mod_sys like 'DVB-T%')" + deleted)
			chaninfo_db = dbconn.fetchall()
			dbconn.close()
			for ch in chaninfo_db:
				verbose(ch[0] + "\t" + ch[1] + "\t" + ch[2] + "\n")
		except Exception as inst:
			print(inst)
			print("Error accessing mythconverg database using Python bindings.  Exiting.")
			sys.exit(1)
	elif not map_file or both:
		# Get database channel info from mysql
		verbose("\nGetting channel info from MythTV database using mysql\n")
		try:
			f = os.popen('mysql -ss ' + options.mysql_args + ' -e \'describe channel deleted\' mythconverg')
			deleted = f.read().splitlines()
			f.close()
			if deleted == []:
				deleted = ''
			else:
				deleted =  ' and c.deleted is NULL'

			f = os.popen('mysql -ss ' + options.mysql_args + ' -e \'select callsign, name, xmltvid from channel c, dtv_multiplex d where c.mplexid is not NULL and c.mplexid = d.mplexid and (d.mod_sys="UNDEFINED" or d.mod_sys like "DVB-T%")' + deleted + '\' mythconverg')
			chaninfo1 = f.read().splitlines()
			f.close()
			for ch in chaninfo1:
				verbose(ch + "\n")
				chaninfo_db.append(ch.split("\t"))
		except Exception as inst:
			print(inst)
			print("Could not access mythconverg database.  Exiting.")
			sys.exit(1)
	return (chaninfo_map, chaninfo_db)

def tune(frontend, fefd, tuning_info):
	feinfo = dvb_frontend_info()
	fcntl.ioctl(fefd, FE_GET_INFO, feinfo)
	verbose('Frontend name: ' + str(feinfo.name, 'utf-8') + '\n')
	verbose('Frontend type: ' + fe_type[feinfo.type] + '\n')
	festatus = dvb_frontend_event()
	fcntl.ioctl(fefd, FE_READ_STATUS, festatus)
	if festatus.status & FE_HAS_LOCK and festatus.parameters.frequency == int(tuning_info['frequency']):
		verbose('Frontend is already locked to frequency: ' + str(festatus.parameters.frequency) + '. Not tuning.\n')
		return
	if feinfo.type != FE_OFDM:
		print('Device', frontend, 'does not appear to be DVB-T. Exiting.')
		sys.exit(1)
	feparams = dvb_frontend_parameters()
	fcntl.ioctl(fefd, FE_GET_FRONTEND, feparams)

	# Convert myth tuning info to dvb api format
	feparams.frequency = int(tuning_info['frequency'])
	if tuning_info['inversion'] == 'a':
		feparams.inversion = 2
	else:
		feparams.inversion = int(tuning_info['inversion'])
	if tuning_info['bandwidth'] == 'a':
		tuning_info['bandwidth'] = 'auto'
	for b in fe_bandwidth:
		if tuning_info['bandwidth'].upper() in b:
			exec('feparams.u.ofdm.bandwidth = ' + b)
	for b in fe_code_rate:
		if tuning_info['hp_code_rate'].replace('/', '_').upper() in b:
			exec('feparams.u.ofdm.code_rate_HP = ' + b)
	for b in fe_code_rate:
		if tuning_info['lp_code_rate'].replace('/', '_').upper() in b:
			exec('feparams.u.ofdm.code_rate_LP = ' + b)
	if tuning_info['transmission_mode'] == 'a':
		tuning_info['transmission_mode'] = 'auto'
	for b in fe_transmit_mode:
		if tuning_info['transmission_mode'].upper() in b:
			exec('feparams.u.ofdm.transmission_mode = ' + b)
	for b in fe_modulation:
		if tuning_info['constellation'].upper() in b:
			exec('feparams.u.ofdm.constellation = ' + b)
	for b in fe_guard_interval:
		if tuning_info['guard_interval'].replace('/', '_').upper() in b:
			exec('feparams.u.ofdm.guard_interval = ' + b)
	for b in fe_hierarchy:
		if tuning_info['hierarchy'].upper() in b:
			exec('feparams.u.ofdm.hierarchy_information = ' + b)

	verbose('\nParameters to be sent to ' + frontend + ':\n')
	verbose('Frequency = ' + str(feparams.frequency) + '\n')
	verbose('Inversion = ' + str(feparams.inversion) + ' = ' + fe_spectral_inversion[feparams.inversion] + '\n')
	verbose('Bandwidth = ' + str(feparams.u.ofdm.bandwidth) + ' = ' + fe_bandwidth[feparams.u.ofdm.bandwidth] + '\n')
	verbose('Transmission mode = ' + str(feparams.u.ofdm.transmission_mode) + ' = ' + fe_transmit_mode[feparams.u.ofdm.transmission_mode] + '\n')
	verbose('HP code rate = ' + str(feparams.u.ofdm.code_rate_HP) + ' = ' + fe_code_rate[feparams.u.ofdm.code_rate_HP] + '\n')
	verbose('LP code rate = ' + str(feparams.u.ofdm.code_rate_LP) + ' = ' + fe_code_rate[feparams.u.ofdm.code_rate_LP] + '\n')
	verbose('Constellation = ' + str(feparams.u.ofdm.constellation) + ' = ' + fe_modulation[feparams.u.ofdm.constellation] + '\n')
	verbose('Guard interval = ' + str(feparams.u.ofdm.guard_interval) + ' = ' + fe_guard_interval[feparams.u.ofdm.guard_interval] + '\n')
	verbose('Hierarchy = ' + str(feparams.u.ofdm.hierarchy_information) + ' = ' + fe_hierarchy[feparams.u.ofdm.hierarchy_information] + '\n')

	# Do it
	fcntl.ioctl(fefd, FE_SET_FRONTEND, feparams)
	i = 0
	locked = False
	while i < 10 and not locked:
		fcntl.ioctl(fefd, FE_READ_STATUS, festatus)
		if festatus.status & FE_HAS_LOCK:
			verbose('Frontend has lock\n')
			locked = True
		else:
			verbose('Waiting for frontend to lock\n')
		time.sleep(1)
		i = i + 1
	if not locked:
		print('Frontend:', frontend, 'did not lock on channel within 10 seconds. Exiting.')
		sys.exit(1)

	fcntl.ioctl(fefd, FE_GET_FRONTEND, feparams)
	verbose('\nParameters read back from device. Might not be accurate?\n') # Why are some of these values wrong? Driver/firmware bugs?
	verbose('Frequency = ' + str(feparams.frequency) + '\n')
	verbose('Inversion = ' + str(feparams.inversion) + ' = ' + getFromList(fe_spectral_inversion, feparams.inversion) + '\n')
	verbose('Bandwidth = ' + str(feparams.u.ofdm.bandwidth) + ' = ' + getFromList(fe_bandwidth, feparams.u.ofdm.bandwidth) + '\n')
	verbose('Transmission mode = ' + str(feparams.u.ofdm.transmission_mode) + ' = ' + getFromList(fe_transmit_mode, feparams.u.ofdm.transmission_mode) + '\n')
	verbose('HP code rate = ' + str(feparams.u.ofdm.code_rate_HP) + ' = ' + getFromList(fe_code_rate, feparams.u.ofdm.code_rate_HP) + '\n')
	verbose('LP code rate = ' + str(feparams.u.ofdm.code_rate_LP) + ' = ' + getFromList(fe_code_rate, feparams.u.ofdm.code_rate_LP) + '\n')
	verbose('Constellation = ' + str(feparams.u.ofdm.constellation) + ' = ' + getFromList(fe_modulation, feparams.u.ofdm.constellation) + '\n')
	verbose('Guard interval = ' + str(feparams.u.ofdm.guard_interval) + ' = ' + getFromList(fe_guard_interval, feparams.u.ofdm.guard_interval) + '\n')
	verbose('Hierarchy = ' + str(feparams.u.ofdm.hierarchy_information) + ' = ' + getFromList(fe_hierarchy, feparams.u.ofdm.hierarchy_information) + '\n')

def getFromList(theList, index):
	if index >= 0 and index < len(theList):
		result = theList[index]
	else:
		result = "Possible bad list index value: " + str(index)
	return result

def verbose(stuff):
	if options.verbosity:
		sys.stdout.write(stuff)
		sys.stdout.flush()

def which(prog):
	for path in os.environ["PATH"].split(os.pathsep):
		f = os.path.join(path, prog)
		if os.path.exists(f) and os.access(f, os.X_OK):
			return f
	return None

def find_pid3(dmxfd, demux_filter):
	verbose("Getting program_map_pid from PAT\n")
	program_map_pid = -1
	demux_filter.pid = 0
	fcntl.ioctl(dmxfd, DMX_SET_FILTER, demux_filter)
	fcntl.ioctl(dmxfd, DMX_START)
	r, w, e = select.select([dmxfd], [] , [], 2)
	if not dmxfd in r:
		dmxfd.close()
		print('Timeout reading from ' + options.demux_device + ', exiting!')
		exit(2)
	buffer = dmxfd.read(3)
	table_id = buffer[0]
	b1 = buffer[1]
	b2 = buffer[2]
	sect_len = ((b1 & 0x0F) << 8) | b2
	buffer += dmxfd.read(sect_len) # The remaining sect_len bytes in the section start immediately after sect_len.
	program_map_pid = ((buffer[14] & 0x0F) << 8) | buffer[15] # Skip to the second program entry
	verbose("program_map_pid = " + str(program_map_pid) + "\n")
	fcntl.ioctl(dmxfd, DMX_STOP)

	verbose("Getting carousel_pid from PMT\n")
	carousel_pid = -1
	demux_filter.pid = program_map_pid
	fcntl.ioctl(dmxfd, DMX_SET_FILTER, demux_filter)
	fcntl.ioctl(dmxfd, DMX_START)
	buffer = dmxfd.read(3)
	table_id = buffer[0]
	b1 = buffer[1]
	b2 = buffer[2]
	sect_len = ((b1 & 0x0F) << 8) | b2
	buffer += dmxfd.read(sect_len)
	program_info_len =((buffer[10] & 0x0F) << 8) | buffer[11]
	p = 11 + 1 + program_info_len
	while p < sect_len - program_info_len - 12:
		stream_type = buffer[p]
		es_len = ((buffer[p+3] & 0x0F) << 8) | buffer[p+4]
		if stream_type == 11:
			carousel_pid = ((buffer[p+1] & 0x1F) << 8) | buffer[p+2]
			verbose("carousel_pid = " + str(carousel_pid) + "\n")
			break
		p = p + 5 + es_len
	fcntl.ioctl(dmxfd, DMX_STOP)
	return carousel_pid

def find_pid():
	f = os.popen("dvbsnoop -nph -ph 2 " + options.n1 + options.extra_args + " 0")
	for line in f.read().split("\n"):
		if line.find("Program_map_PID:") != -1:
			f.close()
			return line.strip().split(" ")[1]
	f.close()
	return -1

def find_pid2(pid):
	f = os.popen("dvbsnoop -nph -ph 2 " + options.n1 + options.extra_args + " " + pid)
	next_one = False
	for line in f.read().split("\n"):
		if line.find("Stream_type: 11") != -1:
			next_one = True
		elif line.find("Elementary_PID:") != -1 and next_one:
			f.close()
			return line.strip().split(" ")[1]
	f.close()
	return -1

def download2(pid, datablocks, module_numbers, dmxfd, demux_filter):
	block_sizes = []
	module_sizes = []
	found_list = False
	finished_download = False

	demux_filter.pid = pid
	fcntl.ioctl(dmxfd, DMX_SET_FILTER, demux_filter)
	fcntl.ioctl(dmxfd, DMX_START)

	while not finished_download:
		buffer = dmxfd.read(3)
		table_id = buffer[0]
		b1 = buffer[1]
		b2 = buffer[2]
		sect_syntax_ind = (b1 & 0x80) >> 7
		private_ind = (b1 & 0x40) >> 6
		sect_len = ((b1 & 0x0F) << 8) | b2
		buffer += dmxfd.read(sect_len)
		message_id = (buffer[10] << 8) | buffer[11]
		if table_id == 60:
			module_id = (buffer[20] << 8) | buffer[21]
			block_no = (buffer[24] << 8) | buffer[25]
			downloaded = False
			if len(datablocks) > 0 and len([blk for blk in datablocks if blk[0] == module_id and blk[1] == block_no]) > 0:
				downloaded = True
			if not downloaded and module_id != 0: # Module 0 does not contain EPG data and is not compressed
				datablocks.append([module_id, block_no, buffer[26:len(buffer)-4]]) # Last 4 bytes are CRC
				if found_list:
					verbose("Blocks left to download = " + str(total_blocks - len(datablocks)) + "        \r")
				else:
					verbose("Started downloading blocks. Waiting for Download Message Block...\r")

		elif table_id == 59 and message_id == 4098 and not found_list:
			found_list = True
			dsmcc_header_len = 12 # only if adaptation_length = 0
			block_size = (buffer[8 + dsmcc_header_len + 4] << 8) | buffer[8 + dsmcc_header_len + 5]
			no_of_modules = (buffer[8 + dsmcc_header_len + 18] << 8) | buffer[8 + dsmcc_header_len + 19]
			verbose("\nblock_size = " + str(block_size) + " no_of_modules = " + str(no_of_modules) + "\n")
			p = 8 + dsmcc_header_len + 20
			total_blocks = 0
			for i in range(0, no_of_modules):
				module_id = (buffer[p] << 8) | buffer[p+1]
				a = array('B', buffer[p+2:p+6])
				module_size = struct.unpack(">I", a)[0]
				module_info_len = buffer[p+7]
				if module_id != 0: # Module 0 does not contain EPG data and is not compressed
					verbose("module_id = " + str(module_id) + " module_size = " + str(module_size) + "\n")
					module_numbers.append(module_id)
					module_sizes.append(module_size)
					total_blocks = total_blocks + module_size//block_size + (module_size % block_size > 0)
				p = p + 8 + module_info_len
			verbose("\nFound Download Message Block. " + str(total_blocks) + " blocks total to download...\n")

		if found_list and len(datablocks) >= total_blocks:
			finished_download = True

	fcntl.ioctl(dmxfd, DMX_STOP)

def download(pid, datablocks, module_numbers):
	block_sizes = []
	module_sizes = []
	no_of_blocks = []
	DDB = False
	DSIorDII = False
	block_list_found = False
	finished_download = False
	store_data = False
	while not finished_download:
		args = ["dvbsnoop", "-nph", "-ph", "2"]
		if options.extra_args != '':
			for a in options.extra_args.split(" "):
				args.append(a)
		args.append(pid)
		f = Popen(args, stdout = PIPE)
		for line in f.stdout:
			if line.find(b"Table_ID: 60") != -1:
				# This is a DDB (Data Download Block)
				DDB = True
				DSIorDII = False
			elif line.find(b"Table_ID: 59") != -1:
				# This is a DSI or DII block
				DSIorDII = True
				DDB = False
			if DSIorDII:
				if line.find(b"blockSize:") != -1:
					# Store the DDB block size
					block_size = int(line.strip().split(b" ")[1])
				elif line.find(b"moduleId:") != -1:
					# Store module number...
					module_num = int(line.strip().split(b" ")[1])
				elif line.find(b"moduleSize:") != -1:
					# ...and module size.
					module_size = int(line.strip().split(b" ")[1])
					gotalready = False
					if len(module_numbers) > 0:
						if module_num in module_numbers:
							gotalready = True
					if not gotalready and module_num < 50 and module_num != 0: # Module 0 does not contain EPG data
						module_numbers.append(module_num)
						module_sizes.append(module_size)
						no_of_blocks.append(module_size//block_size + (module_size % block_size > 0))
			if DDB:
				if line.find(b"moduleId:") != -1:
					# Store the module number
					module_num = int(line.strip().split(b" ")[1])
				elif line.find(b"blockNumber:") != -1:
					# Store the block number
					block_num = int(line.strip().split(b" ")[1])
				elif line.find(b"Block Data:") != -1:
					# Set flag to store the data on the next line
					store_data = True
				elif store_data:
					# Reset the flag nd store the data
					store_data = False
					downloaded = False
					if len(datablocks) > 0 and len([blk for blk in datablocks if blk[0] == module_num and blk[1] == block_num]) > 0:
						downloaded = True
					if not downloaded and module_num != 0: # Module 0 does not contain EPG data
						datablocks.append([module_num, block_num, line.strip()])
						if block_list_found:
							verbose("Blocks left to download = " + str(total_blocks - len(datablocks)) + "	 \r")
						else:
							verbose("Started downloading blocks. Waiting for Download Message Block...\r")

				if len(module_numbers) > 0 and not block_list_found:
					block_list_found = True
					total_blocks = 0
					for b in no_of_blocks:
						total_blocks = total_blocks + b
					verbose("\nFound Download Message Block. " + str(total_blocks) + " blocks total to download...\n")


			if block_list_found and len(datablocks) >= total_blocks:
				finished_download = True

			if finished_download:
				f.terminate()

def build_modules(datablocks, module_numbers):
	"""
		Old algorithm replaced with a balance line to speed it up.
	"""
	verbose("\nBuilding modules\n")
	module_numbers.sort()
	modules = []
	datablocks.sort(key = lambda a: (a[0], a[1]))
	starttime = time.time()
	blki = iter(datablocks)
	blk = next(blki)
	for i in module_numbers:
		bytestring = b''
		try:
			while blk[0] != i:
				blk = next(blki)
		except StopIteration:
			break
		try:
			while blk[0] == i:
				if options.use_dvbsnoop:
					for c in blk[2].split(b" "):
						if len(c) == 2:
							x = struct.pack("B", int(c, 16))
							bytestring += x
				else:
					bytestring += blk[2]
				blk = next(blki)
		except StopIteration:
			pass
		modules.append([i, bytestring])
	stoptime = time.time()

	verbose("Decompressing modules\n")
	for squashed in modules:
		m = squashed[0]
		if len(squashed[1]) > 0:
			try:
				verbose("Expanding module " + str(m) + " from " + str(len(squashed[1])) + " bytes to ")
				unsquashed = zlib.decompress(squashed[1])
				verbose(str(len(unsquashed)) + " bytes\n")
			except Exception as e:
				verbose("\nException on decompress:\n")
				print(e)
				traceback.print_exc()
				verbose("\nLooks like module " + str(m) + " is incomplete. Attempting partial decompression.\n")
				try:
					decom = zlib.decompressobj()
					unsquashed = decom.decompress(squashed[1], 100000)
					verbose("Possible successful partial decompression of module " + str(m) + "\n")
				except Exception as e:
					verbose("\nException on decompress:\n")
					print(e)
					traceback.print_exc()
					verbose("Failed partial decompression of module " + str(m) + "\n")
			build_modules2(unsquashed)
		else:
			verbose("No data downloaded for module " + str(m) + ".\n")

def build_modules2(unsquashed):
	zz = re.finditer(b'BIOP', unsquashed)  # Get a MatchObject containing all instances of "BIOP" in module
	listo = []
	for ll in zz:
		v = ll.start() + 8  # Offset of message_size (32 bit uimsbf)
		listo.append([ll.start(), int(struct.unpack(">I", unsquashed[v:v+4])[0]) + 12])

	next_byte = listo[0][0]  # Following is messy to try and avoid possible spurious instances of the string "BIOP"
	stringo = []			 # and also avoid some non-EPG messages with string "crid://" buried in them.
	for k in listo:
		if k[0] == next_byte:
			end_byte = next_byte + k[1]
			type_id_ofs = next_byte + 13 + ord(unsquashed[next_byte + 12:next_byte + 13]) + 4
			if unsquashed[type_id_ofs:type_id_ofs + 3] == b"fil" and unsquashed[next_byte:next_byte + 130].find(b"crid://") != -1:
				chunk = unsquashed[next_byte:end_byte]
				chunk = re.sub(b'[\x0a\x0d]', b' ', chunk) # Get rid of CR/LF. Not needed in MythTV EPG.
				chunk = re.sub(b'[\x04\x00]', b' ', chunk)
				chunk = re.sub(b'\x1b[Cc]', b' ', chunk)
				crido = chunk.find(b'crid://')  # Find suitable point before which we get rid of \x1c which is used for splitting
				chunk = re.sub(b'\x1c', b'.', chunk[:crido]) + chunk[crido:] # so we don't accidentally split beginning of message
				stringo.extend(chunk.split(b'\x1c'))
			next_byte = end_byte
	unsquashed_modules.append(stringo)

def get_MHEG_data():
	lines = []
	# Loop through all the module files
	for rawlines in unsquashed_modules:
		# Loop through all the BIOP messages
		for line in rawlines:
			if line.startswith(b'BIOP'):
				fields = line.split(b'\x1d')
				l = len(fields) - 1
				temp = b''
				for i in range(0, 5):
					temp = b'\x1d' + fields[l - i].strip() + temp
					if i == 4:
						# Get the date
						t = time.localtime(time.time() - 10 * 24 * 3600) # time 10 days before now - use for Dec/Jan issue
						t2 = time.strptime(str(t[0]) + str(t[1]).rjust(2,'0') + str(t[2]).rjust(2,'0'), "%Y%m%d")
						theday = time.strptime(fields[l - i].strip().decode() + " " + time.strftime("%Y", t2), "%a %d %b %Y")
						# Fix for data crossing from one year to the next
						if theday >= t2:
							thedate = time.strftime("%Y%m%d", theday)
						else:
							thedate = str(t2[0] + 1) + time.strftime("%m%d", theday)
						lines.append(b'DAYdayDAYday' + thedate.encode())
				lines.append(b'BIOP' + temp)
			else:
				lines.append(line)
	return lines

def maketime(thedate):
	if options.timezone:	# If times should be local + offset
		mktimethedate = time.mktime(thedate)
		if time.daylight and time.localtime(time.mktime(thedate)).tm_isdst:
			offset = -time.altzone	# Seconds WEST of UTC. Negative means EAST of UTC.
		else:
			offset = -time.timezone	# Seconds WEST of UTC. Negative means EAST of UTC.
		hh = int(offset/3600.0)	# Divide by float to get correct values for negatives
		mm = round((offset - hh*3600)/60)
		offset  = hh * 100 + mm
		thetime = time.strftime("%Y%m%d%H%M00", time.localtime(time.mktime(thedate)))
		thetime = thetime + ' {0:+5}'.format(offset)
	elif options.UTC:	# If times should be UTC with no offset
		thetime = time.strftime("%Y%m%d%H%M00", time.gmtime(time.mktime(thedate)))
	else:	# If times should be local
		thetime = time.strftime("%Y%m%d%H%M00", time.localtime(time.mktime(thedate)))
	return thetime

def parse_MHEG_data(lines, icons, ratings):
	for line in lines:
		fields = line.split(b'\x1d')
		num_fields = len(fields)
		showrec = []
		if line.startswith(b'DAYdayDAYday'):
			date = [int(line[12:16]), int(line[16:18]), int(line[18:20])]

		# Get channel info if line starts with "BIOP" and contains "crid:" and update channels table if channel not already found
		elif fields[0] == b'BIOP' and line.find(b'crid:') != -1:
			chan = str(fields[2], encoding="utf-8")   # Can be funny characters in channel name
			try:
				i = chanlist.index(chan)
			except:
				chanlist.append(chan)
				verbose(chan + "\n")

		# Build the showrec list to create programme info to add to programs table
		elif num_fields > 9 and fields[1]:
			# TODO: Need to think more about handling daylight savings change
			hh = int(int(fields[1]) / 3600)
			mm = round((int(fields[1]) - (hh * 3600))/60)
			date2 = []
			date2.extend(date)
			date2.extend([hh, mm, 0, 0, 0, -1])
			start_time = maketime(tuple(date2))
			local_start = time.strftime("%Y%m%d%H%M00", time.localtime(time.mktime(tuple(date2))))

			hh = int(int(fields[2]) / 3600)
			mm = round((int(fields[2]) - (hh * 3600))/60)
			del date2[:]
			date2.extend(date)
			date2.extend([hh, mm, 0, 0, 0, -1])
			end_time = maketime(tuple(date2))
			local_end = time.strftime("%Y%m%d%H%M00", time.localtime(time.mktime(tuple(date2))))

			fields[6] = fields[6].lstrip(b"/")
			# unicode the title and description in case there are weird characters in them
			showrec = [start_time, end_time, chan, str(fields[7], encoding="utf-8"), str(fields[8], encoding="utf-8"), str(fields[6], encoding='utf-8')]
			for j in range(8, num_fields):
				if fields[j].startswith(b"/") and fields[j].find(b".png") < 0:
					fields[j] = fields[j].lstrip(b"/")
			for k,v in sorted(icons.items()): # iteritems() sorting is apparently problematic so sort so we're sure of order
				if line.find(k.encode()) != -1:
					flag = v
				else:
					flag = "blank"
				showrec.append(flag)
			rating_found = False
			for k,v in ratings.items():
				if line.find(k.encode()) != -1:
					rating_found = True
					showrec.append(v)
			if rating_found != True:
				showrec.append("no rating")
			showrec.append(str(fields[4], encoding='utf-8'))
			showrec.append(local_start)
			showrec.append(local_end)
			c.execute("insert into programs values (?,?,?,?,?,?,?,?,?,?,?,?,?)", showrec)

def match_channels(chaninfo_db):
	chaninfo_db = list(chaninfo_db)
	for ch in channels:
		if ch[1] in chanlist:
			chanlist.remove(ch[1])
		for chi in chaninfo_db:
			if chi[2] == ch[0]:
				chaninfo_db.remove(chi)
				break
	hits = []
	for chan in chanlist:
		for ch in chaninfo_db:
			matches = difflib.get_close_matches(chan.upper(), [x.upper() for x in ch])
			if len(matches) > 0:
				ratio = 0
				for chan2 in ch:
					ratio = ratio + difflib.SequenceMatcher(None, chan.upper(), chan2.upper()).ratio()
				hits.append([ratio, chan, ch[2]])

	# Sort descending by ratio so best matches come first when iterating the list
	hits.sort(key = lambda x: (-x[0]))

	for hit in hits:
		if hit[2] != '' and hit[2] not in [x[0] for x in channels]:	# Test if we already have a channel with this name. Add it if we don't.
			channels.append([hit[2], hit[1]])
			verbose('"' + hit[1] + '" matched to MythTV xmltv ID "' + hit[2] + '"\n')
			t = (hit[2], hit[1])
			c.execute('update programs set channel=? where channel=?', t)

def map_channels(chaninfo_map):
	for line in chaninfo_map:
		if len(line) == 2:
			chan = line[0]
			xmltvid = line[1]
			channels.append([xmltvid, chan])
			verbose('"' + chan + '" matched to MythTV xmltv ID "' + str(xmltvid) + '"\n')
			t = (xmltvid, chan)
			c.execute('update programs set channel=? where channel=?', t)

def delete_unmapped_channels():
	# Delete programs on channels which have not been mapped or matched.
	c.execute('delete from programs where channel not in (' + ','.join('?'*len([t[0] for t in channels])) + ')', [t[0] for t in channels])

def fix_midnight():
	# Find shows supposedly ending at midnight and check and fix the stop times
	c.execute('select * from programs where local_stop like "%000000"')
	for row in c:
		t = (row['stop'], row['channel'])
		# Look for a show starting at midnight same day, same channel
		c2.execute('select * from programs where start=? and channel=?', t)
		r2 = c2.fetchone()
		if r2 != None:
			if r2['title'] == row['title'] and r2['episode_id'] == row['episode_id']:
				# Correct the program stop time if the show has same title and episode_id
				t = (r2['stop'], r2['channel'], row['stop'])
				c2.execute('update programs set stop=? where channel=? and stop=?', t)
				# Delete the bogus duplicate record which starts at midnight
				t = (r2['channel'], r2['start'], r2['stop'])
				c2.execute('delete from programs where channel=? and start=? and stop=?', t)
		else:
			# Didn't find a show starting after this one so delete it because it is suspect.
			# Cannot be sure about shows supposedly ending at midnight if there is no following
			# show supposedly starting at midnight.
			t = (row['channel'], row['start'], row['stop'])
			c2.execute('delete from programs where channel=? and start=? and stop=?', t)

	# Find shows starting at midnight and delete if start_time not "00:00"
	c.execute('delete from programs where local_start like "%000000" and start_time != "00:00"')

def build_xml():
	# Build the xml doc
	root_element = ET.Element("tv")
	root_element.set("date", maketime(time.localtime()))
	root_element.set("generator-info-name", "mhegepgsnoop.py " + VERSION)
	root_element.set("source-info-name", "DVB-T MHEG Stream")

	# Create the channel elements
	channels.sort(key = lambda a: a[1].lower())
	for r in channels:
		ch = ET.Element("channel")
		display_name = ET.Element("display-name")
		ch.set("id", r[0])
		display_name.text = r[1]
		ch.append(display_name)
		root_element.append(ch)

	# Create the programme elements
	c.execute('select distinct * from programs order by channel collate nocase, start')
	for r in c:
		try:
			prog = ET.Element("programme")
			root_element.append(prog)
			prog.set("channel", r['channel'])
			prog.set("start", r['start'])
			prog.set("stop", r['stop'])
			title = ET.Element("title")
			title.text = r['title']
			prog.append(title)
			desc = ET.Element("desc")
			desc.text = r['desc']
			prog.append(desc)
			if r['episode_id']:
				episode = ET.Element("episode-num")
				episode.text = r[5]
				episode.set("system", "dd_progid")
				prog.append(episode)
			if r['hd_flag'] == "HDTV":
				video = ET.Element("video")
				present = ET.Element("present")
				quality = ET.Element("quality")
				present.text = "yes"
				quality.text = "HDTV"
				video.append(present)
				video.append(quality)
				prog.append(video)
			if r['dolby_flag'] == "dolby":
				audio = ET.Element("audio")
				stereo = ET.Element("stereo")
				stereo.text = "dolby"
				audio.append(stereo)
				prog.append(audio)
			if r['teletext_flag'] == "teletext":
				subtitles = ET.Element("subtitles")
				subtitles.set("type", "teletext")
				prog.append(subtitles)
			if r['rating'] != "no rating":
				rating = ET.Element("rating")
				rating.set("system", "Freeview")
				value = ET.Element("value")
				value.text = r[9]
				rating.append(value)
				prog.append(rating)
		except:
			prog = ET.Element("programme")
			root_element.append(prog)
			prog.set("foobar", "foobar")

	# Write the xml doc to disk
	outfile = open(options.output_file, "w")
	# Add manual declaration and doctype headers because it's tedious to do any other way
	outfile.write('<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE tv SYSTEM "xmltv.dtd">')
#	ET.ElementTree(root_element).write(outfile, encoding="unicode")
	out_str = ET.tostring(root_element, encoding="unicode", method="xml")
	outfile.write(out_str)
	outfile.close()

def prettyup_xml():
	# Pretty up the output so it's easier to read
	# This code edits the file inplace
	for line in fileinput.FileInput(options.output_file, inplace=1):
		line = re.sub("(<[/]*chan)", "\n\t\\1", line)
		line = re.sub("(<displ|<icon|<tit|<desc|<epi|<subt)", "\n\t\t\\1", line)
		line = re.sub("(<[/]*prog)", "\n\t\\1", line)
		line = re.sub("(<[/]*rat)", "\n\t\t\\1", line)
		line = re.sub("(<[/]*aud)", "\n\t\t\\1", line)
		line = re.sub("(<[/]*vid)", "\n\t\t\\1", line)
		line = re.sub("(<val|<pres|<qual|<ster)", "\n\t\t\t\\1", line)
		line = re.sub("(</tv|<tv|<!DOCTYPE)", "\n\\1", line)
		print(line)

def	do_clean_titles(clean_titles):
	verbose('Stripping "' + clean_titles + '" from titles\n')
	regex = re.compile('<title>(' + clean_titles + ')')
	for line in fileinput.FileInput(options.output_file, inplace=1):
		line = regex.sub('<title>', line)
		sys.stdout.write(line) # Use sys.stdout.write to avoid extra new lines

if __name__ == '__main__':
	main()