code:
import wx, threading, traceback, time, os, hashlib, gc, psutil

import lib.Hydrus.Constants as HC
import lib.Hydrus.Globals as HG
import lib.Hydrus.Constants.Local as CC

from lib.GUI import FrameGUI, FrameSplash
from lib.GUI.Dialogs import DialogYesNo
from lib.GUI.Menus import DestroyMenu
from lib.GUI.ScrolledPanels.Management import RepairFileSystemPanel
from lib.GUI.TopLevelWindows import DialogManage

from lib.Handling.Video import FFMPEG_PATH

from lib.Hydrus.API.Local import APIManager
from lib.Hydrus.Caches import BitmapManager, ClientFilesManager, \
	FileViewingStatsManager, LocalBooruCache, ParsingCache, RenderedImageCache, \
	ServicesManager, ShortcutsManager, TagCensorshipManager, TagParentsManager, \
	ThumbnailCache, UndoManager
from lib.Hydrus.Controller import ControllerBase
from lib.Hydrus.Daemons import DAEMONDownloadFiles, DAEMONCheckImportFolders, \
	DAEMONSynchroniseSubscriptions, DAEMONMaintainTrash, DAEMONSynchroniseRepositories, \
	DAEMONCheckExportFolders
from lib.Hydrus.Data import Call, CleanRunningFile, DebugPrint, GenerateKey, GetNow, \
	IsAlreadyRunning, LastShutdownWasBad, Print, RecordRunningStart, RestartProcess, \
	ShowException, ShowText, TimeHasPassed, ToHumanInt
from lib.Hydrus.Data.Local import ReportShutdownException
from lib.Hydrus.DB.Local import DB
from lib.Hydrus.Defaults import GetClientDefaultOptions, SetDefaultBandwidthManagerRules, \
	SetDefaultDomainManagerData, SetDefaultLoginManagerScripts
from lib.Hydrus.Exceptions import InsufficientCredentialsException, ShutdownException, WXDeadWindowException
from lib.Hydrus.Options import ClientOptions
from lib.Hydrus.Paths.Local import GetTempDir
from lib.Hydrus.Serialisable import HG.SerialisableType["CLIENT_API_MANAGER"], HG.SerialisableType["CLIENT_OPTIONS"], \
	HG.SerialisableType["NETWORK_BANDWIDTH_MANAGER"], HG.SerialisableType["NETWORK_SESSION_MANAGER"], \
	HG.SerialisableType["NETWORK_DOMAIN_MANAGER"], HG.SerialisableType["NETWORK_LOGIN_MANAGER"]
from lib.Hydrus.Server.Local import HydrusServiceBooru, HydrusServiceClientAPI
from lib.Hydrus.Tags import TagSiblingsManager
from lib.Hydrus.Threading import DAEMONForegroundWorker, IsThreadShuttingDown, JobKey, \
	WXAwareJob, WXAwareRepeatingJob

from lib.Networking import LocalPortInUse, NetworkEngine
from lib.Networking.Bandwidth import NetworkBandwidthManager
from lib.Networking.Domain import NetworkDomainManager
from lib.Networking.Login import NetworkLoginManager
from lib.Networking.Sessions import NetworkSessionManager

if not HG.twisted_is_broke:
	from twisted.internet import reactor, defer

### CLASSES ###

class LocalController( ControllerBase ):
	def __init__( self, db_dir ):
		self._last_shutdown_was_bad = False
		self._is_booted = False
		self._splash = None

		ControllerBase.__init__( self, db_dir )

		self._name = 'client'

		HG.client_controller = self

		# just to set up some defaults, in case some db update expects something for an odd yaml-loading reason
		self.options = GetClientDefaultOptions()
		self.new_options = ClientOptions()

		HC.options = self.options

		self._page_key_lock = threading.Lock()

		self._thread_slots[ 'watcher_files' ] = ( 0, 15 )
		self._thread_slots[ 'watcher_check' ] = ( 0, 5 )
		self._thread_slots[ 'gallery_files' ] = ( 0, 15 )
		self._thread_slots[ 'gallery_search' ] = ( 0, 5 )

		self._alive_page_keys = set()
		self._closed_page_keys = set()

		self._last_mouse_position = None
		self._menu_open = False
		self._previously_idle = False
		self._idle_started = None

		self.client_files_manager = None
		self.services_manager = None

	def _InitDB( self ):
		return DB( self, self.db_dir, 'client' )

	def _InitTempDir( self ):
		self.temp_dir = GetTempDir()

	def _DestroySplash( self ):
		def wx_code( splash ):
			if splash:
				splash.Hide()
				splash.DestroyLater()

		if self._splash is not None:
			splash = self._splash
			self._splash = None
			wx.CallAfter( wx_code, splash )

	def _GetUPnPServices( self ):
		return self.services_manager.GetServices( ( HC.LOCAL_BOORU, HC.CLIENT_API_SERVICE ) )

	def _ReportShutdownDaemonsStatus( self ):
		names = { daemon.name for daemon in self._daemons if daemon.is_alive() }
		names = list( names )
		names.sort()
		self.pub( 'splash_set_status_subtext', ', '.join( names ) )

	def AcquirePageKey( self ):
		with self._page_key_lock:
			page_key = GenerateKey()
			self._alive_page_keys.add( page_key )

			return page_key

	def CallBlockingToWX( self, win, func, *args, **kwargs ):
		def wx_code( win, job_key ):
			try:
				if win is not None and not win:
					raise WXDeadWindowException( 'Parent Window was destroyed before wx command was called!' )

				result = func( *args, **kwargs )

				job_key.SetVariable( 'result', result )

			except ( WXDeadWindowException, InsufficientCredentialsException, ShutdownException ) as e:
				job_key.SetVariable( 'error', e )

			except Exception as e:
				job_key.SetVariable( 'error', e )

				Print( 'CallBlockingToWX just caught this error:' )
				DebugPrint( traceback.format_exc() )

			finally:
				job_key.Finish()

		job_key = JobKey( cancel_on_shutdown = False )

		job_key.Begin()

		wx.CallAfter( wx_code, win, job_key )

		while not job_key.IsDone():
			if self._model_shutdown:
				raise ShutdownException( 'Application is shutting down!' )

			time.sleep( 0.05 )

		if job_key.HasVariable( 'result' ):
			# result can be None, for wx_code that has no return variable
			result = job_key.GetIfHasVariable( 'result' )

			return result

		error = job_key.GetIfHasVariable( 'error' )

		if error is not None:
			raise error

		raise ShutdownException()

	def CallLaterWXSafe( self, window, initial_delay, func, *args, **kwargs ):
		job_scheduler = self._GetAppropriateJobScheduler( initial_delay )

		call = Call( func, *args, **kwargs )

		job = WXAwareJob( self, job_scheduler, window, initial_delay, call )

		job_scheduler.AddJob( job )

		return job

	def CallRepeatingWXSafe( self, window, initial_delay, period, func, *args, **kwargs ):
		job_scheduler = self._GetAppropriateJobScheduler( period )

		call = Call( func, *args, **kwargs )

		job = WXAwareRepeatingJob( self, job_scheduler, window, initial_delay, period, call )

		job_scheduler.AddJob( job )

		return job

	def CheckAlreadyRunning( self ):
		while IsAlreadyRunning( self.db_dir, 'client' ):
			self.pub( 'splash_set_status_text', 'client already running' )

			def wx_code():
				message = 'It looks like another instance of this client is already running, so this instance cannot start.'
				message += os.linesep * 2
				message += 'If the old instance is closing and does not quit for a _very_ long time, it is usually safe to force-close it from task manager.'

				with DialogYesNo( self._splash, message, 'The client is already running.', yes_label = 'wait a bit, then try again', no_label = 'forget it' ) as dlg:
					if dlg.ShowModal() != wx.ID_YES:
						raise ShutdownException()

			self.CallBlockingToWX( self._splash, wx_code )

			for i in range( 10, 0, -1 ):
				if not IsAlreadyRunning( self.db_dir, 'client' ):
					break

				self.pub( 'splash_set_status_text', 'waiting ' + str( i ) + ' seconds' )

				time.sleep( 1 )

	def CheckMouseIdle( self ):
		mouse_position = wx.GetMousePosition()

		if self._last_mouse_position is None:
			self._last_mouse_position = mouse_position

		elif mouse_position != self._last_mouse_position:
			idle_before_position_update = self.CurrentlyIdle()

			self._timestamps[ 'last_mouse_action' ] = GetNow()
			self._last_mouse_position = mouse_position

			idle_after_position_update = self.CurrentlyIdle()
			move_knocked_us_out_of_idle = ( not idle_before_position_update ) and idle_after_position_update

			if move_knocked_us_out_of_idle:
				self.pub( 'set_status_bar_dirty' )

	def ClosePageKeys( self, page_keys ):
		with self._page_key_lock:
			self._closed_page_keys.update( page_keys )

	def CreateSplash( self ):
		try:
			self._splash = FrameSplash( self )

		except Exception:
			Print( 'There was an error trying to start the splash screen!' )
			Print( traceback.format_exc() )

			raise

	def CurrentlyIdle( self ):
		if HG.force_idle_mode:
			self._idle_started = 0
			return True

		if not TimeHasPassed( self._timestamps[ 'boot' ] + 120 ):
			return False

		idle_normal = self.options[ 'idle_normal' ]
		idle_period = self.options[ 'idle_period' ]
		idle_mouse_period = self.options[ 'idle_mouse_period' ]

		if idle_normal:
			currently_idle = True

			if idle_period is not None:
				if not TimeHasPassed( self._timestamps[ 'last_user_action' ] + idle_period ):
					currently_idle = False

			if idle_mouse_period is not None:
				if not TimeHasPassed( self._timestamps[ 'last_mouse_action' ] + idle_mouse_period ):
					currently_idle = False

		else:
			currently_idle = False

		turning_idle = currently_idle and not self._previously_idle
		self._previously_idle = currently_idle

		if turning_idle:
			self._idle_started = GetNow()
			self.pub( 'wake_daemons' )

		if not currently_idle:
			self._idle_started = None

		return currently_idle

	def CurrentlyVeryIdle( self ):
		if self._idle_started is not None and TimeHasPassed( self._idle_started + 3600 ):
			return True

		return False

	def DoIdleShutdownWork( self ):
		stop_time = GetNow() + ( self.options[ 'idle_shutdown_max_minutes' ] * 60 )

		self.MaintainDB( only_if_idle = False, stop_time = stop_time )

		if not self.options[ 'pause_repo_sync' ]:
			services = self.services_manager.GetServices( HC.REPOSITORIES )

			for service in services:
				if TimeHasPassed( stop_time ):
					return

				service.SyncProcessUpdates( only_when_idle = False, stop_time = stop_time )

		self.Write( 'last_shutdown_work_time', GetNow() )

	def Exit( self ):
		if HG.emergency_exit:
			self.ShutdownView()
			self.ShutdownModel()

			CleanRunningFile( self.db_dir, 'client' )

		else:
			try:
				last_shutdown_work_time = self.Read( 'last_shutdown_work_time' )
				shutdown_work_period = self.new_options.GetInteger( 'shutdown_work_period' )
				we_can_shutdown_work = TimeHasPassed( last_shutdown_work_time + shutdown_work_period )
				idle_shutdown_action = self.options[ 'idle_shutdown' ]

				if we_can_shutdown_work and idle_shutdown_action in ( CC.IDLE_ON_SHUTDOWN, CC.IDLE_ON_SHUTDOWN_ASK_FIRST ):
					idle_shutdown_max_minutes = self.options[ 'idle_shutdown_max_minutes' ]

					time_to_stop = GetNow() + ( idle_shutdown_max_minutes * 60 )

					work_to_do = self.GetIdleShutdownWorkDue( time_to_stop )

					if len( work_to_do ) > 0:
						if idle_shutdown_action == CC.IDLE_ON_SHUTDOWN_ASK_FIRST:
							text = 'Is now a good time for the client to do up to ' + ToHumanInt( idle_shutdown_max_minutes ) + ' minutes\' maintenance work? (Will auto-no in 15 seconds)'
							text += os.linesep * 2
							text += 'The outstanding jobs appear to be:'
							text += os.linesep * 2
							text += os.linesep.join( work_to_do )

							with DialogYesNo( self._splash, text, title = 'Maintenance is due' ) as dlg_yn:
								job = self.CallLaterWXSafe( dlg_yn, 15, dlg_yn.EndModal, wx.ID_NO )

								try:
									if dlg_yn.ShowModal() == wx.ID_YES:
										HG.do_idle_shutdown_work = True

									else:
										# if they said no, don't keep asking
										self.Write( 'last_shutdown_work_time', GetNow() )

								finally:
									job.Cancel()

						else:
							HG.do_idle_shutdown_work = True

				self.CallToThreadLongRunning( self.THREADExitEverything )

			except Exception:
				self._DestroySplash()

				DebugPrint( traceback.format_exc() )

				HG.emergency_exit = True

				self.Exit()

	def GetApp( self ):
		return self._app

	def GetClipboardText( self ):
		if wx.TheClipboard.Open():
			data = wx.TextDataObject()

			wx.TheClipboard.GetData( data )
			wx.TheClipboard.Close()

			text = data.GetText()

			return text

		else:
			raise Exception( 'I could not get permission to access the clipboard.' )

	def GetCommandFromShortcut( self, shortcut_names, shortcut ):
		return self._shortcuts_manager.GetCommand( shortcut_names, shortcut )

	def GetGUI( self ):
		return self.gui

	def GetIdleShutdownWorkDue( self, time_to_stop ):
		work_to_do = []
		work_to_do.extend( self.Read( 'maintenance_due', time_to_stop ) )

		services = self.services_manager.GetServices( HC.REPOSITORIES )

		for service in services:
			if service.CanDoIdleShutdownWork():
				work_to_do.append( service.GetName() + ' repository processing' )

		return work_to_do

	def GetNewOptions( self ):
		return self.new_options

	def InitClientFilesManager( self ):
		def wx_code( missing_locations ):
			with DialogManage( None, 'repair file system' ) as dlg:
				panel = RepairFileSystemPanel( dlg, missing_locations )

				dlg.SetPanel( panel )

				if dlg.ShowModal() == wx.ID_OK:
					self.client_files_manager = ClientFilesManager( self )
					missing_locations = self.client_files_manager.GetMissing()

				else:
					raise ShutdownException( 'File system failed, user chose to quit.' )

			return missing_locations

		self.client_files_manager = ClientFilesManager( self )
		missing_locations = self.client_files_manager.GetMissing()

		while len( missing_locations ) > 0:
			missing_locations = self.CallBlockingToWX( self._splash, wx_code, missing_locations )

	def InitModel( self ):
		self.pub( 'splash_set_title_text', 'booting db\u2026' )

		ControllerBase.ControllerBase.InitModel( self )

		self.pub( 'splash_set_status_text', 'initialising managers' )
		self.pub( 'splash_set_status_subtext', 'services' )

		self.services_manager = ServicesManager( self )

		self.pub( 'splash_set_status_subtext', 'options' )

		self.options = self.Read( 'options' )
		self.new_options = self.Read( 'serialisable', HG.SerialisableType["CLIENT_OPTIONS"] )

		HC.options = self.options

		if self.new_options.GetBoolean( 'use_system_ffmpeg' ):
			if FFMPEG_PATH.startswith( HC.BIN_DIR ):
				import lib.Handling.Video

				lib.Handling.Video.FFMPEG_PATH = os.path.basename( FFMPEG_PATH )

		self.pub( 'splash_set_status_subtext', 'client files' )

		self.InitClientFilesManager()

		#

		self.pub( 'splash_set_status_subtext', 'network' )

		self.parsing_cache = ParsingCache()

		client_api_manager = self.Read( 'serialisable', HG.SerialisableType["CLIENT_API_MANAGER"] )

		if client_api_manager is None:
			client_api_manager = APIManager()
			client_api_manager._dirty = True

			wx.SafeShowMessage( 'Problem loading object', 'Your client api manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )

		self.client_api_manager = client_api_manager

		bandwidth_manager = self.Read( 'serialisable', HG.SerialisableType["NETWORK_BANDWIDTH_MANAGER"] )

		if bandwidth_manager is None:
			bandwidth_manager = NetworkBandwidthManager()
			SetDefaultBandwidthManagerRules( bandwidth_manager )
			bandwidth_manager._dirty = True

			wx.SafeShowMessage( 'Problem loading object', 'Your bandwidth manager was missing on boot! I have recreated a new empty one with default rules. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )

		session_manager = self.Read( 'serialisable', HG.SerialisableType["NETWORK_SESSION_MANAGER"] )

		if session_manager is None:
			session_manager = NetworkSessionManager()
			session_manager._dirty = True

			wx.SafeShowMessage( 'Problem loading object', 'Your session manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )

		domain_manager = self.Read( 'serialisable', HG.SerialisableType["NETWORK_DOMAIN_MANAGER"] )

		if domain_manager is None:
			domain_manager = NetworkDomainManager()
			SetDefaultDomainManagerData( domain_manager )
			domain_manager._dirty = True

			wx.SafeShowMessage( 'Problem loading object', 'Your domain manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )

		domain_manager.Initialise()

		login_manager = self.Read( 'serialisable', HG.SerialisableType["NETWORK_LOGIN_MANAGER"] )

		if login_manager is None:
			login_manager = NetworkLoginManager()
			SetDefaultLoginManagerScripts( login_manager )
			login_manager._dirty = True

			wx.SafeShowMessage( 'Problem loading object', 'Your login manager was missing on boot! I have recreated a new empty one. Please check that your hard drive and client are ok and let the hydrus dev know the details if there is a mystery.' )

		login_manager.Initialise()

		self.network_engine = NetworkEngine( self, bandwidth_manager, session_manager, domain_manager, login_manager )

		self.CallToThreadLongRunning( self.network_engine.MainLoop )

		#

		self._shortcuts_manager = ShortcutsManager( self )

		self.local_booru_manager = LocalBooruCache( self )

		self.file_viewing_stats_manager = FileViewingStatsManager( self )

		self.pub( 'splash_set_status_subtext', 'tag censorship' )

		self.tag_censorship_manager = TagCensorshipManager( self )

		self.pub( 'splash_set_status_subtext', 'tag siblings' )

		self.tag_siblings_manager = TagSiblingsManager( self )

		self.pub( 'splash_set_status_subtext', 'tag parents' )

		self.tag_parents_manager = TagParentsManager( self )
		self._managers[ 'undo' ] = UndoManager( self )

		def wx_code():
			self._caches[ 'images' ] = RenderedImageCache( self )
			self._caches[ 'thumbnail' ] = ThumbnailCache( self )
			self.bitmap_manager = BitmapManager( self )

			CC.GlobalBMPs.STATICInitialise()

		self.pub( 'splash_set_status_subtext', 'image caches' )

		self.CallBlockingToWX( self._splash, wx_code )

		self.sub( self, 'ToClipboard', 'clipboard' )
		self.sub( self, 'RestartClientServerService', 'restart_client_server_service' )

	def InitView( self ):
		if self.options[ 'password' ] is not None:
			self.pub( 'splash_set_status_text', 'waiting for password' )

			def wx_code_password():
				while True:
					with wx.PasswordEntryDialog( self._splash, 'Enter your password', 'Enter password' ) as dlg:
						if dlg.ShowModal() == wx.ID_OK:
							password_bytes = bytes( dlg.GetValue(), 'utf-8' )

							if hashlib.sha256( password_bytes ).digest() == self.options[ 'password' ]:
								break

						else:
							raise InsufficientCredentialsException( 'Bad password check' )

			self.CallBlockingToWX( self._splash, wx_code_password )

		self.pub( 'splash_set_title_text', 'booting gui\u2026' )

		def wx_code_gui():
			self.gui = FrameGUI( self )
			self.ResetIdleTimer()

		self.CallBlockingToWX( self._splash, wx_code_gui )

		# ShowText will now popup as a message, as popup message manager has overwritten the hooks

		ControllerBase.ControllerBase.InitView( self )

		self._listening_services = {}

		self.RestartClientServerService( CC.LOCAL_BOORU_SERVICE_KEY )
		self.RestartClientServerService( CC.CLIENT_API_SERVICE_KEY )

		if not HG.no_daemons:
			self._daemons.append( DAEMONForegroundWorker( self, 'DownloadFiles', DAEMONDownloadFiles, ( 'notify_new_downloads', 'notify_new_permissions' ) ) )
			self._daemons.append( DAEMONForegroundWorker( self, 'SynchroniseSubscriptions', DAEMONSynchroniseSubscriptions, ( 'notify_restart_subs_sync_daemon', 'notify_new_subscriptions' ), period = 4 * 3600, init_wait = 60, pre_call_wait = 3 ) )
			self._daemons.append( DAEMONForegroundWorker( self, 'MaintainTrash', DAEMONMaintainTrash, init_wait = 120 ) )
			self._daemons.append( DAEMONForegroundWorker( self, 'SynchroniseRepositories', DAEMONSynchroniseRepositories, ( 'notify_restart_repo_sync_daemon', 'notify_new_permissions' ), period = 4 * 3600, pre_call_wait = 1 ) )

		job = self.CallRepeating( 5.0, 180.0, DAEMONCheckImportFolders )
		job.WakeOnPubSub( 'notify_restart_import_folders_daemon' )
		job.WakeOnPubSub( 'notify_new_import_folders' )
		job.ShouldDelayOnWakeup( True )
		self._daemon_jobs[ 'import_folders' ] = job

		job = self.CallRepeating( 5.0, 180.0, DAEMONCheckExportFolders )
		job.WakeOnPubSub( 'notify_restart_export_folders_daemon' )
		job.WakeOnPubSub( 'notify_new_export_folders' )
		job.ShouldDelayOnWakeup( True )
		self._daemon_jobs[ 'export_folders' ] = job

		job = self.CallRepeating( 0.0, 30.0, self.SaveDirtyObjects )
		job.WakeOnPubSub( 'important_dirt_to_clean' )
		self._daemon_jobs[ 'save_dirty_objects' ] = job

		job = self.CallRepeating( 5.0, 3600.0, self.SynchroniseAccounts )
		job.ShouldDelayOnWakeup( True )
		job.WakeOnPubSub( 'notify_unknown_accounts' )
		self._daemon_jobs[ 'synchronise_accounts' ] = job

		job = self.CallRepeatingWXSafe( self, 10.0, 10.0, self.CheckMouseIdle )
		self._daemon_jobs[ 'check_mouse_idle' ] = job

		if self.db.IsFirstStart():
			message = 'Hi, this looks like the first time you have started the hydrus client.'
			message += os.linesep * 2
			message += 'Don\'t forget to check out the help if you haven\'t already--it has an extensive \'getting started\' section.'
			message += os.linesep * 2
			message += 'To dismiss popup messages like this, right-click them.'

			ShowText( message )

		if self.db.IsDBUpdated():
			ShowText( 'The client has updated to version ' + str( HC.SOFTWARE_VERSION ) + '!' )

		for message in self.db.GetInitialMessages():
			ShowText( message )

	def IsBooted( self ):
		return self._is_booted

	def LastShutdownWasBad( self ):
		return self._last_shutdown_was_bad

	def MaintainDB( self, only_if_idle = True, stop_time = None ):
		if only_if_idle and not self.GoodTimeToDoBackgroundWork():
			return

		if self.new_options.GetBoolean( 'maintain_similar_files_duplicate_pairs_during_idle' ):
			phashes_stop_time = stop_time

			if phashes_stop_time is None:
				phashes_stop_time = GetNow() + 15

			self.WriteSynchronous( 'maintain_similar_files_phashes', stop_time = phashes_stop_time )

			tree_stop_time = stop_time

			if tree_stop_time is None:
				tree_stop_time = GetNow() + 30

			self.WriteSynchronous( 'maintain_similar_files_tree', stop_time = tree_stop_time, abandon_if_other_work_to_do = True )

			search_distance = self.new_options.GetInteger( 'similar_files_duplicate_pairs_search_distance' )
			search_stop_time = stop_time

			if search_stop_time is None:
				search_stop_time = GetNow() + 60

			self.WriteSynchronous( 'maintain_similar_files_duplicate_pairs', search_distance, stop_time = search_stop_time, abandon_if_other_work_to_do = True )

		if stop_time is None or not TimeHasPassed( stop_time ):
			self.WriteSynchronous( 'maintain_file_reparsing', stop_time = stop_time )

		if stop_time is None or not TimeHasPassed( stop_time ):
			self.WriteSynchronous( 'vacuum', stop_time = stop_time )

		if stop_time is None or not TimeHasPassed( stop_time ):
			self.WriteSynchronous( 'analyze', stop_time = stop_time )

		if stop_time is None or not TimeHasPassed( stop_time ):
			if TimeHasPassed( self._timestamps[ 'last_service_info_cache_fatten' ] + ( 60 * 20 ) ):
				self.pub( 'splash_set_status_text', 'fattening service info' )

				services = self.services_manager.GetServices()

				for service in services:
					self.pub( 'splash_set_status_subtext', service.GetName() )

					try:
						self.Read( 'service_info', service.GetServiceKey() )
					except Exception:
						pass # sometimes this breaks when a service has just been removed and the client is closing, so ignore the error

				self._timestamps[ 'last_service_info_cache_fatten' ] = GetNow()

	def MaintainMemoryFast( self ):
		ControllerBase.ControllerBase.MaintainMemoryFast( self )
		self.parsing_cache.CleanCache()

	def MaintainMemorySlow( self ):
		ControllerBase.ControllerBase.MaintainMemorySlow( self )

		if TimeHasPassed( self._timestamps[ 'last_page_change' ] + 30 * 60 ):
			self.pub( 'delete_old_closed_pages' )
			self._timestamps[ 'last_page_change' ] = GetNow()

		disk_cache_maintenance_mb = self.new_options.GetNoneableInteger( 'disk_cache_maintenance_mb' )

		if disk_cache_maintenance_mb is not None and not self._view_shutdown:
			cache_period = 3600
			disk_cache_stop_time = GetNow() + 2

			if TimeHasPassed( self._timestamps[ 'last_disk_cache_population' ] + cache_period ):
				self.Read( 'load_into_disk_cache', stop_time = disk_cache_stop_time, caller_limit = disk_cache_maintenance_mb * 1024 * 1024 )
				self._timestamps[ 'last_disk_cache_population' ] = GetNow()

	def MenuIsOpen( self ):
		return self._menu_open

	def PageAlive( self, page_key ):
		with self._page_key_lock:
			return page_key in self._alive_page_keys

	def PageClosedButNotDestroyed( self, page_key ):
		with self._page_key_lock:
			return page_key in self._closed_page_keys

	def PopupMenu( self, window, menu ):
		if menu.GetMenuItemCount() > 0:
			self._menu_open = True

			window.PopupMenu( menu )

			self._menu_open = False

		DestroyMenu( window, menu )

	def PrepStringForDisplay( self, text ):
		return text.lower()

	def ProcessPubSub( self ):
		self.CallBlockingToWX( None, self._pubsub.Process )

	def RefreshServices( self ):
		self.services_manager.RefreshServices()

	def ReleasePageKey( self, page_key ):
		with self._page_key_lock:
			self._alive_page_keys.discard( page_key )
			self._closed_page_keys.discard( page_key )

	def ResetPageChangeTimer( self ):
		self._timestamps[ 'last_page_change' ] = GetNow()

	def RestartClientServerService( self, service_key ):
		service = self.services_manager.GetService( service_key )
		service_type = service.GetServiceType()

		name = service.GetName()

		port = service.GetPort()
		allow_non_local_connections = service.AllowsNonLocalConnections()

		def TWISTEDRestartServer():
			def StartServer( *args, **kwargs ):
				try:
					if LocalPortInUse( port ):
						text = 'The client\'s {} could not start because something was already bound to port {}.'.format( name, port )
						text += os.linesep * 2
						text += 'This usually means another hydrus client is already running and occupying that port. It could be a previous instantiation of this client that has yet to completely shut itself down.'
						text += os.linesep * 2
						text += 'You can change the port this service tries to host on under services->manage services.'

						ShowText( text )

						return

					if service_type == HC.LOCAL_BOORU:
						twisted_server = HydrusServiceBooru( service, allow_non_local_connections = allow_non_local_connections )

					elif service_type == HC.CLIENT_API_SERVICE:
						twisted_server = HydrusServiceClientAPI( service, allow_non_local_connections = allow_non_local_connections )

					listening_connection = reactor.listenTCP( port, twisted_server )

					self._listening_services[ service_key ] = listening_connection

					if not LocalPortInUse( port ):
						text = 'Tried to bind port ' + str( port ) + ' for the local booru, but it appeared to fail. It could be a firewall or permissions issue, or perhaps another program was quietly already using it.'

						ShowText( text )

				except Exception as e:
					wx.CallAfter( ShowException, e )

			if service_key in self._listening_services:
				listening_connection = self._listening_services[ service_key ]

				del self._listening_services[ service_key ]

				deferred = defer.maybeDeferred( listening_connection.stopListening )

				if port is not None:
					deferred.addCallback( StartServer )

			else:
				if port is not None:
					StartServer()

		if HG.twisted_is_broke:
			ShowText( 'Twisted failed to import, so could not start the {}! Please contact hydrus dev!'.format( name ) )

		else:
			reactor.callFromThread( TWISTEDRestartServer )

	def RestoreDatabase( self ):
		# restore_intro = ''

		with wx.DirDialog( self.gui, 'Select backup location.' ) as dlg:
			if dlg.ShowModal() == wx.ID_OK:
				path = dlg.GetPath()

				text = 'Are you sure you want to restore a backup from "' + path + '"?'
				text += os.linesep * 2
				text += 'Everything in your current database will be deleted!'
				text += os.linesep * 2
				text += 'The gui will shut down, and then it will take a while to complete the restore. Once it is done, the client will restart.'

				with DialogYesNo( self.gui, text ) as dlg_yn:
					if dlg_yn.ShowModal() == wx.ID_YES:
						def THREADRestart():
							while not self.db.LoopIsFinished():
								time.sleep( 0.1 )

							self.db.RestoreBackup( path )

							while not HG.shutdown_complete:
								time.sleep( 0.1 )

							RestartProcess()

						self.CallToThreadLongRunning( THREADRestart )

						wx.CallAfter( self.gui.Exit )

	def Run( self ):
		self._app = wx.App()

		self._app.locale = wx.Locale( wx.LANGUAGE_DEFAULT ) # Very important to init this here and keep it non garbage collected

		# do not import locale here and try anything clever--assume that bad locale formatting is due to OS-level mess-up, not mine
		# wx locale is supposed to set it all up nice, so if someone's doesn't, explore that and find the external solution

		Print( 'booting controller\u2026' )

		self.frame_icon = wx.Icon( os.path.join( HC.STATIC_DIR, 'hydrus_32_non-transparent.png' ), wx.BITMAP_TYPE_PNG )

		self.CreateSplash()

		self.CallToThreadLongRunning( self.THREADBootEverything )

		self._app.MainLoop()

		DebugPrint( 'shutting down controller\u2026' )

	def SaveDirtyObjects( self ):
		with HG.dirty_object_lock:
			dirty_services = [ service for service in self.services_manager.GetServices() if service.IsDirty() ]

			if len( dirty_services ) > 0:
				self.WriteSynchronous( 'dirty_services', dirty_services )

			if self.client_api_manager.IsDirty():
				self.WriteSynchronous( 'serialisable', self.client_api_manager )

				self.client_api_manager.SetClean()

			if self.network_engine.bandwidth_manager.IsDirty():
				self.WriteSynchronous( 'serialisable', self.network_engine.bandwidth_manager )

				self.network_engine.bandwidth_manager.SetClean()

			if self.network_engine.domain_manager.IsDirty():
				self.WriteSynchronous( 'serialisable', self.network_engine.domain_manager )

				self.network_engine.domain_manager.SetClean()

			if self.network_engine.login_manager.IsDirty():
				self.WriteSynchronous( 'serialisable', self.network_engine.login_manager )

				self.network_engine.login_manager.SetClean()

			if self.network_engine.session_manager.IsDirty():
				self.WriteSynchronous( 'serialisable', self.network_engine.session_manager )

				self.network_engine.session_manager.SetClean()

	def SetServices( self, services ):
		with HG.dirty_object_lock:
			upnp_services = [ service for service in services if service.GetServiceType() in ( HC.LOCAL_BOORU, HC.CLIENT_API_SERVICE ) ]

			self.CallToThread( self.services_upnp_manager.SetServices, upnp_services )

			self.WriteSynchronous( 'update_services', services )

			self.services_manager.RefreshServices()

	def ShutdownModel( self ):
		if not HG.emergency_exit:
			self.file_viewing_stats_manager.Flush()

			self.SaveDirtyObjects()

		ControllerBase.ControllerBase.ShutdownModel( self )

	def ShutdownView( self ):
		if not HG.emergency_exit:
			self.pub( 'splash_set_status_text', 'waiting for daemons to exit' )

			self._ShutdownDaemons()

			if HG.do_idle_shutdown_work:
				try:
					self.DoIdleShutdownWork()

				except Exception:
					ReportShutdownException()

		ControllerBase.ControllerBase.ShutdownView( self )

	def SynchroniseAccounts( self ):
		services = self.services_manager.GetServices( HC.RESTRICTED_SERVICES )

		for service in services:
			if IsThreadShuttingDown():
				return

			service.SyncAccount()

	def SystemBusy( self ):
		if HG.force_idle_mode:
			return False

		max_cpu = self.options[ 'idle_cpu_max' ]

		if max_cpu is None:
			self._system_busy = False

		else:
			if TimeHasPassed( self._timestamps[ 'last_cpu_check' ] + 60 ):
				cpu_times = psutil.cpu_percent( percpu = True )

				if True in ( cpu_time > max_cpu for cpu_time in cpu_times ):
					self._system_busy = True

				else:
					self._system_busy = False

				self._timestamps[ 'last_cpu_check' ] = GetNow()

		return self._system_busy

	def THREADBootEverything( self ):
		try:
			self.CheckAlreadyRunning()

			self._last_shutdown_was_bad = LastShutdownWasBad( self.db_dir, 'client' )

			RecordRunningStart( self.db_dir, 'client' )

			self.InitModel()

			self.InitView()

			self._is_booted = True

		except ( InsufficientCredentialsException, ShutdownException ) as e:
			Print( e )

			HG.emergency_exit = True

			self.Exit()

		except Exception:
			text = 'A serious error occurred while trying to start the program. The error will be shown next in a window. More information may have been written to client.log.'

			DebugPrint( 'If the db crashed, another error may be written just above ^.' )
			DebugPrint( text )

			DebugPrint( traceback.format_exc() )

			wx.SafeShowMessage( 'boot error', text )
			wx.SafeShowMessage( 'boot error', traceback.format_exc() )

			HG.emergency_exit = True

			self.Exit()

		finally:
			self._DestroySplash()

	def THREADExitEverything( self ):
		try:
			gc.collect()

			self.pub( 'splash_set_title_text', 'shutting down gui\u2026' )

			self.ShutdownView()

			self.pub( 'splash_set_title_text', 'shutting down db\u2026' )

			self.ShutdownModel()

			self.pub( 'splash_set_title_text', 'cleaning up\u2026' )

			CleanRunningFile( self.db_dir, 'client' )

		except ( InsufficientCredentialsException, ShutdownException ):
			pass

		except Exception:
			ReportShutdownException()

		finally:
			self._DestroySplash()

	def ToClipboard( self, data_type, data ):
		# need this cause can't do it in a non-gui thread

		if data_type == 'paths':
			paths = data

			if wx.TheClipboard.Open():
				data = wx.DataObjectComposite()

				file_data = wx.FileDataObject()

				for path in paths:
					file_data.AddFile( path )

				text_data = wx.TextDataObject( os.linesep.join( paths ) )

				data.Add( file_data, True )
				data.Add( text_data, False )

				wx.TheClipboard.SetData( data )

				wx.TheClipboard.Close()

			else:
				wx.MessageBox( 'Could not get permission to access the clipboard!' )

		elif data_type == 'text':
			text = data

			if wx.TheClipboard.Open():
				data = wx.TextDataObject( text )

				wx.TheClipboard.SetData( data )

				wx.TheClipboard.Close()

			else:
				wx.MessageBox( 'I could not get permission to access the clipboard.' )

		elif data_type == 'bmp':
			media = data

			image_renderer = self.GetCache( 'images' ).GetImageRenderer( media )

			def CopyToClipboard():
				if wx.TheClipboard.Open():
					wx_bmp = image_renderer.GetWXBitmap()

					data = wx.BitmapDataObject( wx_bmp )

					wx.TheClipboard.SetData( data )

					wx.TheClipboard.Close()

				else:
					wx.MessageBox( 'I could not get permission to access the clipboard.' )

			def THREADWait():
				# have to do this in thread, because the image needs the wx event queue to render

				start_time = time.time()

				while not image_renderer.IsReady():
					if TimeHasPassed( start_time + 15 ):
						raise Exception( 'The image did not render in fifteen seconds, so the attempt to copy it to the clipboard was abandoned.' )

					time.sleep( 0.1 )

				wx.CallAfter( CopyToClipboard )

			self.CallToThread( THREADWait )

	def UnclosePageKeys( self, page_keys ):
		with self._page_key_lock:
			self._closed_page_keys.difference_update( page_keys )

	def WaitUntilViewFree( self ):
		self.WaitUntilModelFree()

		self.WaitUntilThumbnailsFree()

	def WaitUntilThumbnailsFree( self ):
		while True:
			if self._view_shutdown:
				raise ShutdownException( 'Application shutting down!' )

			elif not self._caches[ 'thumbnail' ].DoingWork():
				return

			else:
				time.sleep( 0.00001 )

	def Write( self, action, *args, **kwargs ):
		if action == 'content_updates':
			self._managers[ 'undo' ].AddCommand( 'content_updates', *args, **kwargs )

		return ControllerBase.Write( self, action, *args, **kwargs )


code:
import traceback, time

import lib.Hydrus.Constants as HC
import lib.Hydrus.Globals as HG

from lib.Hydrus.Controller import ControllerBase
from lib.Hydrus.Data import CleanRunningFile, GetNow, Print, RecordRunningStart
from lib.Hydrus.DB.Remote import DB
from lib.Hydrus.Server import HydrusServiceAdmin, HydrusServiceRepositoryFile, HydrusServiceRepositoryTag
from lib.Hydrus.Sessions import HydrusSessionManagerServer

from lib.Networking import LocalPortInUse

if not HG.twisted_is_broke:
	from twisted.internet import reactor, defer, ssl

### CLASSES ###

class RemoteController( ControllerBase ):
	def __init__( self, db_dir ):
		ControllerBase.__init__( self, db_dir )
		self._name = 'server'
		self._shutdown = False
		HG.server_controller = self

	def _GetUPnPServices( self ):
		return self._services

	def _InitDB( self ):
		return DB( self, self.db_dir, 'server' )

	def StartService( self, service ):
		def TWISTEDDoIt():
			service_key = service.GetServiceKey()
			service_type = service.GetServiceType()

			def Start( *args, **kwargs ):
				try:
					port = service.GetPort()

					if LocalPortInUse( port ):
						raise Exception( 'Something was already bound to port ' + str( port ) )

					if service_type == HC.SERVER_ADMIN:
						http_factory = HydrusServiceAdmin( service )

					elif service_type == HC.FILE_REPOSITORY:
						http_factory = HydrusServiceRepositoryFile( service )

					elif service_type == HC.TAG_REPOSITORY:
						http_factory = HydrusServiceRepositoryTag( service )

					else:
						return

					( ssl_cert_path, ssl_key_path ) = self.db.GetSSLPaths()
					sslmethod = ssl.SSL.TLSv1_2_METHOD

					context_factory = ssl.DefaultOpenSSLContextFactory( ssl_key_path, ssl_cert_path, sslmethod )

					self._service_keys_to_connected_ports[ service_key ] = reactor.listenSSL( port, http_factory, context_factory )

					if not LocalPortInUse( port ):
						raise Exception( 'Tried to bind port ' + str( port ) + ' but it failed.' )

				except Exception:
					Print( traceback.format_exc() )

			if service_key in self._service_keys_to_connected_ports:
				deferred = defer.maybeDeferred( self._service_keys_to_connected_ports[ service_key ].stopListening )
				deferred.addCallback( Start )

			else:
				Start()

		reactor.callFromThread( TWISTEDDoIt )

	def StopService( self, service_key ):
		def TWISTEDDoIt():
			defer.maybeDeferred( self._service_keys_to_connected_ports[ service_key ].stopListening )
			del self._service_keys_to_connected_ports[ service_key ]

		reactor.callFromThread( TWISTEDDoIt )

	def DeleteOrphans( self ):
		self.WriteSynchronous( 'delete_orphans' )

	def Exit( self ):
		Print( 'Shutting down daemons and services\u2026' )
		self.ShutdownView()

		Print( 'Shutting down db\u2026' )
		self.ShutdownModel()

		CleanRunningFile( self.db_dir, 'server' )

	def GetFilesDir( self ):
		return self.db.GetFilesDir()

	def GetServices( self ):
		return list( self._services )

	def InitModel( self ):
		ControllerBase.InitModel( self )

		self._services = self.Read( 'services' )

		[ self._admin_service ] = [ service for service in self._services if service.GetServiceType() == HC.SERVER_ADMIN ]

		self.server_session_manager = HydrusSessionManagerServer()

		self._service_keys_to_connected_ports = {}

	def InitView( self ):
		ControllerBase.InitView( self )

		port = self._admin_service.GetPort()

		if LocalPortInUse( port ):
			Print( 'Something is already bound to port ' + str( port ) + ', so your administration service cannot be started. Please quit the server and retry once the port is clear.' )

		else:
			for service in self._services:
				self.StartService( service )

		#

		job = self.CallRepeating( 5.0, 600.0, self.SyncRepositories )
		self._daemon_jobs[ 'sync_repositories' ] = job

		job = self.CallRepeating( 0.0, 30.0, self.SaveDirtyObjects )
		self._daemon_jobs[ 'save_dirty_objects' ] = job

		job = self.CallRepeating( 0.0, 86400.0, self.DeleteOrphans )
		self._daemon_jobs[ 'delete_orphans' ] = job

	def JustWokeFromSleep( self ):
		return False

	def MaintainDB( self, stop_time = None ):
		stop_time = GetNow() + 10
		self.WriteSynchronous( 'analyze', stop_time )

	def ReportDataUsed( self, num_bytes ):
		self._admin_service.ServerReportDataUsed( num_bytes )

	def ReportRequestUsed( self ):
		self._admin_service.ServerReportRequestUsed()

	def Run( self ):
		RecordRunningStart( self.db_dir, 'server' )
		Print( 'Initialising db\u2026' )

		self.InitModel()

		Print( 'Initialising daemons and services\u2026' )

		self.InitView()

		Print( 'Server is running. Press Ctrl+C to quit.' )

		try:
			while not self._model_shutdown and not self._shutdown:
				time.sleep( 1 )

		except KeyboardInterrupt:
			Print( 'Received a keyboard interrupt\u2026' )

		Print( 'Shutting down controller\u2026' )

		self.Exit()

	def SaveDirtyObjects( self ):
		with HG.dirty_object_lock:
			dirty_services = [ service for service in self._services if service.IsDirty() ]
			dirty_accounts = self.server_session_manager.GetDirtyAccounts()

			if len( dirty_services ) > 0:
				self.WriteSynchronous( 'dirty_services', dirty_services )

			if len(  ) > 0:
				self.WriteSynchronous( 'dirty_accounts', dirty_accounts )

	def ServerBandwidthOK( self ):
		return self._admin_service.ServerBandwidthOK()

	def SetServices( self, services ): # doesn't need the dirty_object_lock because the caller takes it
		self._services = services

		self.CallToThread( self.services_upnp_manager.SetServices, self._services )

		[ self._admin_service ] = [ service for service in self._services if service.GetServiceType() == HC.SERVER_ADMIN ]

		current_service_keys = set( self._service_keys_to_connected_ports.keys() )
		future_service_keys = set( [ service.GetServiceKey() for service in self._services ] )
		stop_service_keys = current_service_keys.difference( future_service_keys )

		for service_key in stop_service_keys:
			self.StopService( service_key )

		for service in self._services:
			self.StartService( service )

	def ShutdownView( self ):
		for service in self._services:
			service_key = service.GetServiceKey()

			if service_key in self._service_keys_to_connected_ports:
				self.StopService( service_key )

		ControllerBase.ShutdownView( self )

	def ShutdownFromServer( self ):
		Print( 'Received a server shut down request\u2026' )

		self._shutdown = True

	def SyncRepositories( self ):
		if HG.server_busy:
			return

		repositories = [ service for service in self._services if service.GetServiceType() in HC.REPOSITORIES ]

		for service in repositories:
			service.Sync()

license: wtfpl


code:
from .Controller import *

__all__ = [
	"Local",
	"Remote"
]

from . import *


code:
import os, queue, sqlite3, traceback, time, distutils.version

from lib.Hydrus.Data import ShowText, SplitListIntoChunks, TimeHasPassed, Print, \
	DebugPrint, PrintException, GetNow, Profile, JobDatabase
from lib.Hydrus.Exceptions import DBAccessException, DBException, ShutdownException
from lib.Hydrus.Paths import HasSpaceForDBTransaction, GetFreeSpace

### CONSTANTS ###
import lib.Hydrus.Constants as HC
import lib.Hydrus.Globals as HG

CONNECTION_REFRESH_TIME = 60 * 30

### FUNCTIONS ###

def CanVacuum( db_path, stop_time = None ):
	try:
		db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )

		c = db.cursor()

		( page_size, ) = c.execute( 'PRAGMA page_size;' ).fetchone()
		( page_count, ) = c.execute( 'PRAGMA page_count;' ).fetchone()
		( freelist_count, ) = c.execute( 'PRAGMA freelist_count;' ).fetchone()

		db_size = ( page_count - freelist_count ) * page_size

		if stop_time is not None:
			approx_vacuum_speed_mb_per_s = 1048576 * 1

			approx_vacuum_duration = db_size // approx_vacuum_speed_mb_per_s

			time_i_will_have_to_start = stop_time - approx_vacuum_duration

			if TimeHasPassed( time_i_will_have_to_start ):
				return False

		( db_dir, db_filename ) = os.path.split( db_path )

		( has_space, reason ) = HasSpaceForDBTransaction( db_dir, db_size )

		return has_space

	except Exception as e:
		Print( 'Could not determine whether to vacuum or not:' )

		PrintException( e )

		return False

def ReadLargeIdQueryInSeparateChunks( cursor, select_statement, chunk_size ):
	table_name = 'tempbigread' + os.urandom( 32 ).hex()

	cursor.execute( 'CREATE TEMPORARY TABLE ' + table_name + ' ( job_id INTEGER PRIMARY KEY AUTOINCREMENT, temp_id INTEGER );' )

	cursor.execute( 'INSERT INTO ' + table_name + ' ( temp_id ) ' + select_statement ) # given statement should end in semicolon, so we are good

	num_to_do = cursor.rowcount

	if num_to_do is None or num_to_do == -1:
		num_to_do = 0

	i = 0

	while i < num_to_do:
		chunk = [ temp_id for ( temp_id, ) in cursor.execute( 'SELECT temp_id FROM ' + table_name + ' WHERE job_id BETWEEN ? AND ?;', ( i, i + chunk_size - 1 ) ) ]

		yield chunk

		i += chunk_size

	cursor.execute( 'DROP TABLE ' + table_name + ';' )

def VacuumDB( db_path ):
	db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )

	c = db.cursor()

	( previous_journal_mode, ) = c.execute( 'PRAGMA journal_mode;' ).fetchone()

	fast_big_transaction_wal = not distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' )

	if previous_journal_mode == 'wal' and not fast_big_transaction_wal:
		c.execute( 'PRAGMA journal_mode = TRUNCATE;' )

	if HC.PLATFORM_WINDOWS:
		ideal_page_size = 4096

	else:
		ideal_page_size = 1024

	( page_size, ) = c.execute( 'PRAGMA page_size;' ).fetchone()

	if page_size != ideal_page_size:
		c.execute( 'PRAGMA journal_mode = TRUNCATE;' )
		c.execute( 'PRAGMA page_size = ' + str( ideal_page_size ) + ';' )

	c.execute( 'PRAGMA auto_vacuum = 0;' ) # none

	c.execute( 'VACUUM;' )

	if previous_journal_mode == 'wal':
		c.execute( 'PRAGMA journal_mode = WAL;' )

class HydrusDB( object ):
	READ_WRITE_ACTIONS = []
	UPDATE_WAIT = 2

	TRANSACTION_COMMIT_TIME = 10

	def __init__( self, controller, db_dir, db_name ):
		if GetFreeSpace( db_dir ) < 500 * 1048576:
			raise Exception( 'Sorry, it looks like the db partition has less than 500MB, please free up some space.' )

		self._controller = controller
		self._db_dir = db_dir
		self._db_name = db_name

		self._transaction_started = 0
		self._in_transaction = False
		self._transaction_contains_writes = False

		self._connection_timestamp = 0

		main_db_filename = db_name

		if not main_db_filename.endswith( '.db' ):
			main_db_filename += '.db'

		self._db_filenames = {}

		self._db_filenames[ 'main' ] = main_db_filename

		self._InitExternalDatabases()

		if distutils.version.LooseVersion( sqlite3.sqlite_version ) < distutils.version.LooseVersion( '3.11.0' ):
			self._fast_big_transaction_wal = False

		else:
			self._fast_big_transaction_wal = True

		self._is_first_start = False
		self._is_db_updated = False
		self._local_shutdown = False
		self._loop_finished = False
		self._ready_to_serve_requests = False
		self._could_not_initialise = False

		self._jobs = queue.Queue()
		self._pubsubs = []

		self._currently_doing_job = False
		self._current_status = ''
		self._current_job_name = ''

		self._db = None
		self._c = None

		if os.path.exists( os.path.join( self._db_dir, self._db_filenames[ 'main' ] ) ):
			# open and close to clean up in case last session didn't close well

			self._InitDB()
			self._CloseDBCursor()

		self._InitDB()

		self._RepairDB()

		( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone()

		if version > HC.SOFTWARE_VERSION:
			self._ReportOverupdatedDB( version )

		if version < ( HC.SOFTWARE_VERSION - 15 ):
			self._ReportUnderupdatedDB( version )

		if version < HC.SOFTWARE_VERSION - 50:
			raise Exception( 'Your current database version of hydrus ' + str( version ) + ' is too old for this software version ' + str( HC.SOFTWARE_VERSION ) + ' to update. Please try updating with version ' + str( version + 45 ) + ' or earlier first.' )

		while version < HC.SOFTWARE_VERSION:
			time.sleep( self.UPDATE_WAIT )

			try:
				self._BeginImmediate()

			except Exception as e:
				raise DBAccessException( str( e ) )

			try:
				self._UpdateDB( version )

				self._Commit()

				self._is_db_updated = True

			except Exception:
				e = Exception( 'Updating the ' + self._db_name + ' db to version ' + str( version + 1 ) + ' caused this error:' + os.linesep + traceback.format_exc() )

				try:
					self._Rollback()

				except Exception as rollback_e:
					Print( 'When the update failed, attempting to rollback the database failed.' )

					PrintException( rollback_e )

				raise e

			( version, ) = self._c.execute( 'SELECT version FROM version;' ).fetchone()

		self._CloseDBCursor()

		self._controller.CallToThreadLongRunning( self.MainLoop )

		while not self._ready_to_serve_requests:
			time.sleep( 0.1 )

			if self._could_not_initialise:
				raise Exception( 'Could not initialise the db! Error written to the log!' )

	def _AttachExternalDatabases( self ):
		for ( name, filename ) in list( self._db_filenames.items() ):
			if name == 'main':
				continue

			db_path = os.path.join( self._db_dir, self._db_filenames[ name ] )

			self._c.execute( 'ATTACH ? AS ' + name + ';', ( db_path, ) )

	def _BeginImmediate( self ):
		if not self._in_transaction:
			self._c.execute( 'BEGIN IMMEDIATE;' )
			self._c.execute( 'SAVEPOINT hydrus_savepoint;' )

			self._transaction_started = GetNow()
			self._in_transaction = True

	def _CleanUpCaches( self ):
		pass

	def _CloseDBCursor( self ):
		if self._db is not None:
			if self._in_transaction:
				self._Commit()

			self._c.close()
			self._db.close()

			del self._c
			del self._db

			self._db = None
			self._c = None

	def _Commit( self ):
		if self._in_transaction:
			self._c.execute( 'COMMIT;' )

			self._in_transaction = False

		else:
			Print( 'Received a call to commit, but was not in a transaction!' )

	def _CreateDB( self ):
		raise NotImplementedError()

	def _CreateIndex( self, table_name, columns, unique = False ):
		if '.' in table_name:
			table_name_simple = table_name.split( '.' )[1]

		else:
			table_name_simple = table_name

		index_name = table_name + '_' + '_'.join( columns ) + '_index'

		if unique:
			create_phrase = 'CREATE UNIQUE INDEX IF NOT EXISTS '

		else:
			create_phrase = 'CREATE INDEX IF NOT EXISTS '

		on_phrase = ' ON ' + table_name_simple + ' (' + ', '.join( columns ) + ' );'

		statement = create_phrase + index_name + on_phrase

		self._c.execute( statement )

	def _DisplayCatastrophicError( self, text ):
		message = 'The db encountered a serious error! This is going to be written to the log as well, but here it is for a screenshot:'
		message += os.linesep * 2
		message += text

		DebugPrint( message )

	def _GetRowCount( self ):
		row_count = self._c.rowcount

		if row_count == -1:
			return 0
		else:
			return row_count

	def _InitCaches( self ):
		pass

	def _InitDB( self ):
		create_db = False

		db_path = os.path.join( self._db_dir, self._db_filenames[ 'main' ] )

		if not os.path.exists( db_path ):
			create_db = True

			external_db_paths = [ os.path.join( self._db_dir, self._db_filenames[ db_name ] ) for db_name in self._db_filenames if db_name != 'main' ]

			existing_external_db_paths = [ external_db_path for external_db_path in external_db_paths if os.path.exists( external_db_path ) ]

			if len( existing_external_db_paths ) > 0:
				message = 'Although the external files, "{}" do exist, the main database file, "{}", does not! This makes for an invalid database, and the program will now quit. Please contact hydrus_dev if you do not know how this happened or need help recovering from hard drive failure.'

				message = message.format( ', '.join( existing_external_db_paths ), db_path )

				raise DBException( message )

		self._InitDBCursor()

		result = self._c.execute( 'SELECT 1 FROM sqlite_master WHERE type = ? AND name = ?;', ( 'table', 'version' ) ).fetchone()

		if result is None:
			create_db = True

		if create_db:
			self._is_first_start = True

			self._CreateDB()

			self._Commit()

			self._BeginImmediate()

	def _InitDBCursor( self ):
		self._CloseDBCursor()

		db_path = os.path.join( self._db_dir, self._db_filenames[ 'main' ] )

		# db_just_created = not os.path.exists( db_path )

		self._db = sqlite3.connect( db_path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )

		self._connection_timestamp = GetNow()

		self._c = self._db.cursor()

		if HG.no_db_temp_files:
			self._c.execute( 'PRAGMA temp_store = 2;' ) # use memory for temp store exclusively

		self._c.execute( 'PRAGMA main.cache_size = -10000;' )

		self._c.execute( 'ATTACH ":memory:" AS mem;' )

		self._AttachExternalDatabases()

		db_names = [ name for ( index, name, path ) in self._c.execute( 'PRAGMA database_list;' ) if name not in ( 'mem', 'temp' ) ]

		for db_name in db_names:
			self._c.execute( 'PRAGMA ' + db_name + '.cache_size = -10000;' )

			if HG.no_wal:
				self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = TRUNCATE;' )

				self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 2;' )

				self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()

			else:
				self._c.execute( 'PRAGMA ' + db_name + '.journal_mode = WAL;' )

				# if this is set to 1, transactions are not immediately synced to the journal and can be undone following a power-loss
				# if set to 2, all transactions are synced
				# either way, transactions are atomically consistent, but let's not mess around when power-cut during heavy file import or w/e
				self._c.execute( 'PRAGMA ' + db_name + '.synchronous = 2;' )

				try:
					self._c.execute( 'SELECT * FROM ' + db_name + '.sqlite_master;' ).fetchone()

				except sqlite3.OperationalError as e:
					message = 'The database failed to read some  You may need to run the program in no-wal mode using the --no_wal command parameter. Full error information:'
					message += os.linesep * 2
					message += str( e )

					DebugPrint( message )

					raise DBAccessException( message )

		try:
			self._BeginImmediate()

		except Exception as e:
			raise DBAccessException( str( e ) )

	def _InitDiskCache( self ):
		pass

	def _InitExternalDatabases( self ):
		pass

	def _ManageDBError( self, job, e ):
		raise NotImplementedError()

	def _ProcessJob( self, job ):
		job_type = job.GetType()

		( action, args, kwargs ) = job.GetCallableTuple()

		try:
			if job_type in ( 'read_write', 'write' ):
				self._current_status = 'db write locked'

				self._transaction_contains_writes = True

			else:
				self._current_status = 'db read locked'

			self.publish_status_update()

			if job_type in ( 'read', 'read_write' ):
				result = self._Read( action, *args, **kwargs )

			elif job_type in ( 'write' ):
				result = self._Write( action, *args, **kwargs )

			if self._transaction_contains_writes and TimeHasPassed( self._transaction_started + self.TRANSACTION_COMMIT_TIME ):
				self._current_status = 'db committing'

				self.publish_status_update()

				self._Commit()

				self._BeginImmediate()

				self._transaction_contains_writes = False

			else:
				self._Save()

			for ( topic, args, kwargs ) in self._pubsubs:
				self._controller.pub( topic, *args, **kwargs )

			if job.IsSynchronous():
				job.PutResult( result )

		except Exception as e:
			self._ManageDBError( job, e )

			try:
				self._Rollback()

			except Exception as rollback_e:
				Print( 'When the transaction failed, attempting to rollback the database failed. Please restart the client as soon as is convenient.' )

				self._in_transaction = False

				self._CloseDBCursor()

				self._InitDBCursor()

				PrintException( rollback_e )

		finally:
			self._pubsubs = []

			self._current_status = ''

			self.publish_status_update()

	def _Read( self, action, *args, **kwargs ):
		raise NotImplementedError()

	def _RepairDB( self ):
		pass

	def _ReportOverupdatedDB( self, version ):
		pass

	def _ReportUnderupdatedDB( self, version ):
		pass

	def _ReportStatus( self, text ):
		Print( text )

	def _Rollback( self ):
		if self._in_transaction:
			self._c.execute( 'ROLLBACK TO hydrus_savepoint;' )

		else:
			Print( 'Received a call to rollback, but was not in a transaction!' )

	def _Save( self ):
		self._c.execute( 'RELEASE hydrus_savepoint;' )

		self._c.execute( 'SAVEPOINT hydrus_savepoint;' )

	def _SelectFromList( self, select_statement, xs ):
		# issue here is that doing a simple blah_id = ? is real quick and cacheable but doing a lot of fetchone()s is slow
		# blah_id IN ( 1, 2, 3 ) is fast to execute but not cacheable and doing the str() list splay takes time so there is initial lag
		# doing the temporaryintegertable trick works well for gigantic lists you refer to frequently but it is super laggy when you sometimes are only selecting four things
		# blah_id IN ( ?, ?, ? ) is fast and cacheable but there's a small limit (1024 is too many) to the number of params sql can handle
		# so lets do the latter but break it into 256-strong chunks to get a good medium

		# this will take a select statement with %s like so:
		# SELECT blah_id, blah FROM blahs WHERE blah_id IN %s;

		MAX_CHUNK_SIZE = 256

		# do this just so we aren't always reproducing this long string for gigantic lists
		# and also so we aren't overmaking it when this gets spammed with a lot of len() == 1 calls
		if len( xs ) >= MAX_CHUNK_SIZE:
			max_statement = select_statement % ( '(' + ','.join( '?' * MAX_CHUNK_SIZE ) + ')' )

		for chunk in SplitListIntoChunks( xs, MAX_CHUNK_SIZE ):
			if len( chunk ) == MAX_CHUNK_SIZE:
				chunk_statement = max_statement

			else:
				chunk_statement = select_statement % ( '(' + ','.join( '?' * len( chunk ) ) + ')' )

			for row in self._c.execute( chunk_statement, chunk ):
				yield row

	def _SelectFromListFetchAll( self, select_statement, xs ):
		return [ row for row in self._SelectFromList( select_statement, xs ) ]

	def _ShrinkMemory( self ):
		self._c.execute( 'PRAGMA shrink_memory;' )

	def _STI( self, iterable_cursor ):
		# strip singleton tuples to an iterator

		return ( item for ( item, ) in iterable_cursor )

	def _STL( self, iterable_cursor ):
		# strip singleton tuples to a list

		return [ item for ( item, ) in iterable_cursor ]

	def _STS( self, iterable_cursor ):
		# strip singleton tuples to a set

		return { item for ( item, ) in iterable_cursor }

	def _UpdateDB( self, version ):
		raise NotImplementedError()

	def _Write( self, action, *args, **kwargs ):
		raise NotImplementedError()

	def pub_after_job( self, topic, *args, **kwargs ):
		if len( args ) == 0 and len( kwargs ) == 0:
			if ( topic, args, kwargs ) in self._pubsubs:
				return

		self._pubsubs.append( ( topic, args, kwargs ) )

	def publish_status_update( self ):
		pass

	def CurrentlyDoingJob( self ):
		return self._currently_doing_job

	def GetApproxTotalFileSize( self ):
		total = 0

		for filename in list( self._db_filenames.values() ):
			path = os.path.join( self._db_dir, filename )

			total += os.path.getsize( path )

		return total

	def GetStatus( self ):
		return ( self._current_status, self._current_job_name )

	def IsDBUpdated( self ):
		return self._is_db_updated

	def IsFirstStart( self ):
		return self._is_first_start

	def LoopIsFinished( self ):
		return self._loop_finished

	def JobsQueueEmpty( self ):
		return self._jobs.empty()

	def MainLoop( self ):
		try:
			self._InitDBCursor() # have to reinitialise because the thread id has changed

			self._InitDiskCache()

			self._InitCaches()

		except Exception:
			self._DisplayCatastrophicError( traceback.format_exc() )

			self._could_not_initialise = True

			return

		self._ready_to_serve_requests = True

		error_count = 0

		while not ( ( self._local_shutdown or self._controller.ModelIsShutdown() ) and self._jobs.empty() ):
			try:
				job = self._jobs.get( timeout = 1 )

				self._currently_doing_job = True
				self._current_job_name = job.ToString()

				self.publish_status_update()

				try:
					if HG.db_report_mode:
						summary = 'Running ' + job.ToString()

						ShowText( summary )

					if HG.db_profile_mode:
						summary = 'Profiling ' + job.ToString()

						ShowText( summary )

						Profile( summary, 'self._ProcessJob( job )', globals(), locals() )

					else:
						self._ProcessJob( job )

					error_count = 0

				except Exception:
					error_count += 1

					if error_count > 5:
						raise

					self._jobs.put( job ) # couldn't lock db; put job back on queue

					time.sleep( 5 )

				self._currently_doing_job = False
				self._current_job_name = ''

				self.publish_status_update()

			except queue.Empty:
				if self._transaction_contains_writes and TimeHasPassed( self._transaction_started + self.TRANSACTION_COMMIT_TIME ):
					self._Commit()

					self._BeginImmediate()

					self._transaction_contains_writes = False
			# just to clear out the journal files
			if TimeHasPassed( self._connection_timestamp + CONNECTION_REFRESH_TIME ):
				self._InitDBCursor()

		self._CleanUpCaches()

		self._CloseDBCursor()

		self._loop_finished = True

	def Read( self, action, *args, **kwargs ):
		if action in self.READ_WRITE_ACTIONS:
			job_type = 'read_write'

		else:
			job_type = 'read'

		synchronous = True

		job = JobDatabase( job_type, synchronous, action, *args, **kwargs )

		if self._controller.ModelIsShutdown():
			raise ShutdownException( 'Application has shut down!' )

		self._jobs.put( job )

		return job.GetResult()

	def ReadyToServeRequests( self ):
		return self._ready_to_serve_requests

	def Shutdown( self ):
		self._local_shutdown = True

	def Write( self, action, synchronous, *args, **kwargs ):
		job_type = 'write'

		job = JobDatabase( job_type, synchronous, action, *args, **kwargs )

		if self._controller.ModelIsShutdown():
			raise ShutdownException( 'Application has shut down!' )

		self._jobs.put( job )

		if synchronous:
			return job.GetResult()

class TemporaryIntegerTable( object ):
	def __init__( self, cursor, integer_iterable, column_name ):
		self._cursor = cursor
		self._integer_iterable = integer_iterable
		self._column_name = column_name

		self._table_name = 'mem.tempint' + os.urandom( 32 ).hex()

	def __enter__( self ):
		self._cursor.execute( 'CREATE TABLE ' + self._table_name + ' ( ' + self._column_name + ' INTEGER PRIMARY KEY );' )

		self._cursor.executemany( 'INSERT INTO ' + self._table_name + ' ( ' + self._column_name + ' ) VALUES ( ? );', ( ( i, ) for i in self._integer_iterable ) )

		return self._table_name

	def __exit__( self, exc_type, exc_val, exc_tb ):
		self._cursor.execute( 'DROP TABLE ' + self._table_name + ';' )

		return False