code:
import json, hashlib, os, threading, time, traceback, wx

from lib.Hydrus.Threading import JobKey
from lib.Hydrus.Data import ContentUpdate, ConvertValueRangeToPrettyString, \
	GetNow, Print, ShowException, ShowText, TimeHasPassed, TimestampToPrettyTimeDelta
from lib.Hydrus.Exceptions import BandwidthException, CancelledException, \
	MissingCredentialsException, NetworkException, NetworkVersionException, \
	NotFoundException, ServerBusyException, SessionException, ShutdownException, \
	InsufficientCredentialsException
from lib.Hydrus.Serialisable import CreateFromNetworkBytes, SerialisableDictionary

from lib.Networking import Account, BandwidthRules, BandwidthTracker, \
	NetworkContentUpdate, Credentials, DumpHydrusArgsToNetworkBytes, \
	DumpToGETQuery, DefinitionsUpdate, Metadata, ParseNetworkBytesToParsedHydrusArgs
from lib.Networking.Contexts import NetworkContext
from lib.Networking.Job import NetworkJob, NetworkJobHydrus

from lib.Actions.Import import THREADDownloadURLs, THREADDownloadURL

### CONSTANTS ###

from lib.Hydrus.Ratings import CIRCLE, DEFAULT_LIKE_COLOURS, DEFAULT_NUMERICAL_COLOURS

import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG
import lib.Hydrus.Constants as HC

### FUNCTIONS ###

def GenerateDefaultServiceDictionary( service_type ):
	dictionary = SerialisableDictionary()

	if service_type in HC.REMOTE_SERVICES:
		dictionary[ 'credentials' ] = Credentials( 'hostname', 45871 )
		dictionary[ 'no_requests_reason' ] = ''
		dictionary[ 'no_requests_until' ] = 0

		if service_type in HC.RESTRICTED_SERVICES:
			dictionary[ 'account' ] = Account.GenerateSerialisableTupleFromAccount( Account.GenerateUnknownAccount() )
			dictionary[ 'next_account_sync' ] = 0

			if service_type in HC.REPOSITORIES:
				dictionary[ 'metadata' ] = Metadata()
				dictionary[ 'paused' ] = False
				dictionary[ 'tag_archive_sync' ] = []

		if service_type == HC.IPFS:
			dictionary[ 'credentials' ] = Credentials( '127.0.0.1', 5001 )
			dictionary[ 'multihash_prefix' ] = ''

	if service_type == HC.LOCAL_TAG:
		dictionary[ 'tag_archive_sync' ] = []

	if service_type in ( HC.LOCAL_BOORU, HC.CLIENT_API_SERVICE ):
		dictionary[ 'port' ] = None
		dictionary[ 'upnp_port' ] = None
		dictionary[ 'bandwidth_tracker' ] = BandwidthTracker()
		dictionary[ 'bandwidth_rules' ] = BandwidthRules()

		dictionary[ 'support_cors' ] = False
		dictionary[ 'log_requests' ] = False

		if service_type == HC.LOCAL_BOORU:
			allow_non_local_connections = True

		elif service_type == HC.CLIENT_API_SERVICE:
			allow_non_local_connections = False

		dictionary[ 'allow_non_local_connections' ] = allow_non_local_connections

	if service_type in HC.RATINGS_SERVICES:
		dictionary[ 'shape' ] = CIRCLE
		dictionary[ 'colours' ] = []

		if service_type == HC.LOCAL_RATING_LIKE:
			dictionary[ 'colours' ] = list( DEFAULT_LIKE_COLOURS.items() )

		elif service_type == HC.LOCAL_RATING_NUMERICAL:
			dictionary[ 'colours' ] = list( DEFAULT_NUMERICAL_COLOURS.items() )
			dictionary[ 'num_stars' ] = 5
			dictionary[ 'allow_zero' ]= True

	return dictionary

def GenerateService( service_key, service_type, name, dictionary = None ):
	if dictionary is None:
		dictionary = GenerateDefaultServiceDictionary( service_type )

	if service_type == HC.LOCAL_TAG:
		cl = ServiceLocalTag

	elif service_type == HC.LOCAL_RATING_LIKE:
		cl = ServiceLocalRatingLike

	elif service_type == HC.LOCAL_RATING_NUMERICAL:
		cl = ServiceLocalRatingNumerical

	elif service_type in HC.REPOSITORIES:
		cl = ServiceRepository

	elif service_type in HC.RESTRICTED_SERVICES:
		cl = ServiceRestricted

	elif service_type == HC.IPFS:
		cl = ServiceIPFS

	elif service_type in HC.REMOTE_SERVICES:
		cl = ServiceRemote

	elif service_type == HC.LOCAL_BOORU:
		cl = ServiceLocalBooru

	elif service_type == HC.CLIENT_API_SERVICE:
		cl = ServiceClientAPI

	else:
		cl = Service

	return cl( service_key, service_type, name, dictionary )

### CLASSES ###

class Service( object ):
	def __init__( self, service_key, service_type, name, dictionary = None ):

		if dictionary is None:
			dictionary = GenerateDefaultServiceDictionary( service_type )

		self._service_key = service_key
		self._service_type = service_type
		self._name = name

		self._dirty = False
		self._lock = threading.Lock()

		self._LoadFromDictionary( dictionary )

	def __hash__( self ):
		return self._service_key.__hash__()

	def _CheckFunctional( self ):
		pass

	def _GetSerialisableDictionary( self ):
		return SerialisableDictionary()

	def _LoadFromDictionary( self, dictionary ):
		default_dictionary = GenerateDefaultServiceDictionary( self._service_type )

		for ( key, value ) in default_dictionary.items():
			if key not in dictionary:
				dictionary[ key ] = value

	def _SetDirty( self ):
		self._dirty = True

		HG.client_controller.pub( 'service_updated', self )

	def CheckFunctional( self ):
		with self._lock:
			self._CheckFunctional()

	def Duplicate( self ):
		with self._lock:
			dictionary = self._GetSerialisableDictionary()

			duplicate = GenerateService( self._service_key, self._service_type, self._name, dictionary )

			return duplicate

	def GetSerialisableDictionary( self ):
		with self._lock:
			self._dirty = False

			return self._GetSerialisableDictionary()

	def GetName( self ):
		with self._lock:
			return self._name

	def GetServiceKey( self ):
		with self._lock:
			return self._service_key

	def GetServiceType( self ):
		with self._lock:
			return self._service_type

	def GetStatusString( self ):
		with self._lock:
			try:
				self._CheckFunctional()

				return 'service is functional'

			except Exception as e:
				return str( e )

	def IsDirty( self ):
		with self._lock:
			return self._dirty

	def IsFunctional( self ):
		with self._lock:
			try:
				self._CheckFunctional()

				return True

			except Exception:
				return False

	def SetClean( self ):
		with self._lock:
			self._dirty = False

	def SetName( self, name ):
		with self._lock:
			self._name = name

	def ToTuple( self ):
		dictionary = self._GetSerialisableDictionary()

		return ( self._service_key, self._service_type, self._name, dictionary )

class ServiceLocalServerService( Service ):
	def _CheckFunctional( self ):

		if not self._bandwidth_rules.CanStartRequest( self._bandwidth_tracker ):
			raise BandwidthException( 'bandwidth exceeded' )

		Service._CheckFunctional( self )

	def _GetSerialisableDictionary( self ):
		dictionary = Service._GetSerialisableDictionary( self )

		dictionary[ 'port' ] = self._port
		dictionary[ 'upnp_port' ] = self._upnp_port
		dictionary[ 'allow_non_local_connections' ] = self._allow_non_local_connections
		dictionary[ 'support_cors' ] = self._support_cors
		dictionary[ 'log_requests' ] = self._log_requests
		dictionary[ 'bandwidth_tracker' ] = self._bandwidth_tracker
		dictionary[ 'bandwidth_rules' ] = self._bandwidth_rules

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		Service._LoadFromDictionary( self, dictionary )

		self._port = dictionary[ 'port' ]
		self._upnp_port = dictionary[ 'upnp_port' ]
		self._allow_non_local_connections = dictionary[ 'allow_non_local_connections' ]
		self._support_cors = dictionary[ 'support_cors' ]
		self._log_requests = dictionary[ 'log_requests' ]
		self._bandwidth_tracker = dictionary[ 'bandwidth_tracker' ]
		self._bandwidth_rules = dictionary[ 'bandwidth_rules' ]

		# this should support the same serverservice interface so we can just toss it at the regular serverengine and all the bandwidth will work ok

	def AllowsNonLocalConnections( self ):
		with self._lock:
			return self._allow_non_local_connections

	def BandwidthOK( self ):
		with self._lock:
			return self._bandwidth_rules.CanStartRequest( self._bandwidth_tracker )

	def GetUPnPPort( self ):
		with self._lock:
			return self._upnp_port

	def GetPort( self ):
		with self._lock:
			return self._port

	def LogsRequests( self ):
		with self._lock:
			return self._log_requests

	def ReportDataUsed( self, num_bytes ):
		with self._lock:
			self._bandwidth_tracker.ReportDataUsed( num_bytes )

	def ReportRequestUsed( self ):
		with self._lock:
			self._bandwidth_tracker.ReportRequestUsed()

	def SupportsCORS( self ):
		with self._lock:
			return self._support_cors

class ServiceLocalBooru( ServiceLocalServerService ):
	pass

class ServiceClientAPI( ServiceLocalServerService ):
	pass

class ServiceLocalTag( Service ):
	def _GetSerialisableDictionary( self ):

		dictionary = Service._GetSerialisableDictionary( self )

		dictionary[ 'tag_archive_sync' ] = list( self._tag_archive_sync.items() )

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		Service._LoadFromDictionary( self, dictionary )

		self._tag_archive_sync = dict( dictionary[ 'tag_archive_sync' ] )

	def GetTagArchiveSync( self ):
		with self._lock:
			return self._tag_archive_sync

class ServiceLocalRating( Service ):
	def _GetSerialisableDictionary( self ):

		dictionary = Service._GetSerialisableDictionary( self )

		dictionary[ 'shape' ] = self._shape
		dictionary[ 'colours' ] = list( self._colours.items() )

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		Service._LoadFromDictionary( self, dictionary )

		self._shape = dictionary[ 'shape' ]
		self._colours = dict( dictionary[ 'colours' ] )

	def GetColour( self, rating_state ):
		with self._lock:
			return self._colours[ rating_state ]

	def GetShape( self ):
		with self._lock:
			return self._shape

class ServiceLocalRatingLike( ServiceLocalRating ):
	pass

class ServiceLocalRatingNumerical( ServiceLocalRating ):
	def _GetSerialisableDictionary( self ):
		dictionary = ServiceLocalRating._GetSerialisableDictionary( self )

		dictionary[ 'num_stars' ] = self._num_stars
		dictionary[ 'allow_zero' ] = self._allow_zero

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServiceLocalRating._LoadFromDictionary( self, dictionary )

		self._num_stars = dictionary[ 'num_stars' ]
		self._allow_zero = dictionary[ 'allow_zero' ]

	def AllowZero( self ):
		with self._lock:
			return self._allow_zero

	def GetNumStars( self ):
		with self._lock:
			return self._num_stars

class ServiceRemote( Service ):
	def __init__( self, service_key, service_type, name, dictionary = None ):
		Service.__init__( self, service_key, service_type, name, dictionary = dictionary )

		self.network_context = NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, self._service_key )

	def _DelayFutureRequests( self, reason, duration = None ):
		if duration is None:
			duration = self._GetErrorWaitPeriod()

		next_no_requests_until = GetNow() + duration

		if next_no_requests_until > self._no_requests_until:
			self._no_requests_reason = reason
			self._no_requests_until = GetNow() + duration

		self._SetDirty()

	def _GetBaseURL( self ):
		( host, port ) = self._credentials.GetAddress()

		base_url = 'https://' + host + ':' + str( port ) + '/'

		return base_url

	def _GetErrorWaitPeriod( self ):
		return 3600 * 4

	def _CheckFunctional( self, including_external_communication = True, including_bandwidth = True ):
		if including_external_communication:
			self._CheckCanCommunicateExternally( including_bandwidth = including_bandwidth )

		Service._CheckFunctional( self )

	def _CheckCanCommunicateExternally( self, including_bandwidth = True ):
		if not TimeHasPassed( self._no_requests_until ):
			raise InsufficientCredentialsException( self._no_requests_reason + ' - next request ' + TimestampToPrettyTimeDelta( self._no_requests_until ) )

		if including_bandwidth:
			example_nj = NetworkJobHydrus( self._service_key, 'GET', self._GetBaseURL() )

			can_start = HG.client_controller.network_engine.bandwidth_manager.CanDoWork( example_nj.GetNetworkContexts(), threshold = 60 )

			if not can_start:
				raise BandwidthException( 'bandwidth exceeded' )

	def _GetSerialisableDictionary( self ):
		dictionary = Service._GetSerialisableDictionary( self )

		dictionary[ 'credentials' ] = self._credentials
		dictionary[ 'no_requests_reason' ] = self._no_requests_reason
		dictionary[ 'no_requests_until' ] = self._no_requests_until

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		Service._LoadFromDictionary( self, dictionary )

		self._credentials = dictionary[ 'credentials' ]
		self._no_requests_reason = dictionary[ 'no_requests_reason' ]
		self._no_requests_until = dictionary[ 'no_requests_until' ]

	def DelayFutureRequests( self, reason, duration = None ):
		with self._lock:
			self._DelayFutureRequests( reason, duration = None )

	def GetBandwidthCurrentMonthSummary( self ):
		with self._lock:
			return HG.client_controller.network_engine.bandwidth_manager.GetCurrentMonthSummary( self.network_context )

	def GetBandwidthStringsAndGaugeTuples( self ):
		with self._lock:
			return HG.client_controller.network_engine.bandwidth_manager.GetBandwidthStringsAndGaugeTuples( self.network_context )

	def GetBaseURL( self ):
		with self._lock:
			return self._GetBaseURL()

	def GetCredentials( self ):
		with self._lock:
			return self._credentials

	def SetCredentials( self, credentials ):
		with self._lock:
			self._credentials = credentials

			self._SetDirty()

class ServiceRestricted( ServiceRemote ):
	def _DealWithAccountError( self ):
		account_key = self._account.GetAccountKey()

		self._account = Account.GenerateUnknownAccount( account_key )

		self._next_account_sync = GetNow()

		self._SetDirty()

		HG.client_controller.pub( 'important_dirt_to_clean' )

	def _DealWithFundamentalNetworkError( self ):
		account_key = self._account.GetAccountKey()

		self._account = Account.GenerateUnknownAccount( account_key )

		self._next_account_sync = GetNow() + HC.UPDATE_DURATION * 10

		self._SetDirty()

		HG.client_controller.pub( 'important_dirt_to_clean' )

	def _GetErrorWaitPeriod( self ):
		if self._account.HasPermission( HC.CONTENT_TYPE_ACCOUNTS, HC.PERMISSION_ACTION_OVERRULE ):
			return 900

		else:
			return HC.UPDATE_DURATION

	def _CheckFunctional( self, including_external_communication = True, including_bandwidth = True, including_account = True ):
		if including_account:
			self._account.CheckFunctional()

		ServiceRemote._CheckFunctional( self, including_external_communication = including_external_communication, including_bandwidth = including_bandwidth )

	def _CheckCanCommunicateExternally( self, including_bandwidth = True ):
		if not self._credentials.HasAccessKey():
			raise MissingCredentialsException( 'this service has no access key set' )

		ServiceRemote._CheckCanCommunicateExternally( self, including_bandwidth = including_bandwidth )

	def _GetSerialisableDictionary( self ):
		dictionary = ServiceRemote._GetSerialisableDictionary( self )

		dictionary[ 'account' ] = Account.GenerateSerialisableTupleFromAccount( self._account )
		dictionary[ 'next_account_sync' ] = self._next_account_sync

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServiceRemote._LoadFromDictionary( self, dictionary )

		self._account = Account.GenerateAccountFromSerialisableTuple( dictionary[ 'account' ] )
		self._next_account_sync = dictionary[ 'next_account_sync' ]

	def CheckFunctional( self, including_external_communication = True, including_bandwidth = True, including_account = True ):
		with self._lock:
			self._CheckFunctional( including_external_communication = including_external_communication, including_bandwidth = including_bandwidth, including_account = including_account )

	def GetAccount( self ):
		with self._lock:
			return self._account

	def GetNextAccountSyncStatus( self ):
		if TimeHasPassed( self._next_account_sync ):
			s = 'imminently'

		else:
			s = TimestampToPrettyTimeDelta( self._next_account_sync )

		return 'next account sync ' + s

	def HasPermission( self, content_type, action ):
		with self._lock:
			return self._account.HasPermission( content_type, action )

	def IsDirty( self ):
		if ServiceRemote.IsDirty( self ):
			return True

		with self._lock:
			return self._account.IsDirty()

	def IsFunctional( self, including_external_communication = True, including_bandwidth = True, including_account = True ):
		with self._lock:
			try:
				self._CheckFunctional( including_external_communication = including_external_communication, including_bandwidth = including_bandwidth, including_account = including_account )

				return True

			except Exception:
				return False

	def Request( self, method, command, request_args = None, request_headers = None, report_hooks = None, temp_path = None ):
		if request_args is None:
			request_args = {}
		if request_headers is None:
			request_headers = {}
		if report_hooks is None:
			report_hooks = []

		try:
			if method == HC.GET:
				query = DumpToGETQuery( request_args )

				body = ''

				content_type = None

			elif method == HC.POST:
				query = ''

				if command == 'file':
					content_type = HC.APPLICATION_OCTET_STREAM

					body = request_args[ 'file' ]

					del request_args[ 'file' ]

				else:
					content_type = HC.APPLICATION_JSON

					body = DumpHydrusArgsToNetworkBytes( request_args )

			if query != '':
				command_and_query = command + '?' + query

			else:
				command_and_query = command

			url = self.GetBaseURL() + command_and_query

			if method == HC.GET:
				method = 'GET'

			elif method == HC.POST:
				method = 'POST'

			network_job = NetworkJobHydrus( self._service_key, method, url, body = body, temp_path = temp_path )

			if command in ( '', 'access_key', 'access_key_verification' ):
				# don't try to establish a session key for these requests
				network_job.SetForLogin( True )

				if command == 'access_key_verification':
					network_job.AddAdditionalHeader( 'Hydrus-Key', self._credentials.GetAccessKey().hex() )

			if content_type is not None:
				network_job.AddAdditionalHeader( 'Content-Type', HC.mime_string_lookup[ content_type ] )

			network_job.SetDeathTime( GetNow() + 30 ) # we don't want to wait on logins during shutdown and so on

			HG.client_controller.network_engine.AddJob( network_job )

			network_job.WaitUntilDone()

			network_bytes = network_job.GetContentBytes()

			content_type = network_job.GetContentType()

			if content_type == 'application/json':
				parsed_args = ParseNetworkBytesToParsedHydrusArgs( network_bytes )

				if command == 'account' and 'account' in parsed_args:
					data_used = network_job.GetTotalDataUsed()

					account = parsed_args[ 'account' ]

					# because the account was one behind when it was serialised! mostly do this just to sync up nicely with the service bandwidth tracker
					account.ReportDataUsed( data_used )
					account.ReportRequestUsed()

				response = parsed_args

			else:
				response = network_bytes

			return response

		except Exception as e:
			with self._lock:
				if isinstance( e, ServerBusyException ):
					self._DelayFutureRequests( 'server was busy', 5 * 60 )

				elif isinstance( e, SessionException ):
					HG.client_controller.network_engine.session_manager.ClearSession( self.network_context )

				elif isinstance( e, ( MissingCredentialsException, InsufficientCredentialsException ) ):
					self._DealWithAccountError()

				elif isinstance( e, NetworkVersionException ):
					self._DealWithFundamentalNetworkError()

				elif isinstance( e, NotFoundException ):
					self._DelayFutureRequests( 'got an unexpected 404', HC.UPDATE_DURATION )

				elif isinstance( e, BandwidthException ):
					self._DelayFutureRequests( 'service has exceeded bandwidth', HC.UPDATE_DURATION * 5 )

				else:
					self._DelayFutureRequests( str( e ) )

			raise

	def SetClean( self ):
		ServiceRemote.SetClean( self )

		self._account.SetClean()

	def SyncAccount( self, force = False ):
		with self._lock:
			name = self._name

			if force:
				do_it = True

				if force:
					self._no_requests_until = 0

			else:
				do_it = TimeHasPassed( self._next_account_sync )

				if do_it:
					try:
						self._CheckFunctional( including_account = False )

					except Exception:
						do_it = False

						self._next_account_sync = GetNow() + HC.UPDATE_DURATION

						self._SetDirty()

		if do_it:
			try:
				response = self.Request( HC.GET, 'account' )

				with self._lock:
					self._account = response[ 'account' ]

					if force:
						self._no_requests_until = 0

				HG.client_controller.pub( 'notify_new_permissions' )

			except ( CancelledException, NetworkException ) as e:
				Print( 'Failed to refresh account for ' + name + ':' )

				Print( e )

				if force:
					raise

			except ShutdownException:
				return

			except Exception:
				Print( 'Failed to refresh account for ' + name + ':' )

				Print( traceback.format_exc() )

				if force:
					raise

			finally:
				with self._lock:
					self._next_account_sync = GetNow() + HC.UPDATE_DURATION * 5

					self._SetDirty()

					HG.client_controller.pub( 'important_dirt_to_clean' )

class ServiceRepository( ServiceRestricted ):
	def __init__( self, service_key, service_type, name, dictionary = None ):
		ServiceRestricted.__init__( self, service_key, service_type, name, dictionary = dictionary )

		self._sync_lock = threading.Lock()

	def _CanSyncDownload( self ):
		try:
			self._CheckFunctional()

			return True

		except Exception:
			return False

	def _CanSyncProcess( self ):
		try:
			self._CheckFunctional( including_external_communication = False, including_bandwidth = False, including_account = False )

			return True

		except Exception:
			return False

	def _CheckFunctional( self, including_external_communication = True, including_bandwidth = True, including_account = True ):
		if self._paused:
			raise InsufficientCredentialsException( 'Repository is paused!' )

		if HG.client_controller.options[ 'pause_repo_sync' ]:
			raise InsufficientCredentialsException( 'All repositories are paused!' )

		ServiceRestricted._CheckFunctional( self, including_external_communication = including_external_communication, including_bandwidth = including_bandwidth, including_account = including_account )

	def _GetSerialisableDictionary( self ):
		dictionary = ServiceRestricted._GetSerialisableDictionary( self )

		dictionary[ 'metadata' ] = self._metadata
		dictionary[ 'paused' ] = self._paused
		dictionary[ 'tag_archive_sync' ] = list( self._tag_archive_sync.items() )

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServiceRestricted._LoadFromDictionary( self, dictionary )

		self._metadata = dictionary[ 'metadata' ]
		self._paused = dictionary[ 'paused' ]
		self._tag_archive_sync = dict( dictionary[ 'tag_archive_sync' ] )

	def CanDoIdleShutdownWork( self ):
		with self._lock:
			if not self._CanSyncProcess():
				return False

			service_key = self._service_key

		( download_value, processing_value, range ) = HG.client_controller.Read( 'repository_progress', service_key )

		return processing_value < range

	def GetNextUpdateDueString( self ):
		with self._lock:
			return self._metadata.GetNextUpdateDueString( from_client = True )

	def GetTagArchiveSync( self ):
		with self._lock:
			return self._tag_archive_sync

	def GetUpdateHashes( self ):
		with self._lock:
			return self._metadata.GetUpdateHashes()

	def GetUpdateInfo( self ):
		with self._lock:
			return self._metadata.GetUpdateInfo()

	def IsPaused( self ):
		with self._lock:
			return self._paused

	def PausePlay( self ):
		with self._lock:
			self._paused = not self._paused

			self._SetDirty()

			HG.client_controller.pub( 'important_dirt_to_clean' )

			if not self._paused:
				HG.client_controller.pub( 'notify_new_permissions' )

	def Reset( self ):
		with self._lock:
			self._no_requests_reason = ''
			self._no_requests_until = 0

			self._account = Account.GenerateUnknownAccount()
			self._next_account_sync = 0

			self._metadata = Metadata()

			self._SetDirty()

			HG.client_controller.pub( 'important_dirt_to_clean' )

			HG.client_controller.Write( 'reset_repository', self )

	def Sync( self, only_process_when_idle = False, stop_time = None ):
		with self._sync_lock: # to stop sync_now button clicks from stomping over the regular daemon and vice versa

			try:
				self.SyncDownloadMetadata()

				self.SyncDownloadUpdates( stop_time )

				self.SyncProcessUpdates( only_process_when_idle, stop_time )

				self.SyncThumbnails( stop_time )

			except Exception as e:
				self._DelayFutureRequests( str( e ) )

				ShowText( 'The service "' + self._name + '" encountered an error while trying to sync! It will not do any work for a little while. Please elevate this to hydrus dev if a fix is not obvious.' )

				ShowException( e )

			finally:
				if self.IsDirty():
					HG.client_controller.pub( 'important_dirt_to_clean' )

	def SyncDownloadUpdates( self, stop_time ):
		with self._lock:
			if not self._CanSyncDownload():
				return

			name = self._name
			service_key = self._service_key

		update_hashes = HG.client_controller.Read( 'missing_repository_update_hashes', service_key )

		if len( update_hashes ) > 0:
			job_key = JobKey( cancellable = True, stop_time = stop_time )

			try:
				job_key.SetVariable( 'popup_title', name + ' sync: downloading updates' )

				HG.client_controller.pub( 'message', job_key )

				for ( i, update_hash ) in enumerate( update_hashes ):
					status = 'update ' + ConvertValueRangeToPrettyString( i + 1, len( update_hashes ) )

					HG.client_controller.pub( 'splash_set_status_text', status, print_to_log = False )
					job_key.SetVariable( 'popup_text_1', status )
					job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( update_hashes ) ) )

					with self._lock:
						if not self._CanSyncDownload():
							return

					( i_paused, should_quit ) = job_key.WaitIfNeeded()

					if should_quit:
						with self._lock:
							self._DelayFutureRequests( 'download was recently cancelled', 3 * 60 )

						return

					try:
						update_network_string = self.Request( HC.GET, 'update', { 'update_hash' : update_hash } )

					except CancelledException as e:
						self._DelayFutureRequests( str( e ) )

						return

					except NetworkException as e:
						Print( 'Attempting to download an update for ' + name + ' resulted in a network error:' )

						Print( e )

						return

					update_network_string_hash = hashlib.sha256( update_network_string ).digest()

					if update_network_string_hash != update_hash:
						# this is the weird update problem, seems to be network related
						# throwing a whole hullabaloo about it only caused problems, as the real fix was 'unpause it, try again'

						with self._lock:
							self._DelayFutureRequests( 'had an unusual update response' )

						return

					try:
						update = CreateFromNetworkBytes( update_network_string )

					except Exception as e:
						with self._lock:
							self._paused = True

							self._DealWithFundamentalNetworkError()

						message = 'Update ' + update_hash.hex() + ' downloaded from the ' + self._name + ' repository failed to load! This is a serious error!'
						message += os.linesep * 2
						message += 'The repository has been paused for now. Please look into what could be wrong and report this to the hydrus dev.'

						ShowText( message )

						ShowException( e )

						return

					if isinstance( update, DefinitionsUpdate ):
						mime = HC.APPLICATION_HYDRUS_UPDATE_DEFINITIONS

					elif isinstance( update, NetworkContentUpdate ):
						mime = HC.APPLICATION_HYDRUS_UPDATE_CONTENT

					else:
						with self._lock:
							self._paused = True

							self._DealWithFundamentalNetworkError()

						message = 'Update ' + update_hash.hex() + ' downloaded from the ' + self._name + ' was not a valid update--it was a ' + repr( update ) + '! This is a serious error!'
						message += os.linesep * 2
						message += 'The repository has been paused for now. Please look into what could be wrong and report this to the hydrus dev.'

						ShowText( message )

						return

					try:
						HG.client_controller.WriteSynchronous( 'import_update', update_network_string, update_hash, mime )

					except Exception as e:
						with self._lock:
							self._paused = True

							self._DealWithFundamentalNetworkError()

						message = 'While downloading updates for the ' + self._name + ' repository, one failed to import! The error follows:'

						ShowText( message )

						ShowException( e )

						return

				job_key.SetVariable( 'popup_text_1', 'finished' )
				job_key.DeleteVariable( 'popup_gauge_1' )

			finally:
				job_key.Finish()
				job_key.Delete( 5 )

	def SyncDownloadMetadata( self ):
		with self._lock:
			if not self._CanSyncDownload():
				return

			do_it = self._metadata.UpdateDue( from_client = True )

			next_update_index = self._metadata.GetNextUpdateIndex()

			service_key = self._service_key

			name = self._name

		if do_it:
			try:
				response = self.Request( HC.GET, 'metadata', { 'since' : next_update_index } )

				metadata_slice = response[ 'metadata_slice' ]

			except CancelledException as e:
				self._DelayFutureRequests( str( e ) )

				return

			except NetworkException as e:
				Print( 'Attempting to download metadata for ' + name + ' resulted in a network error:' )

				Print( e )

				return

			HG.client_controller.WriteSynchronous( 'associate_repository_update_hashes', service_key, metadata_slice )

			with self._lock:
				self._metadata.UpdateFromSlice( metadata_slice )

				self._SetDirty()

	def SyncProcessUpdates( self, only_when_idle = False, stop_time = None ):
		with self._lock:
			if not self._CanSyncProcess():
				return

		if only_when_idle and not HG.client_controller.CurrentlyIdle():
			return

		try:
			( did_some_work, did_everything ) = HG.client_controller.WriteSynchronous( 'process_repository', self._service_key, only_when_idle, stop_time )

			if did_some_work:
				with self._lock:
					self._SetDirty()

					if self._service_type == HC.TAG_REPOSITORY:
						HG.client_controller.pub( 'notify_new_force_refresh_tags_data' )

			if not did_everything:
				time.sleep( 3 ) # stop spamming of repo sync daemon from bringing this up again too quick

		except ShutdownException:
			return

		except Exception as e:
			with self._lock:
				message = 'Failed to process updates for the ' + self._name + ' repository! The error follows:'

				ShowText( message )

				ShowException( e )

				self._paused = True

				self._SetDirty()

			HG.client_controller.pub( 'important_dirt_to_clean' )

	def SyncThumbnails( self, stop_time ):
		with self._lock:
			if not self._CanSyncDownload():
				return

			name = self._name
			service_key = self._service_key

		thumbnail_hashes = HG.client_controller.Read( 'missing_thumbnail_hashes', service_key )

		num_to_do = len( thumbnail_hashes )

		if num_to_do > 0:
			client_files_manager = HG.client_controller.client_files_manager

			job_key = JobKey( cancellable = True, stop_time = stop_time )

			try:
				job_key.SetVariable( 'popup_title', name + ' sync: downloading thumbnails' )

				HG.client_controller.pub( 'message', job_key )

				for ( i, thumbnail_hash ) in enumerate( thumbnail_hashes ):
					status = 'thumbnail ' + ConvertValueRangeToPrettyString( i + 1, num_to_do )

					HG.client_controller.pub( 'splash_set_status_text', status, print_to_log = False )
					job_key.SetVariable( 'popup_text_1', status )
					job_key.SetVariable( 'popup_gauge_1', ( i + 1, num_to_do ) )

					with self._lock:
						if not self._CanSyncDownload():
							break

					( i_paused, should_quit ) = job_key.WaitIfNeeded()

					if should_quit:
						with self._lock:
							self._DelayFutureRequests( 'download was recently cancelled', 3 * 60 )

						return

					try:
						thumbnail_bytes = self.Request( HC.GET, 'thumbnail', { 'hash' : thumbnail_hash } )

					except CancelledException as e:
						self._DelayFutureRequests( str( e ) )

						return

					except NetworkException as e:
						Print( 'Attempting to download a thumbnail for ' + name + ' resulted in a network error:' )

						Print( e )

						return

					client_files_manager.AddThumbnailFromBytes( thumbnail_hash, thumbnail_bytes )

				job_key.SetVariable( 'popup_text_1', 'finished' )
				job_key.DeleteVariable( 'popup_gauge_1' )

			finally:
				job_key.Finish()
				job_key.Delete( 5 )

class ServiceIPFS( ServiceRemote ):
	def _GetSerialisableDictionary( self ):
		dictionary = ServiceRemote._GetSerialisableDictionary( self )

		dictionary[ 'multihash_prefix' ] = self._multihash_prefix

		return dictionary

	def _LoadFromDictionary( self, dictionary ):
		ServiceRemote._LoadFromDictionary( self, dictionary )

		self._multihash_prefix = dictionary[ 'multihash_prefix' ]

	def _ConvertMultihashToURLTree( self, name, size, multihash ):
		api_base_url = self._GetAPIBaseURL()

		links_url = api_base_url + 'object/links/' + multihash

		network_job = NetworkJob( 'GET', links_url )

		network_job.OverrideBandwidth()

		HG.client_controller.network_engine.AddJob( network_job )

		network_job.WaitUntilDone()

		parsing_text = network_job.GetContentText()

		links_json = json.loads( parsing_text )

		is_directory = False

		if 'Links' in links_json:
			for link in links_json[ 'Links' ]:
				if link[ 'Name' ] != '':
					is_directory = True

		if is_directory:
			children = []

			for link in links_json[ 'Links' ]:
				subname = link[ 'Name' ]
				subsize = link[ 'Size' ]
				submultihash = link[ 'Hash' ]

				children.append( self._ConvertMultihashToURLTree( subname, subsize, submultihash ) )

			if size is None:
				size = sum( ( subsize for ( type_gumpf, subname, subsize, submultihash ) in children ) )

			return ( 'directory', name, size, children )

		else:
			url = api_base_url + 'cat/' + multihash

			return ( 'file', name, size, url )

	def _GetAPIBaseURL( self ):
		( host, port ) = self._credentials.GetAddress()

		api_base_url = 'http://' + host + ':' + str( port ) + '/api/v0/'

		return api_base_url

	def GetDaemonVersion( self ):
		with self._lock:
			api_base_url = self._GetAPIBaseURL()

		url = api_base_url + 'version'

		network_job = NetworkJob( 'GET', url )

		network_job.OverrideBandwidth()

		HG.client_controller.network_engine.AddJob( network_job )

		network_job.WaitUntilDone()

		parsing_text = network_job.GetContentText()

		j = json.loads( parsing_text )

		return j[ 'Version' ]

	def GetMultihashPrefix( self ):
		with self._lock:
			return self._multihash_prefix

	def ImportFile( self, multihash ):
		def on_wx_select_tree( job_key, url_tree ):
			import lib.GUI.Dialogs

			with lib.GUI.Dialogs.DialogSelectFromURLTree( HG.client_controller.gui, url_tree ) as dlg:
				if dlg.ShowModal() == wx.ID_OK:
					urls = dlg.GetURLs()

					if len( urls ) > 0:
						HG.client_controller.CallToThread( THREADDownloadURLs, job_key, urls, multihash )

		def off_wx():
			job_key = JobKey( pausable = True, cancellable = True )

			job_key.SetVariable( 'popup_text_1', 'Looking up multihash information' )

			HG.client_controller.pub( 'message', job_key )

			with self._lock:
				url_tree = self._ConvertMultihashToURLTree( multihash, None, multihash )

			if url_tree[0] == 'file':
				url = url_tree[3]

				HG.client_controller.CallToThread( THREADDownloadURL, job_key, url, multihash )

			else:
				job_key.SetVariable( 'popup_text_1', 'Waiting for user selection' )

				wx.CallAfter( on_wx_select_tree, job_key, url_tree )

		HG.client_controller.CallToThread( off_wx )

	def PinDirectory( self, hashes, note ):
		job_key = JobKey( pausable = True, cancellable = True )

		job_key.SetVariable( 'popup_title', 'creating ipfs directory on ' + self._name )

		HG.client_controller.pub( 'message', job_key )

		try:
			file_info = []

			hashes = list( hashes )

			hashes.sort()

			for ( i, hash ) in enumerate( hashes ):
				( i_paused, should_quit ) = job_key.WaitIfNeeded()

				if should_quit:
					return

				job_key.SetVariable( 'popup_text_1', 'pinning files: ' + ConvertValueRangeToPrettyString( i + 1, len( hashes ) ) )
				job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( hashes ) ) )

				( media_result, ) = HG.client_controller.Read( 'media_results', ( hash, ) )

				mime = media_result.GetMime()

				result = HG.client_controller.Read( 'service_filenames', self._service_key, { hash } )

				if len( result ) == 0:
					multihash = self.PinFile( hash, mime )

				else:
					( multihash, ) = result

				file_info.append( ( hash, mime, multihash ) )

			with self._lock:
				api_base_url = self._GetAPIBaseURL()

			url = api_base_url + 'object/new?arg=unixfs-dir'

			network_job = NetworkJob( 'GET', url )

			network_job.OverrideBandwidth()

			HG.client_controller.network_engine.AddJob( network_job )

			network_job.WaitUntilDone()

			parsing_text = network_job.GetContentText()

			response_json = json.loads( parsing_text )

			for ( i, ( hash, mime, multihash ) ) in enumerate( file_info ):
				( i_paused, should_quit ) = job_key.WaitIfNeeded()

				if should_quit:
					return

				job_key.SetVariable( 'popup_text_1', 'creating directory: ' + ConvertValueRangeToPrettyString( i + 1, len( file_info ) ) )
				job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( file_info ) ) )

				object_multihash = response_json[ 'Hash' ]

				filename = hash.hex() + HC.mime_ext_lookup[ mime ]

				url = api_base_url + 'object/patch/add-link?arg=' + object_multihash + '&arg=' + filename + '&arg=' + multihash

				network_job = NetworkJob( 'GET', url )

				network_job.OverrideBandwidth()

				HG.client_controller.network_engine.AddJob( network_job )

				network_job.WaitUntilDone()

				parsing_text = network_job.GetContentText()

				response_json = json.loads( parsing_text )

			directory_multihash = response_json[ 'Hash' ]

			url = api_base_url + 'pin/add?arg=' + directory_multihash

			network_job = NetworkJob( 'GET', url )

			network_job.OverrideBandwidth()

			HG.client_controller.network_engine.AddJob( network_job )

			network_job.WaitUntilDone()

			content_update_row = ( hashes, directory_multihash, note )

			content_updates = [ ContentUpdate( HC.CONTENT_TYPE_DIRECTORIES, HC.CONTENT_UPDATE_ADD, content_update_row ) ]

			HG.client_controller.WriteSynchronous( 'content_updates', { self._service_key : content_updates } )

			job_key.SetVariable( 'popup_text_1', 'done!' )
			job_key.DeleteVariable( 'popup_gauge_1' )

			with self._lock:
				text = self._multihash_prefix + directory_multihash

			job_key.SetVariable( 'popup_clipboard', ( 'copy multihash to clipboard', text ) )

			job_key.Finish()

			return directory_multihash

		except Exception as e:
			ShowException( e )

			job_key.SetVariable( 'popup_text_1', 'error' )
			job_key.DeleteVariable( 'popup_gauge_1' )

			job_key.Cancel()

	def PinFile( self, hash, mime ):
		mime_string = HC.mime_string_lookup[ mime ]

		with self._lock:
			api_base_url = self._GetAPIBaseURL()

		url = api_base_url + 'add'

		client_files_manager = HG.client_controller.client_files_manager

		path = client_files_manager.GetFilePath( hash, mime )

		files = { 'path' : ( hash.hex(), open( path, 'rb' ), mime_string ) }

		network_job = NetworkJob( 'GET', url )

		network_job.SetFiles( files )

		network_job.OverrideBandwidth()

		HG.client_controller.network_engine.AddJob( network_job )

		network_job.WaitUntilDone()

		parsing_text = network_job.GetContentText()

		j = json.loads( parsing_text )

		multihash = j[ 'Hash' ]

		( media_result, ) = HG.client_controller.Read( 'media_results', ( hash, ) )

		file_info_manager = media_result.GetFileInfoManager()

		content_update_row = ( file_info_manager, multihash )

		content_updates = [ ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ADD, content_update_row ) ]

		HG.client_controller.WriteSynchronous( 'content_updates', { self._service_key : content_updates } )

		return multihash

	def UnpinDirectory( self, multihash ):
		with self._lock:
			api_base_url = self._GetAPIBaseURL()

		url = api_base_url + 'pin/rm/' + multihash

		network_job = NetworkJob( 'GET', url )

		network_job.OverrideBandwidth()

		HG.client_controller.network_engine.AddJob( network_job )

		network_job.WaitUntilDone()

		content_updates = [ ContentUpdate( HC.CONTENT_TYPE_DIRECTORIES, HC.CONTENT_UPDATE_DELETE, multihash ) ]

		HG.client_controller.WriteSynchronous( 'content_updates', { self._service_key : content_updates } )

	def UnpinFile( self, hash, multihash ):
		with self._lock:
			api_base_url = self._GetAPIBaseURL()

		url = api_base_url + 'pin/rm/' + multihash

		try:
			network_job = NetworkJob( 'GET', url )

			network_job.OverrideBandwidth()

			HG.client_controller.network_engine.AddJob( network_job )

			network_job.WaitUntilDone()

		except NetworkException as e:
			if 'not pinned' not in str( e ):
				raise

		content_updates = [ ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_DELETE, { hash } ) ]

		HG.client_controller.WriteSynchronous( 'content_updates', { self._service_key : content_updates } )


code:
from .Services import *

__all__ = [
	"Local"
]

from . import *

license: wtfpl


code:
import collections, threading

from lib.Hydrus.Exceptions import SessionException
from lib.Hydrus.Data import GenerateKey, GetNow, TimeHasPassed

### CONSTANTS ###

import lib.Hydrus.Globals as HG

HYDRUS_SESSION_LIFETIME = 30 * 86400

### CLASSES ###

class HydrusSessionManagerServer( object ):
	def __init__( self ):
		self._lock = threading.Lock()

		self.RefreshAllAccounts()

		HG.controller.sub( self, 'RefreshAccounts', 'update_session_accounts' )
		HG.controller.sub( self, 'RefreshAllAccounts', 'update_all_session_accounts' )

	def AddSession( self, service_key, access_key ):
		with self._lock:
			account_key = HG.controller.Read( 'account_key_from_access_key', service_key, access_key )

			account_keys_to_accounts = self._service_keys_to_account_keys_to_accounts[ service_key ]

			if account_key not in account_keys_to_accounts:
				account = HG.controller.Read( 'account', service_key, account_key )

				account_keys_to_accounts[ account_key ] = account

			session_key = GenerateKey()

			now = GetNow()

			expires = now + HYDRUS_SESSION_LIFETIME

			HG.controller.Write( 'session', session_key, service_key, account_key, expires )

			self._service_keys_to_session_keys_to_sessions[ service_key ][ session_key ] = ( account_key, expires )

			return ( session_key, expires )

	def GetAccount( self, service_key, session_key ):
		with self._lock:
			session_keys_to_sessions = self._service_keys_to_session_keys_to_sessions[ service_key ]

			if session_key in session_keys_to_sessions:
				( account_key, expires ) = session_keys_to_sessions[ session_key ]

				if TimeHasPassed( expires ):
					del session_keys_to_sessions[ session_key ]

				else:
					account = self._service_keys_to_account_keys_to_accounts[ service_key ][ account_key ]

					return account

			raise SessionException( 'Did not find that session! Try again!' )

	def GetDirtyAccounts( self ):
		with self._lock:
			service_keys_to_dirty_accounts = {}

			for ( service_key, account_keys_to_accounts ) in list( self._service_keys_to_account_keys_to_accounts.items() ):
				dirty_accounts = [ account_key for account_key in list( account_keys_to_accounts.values() ) if account_key.IsDirty() ]

				if len( dirty_accounts ) > 0:
					service_keys_to_dirty_accounts[ service_key ] = dirty_accounts

			return service_keys_to_dirty_accounts

	def RefreshAccounts( self, service_key, account_keys = None ):
		with self._lock:
			account_keys_to_accounts = self._service_keys_to_account_keys_to_accounts[ service_key ]

			if account_keys is None:
				account_keys = list( account_keys_to_accounts.keys() )

			for account_key in account_keys:
				account = HG.controller.Read( 'account', service_key, account_key )

				account_keys_to_accounts[ account_key ] = account

	def RefreshAllAccounts( self, service_key = None ):
		with self._lock:
			if service_key is None:
				self._service_keys_to_session_keys_to_sessions = collections.defaultdict( dict )

				self._service_keys_to_account_keys_to_accounts = collections.defaultdict( dict )

				existing_sessions = HG.controller.Read( 'sessions' )

			else:
				del self._service_keys_to_session_keys_to_sessions[ service_key ]

				del self._service_keys_to_account_keys_to_accounts[ service_key ]

				existing_sessions = HG.controller.Read( 'sessions', service_key )

			for ( session_key, service_key, account, expires ) in existing_sessions:
				account_key = account.GetAccountKey()

				self._service_keys_to_session_keys_to_sessions[ service_key ][ session_key ] = ( account_key, expires )

				self._service_keys_to_account_keys_to_accounts[ service_key ][ account_key ] = account

	def UpdateAccounts( self, service_key, accounts ):
		with self._lock:
			account_keys_to_accounts = self._service_keys_to_account_keys_to_accounts[ service_key ]

			for account in accounts:
				account_keys_to_accounts[ account.GetAccountKey() ] = account


code:
import os
import sqlite3

HASH_TYPE_MD5 = 0 # 16 bytes long
HASH_TYPE_SHA1 = 1 # 20 bytes long
HASH_TYPE_SHA256 = 2 # 32 bytes long
HASH_TYPE_SHA512 = 3 # 64 bytes long

# Please feel free to use this file however you wish.
# None of this is thread-safe, though, so don't try to do anything clever.

# If you want to make a new tag archive for use in hydrus, you want to do something like:
# import HydrusTagArchive
# hta = lib.Hydrus.Tags.Archive.HydrusTagArchive( 'my_little_archive.db' )
# hta.SetHashType( lib.Hydrus.Tags.Archive.HASH_TYPE_MD5 )
# hta.BeginBigJob()
# for ( hash, tags ) in my_complex_mappings_generator:
 	hta.AddMappings( hash, tags )
# -or-
# for ( hash, tag ) in my_simple_mapping_generator:
 	hta.AddMapping( hash, tag )
# hta.CommitBigJob()
# del hta

# If you are only adding a couple tags, you can exclude the BigJob stuff. It just makes millions of sequential writes more efficient.

# Also, this manages hashes as bytes, not hex, so if you have something like:
# hash = ab156e87c5d6e215ab156e87c5d6e215

# Then go hash = bytes.fromhex( hash ) before you pass it to Add/Get/Has/SetMappings

# If you have tags that are namespaced like hydrus (e.g. series:ghost in the shell), then check out:
# GetNamespaces
# DeleteNamespaces
# and
# RebuildNamespaces

# RebuildNamespaces takes namespaces_to_exclude, if you want to curate your namespaces a little better.

# If your GetNamespaces gives garbage, then just hit DeleteNamespaces. I'll be using the result of GetNamespaces to populate
# the tag import options widget when people sync with these archives.

# And also feel free to contact me directly at hydrus.admin@gmail.com if you need help.

### CLASSES ###

class HydrusTagArchive( object ):
	def __init__( self, path ):
		self._path = path

		if not os.path.exists( self._path ):
			create_db = True
		else:
			create_db = False

		self._InitDBCursor()

		if create_db:
			self._InitDB()

		self._namespaces = { namespace for ( namespace, ) in self._c.execute( 'SELECT namespace FROM namespaces;' ) }
		self._namespaces.add( '' )

	def _AddMappings( self, hash_id, tag_ids ):
		self._c.executemany( 'INSERT OR IGNORE INTO mappings ( hash_id, tag_id ) VALUES ( ?, ? );', ( ( hash_id, tag_id ) for tag_id in tag_ids ) )

	def _InitDB( self ):
		self._c.execute( 'CREATE TABLE hash_type ( hash_type INTEGER );', )

		self._c.execute( 'CREATE TABLE hashes ( hash_id INTEGER PRIMARY KEY, hash BLOB_BYTES );' )
		self._c.execute( 'CREATE UNIQUE INDEX hashes_hash_index ON hashes ( hash );' )

		self._c.execute( 'CREATE TABLE mappings ( hash_id INTEGER, tag_id INTEGER, PRIMARY KEY ( hash_id, tag_id ) );' )
		self._c.execute( 'CREATE INDEX mappings_hash_id_index ON mappings ( hash_id );' )

		self._c.execute( 'CREATE TABLE namespaces ( namespace TEXT );' )

		self._c.execute( 'CREATE TABLE tags ( tag_id INTEGER PRIMARY KEY, tag TEXT );' )
		self._c.execute( 'CREATE UNIQUE INDEX tags_tag_index ON tags ( tag );' )

	def _InitDBCursor( self ):
		self._db = sqlite3.connect( self._path, isolation_level = None, detect_types = sqlite3.PARSE_DECLTYPES )

		self._c = self._db.cursor()

	def _GetHashId( self, hash, read_only = False ):
		result = self._c.execute( 'SELECT hash_id FROM hashes WHERE hash = ?;', ( sqlite3.Binary( hash ), ) ).fetchone()

		if result is None:
			if read_only:
				raise Exception()

			self._c.execute( 'INSERT INTO hashes ( hash ) VALUES ( ? );', ( sqlite3.Binary( hash ), ) )

			hash_id = self._c.lastrowid

		else:
			( hash_id, ) = result

		return hash_id

	def _GetTagId( self, tag ):
		if ':' in tag:
			( namespace, subtag ) = tag.split( ':', 1 )

			if namespace != '' and namespace not in self._namespaces:
				self._c.execute( 'INSERT INTO namespaces ( namespace ) VALUES ( ? );', ( namespace, ) )

				self._namespaces.add( namespace )

		result = self._c.execute( 'SELECT tag_id FROM tags WHERE tag = ?;', ( tag, ) ).fetchone()

		if result is None:
			self._c.execute( 'INSERT INTO tags ( tag ) VALUES ( ? );', ( tag, ) )

			tag_id = self._c.lastrowid

		else:
			( tag_id, ) = result

		return tag_id

	def BeginBigJob( self ):
		self._c.execute( 'BEGIN IMMEDIATE;' )

	def CommitBigJob( self ):
		self._c.execute( 'COMMIT;' )
		self._c.execute( 'VACUUM;' )

	def AddMapping( self, hash, tag ):
		hash_id = self._GetHashId( hash )
		tag_id = self._GetTagId( tag )

		self._c.execute( 'INSERT OR IGNORE INTO mappings ( hash_id, tag_id ) VALUES ( ?, ? );', ( hash_id, tag_id ) )

	def AddMappings( self, hash, tags ):
		hash_id = self._GetHashId( hash )

		tag_ids = [ self._GetTagId( tag ) for tag in tags ]

		self._AddMappings( hash_id, tag_ids )

	def DeleteMapping( self, hash, tag ):
		hash_id = self._GetHashId( hash )
		tag_id = self._GetTagId( tag )

		self._c.execute( 'DELETE FROM mappings WHERE hash_id = ? AND tag_id = ?;', ( hash_id, tag_id ) )

	def DeleteMappings( self, hash ):
		self.DeleteTags( hash )

	def DeleteTags( self, hash ):
		try:
			hash_id = self._GetHashId( hash, read_only = True )
		except Exception:
			return

		self._c.execute( 'DELETE FROM mappings WHERE hash_id = ?;', ( hash_id, ) )

	def DeleteNamespaces( self ):
		self._namespaces = {}
		self._namespaces.add( '' )

		self._c.execute( 'DELETE FROM namespaces;' )

	def GetHashType( self ):
		result = self._c.execute( 'SELECT hash_type FROM hash_type;' ).fetchone()

		if result is None:
			result = self._c.execute( 'SELECT hash FROM hashes;' ).fetchone()

			if result is None:
				raise Exception( 'This archive has no hash type set, and as it has no files, no hash type guess can be made.' )

			( hash, ) = result

			hash_len = len( hash )

			len_to_hash_type = {}

			len_to_hash_type[ 16 ] = HASH_TYPE_MD5
			len_to_hash_type[ 20 ] = HASH_TYPE_SHA1
			len_to_hash_type[ 32 ] = HASH_TYPE_SHA256
			len_to_hash_type[ 64 ] = HASH_TYPE_SHA512

			if hash_len in len_to_hash_type:
				self.SetHashType( len_to_hash_type[ hash_len ] )

			else:
				raise Exception()

			return self.GetHashType()

		else:
			( hash_type, ) = result

			return hash_type

	def GetMappings( self, hash ):
		return self.GetTags( hash )

	def GetName( self ):
		filename = os.path.basename( self._path )

		if '.' in filename:
			filename = filename.split( '.', 1 )[0]

		return filename

	def GetNamespaces( self ):
		return self._namespaces

	def GetTags( self, hash ):
		try:
			hash_id = self._GetHashId( hash, read_only = True )
		except Exception:
			return []

		result = { tag for ( tag, ) in self._c.execute( 'SELECT tag FROM mappings NATURAL JOIN tags WHERE hash_id = ?;', ( hash_id, ) ) }

		return result

	def HasHash( self, hash ):
		try:
			self._GetHashId( hash, read_only = True )

			return True

		except Exception:
			return False

	def HasHashTypeSet( self ):
		result = self._c.execute( 'SELECT hash_type FROM hash_type;' ).fetchone()

		return result is not None

	def IterateHashes( self ):
		for ( hash, ) in self._c.execute( 'SELECT hash FROM hashes;' ):
			yield hash

	def IterateMappings( self ):
		hash_ids = [ hash_id for ( hash_id, ) in self._c.execute( 'SELECT hash_id FROM hashes;' ) ]

		for hash_id in hash_ids:
			( hash, ) = self._c.execute( 'SELECT hash FROM hashes WHERE hash_id = ?;', ( hash_id, ) ).fetchone()

			tags = self.GetTags( hash )

			if len( tags ) > 0:
				yield ( hash, tags )

	def RebuildNamespaces( self, namespaces_to_exclude = set() ):
		self._namespaces = set()
		self._namespaces.add( '' )

		self._c.execute( 'DELETE FROM namespaces;' )

		for ( tag, ) in self._c.execute( 'SELECT tag FROM tags;' ):
			if ':' in tag:
				( namespace, subtag ) = tag.split( ':', 1 )

				if namespace != '' and namespace not in self._namespaces and namespace not in namespaces_to_exclude:
					self._namespaces.add( namespace )

		self._c.executemany( 'INSERT INTO namespaces ( namespace ) VALUES ( ? );', ( ( namespace, ) for namespace in self._namespaces ) )

	def SetHashType( self, hash_type ):
		self._c.execute( 'DELETE FROM hash_type;' )

		self._c.execute( 'INSERT INTO hash_type ( hash_type ) VALUES ( ? );', ( hash_type, ) )

	def SetMappings( self, hash, tags ):
		hash_id = self._GetHashId( hash )

		self._c.execute( 'DELETE FROM mappings WHERE hash_id = ?;', ( hash_id, ) )

		tag_ids = [ self._GetTagId( tag ) for tag in tags ]

		self._AddMappings( hash_id, tag_ids )


code:
import threading
import collections

from lib.Hydrus.Serialisable import SerialisableBase
from lib.Hydrus.Tags import ConvertTagToSortable, SplitTag

### CONSTANTS ###

import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

### FUNCTIONS ###

def ConvertTagSliceToString( tag_slice ):
	if tag_slice == '':
		return 'unnamespaced tags'

	elif tag_slice == ':':
		return 'namespaced tags'

	elif tag_slice.count( ':' ) == 1 and tag_slice.endswith( ':' ):
		namespace = tag_slice[ : -1 ]

		return '\'' + namespace + '\' tags'

	else:
		return tag_slice

def RenderNamespaceForUser( namespace ):
	if namespace == '' or namespace is None:
		return 'unnamespaced'

	else:
		return namespace

def RenderTag( tag, render_for_user ):
	( namespace, subtag ) = SplitTag( tag )

	if namespace == '':
		return subtag

	else:
		if render_for_user:
			new_options = HG.client_controller.new_options

			if new_options.GetBoolean( 'show_namespaces' ):
				connector = new_options.GetString( 'namespace_connector' )

			else:
				return subtag

		else:
			connector = ':'

		return namespace + connector + subtag

def SortTags( sort_by, tags_list, tags_to_count = None ):
	def lexicographic_key( tag ):
		( namespace, subtag ) = SplitTag( tag )

		comparable_namespace = ConvertTagToSortable( namespace )
		comparable_subtag = ConvertTagToSortable( subtag )

		if namespace == '':
			return ( comparable_subtag, comparable_subtag )

		else:
			return ( comparable_namespace, comparable_subtag )

	def incidence_key( tag ):
		if tags_to_count is None:
			return 1

		else:
			return tags_to_count[ tag ]

	def namespace_key( tag ):
		( namespace, subtag ) = SplitTag( tag )

		if namespace == '':
			namespace = '{' # '{' is above 'z' in ascii, so this works for most situations

		return namespace

	def namespace_lexicographic_key( tag ):
		# '{' is above 'z' in ascii, so this works for most situations

		( namespace, subtag ) = SplitTag( tag )

		if namespace == '':
			return ( '{', ConvertTagToSortable( subtag ) )

		else:
			return ( namespace, ConvertTagToSortable( subtag ) )

	if sort_by in ( CC.SORT_BY_INCIDENCE_ASC, CC.SORT_BY_INCIDENCE_DESC, CC.SORT_BY_INCIDENCE_NAMESPACE_ASC, CC.SORT_BY_INCIDENCE_NAMESPACE_DESC ):
		# let's establish a-z here for equal incidence values later
		if sort_by in ( CC.SORT_BY_INCIDENCE_ASC, CC.SORT_BY_INCIDENCE_NAMESPACE_ASC ):
			tags_list.sort( key = lexicographic_key, reverse = True )

			reverse = False

		elif sort_by in ( CC.SORT_BY_INCIDENCE_DESC, CC.SORT_BY_INCIDENCE_NAMESPACE_DESC ):
			tags_list.sort( key = lexicographic_key )

			reverse = True

		tags_list.sort( key = incidence_key, reverse = reverse )

		if sort_by in ( CC.SORT_BY_INCIDENCE_NAMESPACE_ASC, CC.SORT_BY_INCIDENCE_NAMESPACE_DESC ):
			# python list sort is stable, so lets now sort again

			if sort_by == CC.SORT_BY_INCIDENCE_NAMESPACE_ASC:
				reverse = True

			elif sort_by == CC.SORT_BY_INCIDENCE_NAMESPACE_DESC:
				reverse = False

			tags_list.sort( key = namespace_key, reverse = reverse )

	else:
		if sort_by in ( CC.SORT_BY_LEXICOGRAPHIC_DESC, CC.SORT_BY_LEXICOGRAPHIC_NAMESPACE_DESC ):
			reverse = True

		elif sort_by in ( CC.SORT_BY_LEXICOGRAPHIC_ASC, CC.SORT_BY_LEXICOGRAPHIC_NAMESPACE_ASC ):
			reverse = False

		if sort_by in ( CC.SORT_BY_LEXICOGRAPHIC_NAMESPACE_ASC, CC.SORT_BY_LEXICOGRAPHIC_NAMESPACE_DESC ):
			key = namespace_lexicographic_key

		elif sort_by in ( CC.SORT_BY_LEXICOGRAPHIC_ASC, CC.SORT_BY_LEXICOGRAPHIC_DESC ):
			key = lexicographic_key

		tags_list.sort( key = key, reverse = reverse )

class ServiceKeysToTags( SerialisableBase, collections.defaultdict ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["SERVICE_KEYS_TO_TAGS"]
	SERIALISABLE_NAME = 'Service Keys To Tags'
	SERIALISABLE_VERSION = 1

	def __init__( self, *args, **kwargs ):
		collections.defaultdict.__init__( self, set, *args, **kwargs )
		SerialisableBase.__init__( self )

	def _GetSerialisableInfo( self ):
		return [ ( service_key.hex(), list( tags ) ) for ( service_key, tags ) in self.items() ]

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		for ( service_key_hex, tags_list ) in serialisable_info:
			self[ bytes.fromhex( service_key_hex ) ] = set( tags_list )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["SERVICE_KEYS_TO_TAGS"] ] = ServiceKeysToTags

class TagFilter( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["TAG_FILTER"]
	SERIALISABLE_NAME = 'Tag Filter Rules'
	SERIALISABLE_VERSION = 1

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._lock = threading.Lock()

		self._tag_slices_to_rules = {}

	def __eq__( self, other ):
		return self._tag_slices_to_rules == other._tag_slices_to_rules

	def _GetTagSlices( self, tag ):
		( namespace, subtag ) = SplitTag( tag )

		tag_slices = []

		tag_slices.append( tag )

		if namespace != '':
			tag_slices.append( namespace + ':' )
			tag_slices.append( ':' )

		else:
			tag_slices.append( '' )

		return tag_slices

	def _GetSerialisableInfo( self ):
		return list( self._tag_slices_to_rules.items() )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		self._tag_slices_to_rules = dict( serialisable_info )

	def _TagOK( self, tag ):
		tag_slices = self._GetTagSlices( tag )

		blacklist_encountered = False

		for tag_slice in tag_slices:
			if tag_slice in self._tag_slices_to_rules:
				rule = self._tag_slices_to_rules[ tag_slice ]

				if rule == CC.FILTER_WHITELIST:
					return True # there is an exception for this class of tag

				elif rule == CC.FILTER_BLACKLIST: # there is a rule against this class of tag

					blacklist_encountered = True

		if blacklist_encountered: # rule against and no exceptions

			return False

		else:
			return True # no rules against or explicitly for, so permitted

	def AllowsEverything( self ):
		with self._lock:
			for ( tag_slice, rule ) in self._tag_slices_to_rules.items():
				if rule == CC.FILTER_BLACKLIST:
					return False

			return True

	def Filter( self, tags ):
		with self._lock:
			return { tag for tag in tags if self._TagOK( tag ) }

	def GetTagSlicesToRules( self ):
		with self._lock:
			return dict( self._tag_slices_to_rules )

	def SetRule( self, tag_slice, rule ):
		with self._lock:
			self._tag_slices_to_rules[ tag_slice ] = rule

	def ToBlacklistString( self ):
		with self._lock:
			blacklist = []
			whitelist = []

			for ( tag_slice, rule ) in self._tag_slices_to_rules.items():
				if rule == CC.FILTER_BLACKLIST:
					blacklist.append( tag_slice )

				elif rule == CC.FILTER_WHITELIST:
					whitelist.append( tag_slice )

			blacklist.sort()
			whitelist.sort()

			if len( blacklist ) == 0:
				return 'no blacklist set'

			else:
				if set( blacklist ) == { '', ':' }:
					text = 'blacklisting on any tags'

				else:
					text = 'blacklisting on ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in blacklist ) )

				if len( whitelist ) > 0:
					text += ' except ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in whitelist ) )

				return text

	def ToCensoredString( self ):
		with self._lock:
			blacklist = []
			whitelist = []

			for ( tag_slice, rule ) in list( self._tag_slices_to_rules.items() ):
				if rule == CC.FILTER_BLACKLIST:
					blacklist.append( tag_slice )

				elif rule == CC.FILTER_WHITELIST:
					whitelist.append( tag_slice )

			blacklist.sort()
			whitelist.sort()

			if len( blacklist ) == 0:
				return 'all tags allowed'

			else:
				if set( blacklist ) == { '', ':' }:
					text = 'no tags allowed'

				else:
					text = 'all but ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in blacklist ) ) + ' allowed'

				if len( whitelist ) > 0:
					text += ' except ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in whitelist ) )

				return text

	def ToPermittedString( self ):
		with self._lock:
			blacklist = []
			whitelist = []

			for ( tag_slice, rule ) in list( self._tag_slices_to_rules.items() ):
				if rule == CC.FILTER_BLACKLIST:
					blacklist.append( tag_slice )

				elif rule == CC.FILTER_WHITELIST:
					whitelist.append( tag_slice )

			blacklist.sort()
			whitelist.sort()

			if len( blacklist ) == 0:
				return 'all tags'

			else:
				if set( blacklist ) == { '', ':' }:
					if len( whitelist ) == 0:
						text = 'no tags'

					else:
						text = 'only ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in whitelist ) )

				elif set( blacklist ) == { '' }:
					text = 'all namespaced tags'

					if len( whitelist ) > 0:
						text += ' and ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in whitelist ) )

				elif set( blacklist ) == { ':' }:
					text = 'all unnamespaced tags'

					if len( whitelist ) > 0:
						text += ' and ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in whitelist ) )

				else:
					text = 'all tags except ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in blacklist ) )

					if len( whitelist ) > 0:
						text += ' (except ' + ', '.join( ( ConvertTagSliceToString( tag_slice ) for tag_slice in whitelist ) ) + ')'

			return text

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["TAG_FILTER"] ] = TagFilter


code:
import collections, re, os

from lib.Hydrus.Exceptions import SizeException
from lib.Hydrus.Text import re_leading_single_colon, re_leading_space_or_garbage, re_multiple_spaces, re_newlines

### FUNCTIONS ###

def CensorshipMatch( tag, censorships ):
	for censorship in censorships:
		if censorship == '': # '' - all non namespaced tags
			( namespace, subtag ) = SplitTag( tag )

			if namespace == '':
				return True

		elif censorship == ':': # ':' - all namespaced tags
			( namespace, subtag ) = SplitTag( tag )

			if namespace != '':
				return True

		elif ':' in censorship:
			if censorship.endswith( ':' ): # 'series:' - namespaced tags
				( namespace, subtag ) = SplitTag( tag )

				if namespace == censorship[:-1]:
					return True

			else: # 'series:evangelion' - exact match with namespace
				if tag == censorship:
					return True

		else:
			# 'table' - normal tag, or namespaced version of same
			( namespace, subtag ) = SplitTag( tag )

			if subtag == censorship:
				return True

	return False

def CollapseMultipleSortedNumericTagsToMinMax( tags ):
	if len( tags ) <= 2:
		return tags

	else:
		includes_non_numeric_tag = True in ( not isinstance( ConvertTagToSortable( tag ), tuple ) for tag in tags )

		if includes_non_numeric_tag:
			return tags

		else:
			# this list of tags is entirely numeric and may well be something like 1, 2, 3, 4, 5
			# the caller wants to present 1-5 instead, so lets cut out the first and last
			if not isinstance( tags, list ):
				tags = list( tags )

			return [ tags[0], tags[-1] ]

def ConvertTagToSortable( tag ):
	# this copies the human sort in hydrusdata
	convert = lambda text: ( '', int( text ) ) if text.isdecimal() else ( text, 0 )

	return tuple( [ convert( c ) for c in re.split( '([0-9]+)', tag.lower() ) ] )

	# old method

	'''if len( t ) > 0 and t[0].isdecimal():
		# We want to maintain that:
		# 0 < 0a < 0b < 1 ( lexicographic comparison )
		# -and-
		# 2 < 22 ( value comparison )
		# So, if the first bit can be turned into an int, split it into ( int, extra )

		int_component = ''

		i = 0

		for character in t:
			if character.isdecimal():
				int_component += character

			else:
				break

			i += 1

		str_component = t[i:]

		number = int( int_component )

		return ( number, str_component )

	else:
		return t
		'''

def FilterNamespaces( tags, namespaces ):
	processed_tags = collections.defaultdict( set )

	for tag in tags:
		( namespace, subtag ) = SplitTag( tag )

		processed_tags[ namespace ].add( tag )

	result = set()

	for namespace in namespaces:
		if namespace is None:
			result.update( processed_tags[ '' ] )

		else:
			result.update( processed_tags[ namespace ] )

	return result

def SortNumericTags( tags ):
	tags = list( tags )

	tags.sort( key = ConvertTagToSortable )

	return tags

def CheckTagNotEmpty( tag ):
	( namespace, subtag ) = SplitTag( tag )

	if subtag == '':
		raise SizeException( 'Received a zero-length tag!' )

def CleanTag( tag ):
	try:
		if tag is None:
			raise Exception()

		tag = tag[:1024]

		tag = tag.lower()

		tag = re_leading_single_colon.sub( '::', tag ) # Convert anything starting with one colon to start with two i.e. :D -> ::D

		if ':' in tag:
			tag = StripTextOfGumpf( tag ) # need to repeat here to catch 'system:' stuff

			( namespace, subtag ) = SplitTag( tag )

			namespace = StripTextOfGumpf( namespace )
			subtag = StripTextOfGumpf( subtag )

			tag = CombineTag( namespace, subtag )

		else:
			tag = StripTextOfGumpf( tag )

	except Exception as e:
		text = 'Was unable to parse the tag: ' + str( tag )
		text += os.linesep * 2
		text += str( e )

		raise Exception( text )

	return tag

def CleanTags( tags ):
	clean_tags = set()

	for tag in tags:
		if tag is None:
			continue

		tag = CleanTag( tag )

		try:
			CheckTagNotEmpty( tag )

		except SizeException:
			continue

		clean_tags.add( tag )

	return clean_tags

def CombineTag( namespace, subtag ):
	if namespace == '':
		if re_leading_single_colon.search( subtag ) is not None:
			return ':' + subtag

		else:
			return subtag

	else:
		return namespace + ':' + subtag

def SplitTag( tag ):
	if ':' in tag:
		return tag.split( ':', 1 )

	else:
		return ( '', tag )

def StripTextOfGumpf( t ):
	t = re_newlines.sub( '', t )

	t = re_multiple_spaces.sub( ' ', t )

	t = t.strip()

	t = re_leading_space_or_garbage.sub( '', t )

	return t


code:
from .Tags import *

__all__ = [
	"Archive",
	"Local",
]

from . import *


code:
import json, re

try:
	import chardet
	CHARDET_OK = True

except Exception:
	CHARDET_OK = False

### CONSTANTS ###
re_newlines = re.compile( '[\r\n]+' )
re_multiple_spaces = re.compile( '\\s+' )
re_leading_space_or_garbage = re.compile( '^( \\s|-|system: )+' )
re_leading_single_colon = re.compile( '^:(?!:)' )
re_leading_byte_order_mark = re.compile( '^\ufeff' ) # unicode .txt files prepend with this, wew

### FUNCTIONS ###

def HexFilter( text ):
	text = text.lower()

	text = re.sub( '[^0123456789abcdef]', '', text )

	return text

def DeserialiseNewlinedTexts( text ):
	texts = text.splitlines()

	texts = [ StripIOInputLine( line ) for line in texts ]

	texts = [ line for line in texts if line != '' ]

	return texts

def LooksLikeHTML( file_data ):
	# this will false-positive if it is json that contains html, ha ha

	if isinstance( file_data, bytes ):
		search_elements = ( b'<html', b'<HTML' )

	else:
		search_elements = ( '<html', '<HTML' )

	for s_e in search_elements:
		if s_e in file_data:
			return True

	return False

def LooksLikeJSON( file_data ):
	try:
		if isinstance( file_data, bytes ):
			file_data = str( file_data, 'utf-8' )

		json.loads( file_data )

		return True

	except Exception:
		return False

def NonFailingUnicodeDecode( data, encoding ):
	try:
		text = str( data, encoding )

	except UnicodeDecodeError:
		unicode_replacement_character = u'\ufffd'

		text = str( data, encoding, errors = 'replace' )

		error_count = text.count( unicode_replacement_character )

		if CHARDET_OK:
			chardet_result = chardet.detect( data )

			if chardet_result[ 'confidence' ] > 0.85:
				chardet_encoding = chardet_result[ 'encoding' ]

				chardet_text = str( data, chardet_encoding, errors = 'replace' )

				chardet_error_count = chardet_text.count( unicode_replacement_character )

				if chardet_error_count < error_count:
					return ( chardet_text, chardet_encoding )

	return ( text, encoding )

def RemoveNewlines( text ):
	text = re.sub( r'\r|\n', '', text )

	return text

def SortStringsIgnoringCase( list_of_strings ):
	list_of_strings.sort( key = lambda s: s.lower() )

def StripIOInputLine( t ):
	t = re_leading_byte_order_mark.sub( '', t )

	t = t.strip()

	return t


code:
import os, threading, time, wx

from lib.Hydrus.Data import GetNow, GetNowFloat, GenerateKey, TimeHasPassed

from lib.Hydrus.Threading import SchedulableJob, IsThreadShuttingDown, RepeatingJob

### CONSTANTS ###

import lib.Hydrus.Globals as HG

### CLASSES ###

class JobKey( object ):
	def __init__( self, pausable = False, cancellable = False, only_when_idle = False, only_start_if_unbusy = False, stop_time = None, cancel_on_shutdown = True ):
		self._key = GenerateKey()

		self._creation_time = GetNowFloat()

		self._pausable = pausable
		self._cancellable = cancellable
		self._only_when_idle = only_when_idle
		self._only_start_if_unbusy = only_start_if_unbusy
		self._stop_time = stop_time
		self._cancel_on_shutdown = cancel_on_shutdown

		self._start_time = GetNow()

		self._deleted = threading.Event()
		self._deletion_time = None
		self._begun = threading.Event()
		self._done = threading.Event()
		self._cancelled = threading.Event()
		self._paused = threading.Event()

		self._yield_pause_period = 10
		self._next_yield_pause = GetNow() + self._yield_pause_period

		self._bigger_pause_period = 100
		self._next_bigger_pause = GetNow() + self._bigger_pause_period

		self._longer_pause_period = 1000
		self._next_longer_pause = GetNow() + self._longer_pause_period

		self._urls = []
		self._variable_lock = threading.Lock()
		self._variables = dict()

	def __eq__( self, other ):
		return self.__hash__() == other.__hash__()

	def __hash__( self ):
		return self._key.__hash__()

	def __ne__( self, other ):
		return self.__hash__() != other.__hash__()

	def _CheckCancelTests( self ):
		if not self._cancelled.is_set():
			should_cancel = False

			if self._cancel_on_shutdown and IsThreadShuttingDown():
				should_cancel = True

			if self._only_when_idle and not HG.client_controller.CurrentlyIdle():
				should_cancel = True

			if self._stop_time is not None:
				if TimeHasPassed( self._stop_time ):
					should_cancel = True

			if should_cancel:
				self.Cancel()

		if not self._deleted.is_set():
			if self._deletion_time is not None:
				if TimeHasPassed( self._deletion_time ):
					self.Finish()

					self._deleted.set()

	def AddURL( self, url ):
		with self._variable_lock:
			self._urls.append( url )

	def Begin( self ):
		self._begun.set()

	def CanBegin( self ):
		self._CheckCancelTests()

		if self.IsCancelled():
			return False

		if self._only_start_if_unbusy and HG.client_controller.SystemBusy():
			return False

		return True

	def Cancel( self, seconds = None ):
		if seconds is None:
			self._cancelled.set()

			self.Finish()

		else:
			HG.client_controller.CallLater( seconds, self.Cancel )

	def Delete( self, seconds = None ):
		if seconds is None:
			self._deletion_time = GetNow()

		else:
			self._deletion_time = GetNow() + seconds

	def DeleteVariable( self, name ):
		with self._variable_lock:
			if name in self._variables:
				del self._variables[ name ]

		time.sleep( 0.00001 )

	def Finish( self, seconds = None ):
		if seconds is None:
			self._done.set()

		else:
			HG.client_controller.CallLater( seconds, self.Finish )

	def GetCreationTime( self ):
		return self._creation_time

	def GetIfHasVariable( self, name ):
		with self._variable_lock:
			if name in self._variables:
				return self._variables[ name ]

			else:
				return None

	def GetKey( self ):
		return self._key

	def GetURLs( self ):
		with self._variable_lock:
			return list( self._urls )

	def HasVariable( self, name ):
		with self._variable_lock:
			return name in self._variables

	def IsBegun( self ):
		self._CheckCancelTests()

		return self._begun.is_set()

	def IsCancellable( self ):
		self._CheckCancelTests()

		return self._cancellable and not self.IsDone()

	def IsCancelled( self ):
		self._CheckCancelTests()

		return self._cancelled.is_set()

	def IsDeleted( self ):
		self._CheckCancelTests()

		return self._deleted.is_set()

	def IsDone( self ):
		self._CheckCancelTests()

		return self._done.is_set()

	def IsPausable( self ):
		self._CheckCancelTests()

		return self._pausable and not self.IsDone()

	def IsPaused( self ):
		self._CheckCancelTests()

		return self._paused.is_set() and not self.IsDone()

	def IsWorking( self ):
		self._CheckCancelTests()

		return self.IsBegun() and not self.IsDone()

	def PausePlay( self ):
		if self._paused.is_set():
			self._paused.clear()
		else:
			self._paused.set()

	def SetCancellable( self, value ):
		self._cancellable = value

	def SetPausable( self, value ):
		self._pausable = value

	def SetVariable( self, name, value ):
		with self._variable_lock:
			self._variables[ name ] = value

		time.sleep( 0.00001 )

	def TimeRunning( self ):
		return GetNow() - self._start_time

	def ToString( self ):
		stuff_to_print = []

		with self._variable_lock:
			if 'popup_title' in self._variables:
				stuff_to_print.append( self._variables[ 'popup_title' ] )

			if 'popup_text_1' in self._variables:
				stuff_to_print.append( self._variables[ 'popup_text_1' ] )

			if 'popup_text_2' in self._variables:
				stuff_to_print.append( self._variables[ 'popup_text_2' ] )

			if 'popup_traceback' in self._variables:
				stuff_to_print.append( self._variables[ 'popup_traceback' ] )

		stuff_to_print = [ str( s ) for s in stuff_to_print ]

		try:
			return os.linesep.join( stuff_to_print )

		except Exception:
			return repr( stuff_to_print )

	def WaitIfNeeded( self ):
		if TimeHasPassed( self._next_yield_pause ):
			time.sleep( 0.1 )

			self._next_yield_pause = GetNow() + self._yield_pause_period

			if TimeHasPassed( self._next_bigger_pause ):
				time.sleep( 1 )

				self._next_bigger_pause = GetNow() + self._bigger_pause_period

				if TimeHasPassed( self._longer_pause_period ):
					time.sleep( 10 )

					self._next_longer_pause = GetNow() + self._longer_pause_period

		i_paused = False
		should_quit = False

		while self.IsPaused():
			i_paused = True

			time.sleep( 0.1 )

			if self.IsDone():
				break

		if self.IsCancelled():
			should_quit = True

		return ( i_paused, should_quit )

class FileRWLock( object ):
	class RLock( object ):
		def __init__( self, parent ):
			self.parent = parent

		def __enter__( self ):
			while not IsThreadShuttingDown():
				with self.parent.lock:
					# if there are no writers, we can start reading

					if self.parent.num_waiting_writers == 0:
						self.parent.num_readers += 1

						return

				# otherwise wait a bit

				self.parent.read_available_event.wait( 1 )

				self.parent.read_available_event.clear()

		def __exit__( self, exc_type, exc_val, exc_tb ):
			with self.parent.lock:
				self.parent.num_readers -= 1

				do_notify = self.parent.num_readers == 0

			if do_notify:
				self.parent.write_available_event.set()

	class WLock( object ):
		def __init__( self, parent ):
			self.parent = parent

		def __enter__( self ):
			# let all the readers know that we are bumping up to the front of the queue
			with self.parent.lock:
				self.parent.num_waiting_writers += 1

			while not IsThreadShuttingDown():
				with self.parent.lock:
					# if nothing reading or writing atm, sieze the opportunity

					if self.parent.num_readers == 0 and not self.parent.there_is_an_active_writer:
						self.parent.there_is_an_active_writer = True

						return

				# otherwise wait a bit

				self.parent.write_available_event.wait( 1 )

				self.parent.write_available_event.clear()

		def __exit__( self, exc_type, exc_val, exc_tb ):
			with self.parent.lock:
				self.parent.there_is_an_active_writer = False

				self.parent.num_waiting_writers -= 1

				do_read_notify = self.parent.num_waiting_writers == 0 # reading is now available
				do_write_notify = self.parent.num_waiting_writers > 0 # another writer is waiting

			if do_read_notify:
				self.parent.read_available_event.set()

			if do_write_notify:
				self.parent.write_available_event.set()

	def __init__( self ):
		self.read = self.RLock( self )
		self.write = self.WLock( self )

		self.lock = threading.Lock()

		self.read_available_event = threading.Event()
		self.write_available_event = threading.Event()

		self.num_readers = 0
		self.num_waiting_writers = 0
		self.there_is_an_active_writer = False

class WXAwareJob( SchedulableJob ):
	def __init__( self, controller, scheduler, window, initial_delay, work_callable ):
		SchedulableJob.__init__( self, controller, scheduler, initial_delay, work_callable )

		self._window = window

	def _BootWorker( self ):
		def wx_code():
			if not self._window:
				return

			self.Work()

		wx.CallAfter( wx_code )

	def _MyWindowDead( self ):
		return not self._window

	def IsCancelled( self ):
		my_window_dead = self._MyWindowDead()

		if my_window_dead:
			self._is_cancelled.set()

		return SchedulableJob.IsCancelled( self )

	def IsDead( self ):
		return self._MyWindowDead()

class WXAwareRepeatingJob( RepeatingJob ):
	def __init__( self, controller, scheduler, window, initial_delay, period, work_callable ):
		RepeatingJob.__init__( self, controller, scheduler, initial_delay, period, work_callable )

		self._window = window

	def _BootWorker( self ):
		def wx_code():
			if not self._window:
				return

			self.Work()

		wx.CallAfter( wx_code )

	def _MyWindowDead( self ):
		return not self._window

	def IsCancelled( self ):
		my_window_dead = self._MyWindowDead()

		if my_window_dead:
			self._is_cancelled.set()

		return SchedulableJob.IsCancelled( self )

	def IsDead( self ):
		return self._MyWindowDead()


code:
import os, bisect, queue, random, threading, time, traceback

from lib.Hydrus.Data import TimeHasPassed, GetNow, GetNowFloat, \
	TimeDeltaToPrettyTimeDelta, TimeHasPassedFloat, Print, ShowText, \
	ShowException, ToHumanInt, GetTimeDeltaUntilTimeFloat

from lib.Hydrus.Exceptions import ShutdownException

### CONSTANTS ###
import lib.Hydrus.Globals as HG

NEXT_THREAD_CLEAROUT = 0

THREADS_TO_THREAD_INFO = {}
THREAD_INFO_LOCK = threading.Lock()

### FUNCTIONS ###

def CheckIfThreadShuttingDown():
	if IsThreadShuttingDown():
		raise ShutdownException( 'Thread is shutting down!' )

def ClearOutDeadThreads():
	with THREAD_INFO_LOCK:
		all_threads = list( THREADS_TO_THREAD_INFO.keys() )

		for thread in all_threads:
			if not thread.is_alive():
				del THREADS_TO_THREAD_INFO[ thread ]

def GetThreadInfo( thread = None ):
	global NEXT_THREAD_CLEAROUT

	if TimeHasPassed( NEXT_THREAD_CLEAROUT ):
		ClearOutDeadThreads()

		NEXT_THREAD_CLEAROUT = GetNow() + 600

	if thread is None:
		thread = threading.current_thread()

	with THREAD_INFO_LOCK:
		if thread not in THREADS_TO_THREAD_INFO:
			thread_info = {}

			thread_info[ 'shutting_down' ] = False

			THREADS_TO_THREAD_INFO[ thread ] = thread_info

		return THREADS_TO_THREAD_INFO[ thread ]

def IsThreadShuttingDown():
	me = threading.current_thread()

	if isinstance( me, DAEMON ):
		if HG.view_shutdown:
			return True

	else:
		if HG.model_shutdown:
			return True

	thread_info = GetThreadInfo()

	return thread_info[ 'shutting_down' ]

def ShutdownThread( thread ):
	thread_info = GetThreadInfo( thread )

	thread_info[ 'shutting_down' ] = True

class DAEMON( threading.Thread ):
	def __init__( self, controller, name ):
		threading.Thread.__init__( self, name = name )

		self._controller = controller
		self._name = name

		self._event = threading.Event()

		self._controller.sub( self, 'wake', 'wake_daemons' )
		self._controller.sub( self, 'shutdown', 'shutdown' )

	def _DoPreCall( self ):
		if HG.daemon_report_mode:
			ShowText( self._name + ' doing a job.' )

	def GetCurrentJobSummary( self ):
		return 'unknown job'

	def GetName( self ):
		return self._name

	def shutdown( self ):
		ShutdownThread( self )

		self.wake()

	def wake( self ):
		self._event.set()

class DAEMONWorker( DAEMON ):
	def __init__( self, controller, name, callable, topics = None, period = 3600, init_wait = 3, pre_call_wait = 0 ):
		if topics is None:
			topics = []

		DAEMON.__init__( self, controller, name )

		self._callable = callable
		self._topics = topics
		self._period = period
		self._init_wait = init_wait
		self._pre_call_wait = pre_call_wait

		for topic in topics:
			self._controller.sub( self, 'set', topic )

		self.start()

	def _CanStart( self ):
		return self._ControllerIsOKWithIt()

	def _ControllerIsOKWithIt( self ):
		return True

	def _DoAWait( self, wait_time, event_can_wake = True ):
		time_to_start = GetNow() + wait_time

		while not TimeHasPassed( time_to_start ):
			if event_can_wake:
				event_was_set = self._event.wait( 1.0 )

				if event_was_set:
					self._event.clear()

					return

			else:
				time.sleep( 1.0 )

			CheckIfThreadShuttingDown()

	def _WaitUntilCanStart( self ):
		while not self._CanStart():
			time.sleep( 1.0 )

			CheckIfThreadShuttingDown()

	def GetCurrentJobSummary( self ):
		return self._callable

	def run( self ):
		try:
			self._DoAWait( self._init_wait )

			while True:
				CheckIfThreadShuttingDown()

				self._DoAWait( self._pre_call_wait, event_can_wake = False )

				CheckIfThreadShuttingDown()

				self._WaitUntilCanStart()

				CheckIfThreadShuttingDown()

				self._DoPreCall()

				try:
					self._callable( self._controller )

				except ShutdownException:
					return

				except Exception as e:
					ShowText( 'Daemon ' + self._name + ' encountered an exception:' )

					ShowException( e )

				self._DoAWait( self._period )

		except ShutdownException:
			return

	def set( self, *args, **kwargs ):
		self._event.set()

# Big stuff like DB maintenance that we don't want to run while other important stuff is going on, like user interaction or vidya on another process

class DAEMONBackgroundWorker( DAEMONWorker ):
	def _ControllerIsOKWithIt( self ):
		return self._controller.GoodTimeToDoBackgroundWork()

# Big stuff that we want to run when the user sees, but not at the expense of something else, like laggy session load

class DAEMONForegroundWorker( DAEMONWorker ):
	def _ControllerIsOKWithIt( self ):
		return self._controller.GoodTimeToDoForegroundWork()

class THREADCallToThread( DAEMON ):
	def __init__( self, controller, name ):
		DAEMON.__init__( self, controller, name )

		self._callable = None

		self._queue = queue.Queue()

		self._currently_working = True # start off true so new threads aren't used twice by two quick successive calls

	def CurrentlyWorking( self ):
		return self._currently_working

	def GetCurrentJobSummary( self ):
		return self._callable

	def put( self, callable, *args, **kwargs ):
		self._currently_working = True

		self._queue.put( ( callable, args, kwargs ) )

		self._event.set()

	def run( self ):
		try:
			while True:
				while self._queue.empty():
					CheckIfThreadShuttingDown()

					self._event.wait( 10.0 )

					self._event.clear()

				CheckIfThreadShuttingDown()

				self._DoPreCall()

				try:
					( callable, args, kwargs ) = self._queue.get()

					self._callable = ( callable, args, kwargs )

					callable( *args, **kwargs )

					self._callable = None

					del callable

				except ShutdownException:
					return

				except Exception as e:
					Print( traceback.format_exc() )

					ShowException( e )

				finally:
					self._currently_working = False

				time.sleep( 0.00001 )

		except ShutdownException:
			return

class JobScheduler( threading.Thread ):
	def __init__( self, controller ):
		threading.Thread.__init__( self, name = 'Job Scheduler' )

		self._controller = controller

		self._waiting = []

		self._waiting_lock = threading.Lock()

		self._new_job_arrived = threading.Event()

		self._current_job = None

		self._cancel_filter_needed = threading.Event()
		self._sort_needed = threading.Event()

		self._controller.sub( self, 'shutdown', 'shutdown' )

	def _FilterCancelled( self ):
		with self._waiting_lock:
			self._waiting = [ job for job in self._waiting if not job.IsCancelled() ]

	def _GetLoopWaitTime( self ):
		with self._waiting_lock:
			if len( self._waiting ) == 0:
				return 0.2

			next_job = self._waiting[0]

		time_delta_until_due = next_job.GetTimeDeltaUntilDue()

		return min( 1.0, time_delta_until_due )

	def _NoWorkToStart( self ):
		with self._waiting_lock:
			if len( self._waiting ) == 0:
				return True

			next_job = self._waiting[0]

		if next_job.IsDue():
			return False

		else:
			return True

	def _SortWaiting( self ):
		# sort the waiting jobs in ascending order of expected work time
		with self._waiting_lock: # this uses __lt__ to sort
			self._waiting.sort()

	def _StartWork( self ):
		jobs_started = 0

		while True:
			with self._waiting_lock:
				if len( self._waiting ) == 0:
					break

				if jobs_started >= 10: # try to avoid spikes
					break

				next_job = self._waiting[0]

				if next_job.IsDue():
					next_job = self._waiting.pop( 0 )

					if next_job.IsCancelled():
						continue

					if next_job.SlotOK():
						next_job.StartWork()

						jobs_started += 1

					else:
						# delay is automatically set by SlotOK
						bisect.insort( self._waiting, next_job )

				else:
					break # all the rest in the queue are not due

	def AddJob( self, job ):
		with self._waiting_lock:
			bisect.insort( self._waiting, job )

		self._new_job_arrived.set()

	def ClearOutDead( self ):
		with self._waiting_lock:
			self._waiting = [ job for job in self._waiting if not job.IsDead() ]

	def GetName( self ):
		return 'Job Scheduler'

	def GetCurrentJobSummary( self ):
		with self._waiting_lock:
			return ToHumanInt( len( self._waiting ) ) + ' jobs'

	def GetPrettyJobSummary( self ):
		with self._waiting_lock:
			num_jobs = len( self._waiting )

			job_lines = [ repr( job ) for job in self._waiting ]

			lines = [ ToHumanInt( num_jobs ) + ' jobs:' ] + job_lines

			text = os.linesep.join( lines )

			return text

	def JobCancelled( self ):
		self._cancel_filter_needed.set()

	def shutdown( self ):
		ShutdownThread( self )

	def WorkTimesHaveChanged( self ):
		self._sort_needed.set()

	def run( self ):
		while True:
			try:
				while self._NoWorkToStart():
					if IsThreadShuttingDown():
						return

					#

					if self._cancel_filter_needed.is_set():
						self._FilterCancelled()

						self._cancel_filter_needed.clear()

					if self._sort_needed.is_set():
						self._SortWaiting()

						self._sort_needed.clear()

						continue # if some work is now due, let's do it!

					#

					wait_time = self._GetLoopWaitTime()

					self._new_job_arrived.wait( wait_time )

					self._new_job_arrived.clear()

				self._StartWork()

			except ShutdownException:
				return

			except Exception as e:
				Print( traceback.format_exc() )

				ShowException( e )

			time.sleep( 0.00001 )

class SchedulableJob( object ):
	def __init__( self, controller, scheduler, initial_delay, work_callable ):
		self._controller = controller
		self._scheduler = scheduler
		self._work_callable = work_callable

		self._should_delay_on_wakeup = False

		self._next_work_time = GetNowFloat() + initial_delay

		self._thread_slot_type = None

		self._work_lock = threading.Lock()

		self._currently_working = threading.Event()
		self._is_cancelled = threading.Event()

	def __lt__( self, other ): # for the scheduler to do bisect.insort noice
		return self._next_work_time < other._next_work_time

	def __repr__( self ):
		return repr( self.__class__ ) + ': ' + repr( self._work_callable ) + ' next in ' + TimeDeltaToPrettyTimeDelta( self._next_work_time - GetNowFloat() )

	def _BootWorker( self ):
		self._controller.CallToThread( self.Work )

	def Cancel( self ):
		self._is_cancelled.set()

		self._scheduler.JobCancelled()

	def CurrentlyWorking( self ):
		return self._currently_working.is_set()

	def GetTimeDeltaUntilDue( self ):
		return GetTimeDeltaUntilTimeFloat( self._next_work_time )

	def IsCancelled( self ):
		return self._is_cancelled.is_set()

	def IsDead( self ):
		return False

	def IsDue( self ):
		return TimeHasPassedFloat( self._next_work_time )

	def PubSubWake( self, *args, **kwargs ):
		self.Wake()

	def SetThreadSlotType( self, thread_type ):
		self._thread_slot_type = thread_type

	def ShouldDelayOnWakeup( self, value ):
		self._should_delay_on_wakeup = value

	def SlotOK( self ):
		if self._thread_slot_type is not None:
			if HG.controller.AcquireThreadSlot( self._thread_slot_type ):
				return True

			else:
				self._next_work_time = GetNowFloat() + 10 + random.random()

				return False

		return True

	def StartWork( self ):
		if self._is_cancelled.is_set():
			return

		self._currently_working.set()

		self._BootWorker()

	def Wake( self, next_work_time = None ):
		if next_work_time is None:
			next_work_time = GetNowFloat()

		self._next_work_time = next_work_time

		self._scheduler.WorkTimesHaveChanged()

	def WakeOnPubSub( self, topic ):
		HG.controller.sub( self, 'PubSubWake', topic )

	def Work( self ):
		try:
			if self._should_delay_on_wakeup:
				while HG.controller.JustWokeFromSleep():
					if IsThreadShuttingDown():
						return

					time.sleep( 1 )

			with self._work_lock:
				self._work_callable()

		finally:
			if self._thread_slot_type is not None:
				HG.controller.ReleaseThreadSlot( self._thread_slot_type )

			self._currently_working.clear()

class RepeatingJob( SchedulableJob ):
	def __init__( self, controller, scheduler, initial_delay, period, work_callable ):
		SchedulableJob.__init__( self, controller, scheduler, initial_delay, work_callable )

		self._period = period

		self._stop_repeating = threading.Event()

	def Cancel( self ):
		SchedulableJob.Cancel( self )

		self._stop_repeating.set()

	def Delay( self, delay ):
		self._next_work_time = GetNowFloat() + delay

		self._scheduler.WorkTimesHaveChanged()

	def IsFinishedWorking( self ):
		return self._stop_repeating.is_set()

	def SetPeriod( self, period ):
		if period > 10.0:
			period += random.random() # smooth out future spikes if ten of these all fire at the same time

		self._period = period

	def StartWork( self ):
		if self._stop_repeating.is_set():
			return

		SchedulableJob.StartWork( self )

	def Work( self ):
		SchedulableJob.Work( self )

		if not self._stop_repeating.is_set():
			self._next_work_time = GetNowFloat() + self._period

			self._scheduler.AddJob( self )


code:
from .Threading import *

__all__ = [
	"Local",
]

from . import *


code:
### INIT ###
FolderModules = [
	"API",
	"Constants",
	"Controller",
	"DB",
	"Files",
	"Paths",
	"Ratings",
	"Serialisable",
	"Server",
	"Services",
	"Tags",
	"Threading"
]

FileModules = [
	"Caches",
	"Daemons",
	"Data",
	"Defaults",
	"Duplicates",
	"Encryption",
	"Exceptions",
	"Globals",
	"Logger",
	"Media",
	"Options",
	"Sessions",
	"Text"
]

__all__ = FolderModules + FileModules

from . import *