code:
import os, threading, collections, time, traceback, urllib.parse

from lib.Actions.Import import ConvertAllParseResultsToFileSeeds

from lib.Handling import GenerateThumbnailBytes, GetExtraHashesFromPath, GetFileInfo, \
	GetMime
from lib.Handling.Image import ConvertToPngIfBmp, IsDecompressionBomb
from lib.Handling.Image.Local import GenerateShapePerceptualHashes
from lib.Handling.Parsing import GetHashesFromParseResults, GetTagsFromParseResults, \
	GetTimestampFromParseResults, GetURLsFromParseResults, GetHashFromPath

from lib.Hydrus.Data import ContentUpdate, ConvertValueRangeToPrettyString, GenerateKey, \
	GetNow, Print, ToHumanInt
from lib.Hydrus.Data.Local import ConvertServiceKeysToTagsToServiceKeysToContentUpdates
from lib.Hydrus.Exceptions import CancelledException, DecompressionBombException, \
	InsufficientCredentialsException, MimeException, NotFoundException, ShutdownException, \
	URLMatchException, VetoException
from lib.Hydrus.Paths import CleanUpTempPath, MirrorFile
from lib.Hydrus.Paths.Local import GetTempPath
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, SerialisableBase, \
	SerialisableList
from lib.Hydrus.Tags import CleanTags
from lib.Hydrus.Tags.Local import ServiceKeysToTags

from lib.Networking.Domain import ConvertHTTPToHTTPS, GetSearchURLs

### CONSTANTS ###

import lib.Hydrus.Globals as HG
import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC

from lib.Hydrus.Serialisable import HG.SerialisableType["FILE_SEED"], HG.SerialisableType["FILE_SEED_CACHE"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES

FILE_SEED_TYPE_HDD = 0
FILE_SEED_TYPE_URL = 1

### FUNCTIONS ###

def GenerateFileSeedCacheStatus( file_seed_cache ):
	statuses_to_counts = file_seed_cache.GetStatusesToCounts()

	return GenerateStatusesToCountsStatus( statuses_to_counts )

def GenerateFileSeedCachesStatus( file_seed_caches ):
	statuses_to_counts = collections.Counter()

	for file_seed_cache in file_seed_caches:
		statuses_to_counts.update( file_seed_cache.GetStatusesToCounts() )

	return GenerateStatusesToCountsStatus( statuses_to_counts )

def GenerateStatusesToCountsStatus( statuses_to_counts ):
	num_successful_and_new = statuses_to_counts[ CC.STATUS_SUCCESSFUL_AND_NEW ]
	num_successful_but_redundant = statuses_to_counts[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ]
	num_ignored = statuses_to_counts[ CC.STATUS_VETOED ]
	num_deleted = statuses_to_counts[ CC.STATUS_DELETED ]
	num_failed = statuses_to_counts[ CC.STATUS_ERROR ]
	num_skipped = statuses_to_counts[ CC.STATUS_SKIPPED ]
	num_unknown = statuses_to_counts[ CC.STATUS_UNKNOWN ]

	status_strings = []

	num_successful = num_successful_and_new + num_successful_but_redundant

	if num_successful > 0:
		s = ToHumanInt( num_successful ) + ' successful'

		if num_successful_and_new > 0:
			if num_successful_but_redundant > 0:
				s += ' (' + ToHumanInt( num_successful_but_redundant ) + ' already in db)'

		else:
			s += ' (all already in db)'

		status_strings.append( s )

	if num_ignored > 0:
		status_strings.append( ToHumanInt( num_ignored ) + ' ignored' )

	if num_deleted > 0:
		status_strings.append( ToHumanInt( num_deleted ) + ' previously deleted' )

	if num_failed > 0:
		status_strings.append( ToHumanInt( num_failed ) + ' failed' )

	if num_skipped > 0:
		status_strings.append( ToHumanInt( num_skipped ) + ' skipped' )

	status = ', '.join( status_strings )

	#

	total = sum( statuses_to_counts.values() )

	total_processed = total - num_unknown

	#

	simple_status = ''

	if total > 0:
		if num_unknown > 0:
			simple_status += ConvertValueRangeToPrettyString( total_processed, total )

		else:
			simple_status += ToHumanInt( total_processed )

		show_new_on_file_seed_short_summary = HG.client_controller.new_options.GetBoolean( 'show_new_on_file_seed_short_summary' )

		if show_new_on_file_seed_short_summary and num_successful_and_new:
			simple_status += ' - ' + ToHumanInt( num_successful_and_new ) + 'N'

		simple_status_strings = []

		if num_ignored > 0:
			simple_status_strings.append( ToHumanInt( num_ignored ) + 'Ig' )

		show_deleted_on_file_seed_short_summary = HG.client_controller.new_options.GetBoolean( 'show_deleted_on_file_seed_short_summary' )

		if show_deleted_on_file_seed_short_summary and num_deleted > 0:
			simple_status_strings.append( ToHumanInt( num_deleted ) + 'D' )

		if num_failed > 0:
			simple_status_strings.append( ToHumanInt( num_failed ) + 'F' )

		if num_skipped > 0:
			simple_status_strings.append( ToHumanInt( num_skipped ) + 'S' )

		if len( simple_status_strings ) > 0:
			simple_status += ' - ' + ''.join( simple_status_strings )

	return ( status, simple_status, ( total_processed, total ) )

class FileImportJob( object ):
	def __init__( self, temp_path, file_import_options = None ):
		if file_import_options is None:
			file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )

		self._temp_path = temp_path
		self._file_import_options = file_import_options

		self._hash = None
		self._pre_import_status = None

		self._file_info = None
		self._thumbnail = None
		self._phashes = None
		self._extra_hashes = None

	def CheckIsGoodToImport( self ):
		( size, mime, width, height, duration, num_frames, num_words ) = self._file_info

		self._file_import_options.CheckFileIsValid( size, mime, width, height )

	def GetExtraHashes( self ):
		return self._extra_hashes

	def GetFileImportOptions( self ):
		return self._file_import_options

	def GetFileInfo( self ):
		return self._file_info

	def GetHash( self ):
		return self._hash

	def GetMime( self ):
		( size, mime, width, height, duration, num_frames, num_words ) = self._file_info

		return mime

	def GetPreImportStatus( self ):
		return self._pre_import_status

	def GetPHashes( self ):
		return self._phashes

	def GetTempPathAndThumbnail( self ):
		return ( self._temp_path, self._thumbnail )

	def PubsubContentUpdates( self ):
		if self._pre_import_status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
			if self._file_import_options.AutomaticallyArchives():
				service_keys_to_content_updates = { CC.COMBINED_LOCAL_FILE_SERVICE_KEY : [ ContentUpdate( HC.CONTENT_TYPE_FILES, HC.CONTENT_UPDATE_ARCHIVE, set( ( self._hash, ) ) ) ] }

				HG.client_controller.Write( 'content_updates', service_keys_to_content_updates )

	def IsNewToDB( self ):
		if self._pre_import_status == CC.STATUS_UNKNOWN:
			return True

		if self._pre_import_status == CC.STATUS_DELETED:
			if not self._file_import_options.ExcludesDeleted():
				return True

		return False

	def GenerateHashAndStatus( self ):
		ConvertToPngIfBmp( self._temp_path )

		self._hash = GetHashFromPath( self._temp_path )

		( self._pre_import_status, hash, note ) = HG.client_controller.Read( 'hash_status', 'sha256', self._hash, prefix = 'file recognised' )

		return ( self._pre_import_status, self._hash, note )

	def GenerateInfo( self ):
		mime = GetMime( self._temp_path )

		# new_options = HG.client_controller.new_options

		if mime in HC.DECOMPRESSION_BOMB_IMAGES and not self._file_import_options.AllowsDecompressionBombs():
			if IsDecompressionBomb( self._temp_path ):
				raise DecompressionBombException( 'Image seems to be a Decompression Bomb!' )

		self._file_info = GetFileInfo( self._temp_path, mime )

		( size, mime, width, height, duration, num_frames, num_words ) = self._file_info

		if mime in HC.MIMES_WITH_THUMBNAILS:
			bounding_dimensions = HG.client_controller.options[ 'thumbnail_dimensions' ]

			percentage_in = HG.client_controller.new_options.GetInteger( 'video_thumbnail_percentage_in' )

			self._thumbnail = GenerateThumbnailBytes( self._temp_path, bounding_dimensions, mime, width, height, duration, num_frames, percentage_in = percentage_in )

		if mime in HC.MIMES_WE_CAN_PHASH:
			self._phashes = GenerateShapePerceptualHashes( self._temp_path, mime )

		self._extra_hashes = GetExtraHashesFromPath( self._temp_path )

class FileSeed( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["FILE_SEED"]
	SERIALISABLE_NAME = 'File Import'
	SERIALISABLE_VERSION = 3

	def __init__( self, file_seed_type = None, file_seed_data = None ):
		if file_seed_type is None:
			file_seed_type = FILE_SEED_TYPE_URL

		if file_seed_data is None:
			file_seed_data = 'https://big-guys.4u/monica_lewinsky_hott.tiff.exe.vbs'

		SerialisableBase.__init__( self )

		self.file_seed_type = file_seed_type
		self.file_seed_data = file_seed_data

		self.created = GetNow()
		self.modified = self.created
		self.source_time = None
		self.status = CC.STATUS_UNKNOWN
		self.note = ''

		self._referral_url = None

		self._fixed_service_keys_to_tags = ServiceKeysToTags()

		self._urls = set()
		self._tags = set()
		self._hashes = {}

	def __eq__( self, other ):
		return self.__hash__() == other.__hash__()

	def __hash__( self ):
		return ( self.file_seed_type, self.file_seed_data ).__hash__()

	def __ne__( self, other ):
		return self.__hash__() != other.__hash__()

	def _CheckTagsBlacklist( self, tags, tag_import_options ):
		tag_import_options.CheckBlacklist( tags )

	def _GetSerialisableInfo( self ):
		serialisable_fixed_service_keys_to_tags = self._fixed_service_keys_to_tags.GetSerialisableTuple()

		serialisable_urls = list( self._urls )
		serialisable_tags = list( self._tags )
		serialisable_hashes = [ ( hash_type, hash.hex() ) for ( hash_type, hash ) in list( self._hashes.items() ) if hash is not None ]

		return ( self.file_seed_type, self.file_seed_data, self.created, self.modified, self.source_time, self.status, self.note, self._referral_url, serialisable_fixed_service_keys_to_tags, serialisable_urls, serialisable_tags, serialisable_hashes )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self.file_seed_type, self.file_seed_data, self.created, self.modified, self.source_time, self.status, self.note, self._referral_url, serialisable_fixed_service_keys_to_tags, serialisable_urls, serialisable_tags, serialisable_hashes ) = serialisable_info

		self._fixed_service_keys_to_tags = CreateFromSerialisableTuple( serialisable_fixed_service_keys_to_tags )

		self._urls = set( serialisable_urls )
		self._tags = set( serialisable_tags )
		self._hashes = { hash_type : bytes.fromhex( encoded_hash ) for ( hash_type, encoded_hash ) in serialisable_hashes if encoded_hash is not None }

	def _NormaliseAndFilterAssociableURLs( self, urls ):
		normalised_urls = set()

		for url in urls:
			try:
				url = HG.client_controller.network_engine.domain_manager.NormaliseURL( url )

			except URLMatchException:
				continue # not a url--something like "file:///C:/Users/Tall%20Man/Downloads/maxresdefault.jpg" ha ha ha

			normalised_urls.add( url )

		associable_urls = { url for url in normalised_urls if HG.client_controller.network_engine.domain_manager.ShouldAssociateURLWithFiles( url ) }

		return associable_urls

	def _SetupTagImportOptions( self, given_tag_import_options ):
		if given_tag_import_options.IsDefault():
			if self.IsAPostURL():
				tio_lookup_url = self.file_seed_data

			else:
				if self._referral_url is not None:
					tio_lookup_url = self._referral_url

				else:
					tio_lookup_url = self.file_seed_data

			tag_import_options = HG.client_controller.network_engine.domain_manager.GetDefaultTagImportOptionsForURL( tio_lookup_url )

		else:
			tag_import_options = given_tag_import_options

		return tag_import_options

	def _UpdateModified( self ):
		self.modified = GetNow()

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( file_seed_type, file_seed_data, created, modified, source_time, status, note, serialisable_urls, serialisable_tags, serialisable_hashes ) = old_serialisable_info

			referral_url = None

			new_serialisable_info = ( file_seed_type, file_seed_data, created, modified, source_time, status, note, referral_url, serialisable_urls, serialisable_tags, serialisable_hashes )

			return ( 2, new_serialisable_info )

		if version == 2:
			( file_seed_type, file_seed_data, created, modified, source_time, status, note, referral_url, serialisable_urls, serialisable_tags, serialisable_hashes ) = old_serialisable_info

			fixed_service_keys_to_tags = ServiceKeysToTags()

			serialisable_fixed_service_keys_to_tags = fixed_service_keys_to_tags.GetSerialisableTuple()

			new_serialisable_info = ( file_seed_type, file_seed_data, created, modified, source_time, status, note, referral_url, serialisable_fixed_service_keys_to_tags, serialisable_urls, serialisable_tags, serialisable_hashes )

			return ( 3, new_serialisable_info )

	def AddParseResults( self, parse_results, file_import_options ):
		for ( hash_type, hash ) in GetHashesFromParseResults( parse_results ):
			if hash_type not in self._hashes:
				self._hashes[ hash_type ] = hash

		if file_import_options.ShouldAssociateSourceURLs():
			source_urls = GetURLsFromParseResults( parse_results, ( HC.URL_TYPE_SOURCE, ) )

			associable_urls = self._NormaliseAndFilterAssociableURLs( source_urls )

			associable_urls.discard( self.file_seed_data )

			self._urls.update( associable_urls )

		tags = GetTagsFromParseResults( parse_results )

		self._tags.update( tags )

		source_timestamp = GetTimestampFromParseResults( parse_results, HC.TIMESTAMP_TYPE_SOURCE )

		if source_timestamp is not None:
			source_timestamp = min( GetNow() - 30, source_timestamp )

			self.source_time = source_timestamp

		self._UpdateModified()

	def AddTags( self, tags ):
		tags = CleanTags( tags )

		self._tags.update( tags )

		self._UpdateModified()

	def AddURL( self, url ):
		urls = ( url, )

		associable_urls = self._NormaliseAndFilterAssociableURLs( urls )

		associable_urls.discard( self.file_seed_data )

		self._urls.update( associable_urls )

	def CheckPreFetchMetadata( self, tag_import_options ):
		self._CheckTagsBlacklist( self._tags, tag_import_options )

	def DownloadAndImportRawFile( self, file_url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook, override_bandwidth = False ):
		self.AddURL( file_url )

		( os_file_handle, temp_path ) = GetTempPath()

		try:
			if self.file_seed_data != file_url:
				referral_url = self.file_seed_data

			else:
				referral_url = self._referral_url

			status_hook( 'downloading file' )

			network_job = network_job_factory( 'GET', file_url, temp_path = temp_path, referral_url = referral_url )

			if override_bandwidth:
				network_job.OverrideBandwidth( 30 )

			network_job.SetFileImportOptions( file_import_options )

			HG.client_controller.network_engine.AddJob( network_job )

			with network_job_presentation_context_factory( network_job ):
				network_job.WaitUntilDone()

			status_hook( 'importing file' )

			self.Import( temp_path, file_import_options )

		finally:
			CleanUpTempPath( os_file_handle, temp_path )

	def FetchPageMetadata( self, tag_import_options ):
		pass

	def GetExampleNetworkJob( self, network_job_factory ):
		if self.IsAPostURL():
			post_url = self.file_seed_data

			( url_to_check, parser ) = HG.client_controller.network_engine.domain_manager.GetURLToFetchAndParser( post_url )

		else:
			url_to_check = self.file_seed_data

		network_job = network_job_factory( 'GET', url_to_check )

		return network_job

	def GetHash( self ):
		if 'sha256' in self._hashes:
			return self._hashes[ 'sha256' ]

		return None

	def GetPreImportStatusPredictionHash( self, file_import_options ):
		UNKNOWN_DEFAULT = ( CC.STATUS_UNKNOWN, None, '' )

		( status, hash, note ) = UNKNOWN_DEFAULT

		if file_import_options.DoNotCheckHashesBeforeImporting():
			return ( status, hash, note )

		# hashes

		if status == CC.STATUS_UNKNOWN:
			for ( hash_type, found_hash ) in list( self._hashes.items() ):
				( status, hash, note ) = HG.client_controller.Read( 'hash_status', hash_type, found_hash, prefix = 'hash recognised' )

				if status != CC.STATUS_UNKNOWN:
					break

		if status == CC.STATUS_DELETED:
			if not file_import_options.ExcludesDeleted():
				( status, hash, note ) = UNKNOWN_DEFAULT

		return ( status, hash, note )

	def GetPreImportStatusPredictionURL( self, file_import_options, file_url = None ):
		UNKNOWN_DEFAULT = ( CC.STATUS_UNKNOWN, None, '' )

		( status, hash, note ) = UNKNOWN_DEFAULT

		if file_import_options.DoNotCheckKnownURLsBeforeImporting():
			return ( status, hash, note )

		# urls

		urls = set( self._urls )

		if file_url is not None:
			urls.add( file_url )

		if self.file_seed_type == FILE_SEED_TYPE_URL:
			urls.add( self.file_seed_data )

		# unrecognised_url_results = set()

		for url in urls:
			if HG.client_controller.network_engine.domain_manager.URLCanReferToMultipleFiles( url ):
				continue

			# we now only trust url-matched single urls and the post/file urls
			# trusting unmatched source urls was too much of a hassle with too many boorus providing bad source urls like user account pages

			if HG.client_controller.network_engine.domain_manager.URLDefinitelyRefersToOneFile( url ) or url in ( self.file_seed_data, file_url ):
				results = HG.client_controller.Read( 'url_statuses', url )

				if len( results ) == 0: # if no match found, no useful data discovered

					continue

				elif len( results ) > 1: # if more than one file claims this url, it cannot be relied on to guess the file

					continue

				else: # i.e. 1 match found

					( status, hash, note ) = results[0]

					if status != CC.STATUS_UNKNOWN:
						# a known one-file url has given a single clear result. sounds good

						we_have_a_match = True

						if self.file_seed_type == FILE_SEED_TYPE_URL:
							# to double-check, let's see if the file that claims that url has any other interesting urls
							# if the file has another url with the same url class as ours, then this is prob an unreliable 'alternate' source url attribution, and untrustworthy

							my_url = self.file_seed_data

							if url != my_url:
								my_url_match = HG.client_controller.network_engine.domain_manager.GetURLMatch( my_url )

								( media_result, ) = HG.client_controller.Read( 'media_results', ( hash, ) )

								this_files_urls = media_result.GetLocationsManager().GetURLs()

								for this_files_url in this_files_urls:
									if this_files_url != my_url:
										this_url_match = HG.client_controller.network_engine.domain_manager.GetURLMatch( this_files_url )

										if my_url_match == this_url_match:
											# oh no, the file this source url refers to has a different known url in this same domain
											# it is more likely that an edit on this site points to the original elsewhere

											( status, hash, note ) = UNKNOWN_DEFAULT

											we_have_a_match = False

											break

						if we_have_a_match:
							break # if a known one-file url gives a single clear result, that result is reliable

		if status == CC.STATUS_DELETED:
			if not file_import_options.ExcludesDeleted():
				( status, hash, note ) = UNKNOWN_DEFAULT

		return ( status, hash, note )

	def GetSearchFileSeeds( self ):
		if self.file_seed_type == FILE_SEED_TYPE_URL:
			search_urls = GetSearchURLs( self.file_seed_data )

			search_file_seeds = [ FileSeed( FILE_SEED_TYPE_URL, search_url ) for search_url in search_urls ]

		else:
			search_file_seeds = [ self ]

		return search_file_seeds

	def HasHash( self ):
		return self.GetHash() is not None

	def Import( self, temp_path, file_import_options ):
		file_import_job = FileImportJob( temp_path, file_import_options )

		( status, hash, note ) = HG.client_controller.client_files_manager.ImportFile( file_import_job )

		self.SetStatus( status, note = note )
		self.SetHash( hash )

	def ImportPath( self, file_seed_cache, file_import_options, limited_mimes = None ):
		try:
			if self.file_seed_type != FILE_SEED_TYPE_HDD:
				raise VetoException( 'Attempted to import as a path, but I do not think I am a path!' )

			path = self.file_seed_data

			if not os.path.exists( path ):
				raise VetoException( 'Source file does not exist!' )

			if limited_mimes is not None:
				mime = GetMime( path )

				if mime not in limited_mimes:
					raise VetoException( 'Not in allowed mimes!' )

			( os_file_handle, temp_path ) = GetTempPath()

			try:
				copied = MirrorFile( path, temp_path )

				if not copied:
					raise Exception( 'File failed to copy to temp path--see log for error.' )

				self.Import( temp_path, file_import_options )

			finally:
				CleanUpTempPath( os_file_handle, temp_path )

			self.WriteContentUpdates()

		except MimeException as e:
			self.SetStatus( CC.STATUS_ERROR, exception = e )

		except VetoException as e:
			self.SetStatus( CC.STATUS_VETOED, note = str( e ) )

		except Exception as e:
			self.SetStatus( CC.STATUS_ERROR, exception = e )

		file_seed_cache.NotifyFileSeedsUpdated( ( self, ) )

	def IsAPostURL( self ):
		if self.file_seed_type == FILE_SEED_TYPE_URL:
			( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( self.file_seed_data )

			if url_type == HC.URL_TYPE_POST:
				return True

		return False

	def IsDeleted( self ):
		return self.status == CC.STATUS_DELETED

	def Normalise( self ):
		if self.file_seed_type == FILE_SEED_TYPE_URL:
			self.file_seed_data = HG.client_controller.network_engine.domain_manager.NormaliseURL( self.file_seed_data )

	def PredictPreImportStatus( self, file_import_options, tag_import_options, file_url = None ):
		( url_status, url_hash, url_note ) = self.GetPreImportStatusPredictionURL( file_import_options, file_url = file_url )
		( hash_status, hash_hash, hash_note ) = self.GetPreImportStatusPredictionHash( file_import_options )

		url_recognised_and_file_already_in_db = url_status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT
		hash_recognised_and_file_already_in_db = hash_status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT

		# now let's set the prediction

		if hash_status != CC.STATUS_UNKNOWN: # trust hashes over urls m8

			( status, hash, note ) = ( hash_status, hash_hash, hash_note )

		else:
			( status, hash, note ) = ( url_status, url_hash, url_note )

		if self.status == CC.STATUS_UNKNOWN and status != CC.STATUS_UNKNOWN:
			self.status = status

			if hash is not None:
				self._hashes[ 'sha256' ] = hash

			self.note = note

			self._UpdateModified()

		# and make some recommendations

		should_download_file = self.status == CC.STATUS_UNKNOWN

		should_download_metadata = should_download_file # if we want the file, we need the metadata to get the file_url!

		# but if we otherwise still want to force some tags, let's do it
		if not should_download_metadata and tag_import_options.WorthFetchingTags():
			url_override = url_recognised_and_file_already_in_db and tag_import_options.ShouldFetchTagsEvenIfURLKnownAndFileAlreadyInDB()
			hash_override = hash_recognised_and_file_already_in_db and tag_import_options.ShouldFetchTagsEvenIfHashKnownAndFileAlreadyInDB()

			if url_override or hash_override:
				should_download_metadata = True

		return ( should_download_metadata, should_download_file )

	def PresentToPage( self, page_key ):
		hash = self.GetHash()

		if hash is not None:
			( media_result, ) = HG.client_controller.Read( 'media_results', ( hash, ) )

			HG.client_controller.pub( 'add_media_results', page_key, ( media_result, ) )

	def SetFixedServiceKeysToTags( self, service_keys_to_tags ):
		self._fixed_service_keys_to_tags = ServiceKeysToTags( service_keys_to_tags )

	def SetHash( self, hash ):
		if hash is not None:
			self._hashes[ 'sha256' ] = hash

	def SetReferralURL( self, referral_url ):
		self._referral_url = referral_url

	def SetStatus( self, status, note = '', exception = None ):
		if exception is not None:
			first_line = str( exception ).split( os.linesep )[0]

			note = first_line + '\u2026 (Copy note to see full error)'
			note += os.linesep
			note += traceback.format_exc()

			Print( 'Error when processing ' + self.file_seed_data + ' !' )
			Print( traceback.format_exc() )

		self.status = status
		self.note = note

		self._UpdateModified()

	def ShouldPresent( self, file_import_options, in_inbox = None ):
		hash = self.GetHash()

		if hash is not None and self.status in CC.SUCCESSFUL_IMPORT_STATES:
			if in_inbox is None:
				if file_import_options.ShouldPresentIgnorantOfInbox( self.status ):
					return True

				if file_import_options.ShouldNotPresentIgnorantOfInbox( self.status ):
					return False

				in_inbox = HG.client_controller.Read( 'in_inbox', hash )

			if file_import_options.ShouldPresent( self.status, in_inbox ):
				return True

		return False

	def WorksInNewSystem( self ):
		if self.file_seed_type == FILE_SEED_TYPE_URL:
			( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( self.file_seed_data )

			if url_type == HC.URL_TYPE_FILE:
				return True

			if url_type == HC.URL_TYPE_POST and can_parse:
				return True

			if url_type == HC.URL_TYPE_UNKNOWN and self._referral_url is not None: # this is likely be a multi-file child of a post url file_seed

				( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( self._referral_url )

				if url_type == HC.URL_TYPE_POST: # we must have got here through parsing that m8, so let's assume this is an unrecognised file url

					return True

		return False

	def WorkOnURL( self, file_seed_cache, status_hook, network_job_factory, network_job_presentation_context_factory, file_import_options, tag_import_options ):
		did_substantial_work = False

		try:
			( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( self.file_seed_data )

			if url_type not in ( HC.URL_TYPE_POST, HC.URL_TYPE_FILE, HC.URL_TYPE_UNKNOWN ):
				raise VetoException( 'This URL appeared to be a "{}", which is not a File or Post URL!'.format( match_name ) )

			if url_type == HC.URL_TYPE_POST and not can_parse:
				raise VetoException( 'Did not have a parser for this URL!' )

			tag_import_options = self._SetupTagImportOptions( tag_import_options )

			status_hook( 'checking url status' )

			( should_download_metadata, should_download_file ) = self.PredictPreImportStatus( file_import_options, tag_import_options )

			if self.IsAPostURL():
				if should_download_metadata:
					did_substantial_work = True

					post_url = self.file_seed_data

					( url_to_check, parser ) = HG.client_controller.network_engine.domain_manager.GetURLToFetchAndParser( post_url )

					status_hook( 'downloading file page' )

					if self._referral_url not in ( post_url, url_to_check ):
						referral_url = self._referral_url

					else:
						referral_url = None

					network_job = network_job_factory( 'GET', url_to_check, referral_url = referral_url )

					HG.client_controller.network_engine.AddJob( network_job )

					with network_job_presentation_context_factory( network_job ):
						network_job.WaitUntilDone()

					parsing_text = network_job.GetContentText()

					parsing_context = {}

					parsing_context[ 'post_url' ] = post_url
					parsing_context[ 'url' ] = url_to_check

					all_parse_results = parser.Parse( parsing_context, parsing_text )

					if len( all_parse_results ) == 0:
						raise VetoException( 'The parser found nothing in the document!' )

					elif len( all_parse_results ) > 1:
						# multiple child urls generated by a subsidiary page parser

						file_seeds = ConvertAllParseResultsToFileSeeds( all_parse_results, self.file_seed_data, file_import_options )

						for file_seed in file_seeds:
							file_seed._fixed_service_keys_to_tags = self._fixed_service_keys_to_tags.Duplicate()

						try:
							my_index = file_seed_cache.GetFileSeedIndex( self )

							insertion_index = my_index + 1

						except Exception:
							insertion_index = len( file_seed_cache )

						num_urls_added = file_seed_cache.InsertFileSeeds( insertion_index, file_seeds )

						status = CC.STATUS_SUCCESSFUL_AND_NEW
						note = 'Found ' + ToHumanInt( num_urls_added ) + ' new URLs.'

						self.SetStatus( status, note = note )

					else:
						# no subsidiary page parser results, just one

						parse_results = all_parse_results[0]

						self.AddParseResults( parse_results, file_import_options )

						self.CheckPreFetchMetadata( tag_import_options )

						desired_urls = GetURLsFromParseResults( parse_results, ( HC.URL_TYPE_DESIRED, ), only_get_top_priority = True )

						child_urls = []

						if len( desired_urls ) == 0:
							raise VetoException( 'Could not find a file or post URL to download!' )

						elif len( desired_urls ) == 1:
							desired_url = desired_urls[0]

							( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( desired_url )

							if url_type in ( HC.URL_TYPE_FILE, HC.URL_TYPE_UNKNOWN ):
								file_url = desired_url

								( should_download_metadata, should_download_file ) = self.PredictPreImportStatus( file_import_options, tag_import_options, file_url )

								if should_download_file:
									self.DownloadAndImportRawFile( file_url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook, override_bandwidth = True )

							elif url_type == HC.URL_TYPE_POST and can_parse:
								# a pixiv mode=medium page has spawned a mode=manga page, so we need a new file_seed to go pursue that

								child_urls = [ desired_url ]

							else:
								raise VetoException( 'Found a URL--' + desired_url + '--but could not understand/parse it!' )

						else:
							child_urls = desired_urls

						if len( child_urls ) > 0:
							child_file_seeds = []

							for child_url in child_urls:
								duplicate_file_seed = self.Duplicate() # inherits all urls and tags from here

								duplicate_file_seed.file_seed_data = child_url

								duplicate_file_seed.SetReferralURL( self.file_seed_data )

								if self._referral_url is not None:
									duplicate_file_seed.AddURL( self._referral_url )

								child_file_seeds.append( duplicate_file_seed )

							try:
								my_index = file_seed_cache.GetFileSeedIndex( self )

								insertion_index = my_index + 1

							except Exception:
								insertion_index = len( file_seed_cache )

							num_urls_added = file_seed_cache.InsertFileSeeds( insertion_index, child_file_seeds )

							status = CC.STATUS_SUCCESSFUL_AND_NEW
							note = 'Found ' + ToHumanInt( num_urls_added ) + ' new URLs.'

							self.SetStatus( status, note = note )

			else:
				if should_download_file:
					did_substantial_work = True

					file_url = self.file_seed_data

					self.DownloadAndImportRawFile( file_url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook )

			did_substantial_work |= self.WriteContentUpdates( tag_import_options )

		except ShutdownException:
			return False

		except VetoException as e:
			status = CC.STATUS_VETOED

			note = str( e )

			self.SetStatus( status, note = note )

			if isinstance( e, CancelledException ):
				status_hook( 'cancelled!' )

				time.sleep( 2 )

		except InsufficientCredentialsException:
			status = CC.STATUS_VETOED
			note = '403'

			self.SetStatus( status, note = note )

			status_hook( '403' )

			time.sleep( 2 )

		except NotFoundException:
			status = CC.STATUS_VETOED
			note = '404'

			self.SetStatus( status, note = note )

			status_hook( '404' )

			time.sleep( 2 )

		except Exception as e:
			status = CC.STATUS_ERROR

			self.SetStatus( status, exception = e )

			status_hook( 'error!' )

			time.sleep( 3 )

		file_seed_cache.NotifyFileSeedsUpdated( ( self, ) )

		return did_substantial_work

	def WriteContentUpdates( self, tag_import_options = None ):
		did_work = False

		if self.status == CC.STATUS_ERROR:
			return did_work

		hash = self.GetHash()

		if hash is None:
			return did_work

		# changed this to say that urls alone are not 'did work' since all url results are doing this, and when they have no tags, they are usually superfast db hits anyway
		# better to scream through an 'already in db' import list that flicker

		service_keys_to_content_updates = collections.defaultdict( list )

		urls = set( self._urls )

		if self.file_seed_type == FILE_SEED_TYPE_URL:
			urls.add( self.file_seed_data )

		if self._referral_url is not None:
			urls.add( self._referral_url )

		associable_urls = self._NormaliseAndFilterAssociableURLs( urls )

		if len( associable_urls ) > 0:
			content_update = ContentUpdate( HC.CONTENT_TYPE_URLS, HC.CONTENT_UPDATE_ADD, ( associable_urls, ( hash, ) ) )

			service_keys_to_content_updates[ CC.COMBINED_LOCAL_FILE_SERVICE_KEY ].append( content_update )

		if tag_import_options is not None:
			in_inbox = HG.client_controller.Read( 'in_inbox', hash )

			for ( service_key, content_updates ) in tag_import_options.GetServiceKeysToContentUpdates( self.status, in_inbox, hash, set( self._tags ) ).items():
				service_keys_to_content_updates[ service_key ].extend( content_updates )

				did_work = True

		for ( service_key, content_updates ) in ConvertServiceKeysToTagsToServiceKeysToContentUpdates( ( hash, ), self._fixed_service_keys_to_tags ).items():
			service_keys_to_content_updates[ service_key ].extend( content_updates )

			did_work = True

		if len( service_keys_to_content_updates ) > 0:
			HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )

		return did_work

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["FILE_SEED"] ] = FileSeed

class FileSeedCache( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["FILE_SEED_CACHE"]
	SERIALISABLE_NAME = 'Import File Status Cache'
	SERIALISABLE_VERSION = 8

	COMPACT_NUMBER = 250

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._file_seeds = SerialisableList()

		self._file_seeds_to_indices = {}

		self._file_seed_cache_key = GenerateKey()

		self._status_cache = None
		self._status_cache_generation_time = 0

		self._status_dirty = True

		self._lock = threading.Lock()

	def __len__( self ):
		return len( self._file_seeds )

	def _GenerateStatus( self ):
		self._status_cache = GenerateStatusesToCountsStatus( self._GetStatusesToCounts() )
		self._status_cache_generation_time = GetNow()

		self._status_dirty = False

	def _GetFileSeeds( self, status = None ):
		if status is None:
			return list( self._file_seeds )

		else:
			return [ file_seed for file_seed in self._file_seeds if file_seed.status == status ]

	def _GetSerialisableInfo( self ):
		return self._file_seeds.GetSerialisableTuple()

	def _GetSourceTimestamp( self, file_seed ):
		source_timestamp = file_seed.source_time

		if source_timestamp is None:
			# decent fallback compromise
			# -30 since added and 'last check' timestamps are often the same, and this messes up calculations

			source_timestamp = file_seed.created - 30

		return source_timestamp

	def _GetStatusesToCounts( self ):
		statuses_to_counts = collections.Counter()

		for file_seed in self._file_seeds:
			statuses_to_counts[ file_seed.status ] += 1

		return statuses_to_counts

	def _HasFileSeed( self, file_seed ):
		search_file_seeds = file_seed.GetSearchFileSeeds()

		has_file_seed = True in ( search_file_seed in self._file_seeds_to_indices for search_file_seed in search_file_seeds )

		return has_file_seed

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		with self._lock:
			self._file_seeds = CreateFromSerialisableTuple( serialisable_info )

			self._file_seeds_to_indices = { file_seed : index for ( index, file_seed ) in enumerate( self._file_seeds ) }

	def _SetStatusDirty( self ):
		self._status_dirty = True

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			new_serialisable_info = []

			for ( file_seed, file_seed_info ) in old_serialisable_info:
				if 'note' in file_seed_info:
					file_seed_info[ 'note' ] = str( file_seed_info[ 'note' ] )

				new_serialisable_info.append( ( file_seed, file_seed_info ) )

			return ( 2, new_serialisable_info )

		if version in ( 2, 3 ):
			# gelbooru replaced their thumbnail links with this redirect spam
			# 'https://gelbooru.com/redirect.php?s=Ly9nZWxib29ydS5jb20vaW5kZXgucGhwP3BhZ2U9cG9zdCZzPXZpZXcmaWQ9MzY4ODA1OA=='

			# I missed some http ones here, so I've broadened the test and rescheduled it

			new_serialisable_info = []

			for ( file_seed, file_seed_info ) in old_serialisable_info:
				if 'gelbooru.com/redirect.php' in file_seed:
					continue

				new_serialisable_info.append( ( file_seed, file_seed_info ) )

			return ( 4, new_serialisable_info )

		if version == 4:
			def ConvertRegularToRawURL( regular_url ):
				# convert this:
				# http://68.media.tumblr.com/5af0d991f26ef9fdad5a0c743fb1eca2/tumblr_opl012ZBOu1tiyj7vo1_500.jpg
				# to this:
				# http://68.media.tumblr.com/5af0d991f26ef9fdad5a0c743fb1eca2/tumblr_opl012ZBOu1tiyj7vo1_raw.jpg
				# the 500 part can be a bunch of stuff, including letters

				url_components = regular_url.split( '_' )

				last_component = url_components[ -1 ]

				( number_gubbins, file_ext ) = last_component.split( '.' )

				raw_last_component = 'raw.' + file_ext

				url_components[ -1 ] = raw_last_component

				raw_url = '_'.join( url_components )

				return raw_url

			def Remove68Subdomain( long_url ):
				# sometimes the 68 subdomain gives a 404 on the raw url, so:
				# convert this:
				# http://68.media.tumblr.com/5af0d991f26ef9fdad5a0c743fb1eca2/tumblr_opl012ZBOu1tiyj7vo1_raw.jpg
				# to this:
				# http://media.tumblr.com/5af0d991f26ef9fdad5a0c743fb1eca2/tumblr_opl012ZBOu1tiyj7vo1_raw.jpg

				# I am not sure if it is always 68, but let's not assume

				( scheme, rest ) = long_url.split( '://', 1 )

				if rest.startswith( 'media.tumblr.com' ):
					return long_url

				( gumpf, shorter_rest ) = rest.split( '.', 1 )

				shorter_url = scheme + '://' + shorter_rest

				return shorter_url

			new_serialisable_info = []

			good_file_seeds = set()

			for ( file_seed, file_seed_info ) in old_serialisable_info:
				try:
					parse = urllib.parse.urlparse( file_seed )

					if 'media.tumblr.com' in parse.netloc:
						file_seed = Remove68Subdomain( file_seed )

						file_seed = ConvertRegularToRawURL( file_seed )

						file_seed = ConvertHTTPToHTTPS( file_seed )

					if 'pixiv.net' in parse.netloc:
						file_seed = ConvertHTTPToHTTPS( file_seed )

					if file_seed in good_file_seeds: # we hit a dupe, so skip it

						continue

				except Exception:
					pass

				good_file_seeds.add( file_seed )

				new_serialisable_info.append( ( file_seed, file_seed_info ) )

			return ( 5, new_serialisable_info )

		if version == 5:
			new_serialisable_info = []

			for ( file_seed, file_seed_info ) in old_serialisable_info:
				file_seed_info[ 'source_timestamp' ] = None

				new_serialisable_info.append( ( file_seed, file_seed_info ) )

			return ( 6, new_serialisable_info )

		if version == 6:
			new_serialisable_info = []

			for ( file_seed, file_seed_info ) in old_serialisable_info:
				try:
					magic_phrase = '//media.tumblr.com'
					replacement = '//data.tumblr.com'

					if magic_phrase in file_seed:
						file_seed = file_seed.replace( magic_phrase, replacement )

				except Exception:
					pass

				new_serialisable_info.append( ( file_seed, file_seed_info ) )

			return ( 7, new_serialisable_info )

		if version == 7:
			file_seeds = SerialisableList()

			for ( file_seed_text, file_seed_info ) in old_serialisable_info:
				if file_seed_text.startswith( 'http' ):
					file_seed_type = FILE_SEED_TYPE_URL

				else:
					file_seed_type = FILE_SEED_TYPE_HDD

				file_seed = FileSeed( file_seed_type, file_seed_text )

				file_seed.status = file_seed_info[ 'status' ]
				file_seed.created = file_seed_info[ 'added_timestamp' ]
				file_seed.modified = file_seed_info[ 'last_modified_timestamp' ]
				file_seed.source_time = file_seed_info[ 'source_timestamp' ]
				file_seed.note = file_seed_info[ 'note' ]

				file_seeds.append( file_seed )

			new_serialisable_info = file_seeds.GetSerialisableTuple()

			return ( 8, new_serialisable_info )

	def AddFileSeeds( self, file_seeds ):
		if len( file_seeds ) == 0:
			return 0

		new_file_seeds = []

		with self._lock:
			for file_seed in file_seeds:
				if self._HasFileSeed( file_seed ):
					continue

				file_seed.Normalise()

				new_file_seeds.append( file_seed )

				self._file_seeds.append( file_seed )

				self._file_seeds_to_indices[ file_seed ] = len( self._file_seeds ) - 1

			self._SetStatusDirty()

		self.NotifyFileSeedsUpdated( new_file_seeds )

		return len( new_file_seeds )

	def AdvanceFileSeed( self, file_seed ):
		with self._lock:
			if file_seed in self._file_seeds_to_indices:
				index = self._file_seeds_to_indices[ file_seed ]

				if index > 0:
					self._file_seeds.remove( file_seed )

					self._file_seeds.insert( index - 1, file_seed )

				self._file_seeds_to_indices = { file_seed : index for ( index, file_seed ) in enumerate( self._file_seeds ) }

		self.NotifyFileSeedsUpdated( ( file_seed, ) )

	def CanCompact( self, compact_before_this_source_time ):
		with self._lock:
			if len( self._file_seeds ) <= self.COMPACT_NUMBER:
				return False

			for file_seed in self._file_seeds[:-self.COMPACT_NUMBER]:
				if file_seed.status == CC.STATUS_UNKNOWN:
					continue

				if self._GetSourceTimestamp( file_seed ) < compact_before_this_source_time:
					return True

		return False

	def Compact( self, compact_before_this_source_time ):
		with self._lock:
			if len( self._file_seeds ) <= self.COMPACT_NUMBER:
				return

			new_file_seeds = SerialisableList()

			for file_seed in self._file_seeds[:-self.COMPACT_NUMBER]:
				still_to_do = file_seed.status == CC.STATUS_UNKNOWN
				still_relevant = self._GetSourceTimestamp( file_seed ) > compact_before_this_source_time

				if still_to_do or still_relevant:
					new_file_seeds.append( file_seed )

			new_file_seeds.extend( self._file_seeds[-self.COMPACT_NUMBER:] )

			self._file_seeds = new_file_seeds
			self._file_seeds_to_indices = { file_seed : index for ( index, file_seed ) in enumerate( self._file_seeds ) }

			self._SetStatusDirty()

	def DelayFileSeed( self, file_seed ):
		with self._lock:
			if file_seed in self._file_seeds_to_indices:
				index = self._file_seeds_to_indices[ file_seed ]

				if index < len( self._file_seeds ) - 1:
					self._file_seeds.remove( file_seed )

					self._file_seeds.insert( index + 1, file_seed )

				self._file_seeds_to_indices = { file_seed : index for ( index, file_seed ) in enumerate( self._file_seeds ) }

		self.NotifyFileSeedsUpdated( ( file_seed, ) )

	def GetEarliestSourceTime( self ):
		with self._lock:
			if len( self._file_seeds ) == 0:
				return None

			earliest_timestamp = min( ( self._GetSourceTimestamp( file_seed ) for file_seed in self._file_seeds ) )

		return earliest_timestamp

	def GetFileSeedCacheKey( self ):
		return self._file_seed_cache_key

	def GetFileSeedCount( self, status = None ):
		result = 0

		with self._lock:
			if status is None:
				result = len( self._file_seeds )

			else:
				for file_seed in self._file_seeds:
					if file_seed.status == status:
						result += 1

		return result

	def GetFileSeeds( self, status = None ):
		with self._lock:
			return self._GetFileSeeds( status )

	def GetFileSeedIndex( self, file_seed ):
		with self._lock:
			return self._file_seeds_to_indices[ file_seed ]

	def GetHashes( self ):
		with self._lock:
			hashes = [ file_seed.GetHash() for file_seed in self._file_seeds if file_seed.HasHash() ]

		return hashes

	def GetLatestAddedTime( self ):
		with self._lock:
			if len( self._file_seeds ) == 0:
				return 0

			latest_timestamp = max( ( file_seed.created for file_seed in self._file_seeds ) )

		return latest_timestamp

	def GetLatestSourceTime( self ):
		with self._lock:
			if len( self._file_seeds ) == 0:
				return 0

			latest_timestamp = max( ( self._GetSourceTimestamp( file_seed ) for file_seed in self._file_seeds ) )

		return latest_timestamp

	def GetNextFileSeed( self, status ):
		with self._lock:
			for file_seed in self._file_seeds:
				if file_seed.status == status:
					return file_seed

		return None

	def GetNumNewFilesSince( self, since ):
		num_files = 0

		with self._lock:
			for file_seed in self._file_seeds:
				source_timestamp = self._GetSourceTimestamp( file_seed )

				if source_timestamp >= since:
					num_files += 1

		return num_files

	def GetPresentedHashes( self, file_import_options ):
		with self._lock:
			eligible_file_seeds = [ file_seed for file_seed in self._file_seeds if file_seed.HasHash() ]

			file_seed_hashes = [ file_seed.GetHash() for file_seed in eligible_file_seeds ]

			inbox_hashes = HG.client_controller.Read( 'in_inbox', file_seed_hashes )

			hashes = []

			for file_seed in eligible_file_seeds:
				hash = file_seed.GetHash()

				in_inbox = hash in inbox_hashes

				if file_seed.ShouldPresent( file_import_options, in_inbox = in_inbox ):
					hashes.append( hash )

			return hashes

	def GetStatus( self ):
		with self._lock:
			if self._status_dirty:
				self._GenerateStatus()

			return self._status_cache

	def GetStatusGenerationTime( self ):
		with self._lock:
			if self._status_dirty:
				return GetNow()

			return self._status_cache_generation_time

	def GetStatusesToCounts( self ):
		with self._lock:
			return self._GetStatusesToCounts()

	def GetValueRange( self ):
		with self._lock:
			if self._status_dirty:
				self._GenerateStatus()

			( status, simple_status, ( total_processed, total ) ) = self._status_cache

			return ( total_processed, total )

	def HasFileSeed( self, file_seed ):
		with self._lock:
			return self._HasFileSeed( file_seed )

	def InsertFileSeeds( self, index, file_seeds ):
		if len( file_seeds ) == 0:
			return 0

		new_file_seeds = set()

		with self._lock:
			index = min( index, len( self._file_seeds ) )

			for file_seed in file_seeds:
				if self._HasFileSeed( file_seed ) or file_seed in new_file_seeds:
					continue

				file_seed.Normalise()

				new_file_seeds.add( file_seed )

				self._file_seeds.insert( index, file_seed )

				index += 1

			self._file_seeds_to_indices = { file_seed : index for ( index, file_seed ) in enumerate( self._file_seeds ) }

			self._SetStatusDirty()

		self.NotifyFileSeedsUpdated( new_file_seeds )

		return len( new_file_seeds )

	def NotifyFileSeedsUpdated( self, file_seeds ):
		with self._lock:
			self._SetStatusDirty()

		HG.client_controller.pub( 'file_seed_cache_file_seeds_updated', self._file_seed_cache_key, file_seeds )

	def RemoveFileSeeds( self, file_seeds ):
		with self._lock:
			file_seeds_to_delete = set( file_seeds )

			self._file_seeds = SerialisableList( [ file_seed for file_seed in self._file_seeds if file_seed not in file_seeds_to_delete ] )

			self._file_seeds_to_indices = { file_seed : index for ( index, file_seed ) in enumerate( self._file_seeds ) }

			self._SetStatusDirty()

		self.NotifyFileSeedsUpdated( file_seeds_to_delete )

	def RemoveFileSeedsByStatus( self, statuses_to_remove ):
		with self._lock:
			file_seeds_to_delete = [ file_seed for file_seed in self._file_seeds if file_seed.status in statuses_to_remove ]

		self.RemoveFileSeeds( file_seeds_to_delete )

	def RemoveAllButUnknownFileSeeds( self ):
		with self._lock:
			file_seeds_to_delete = [ file_seed for file_seed in self._file_seeds if file_seed.status != CC.STATUS_UNKNOWN ]

		self.RemoveFileSeeds( file_seeds_to_delete )

	def RetryFailures( self ):
		with self._lock:
			failed_file_seeds = self._GetFileSeeds( CC.STATUS_ERROR )

			for file_seed in failed_file_seeds:
				file_seed.SetStatus( CC.STATUS_UNKNOWN )

		self.NotifyFileSeedsUpdated( failed_file_seeds )

	def RetryIgnored( self ):
		with self._lock:
			ignored_file_seeds = self._GetFileSeeds( CC.STATUS_VETOED )

			for file_seed in ignored_file_seeds:
				file_seed.SetStatus( CC.STATUS_UNKNOWN )

		self.NotifyFileSeedsUpdated( ignored_file_seeds )

	def WorkToDo( self ):
		with self._lock:
			if self._status_dirty:
				self._GenerateStatus()

			( status, simple_status, ( total_processed, total ) ) = self._status_cache

			return total_processed < total

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["FILE_SEED_CACHE"] ] = FileSeedCache


code:
import threading, time

from lib.Actions.Import import DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME, \
	GetRepeatingJobInitialDelay, NetworkJobPresentationContext, \
	PageImporterShouldStopWorking, REPEATING_JOB_TYPICAL_PERIOD, \
	UpdateFileSeedCacheWithFileSeeds, WakeRepeatingJob
from lib.Actions.Import.FileSeeds import FileSeedCache, GenerateFileSeedCachesStatus
from lib.Actions.Import.GallerySeeds import GallerySeed, GallerySeedLog
from lib.Actions.Import.Options import FileImportOptions, TagImportOptions

from lib.Hydrus.Data import ConvertTimestampToPrettyExpires, GenerateKey, GetNow, \
	GetNowPrecise, PrintException, ShowException, ShowText, TimeHasPassed, ToHumanInt
from lib.Hydrus.Exceptions import CancelledException, NetworkException, \
	NotFoundException, VetoException
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, \
	HG.SerialisableType["GALLERY_IMPORT"], HG.SerialisableType["MULTIPLE_GALLERY_IMPORT"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES, SerialisableList, SerialisableBase

from lib.Networking.Downloading import ConvertGalleryIdentifierToGUGName, \
	ConvertGalleryIdentifierToGUGKeyAndName
from lib.Networking.Jobs import NetworkJobDownloader

import lib.Hydrus.Globals as HG
import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC

### CLASSES ###

class GalleryImport( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["GALLERY_IMPORT"]
	SERIALISABLE_NAME = 'Gallery Import'
	SERIALISABLE_VERSION = 2

	def __init__( self, query = None, source_name = None, initial_search_urls = None, start_file_queue_paused = False, start_gallery_queue_paused = False ):
		if query is None:
			query = 'samus_aran'

		if source_name is None:
			source_name = 'unknown'

		if initial_search_urls is None:
			initial_search_urls = []

		SerialisableBase.__init__( self )

		self._creation_time = GetNow()
		self._gallery_import_key = GenerateKey()

		self._query = query

		self._source_name = source_name

		self._page_key = 'initialising page key'
		self._publish_to_page = False

		self._current_page_index = 0
		self._num_new_urls_found = 0
		self._num_urls_found = 0

		self._file_limit = HC.options[ 'gallery_file_limit' ]

		self._files_paused = start_file_queue_paused
		self._gallery_paused = start_gallery_queue_paused

		self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )

		self._tag_import_options = TagImportOptions( is_default = True )

		self._gallery_seed_log = GallerySeedLog()

		gallery_seeds = [ GallerySeed( url ) for url in initial_search_urls ]

		self._gallery_seed_log.AddGallerySeeds( gallery_seeds )

		self._file_seed_cache = FileSeedCache()

		self._no_work_until = 0
		self._no_work_until_reason = ''

		self._lock = threading.Lock()

		self._file_status = ''
		self._gallery_status = ''
		self._gallery_status_can_change_timestamp = 0

		self._all_work_finished = False

		self._file_network_job = None
		self._gallery_network_job = None

		self._files_repeating_job = None
		self._gallery_repeating_job = None

		HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
		HG.client_controller.sub( self, 'NotifyGallerySeedsUpdated', 'gallery_seed_log_gallery_seeds_updated' )

	def _AmOverFileLimit( self ):
		if self._file_limit is not None and self._num_new_urls_found >= self._file_limit:
			return True

		return False

	def _DelayWork( self, time_delta, reason ):
		self._no_work_until = GetNow() + time_delta
		self._no_work_until_reason = reason

	def _GetSerialisableInfo( self ):
		serialisable_gallery_import_key = self._gallery_import_key.hex()

		serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
		serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()

		serialisable_gallery_seed_log = self._gallery_seed_log.GetSerialisableTuple()
		serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()

		return ( serialisable_gallery_import_key, self._creation_time, self._query, self._source_name, self._current_page_index, self._num_urls_found, self._num_new_urls_found, self._file_limit, self._gallery_paused, self._files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_seed_log, serialisable_file_seed_cache, self._no_work_until, self._no_work_until_reason )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_gallery_import_key, self._creation_time, self._query, self._source_name, self._current_page_index, self._num_urls_found, self._num_new_urls_found, self._file_limit, self._gallery_paused, self._files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_seed_log, serialisable_file_seed_cache, self._no_work_until, self._no_work_until_reason ) = serialisable_info

		self._gallery_import_key = bytes.fromhex( serialisable_gallery_import_key )

		self._file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )
		self._tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )

		self._gallery_seed_log = CreateFromSerialisableTuple( serialisable_gallery_seed_log )
		self._file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )

	def _FileNetworkJobPresentationContextFactory( self, network_job ):
		def enter_call():
			with self._lock:
				self._file_network_job = network_job

		def exit_call():
			with self._lock:
				self._file_network_job = None

		return NetworkJobPresentationContext( enter_call, exit_call )

	def _GalleryNetworkJobPresentationContextFactory( self, network_job ):
		def enter_call():
			with self._lock:
				self._gallery_network_job = network_job

		def exit_call():
			with self._lock:
				self._gallery_network_job = None

		return NetworkJobPresentationContext( enter_call, exit_call )

	def _NetworkJobFactory( self, *args, **kwargs ):
		network_job = NetworkJobDownloader( self._gallery_import_key, *args, **kwargs )

		return network_job

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( serialisable_gallery_import_key, self._creation_time, self._query, serialisable_gallery_identifier, self._current_page_index, self._num_urls_found, self._num_new_urls_found, self._file_limit, self._gallery_paused, self._files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_seed_log, serialisable_file_seed_cache, self._no_work_until, self._no_work_until_reason ) = old_serialisable_info

			gallery_identifier = CreateFromSerialisableTuple( serialisable_gallery_identifier )

			source_name = ConvertGalleryIdentifierToGUGName( gallery_identifier )

			new_serialisable_info = ( serialisable_gallery_import_key, self._creation_time, self._query, source_name, self._current_page_index, self._num_urls_found, self._num_new_urls_found, self._file_limit, self._gallery_paused, self._files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_seed_log, serialisable_file_seed_cache, self._no_work_until, self._no_work_until_reason )

			return ( 2, new_serialisable_info )

	def _WorkOnFiles( self ):
		file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if file_seed is None:
			return

		did_substantial_work = False

		try:
			def status_hook( text ):
				with self._lock:
					self._file_status = text

			did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, self._tag_import_options )

			with self._lock:
				should_present = self._publish_to_page and file_seed.ShouldPresent( self._file_import_options )

				page_key = self._page_key

			if should_present:
				file_seed.PresentToPage( page_key )

				did_substantial_work = True

		except VetoException as e:
			status = CC.STATUS_VETOED

			note = str( e )

			file_seed.SetStatus( status, note = note )

			if isinstance( e, CancelledException ):
				time.sleep( 2 )

		except NotFoundException:
			status = CC.STATUS_VETOED
			note = '404'

			file_seed.SetStatus( status, note = note )

			time.sleep( 2 )

		except Exception as e:
			status = CC.STATUS_ERROR

			file_seed.SetStatus( status, exception = e )

			time.sleep( 3 )

		finally:
			self._file_seed_cache.NotifyFileSeedsUpdated( ( file_seed, ) )

			with self._lock:
				self._file_network_job = None

		with self._lock:
			self._file_status = ''

		if did_substantial_work:
			time.sleep( DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )

	def _WorkOnGallery( self ):
		gallery_seed = self._gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )

		if gallery_seed is None:
			return

		with self._lock:
			if self._AmOverFileLimit():
				self._gallery_paused = True

				self._gallery_status = ''

				return

			self._gallery_status = 'checking next page'

		def file_seeds_callable( file_seeds ):
			if self._file_limit is None:
				max_new_urls_allowed = None

			else:
				max_new_urls_allowed = self._file_limit - self._num_new_urls_found

			return UpdateFileSeedCacheWithFileSeeds( self._file_seed_cache, file_seeds, max_new_urls_allowed = max_new_urls_allowed )

		def status_hook( text ):
			with self._lock:
				self._gallery_status = text

		def title_hook( text ):
			return

		try:
			( num_urls_added, num_urls_already_in_file_seed_cache, num_urls_total, result_404, added_new_gallery_pages, stop_reason ) = gallery_seed.WorkOnURL( 'download page', self._gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._NetworkJobFactory, self._GalleryNetworkJobPresentationContextFactory, self._file_import_options )

			self._num_new_urls_found += num_urls_added
			self._num_urls_found += num_urls_total

			if num_urls_added > 0:
				WakeRepeatingJob( self._files_repeating_job )

			self._current_page_index += 1

		except NetworkException as e:
			with self._lock:
				delay = HG.client_controller.new_options.GetInteger( 'downloader_network_error_delay' )

				self._DelayWork( delay, str( e ) )

			return

		except Exception as e:
			gallery_seed_status = CC.STATUS_ERROR
			gallery_seed_note = str( e )

			gallery_seed.SetStatus( gallery_seed_status, note = gallery_seed_note )

			PrintException( e )

			with self._lock:
				self._gallery_paused = True

		self._gallery_seed_log.NotifyGallerySeedsUpdated( ( gallery_seed, ) )

		with self._lock:
			self._gallery_status = ''

		return True

	def CanRetryFailed( self ):
		with self._lock:
			return self._file_seed_cache.GetFileSeedCount( CC.STATUS_ERROR ) > 0

	def CurrentlyWorking( self ):
		with self._lock:
			finished = not self._file_seed_cache.WorkToDo()

			return not finished and not self._files_paused

	def FilesPaused( self ):
		with self._lock:
			return self._files_paused

	def GalleryFinished( self ):
		with self._lock:
			return not self._gallery_seed_log.WorkToDo()

	def GalleryPaused( self ):
		with self._lock:
			return self._gallery_paused

	def GetCreationTime( self ):
		with self._lock:
			return self._creation_time

	def GetCurrentAction( self ):
		with self._lock:
			if self._file_status != '':
				return self._file_status

			elif self._gallery_status != '':
				return self._gallery_status

			elif not self._gallery_seed_log.WorkToDo() and not self._file_seed_cache.WorkToDo():
				return 'done!'

			else:
				return ''

	def GetFileImportOptions( self ):
		with self._lock:
			return self._file_import_options

	def GetFileSeedCache( self ):
		with self._lock:
			return self._file_seed_cache

	def GetFileLimit( self ):
		with self._lock:
			return self._file_limit

	def GetGalleryImportKey( self ):
		with self._lock:
			return self._gallery_import_key

	def GetGallerySeedLog( self ):
		with self._lock:
			return self._gallery_seed_log

	def GetGalleryStatus( self ):
		with self._lock:
			if TimeHasPassed( self._no_work_until ):
				gallery_status = self._gallery_status

			else:
				no_work_text = ConvertTimestampToPrettyExpires( self._no_work_until ) + ': ' + self._no_work_until_reason

				gallery_status = no_work_text

			return gallery_status

	def GetHashes( self ):
		with self._lock:
			return self._file_seed_cache.GetHashes()

	def GetNetworkJobs( self ):
		with self._lock:
			return ( self._file_network_job, self._gallery_network_job )

	def GetNewHashes( self ):
		with self._lock:
			file_import_options = FileImportOptions()

			file_import_options.SetPresentationOptions( True, False, False )

			return self._file_seed_cache.GetPresentedHashes( file_import_options )

	def GetOptions( self ):
		with self._lock:
			return ( self._file_import_options, self._tag_import_options, self._file_limit )

	def GetPresentedHashes( self ):
		with self._lock:
			return self._file_seed_cache.GetPresentedHashes( self._file_import_options )

	def GetQueryText( self ):
		with self._lock:
			return self._query

	def GetSourceName( self ):
		with self._lock:
			return self._source_name

	def GetStatus( self ):
		with self._lock:
			if TimeHasPassed( self._no_work_until ):
				gallery_status = self._gallery_status
				file_status = self._file_status

			else:
				no_work_text = ConvertTimestampToPrettyExpires( self._no_work_until ) + ': ' + self._no_work_until_reason

				gallery_status = no_work_text
				file_status = no_work_text

			return ( gallery_status, file_status, self._files_paused, self._gallery_paused )

	def GetTagImportOptions( self ):
		with self._lock:
			return self._tag_import_options

	def GetValueRange( self ):
		with self._lock:
			return self._file_seed_cache.GetValueRange()

	def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
		if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
			WakeRepeatingJob( self._files_repeating_job )

	def NotifyGallerySeedsUpdated( self, gallery_seed_log_key, gallery_seeds ):
		if gallery_seed_log_key == self._gallery_seed_log.GetGallerySeedLogKey():
			WakeRepeatingJob( self._gallery_repeating_job )

	def PausePlayFiles( self ):
		with self._lock:
			self._files_paused = not self._files_paused

			WakeRepeatingJob( self._files_repeating_job )

	def PausePlayGallery( self ):
		with self._lock:
			self._gallery_paused = not self._gallery_paused

			WakeRepeatingJob( self._gallery_repeating_job )

	def PublishToPage( self, publish_to_page ):
		with self._lock:
			self._publish_to_page = publish_to_page

	def Repage( self, page_key ):
		with self._lock:
			self._page_key = page_key

	def RetryFailed( self ):
		with self._lock:
			self._file_seed_cache.RetryFailures()

	def SetFileLimit( self, file_limit ):
		with self._lock:
			self._file_limit = file_limit

	def SetFileImportOptions( self, file_import_options ):
		with self._lock:
			self._file_import_options = file_import_options

	def SetFileSeedCache( self, file_seed_cache ):
		with self._lock:
			self._file_seed_cache = file_seed_cache

	def SetGallerySeedLog( self, gallery_seed_log ):
		with self._lock:
			self._gallery_seed_log = gallery_seed_log

	def SetTagImportOptions( self, tag_import_options ):
		with self._lock:
			self._tag_import_options = tag_import_options

	def Start( self, page_key, publish_to_page ):
		self._page_key = page_key
		self._publish_to_page = publish_to_page

		self._files_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles )
		self._gallery_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnGallery )

		self._files_repeating_job.SetThreadSlotType( 'gallery_files' )
		self._gallery_repeating_job.SetThreadSlotType( 'gallery_search' )

	def REPEATINGWorkOnFiles( self ):
		with self._lock:
			if PageImporterShouldStopWorking( self._page_key ):
				self._files_repeating_job.Cancel()

				return

			files_paused = self._files_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )
			work_pending = self._file_seed_cache.WorkToDo() and not files_paused
			no_delays = TimeHasPassed( self._no_work_until )
			page_shown = not HG.client_controller.PageClosedButNotDestroyed( self._page_key )
			network_engine_good = not HG.client_controller.network_engine.IsBusy()

			ok_to_work = work_pending and no_delays and page_shown and network_engine_good

		while ok_to_work:
			try:
				self._WorkOnFiles()

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( self._page_key ):
					self._files_repeating_job.Cancel()

					return

				files_paused = self._files_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )
				work_pending = self._file_seed_cache.WorkToDo() and not files_paused
				no_delays = TimeHasPassed( self._no_work_until )
				page_shown = not HG.client_controller.PageClosedButNotDestroyed( self._page_key )
				network_engine_good = not HG.client_controller.network_engine.IsBusy()

				ok_to_work = work_pending and no_delays and page_shown and network_engine_good

	def REPEATINGWorkOnGallery( self ):
		with self._lock:
			if PageImporterShouldStopWorking( self._page_key ):
				self._gallery_repeating_job.Cancel()

				return

			gallery_paused = self._gallery_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )

			work_pending = self._gallery_seed_log.WorkToDo() and not gallery_paused
			no_delays = TimeHasPassed( self._no_work_until )
			page_shown = not HG.client_controller.PageClosedButNotDestroyed( self._page_key )
			network_engine_good = not HG.client_controller.network_engine.IsBusy()

			ok_to_work = work_pending and no_delays and page_shown and network_engine_good

		while ok_to_work:
			try:
				self._WorkOnGallery()

				time.sleep( 1 )

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( self._page_key ):
					self._gallery_repeating_job.Cancel()

					return

				gallery_paused = self._gallery_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )

				work_pending = self._gallery_seed_log.WorkToDo() and not gallery_paused
				no_delays = TimeHasPassed( self._no_work_until )
				page_shown = not HG.client_controller.PageClosedButNotDestroyed( self._page_key )
				network_engine_good = not HG.client_controller.network_engine.IsBusy()

				ok_to_work = work_pending and no_delays and page_shown and network_engine_good

		with self._lock:
			self._gallery_status = ''

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["GALLERY_IMPORT"] ] = GalleryImport

class MultipleGalleryImport( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["MULTIPLE_GALLERY_IMPORT"]
	SERIALISABLE_NAME = 'Multiple Gallery Import'
	SERIALISABLE_VERSION = 7

	def __init__( self, gug_key_and_name = None ):
		if gug_key_and_name is None:
			gug_key_and_name = ( GenerateKey(), 'unknown source' )

		SerialisableBase.__init__( self )

		self._lock = threading.Lock()

		self._page_key = 'initialising page key'

		self._gug_key_and_name = gug_key_and_name

		self._highlighted_gallery_import_key = None

		# new_options = HG.client_controller.new_options

		self._file_limit = HC.options[ 'gallery_file_limit' ]

		self._start_file_queues_paused = False
		self._start_gallery_queues_paused = False
		self._merge_simultaneous_pends_to_one_importer = False

		self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
		self._tag_import_options = TagImportOptions( is_default = True )

		self._gallery_imports = SerialisableList()

		self._gallery_import_keys_to_gallery_imports = {}

		self._status_dirty = True
		self._status_cache = None
		self._status_cache_generation_time = 0

		self._last_time_imports_changed = GetNowPrecise()

		self._last_pubbed_value_range = ( 0, 0 )
		self._next_pub_value_check_time = 0

		self._importers_repeating_job = None

	def _AddGalleryImport( self, gallery_import ):
		gallery_import.PublishToPage( False )
		gallery_import.Repage( self._page_key )

		self._gallery_imports.append( gallery_import )

		self._last_time_imports_changed = GetNowPrecise()

		gallery_import_key = gallery_import.GetGalleryImportKey()

		self._gallery_import_keys_to_gallery_imports[ gallery_import_key ] = gallery_import

		if len( self._gallery_imports ) == 1:
			self._highlighted_gallery_import_key = gallery_import_key

	def _GetSerialisableInfo( self ):
		( gug_key, gug_name ) = self._gug_key_and_name

		serialisable_gug_key_and_name = ( gug_key.hex(), gug_name )

		if self._highlighted_gallery_import_key is None:
			serialisable_highlighted_gallery_import_key = self._highlighted_gallery_import_key

		else:
			serialisable_highlighted_gallery_import_key = self._highlighted_gallery_import_key.hex()

		pend_options = ( self._start_file_queues_paused, self._start_gallery_queues_paused, self._merge_simultaneous_pends_to_one_importer )

		serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
		serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()

		serialisable_gallery_imports = self._gallery_imports.GetSerialisableTuple()

		return ( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, self._file_limit, pend_options, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, self._file_limit, pend_options, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports ) = serialisable_info

		( serialisable_gug_key, gug_name ) = serialisable_gug_key_and_name

		self._gug_key_and_name = ( bytes.fromhex( serialisable_gug_key ), gug_name )

		if serialisable_highlighted_gallery_import_key is None:
			self._highlighted_gallery_import_key = None

		else:
			self._highlighted_gallery_import_key = bytes.fromhex( serialisable_highlighted_gallery_import_key )

		( self._start_file_queues_paused, self._start_gallery_queues_paused, self._merge_simultaneous_pends_to_one_importer ) = pend_options

		self._file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )
		self._tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )

		self._gallery_imports = CreateFromSerialisableTuple( serialisable_gallery_imports )

		self._gallery_import_keys_to_gallery_imports = { gallery_import.GetGalleryImportKey() : gallery_import for gallery_import in self._gallery_imports }

	def _RegenerateStatus( self ):
		file_seed_caches = [ gallery_import.GetFileSeedCache() for gallery_import in self._gallery_imports ]

		self._status_cache = GenerateFileSeedCachesStatus( file_seed_caches )

		self._status_dirty = False
		self._status_cache_generation_time = GetNow()

	def _RemoveGalleryImport( self, gallery_import_key ):
		if gallery_import_key not in self._gallery_import_keys_to_gallery_imports:
			return

		gallery_import = self._gallery_import_keys_to_gallery_imports[ gallery_import_key ]

		gallery_import.PublishToPage( False )
		gallery_import.Repage( 'dead page key' )

		self._gallery_imports.remove( gallery_import )

		self._last_time_imports_changed = GetNowPrecise()

		del self._gallery_import_keys_to_gallery_imports[ gallery_import_key ]

	def _SetDirty( self ):
		self._status_dirty = True

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_current_query_stuff, pending_queries, get_tags_if_url_recognised_and_file_redundant, file_limit, gallery_paused, files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_file_seed_cache ) = old_serialisable_info

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_current_query_stuff, pending_queries, file_limit, gallery_paused, files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_file_seed_cache )

			return ( 2, new_serialisable_info )

		if version == 2:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_current_query_stuff, pending_queries, file_limit, gallery_paused, files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_file_seed_cache ) = old_serialisable_info

			gallery_seed_log = GallerySeedLog()

			serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_current_query_stuff, pending_queries, file_limit, gallery_paused, files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_seed_log, serialisable_file_seed_cache )

			return ( 3, new_serialisable_info )

		if version == 3:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_current_query_stuff, pending_queries, file_limit, gallery_paused, files_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_seed_log, serialisable_file_seed_cache ) = old_serialisable_info

			( current_query, current_query_num_new_urls, serialisable_current_gallery_stream_identifier, current_gallery_stream_identifier_page_index, serialisable_current_gallery_stream_identifier_found_urls, serialisable_pending_gallery_stream_identifiers ) = serialisable_current_query_stuff

			highlighted_gallery_import_key = None

			serialisable_highlighted_gallery_import_key = highlighted_gallery_import_key

			gallery_imports = SerialisableList()

			file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )
			tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )

			gallery_seed_log = CreateFromSerialisableTuple( serialisable_gallery_seed_log )
			file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )

			if len( file_seed_cache ) > 0:
				current_query = 'queue brought from old page'

				gallery_import = GalleryImport( query = current_query, source_name = 'updated from old system', initial_search_urls = [] )

				gallery_import.PausePlayGallery()
				gallery_import.PausePlayFiles()

				gallery_import.SetFileLimit( file_limit )

				gallery_import.SetFileImportOptions( file_import_options )
				gallery_import.SetTagImportOptions( tag_import_options )

				gallery_import.SetFileSeedCache( file_seed_cache )
				gallery_import.SetGallerySeedLog( gallery_seed_log )

				gallery_imports.append( gallery_import )

			serialisable_gallery_imports = gallery_imports.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_highlighted_gallery_import_key, file_limit, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports )

			return ( 4, new_serialisable_info )

		if version == 4:
			( serialisable_gallery_identifier, serialisable_highlighted_gallery_import_key, file_limit, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports ) = old_serialisable_info

			gallery_identifier = CreateFromSerialisableTuple( serialisable_gallery_identifier )

			( gug_key, gug_name ) = ConvertGalleryIdentifierToGUGKeyAndName( gallery_identifier )

			serialisable_gug_key_and_name = ( GenerateKey().hex(), gug_name )

			new_serialisable_info = ( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, file_limit, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports )

			return ( 5, new_serialisable_info )

		if version == 5:
			( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, file_limit, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports ) = old_serialisable_info

			start_file_queues_paused = False
			start_gallery_queues_paused = False

			new_serialisable_info = ( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, file_limit, start_file_queues_paused, start_gallery_queues_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports )

			return ( 6, new_serialisable_info )

		if version == 6:
			( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, file_limit, start_file_queues_paused, start_gallery_queues_paused, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports ) = old_serialisable_info

			merge_simultaneous_pends_to_one_importer = False

			pend_options = ( start_file_queues_paused, start_gallery_queues_paused, merge_simultaneous_pends_to_one_importer )

			new_serialisable_info = ( serialisable_gug_key_and_name, serialisable_highlighted_gallery_import_key, file_limit, pend_options, serialisable_file_import_options, serialisable_tag_import_options, serialisable_gallery_imports )

			return ( 7, new_serialisable_info )

	def CurrentlyWorking( self ):
		with self._lock:
			return True in ( gallery_import.CurrentlyWorking() for gallery_import in self._gallery_imports )

	def GetFileLimit( self ):
		with self._lock:
			return self._file_limit

	def GetFileImportOptions( self ):
		with self._lock:
			return self._file_import_options

	def GetGalleryImports( self ):
		with self._lock:
			return list( self._gallery_imports )

	def GetGUGKeyAndName( self ):
		with self._lock:
			return self._gug_key_and_name

	def GetHighlightedGalleryImport( self ):
		with self._lock:
			if self._highlighted_gallery_import_key is not None:
				if self._highlighted_gallery_import_key in self._gallery_import_keys_to_gallery_imports:
					return self._gallery_import_keys_to_gallery_imports[ self._highlighted_gallery_import_key ]

				self._highlighted_gallery_import_key = None

			return None

	def GetInitialSearchText( self ):
		return HG.client_controller.network_engine.domain_manager.GetInitialSearchText( self._gug_key_and_name )

	def GetLastTimeImportsChanged( self ):
		with self._lock:
			return self._last_time_imports_changed

	def GetNumGalleryImports( self ):
		with self._lock:
			return len( self._gallery_imports )

	def GetQueueStartSettings( self ):
		with self._lock:
			return ( self._start_file_queues_paused, self._start_gallery_queues_paused, self._merge_simultaneous_pends_to_one_importer )

	def GetTagImportOptions( self ):
		with self._lock:
			return self._tag_import_options

	def GetTotalStatus( self ):
		with self._lock:
			if self._status_dirty:
				self._RegenerateStatus()

			return self._status_cache

	def GetValueRange( self ):
		with self._lock:
			total_value = 0
			total_range = 0

			for gallery_import in self._gallery_imports:
				( value, range ) = gallery_import.GetValueRange()

				if value != range:
					total_value += value
					total_range += range

			return ( total_value, total_range )

	def PendQueries( self, query_texts ):
		created_importers = []

		with self._lock:
			gug = HG.client_controller.network_engine.domain_manager.GetGUG( self._gug_key_and_name )

			if gug is None:
				ShowText( 'Could not find a Gallery URL Generator for "' + self._gug_key_and_name[1] + '"!' )

				return created_importers

			self._gug_key_and_name = gug.GetGUGKeyAndName() # just a refresher, to keep up with any changes

			groups_of_query_data = []

			for query_text in query_texts:
				initial_search_urls = gug.GenerateGalleryURLs( query_text )

				if len( initial_search_urls ) == 0:
					ShowText( 'The Gallery URL Generator "' + self._gug_key_and_name[1] + '" did not produce any URLs!' )

					return created_importers

				groups_of_query_data.append( ( query_text, initial_search_urls ) )

			if self._merge_simultaneous_pends_to_one_importer and len( groups_of_query_data ) > 1:
				# flatten these groups down to one

				all_search_urls_flat = []

				for ( query_text, initial_search_urls ) in groups_of_query_data:
					all_search_urls_flat.extend( initial_search_urls )

				query_text = ToHumanInt( len( groups_of_query_data ) ) + ' queries'

				groups_of_query_data = [ ( query_text, all_search_urls_flat ) ]

			for ( query_text, initial_search_urls ) in groups_of_query_data:
				gallery_import = GalleryImport( query = query_text, source_name = self._gug_key_and_name[1], initial_search_urls = initial_search_urls, start_file_queue_paused = self._start_file_queues_paused, start_gallery_queue_paused = self._start_gallery_queues_paused )

				gallery_import.SetFileLimit( self._file_limit )

				gallery_import.SetFileImportOptions( self._file_import_options )
				gallery_import.SetTagImportOptions( self._tag_import_options )

				publish_to_page = False

				gallery_import.Start( self._page_key, publish_to_page )

				self._AddGalleryImport( gallery_import )

				created_importers.append( gallery_import )

			WakeRepeatingJob( self._importers_repeating_job )

			self._SetDirty()

		return created_importers

	def RemoveGalleryImport( self, gallery_import_key ):
		with self._lock:
			self._RemoveGalleryImport( gallery_import_key )

			self._SetDirty()

	def SetFileLimit( self, file_limit ):
		with self._lock:
			self._file_limit = file_limit

	def SetFileImportOptions( self, file_import_options ):
		with self._lock:
			self._file_import_options = file_import_options

	def SetGUGKeyAndName( self, gug_key_and_name ):
		with self._lock:
			self._gug_key_and_name = gug_key_and_name

	def SetHighlightedGalleryImport( self, highlighted_gallery_import ):
		with self._lock:
			if highlighted_gallery_import is None:
				self._highlighted_gallery_import_key = None

			else:
				self._highlighted_gallery_import_key = highlighted_gallery_import.GetGalleryImportKey()

				highlighted_gallery_import.PublishToPage( True )

	def SetQueueStartSettings( self, start_file_queues_paused, start_gallery_queues_paused, merge_simultaneous_pends_to_one_importer ):
		with self._lock:
			self._start_file_queues_paused = start_file_queues_paused
			self._start_gallery_queues_paused = start_gallery_queues_paused
			self._merge_simultaneous_pends_to_one_importer = merge_simultaneous_pends_to_one_importer

	def SetTagImportOptions( self, tag_import_options ):
		with self._lock:
			self._tag_import_options = tag_import_options

	def Start( self, page_key ):
		with self._lock:
			self._page_key = page_key

		# set a 2s period so the page value/range is breddy snappy
		self._importers_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), 2.0, self.REPEATINGWorkOnImporters )

		for gallery_import in self._gallery_imports:
			publish_to_page = gallery_import.GetGalleryImportKey() == self._highlighted_gallery_import_key

			gallery_import.Start( page_key, publish_to_page )

	def REPEATINGWorkOnImporters( self ):
		with self._lock:
			if PageImporterShouldStopWorking( self._page_key ):
				self._importers_repeating_job.Cancel()

				return

			if not self._status_dirty: # if we think we are clean

				for gallery_import in self._gallery_imports:
					file_seed_cache = gallery_import.GetFileSeedCache()

					if file_seed_cache.GetStatusGenerationTime() > self._status_cache_generation_time: # has there has been an update?

						self._SetDirty()

						break

		if TimeHasPassed( self._next_pub_value_check_time ):
			self._next_pub_value_check_time = GetNow() + 5

			current_value_range = self.GetValueRange()

			if current_value_range != self._last_pubbed_value_range:
				self._last_pubbed_value_range = current_value_range

				HG.client_controller.pub( 'refresh_page_name', self._page_key )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["MULTIPLE_GALLERY_IMPORT"] ] = MultipleGalleryImport

license: wtfpl