code:
import itertools, os, threading, time, traceback, collections

from lib.Actions.Import import ConvertAllParseResultsToFileSeeds

from lib.Handling.Parsing import GetTitleFromAllParseResults, \
	GetURLsFromParseResults
from lib.Hydrus.Data import DedupeList, GenerateKey, GetNow, Print, ToHumanInt
from lib.Hydrus.Exceptions import CancelledException, InsufficientCredentialsException, \
	NetworkException, NotFoundException, ShutdownException, VetoException
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, \
	SerialisableBase, SerialisableList
from lib.Hydrus.Tags.Local import ServiceKeysToTags

### CONSTANTS ###

import lib.Hydrus.Globals as HG
import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC

from lib.Hydrus.Serialisable import HG.SerialisableType["GALLERY_SEED_LOG"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES, HG.SerialisableType["GALLERY_SEED"]

### FUNCTIONS ###

def GenerateGallerySeedLogStatus( statuses_to_counts ):
	num_successful = statuses_to_counts[ CC.STATUS_SUCCESSFUL_AND_NEW ]
	num_ignored = statuses_to_counts[ CC.STATUS_VETOED ]
	num_failed = statuses_to_counts[ CC.STATUS_ERROR ]
	num_skipped = statuses_to_counts[ CC.STATUS_SKIPPED ]
	num_unknown = statuses_to_counts[ CC.STATUS_UNKNOWN ]

	# add some kind of '(512 files found (so far))', which may be asking too much here
	# might be this is complicated and needs to be (partly) done in the object, which will know if it is paused or whatever.

	status_strings = []

	if num_successful > 0:
		s = ToHumanInt( num_successful ) + ' successful'

		status_strings.append( s )

	if num_ignored > 0:
		status_strings.append( ToHumanInt( num_ignored ) + ' ignored' )

	if num_failed > 0:
		status_strings.append( ToHumanInt( num_failed ) + ' failed' )

	if num_skipped > 0:
		status_strings.append( ToHumanInt( num_skipped ) + ' skipped' )

	if num_unknown > 0:
		status_strings.append( ToHumanInt( num_unknown ) + ' pending' )

	status = ', '.join( status_strings )

	total = sum( statuses_to_counts.values() )

	total_processed = total - num_unknown

	return ( status, ( total_processed, total ) )

class GallerySeed( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["GALLERY_SEED"]
	SERIALISABLE_NAME = 'Gallery Log Entry'
	SERIALISABLE_VERSION = 2

	def __init__( self, url = None, can_generate_more_pages = True ):
		if url is None:
			url = 'https://nostrils-central.cx/index.php?post=s&tag=hyper_nostrils&page=3'

		else:
			url = HG.client_controller.network_engine.domain_manager.NormaliseURL( url )

		SerialisableBase.__init__( self )

		self.url = url
		self._can_generate_more_pages = can_generate_more_pages

		self._fixed_service_keys_to_tags = ServiceKeysToTags()

		self.created = GetNow()
		self.modified = self.created
		self.status = CC.STATUS_UNKNOWN
		self.note = ''

		self._referral_url = None

		self._force_next_page_url_generation = False

	def __eq__( self, other ):
		return self.__hash__() == other.__hash__()

	def __hash__( self ):
		return ( self.url, self.created ).__hash__()

	def __ne__( self, other ):
		return self.__hash__() != other.__hash__()

	def _GetSerialisableInfo( self ):
		serialisable_fixed_service_keys_to_tags = self._fixed_service_keys_to_tags.GetSerialisableTuple()

		return ( self.url, self._can_generate_more_pages, serialisable_fixed_service_keys_to_tags, self.created, self.modified, self.status, self.note, self._referral_url )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self.url, self._can_generate_more_pages, serialisable_fixed_service_keys_to_tags, self.created, self.modified, self.status, self.note, self._referral_url ) = serialisable_info

		self._fixed_service_keys_to_tags = CreateFromSerialisableTuple( serialisable_fixed_service_keys_to_tags )

	def _UpdateModified( self ):
		self.modified = GetNow()

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( url, can_generate_more_pages, created, modified, status, note, referral_url ) = old_serialisable_info

			fixed_service_keys_to_tags = ServiceKeysToTags()

			serialisable_fixed_service_keys_to_tags = fixed_service_keys_to_tags.GetSerialisableTuple()

			new_serialisable_info = ( url, can_generate_more_pages, serialisable_fixed_service_keys_to_tags, created, modified, status, note, referral_url )

			return ( 2, new_serialisable_info )

	def ForceNextPageURLGeneration( self ):
		self._force_next_page_url_generation = True

	def GenerateRestartedDuplicate( self, can_generate_more_pages ):
		gallery_seed = GallerySeed( url = self.url, can_generate_more_pages = can_generate_more_pages )

		if can_generate_more_pages:
			gallery_seed.ForceNextPageURLGeneration()

		return gallery_seed

	def GetExampleNetworkJob( self, network_job_factory ):
		( url_to_check, parser ) = HG.client_controller.network_engine.domain_manager.GetURLToFetchAndParser( self.url )

		network_job = network_job_factory( 'GET', url_to_check )

		return network_job

	def SetFixedServiceKeysToTags( self, service_keys_to_tags ):
		self._fixed_service_keys_to_tags = ServiceKeysToTags( service_keys_to_tags )

	def SetReferralURL( self, referral_url ):
		self._referral_url = referral_url

	def SetStatus( self, status, note = '', exception = None ):
		if exception is not None:
			first_line = str( exception ).split( os.linesep )[0]

			note = first_line + '\u2026 (Copy note to see full error)'
			note += os.linesep
			note += traceback.format_exc()

			Print( 'Error when processing ' + self.url + ' !' )
			Print( traceback.format_exc() )

		self.status = status
		self.note = note

		self._UpdateModified()

	def WorksInNewSystem( self ):
		( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( self.url )

		if url_type == HC.URL_TYPE_GALLERY and can_parse:
			return True

		return False

	def WorkOnURL( self, gallery_token_name, gallery_seed_log, file_seeds_callable, status_hook, title_hook, network_job_factory, network_job_presentation_context_factory, file_import_options, gallery_urls_seen_before = None ):
		if gallery_urls_seen_before is None:
			gallery_urls_seen_before = set()

		gallery_urls_seen_before.add( self.url )

		# maybe something like 'append urls' vs 'reverse-prepend' for subs or something

		# should also take--and populate--a set of urls we have seen this 'run', so we can bomb out if next_gallery_url ends up in some loop

		num_urls_added = 0
		num_urls_already_in_file_seed_cache = 0
		num_urls_total = 0
		result_404 = False
		added_new_gallery_pages = False
		stop_reason = ''

		try:
			( url_type, match_name, can_parse ) = HG.client_controller.network_engine.domain_manager.GetURLParseCapability( self.url )

			if url_type not in ( HC.URL_TYPE_GALLERY, HC.URL_TYPE_WATCHABLE ):
				raise VetoException( 'Did not recognise this as a gallery or watchable URL!' )

			if not can_parse:
				raise VetoException( 'Did not have a parser for this URL!' )

			( url_to_check, parser ) = HG.client_controller.network_engine.domain_manager.GetURLToFetchAndParser( self.url )

			status_hook( 'downloading gallery page' )

			if self._referral_url not in ( self.url, url_to_check ):
				referral_url = self._referral_url

			else:
				referral_url = None

			network_job = network_job_factory( 'GET', url_to_check, referral_url = referral_url )

			network_job.SetGalleryToken( gallery_token_name )

			network_job.OverrideBandwidth( 30 )

			HG.client_controller.network_engine.AddJob( network_job )

			with network_job_presentation_context_factory( network_job ):
				network_job.WaitUntilDone()

			parsing_text = network_job.GetContentText()

			parsing_context = {}

			parsing_context[ 'gallery_url' ] = self.url
			parsing_context[ 'url' ] = url_to_check

			all_parse_results = parser.Parse( parsing_context, parsing_text )

			if len( all_parse_results ) == 0:
				raise VetoException( 'The parser found nothing in the document!' )

			title = GetTitleFromAllParseResults( all_parse_results )

			if title is not None:
				title_hook( title )

			file_seeds = ConvertAllParseResultsToFileSeeds( all_parse_results, self.url, file_import_options )

			for file_seed in file_seeds:
				file_seed.SetFixedServiceKeysToTags( self._fixed_service_keys_to_tags )

			num_urls_total = len( file_seeds )

			( num_urls_added, num_urls_already_in_file_seed_cache, can_search_for_more_files, stop_reason ) = file_seeds_callable( file_seeds )

			status = CC.STATUS_SUCCESSFUL_AND_NEW

			note = ToHumanInt( num_urls_added ) + ' new urls found'

			if num_urls_already_in_file_seed_cache > 0:
				note += ' (' + ToHumanInt( num_urls_already_in_file_seed_cache ) + ' of page already in)'

			if not can_search_for_more_files:
				note += ' - ' + stop_reason

			if parser.CanOnlyGenerateGalleryURLs() or self._force_next_page_url_generation:
				can_add_more_gallery_urls = True

			else:
				# only keep searching if we found any files, otherwise this could be a blank results page with another stub page
				can_add_more_gallery_urls = num_urls_added > 0 and can_search_for_more_files

			if self._can_generate_more_pages and can_add_more_gallery_urls:
				flattened_results = list( itertools.chain.from_iterable( all_parse_results ) )

				next_page_urls = GetURLsFromParseResults( flattened_results, ( HC.URL_TYPE_NEXT, ), only_get_top_priority = True )

				if self.url in next_page_urls:
					next_page_urls.remove( self.url )

				if url_to_check in next_page_urls:
					next_page_urls.remove( url_to_check )

				if len( next_page_urls ) > 0:
					next_page_generation_phrase = ' next gallery pages found'

				else:
					# we have failed to parse a next page url, but we would still like one, so let's see if the url match can provide one

					url_match = HG.client_controller.network_engine.domain_manager.GetURLMatch( url_to_check )

					if url_match is not None and url_match.CanGenerateNextGalleryPage():
						try:
							next_page_url = url_match.GetNextGalleryPage( url_to_check )

							next_page_urls = [ next_page_url ]

						except Exception:
							note += ' - Attempted to generate a next gallery page url, but failed!'
							note += os.linesep
							note += traceback.format_exc()

					next_page_generation_phrase = ' next gallery pages extrapolated from url class'

				if len( next_page_urls ) > 0:
					next_page_urls = DedupeList( next_page_urls )

					new_next_page_urls = [ next_page_url for next_page_url in next_page_urls if next_page_url not in gallery_urls_seen_before ]

					duplicate_next_page_urls = gallery_urls_seen_before.intersection( new_next_page_urls )

					num_new_next_page_urls = len( new_next_page_urls )
					num_dupe_next_page_urls = len( duplicate_next_page_urls )

					if num_new_next_page_urls > 0:
						next_gallery_seeds = [ GallerySeed( next_page_url ) for next_page_url in new_next_page_urls ]

						for next_gallery_seed in next_gallery_seeds:
							next_gallery_seed.SetFixedServiceKeysToTags( self._fixed_service_keys_to_tags )

						gallery_seed_log.AddGallerySeeds( next_gallery_seeds )

						added_new_gallery_pages = True

						gallery_urls_seen_before.update( new_next_page_urls )

						if num_dupe_next_page_urls == 0:
							note += ' - ' + ToHumanInt( num_new_next_page_urls ) + next_page_generation_phrase

						else:
							note += ' - ' + ToHumanInt( num_new_next_page_urls ) + next_page_generation_phrase + ', but ' + ToHumanInt( num_dupe_next_page_urls ) + ' had already been visited this run and were not added'

					else:
						note += ' - ' + ToHumanInt( num_dupe_next_page_urls ) + next_page_generation_phrase + ', but they had already been visited this run and were not added'

			self.SetStatus( status, note = note )

		except ShutdownException:
			pass

		except VetoException as e:
			status = CC.STATUS_VETOED

			note = str( e )

			self.SetStatus( status, note = note )

			if isinstance( e, CancelledException ):
				status_hook( 'cancelled!' )

				time.sleep( 2 )

		except InsufficientCredentialsException:
			status = CC.STATUS_VETOED
			note = '403'

			self.SetStatus( status, note = note )

			status_hook( '403' )

			time.sleep( 2 )

			result_404 = True

		except NotFoundException:
			status = CC.STATUS_VETOED
			note = '404'

			self.SetStatus( status, note = note )

			status_hook( '404' )

			time.sleep( 2 )

			result_404 = True

		except Exception as e:
			status = CC.STATUS_ERROR

			self.SetStatus( status, exception = e )

			status_hook( 'error!' )

			time.sleep( 3 )

			if isinstance( e, NetworkException ): # so the larger queue can set a delaywork or whatever

				raise

		gallery_seed_log.NotifyGallerySeedsUpdated( ( self, ) )

		return ( num_urls_added, num_urls_already_in_file_seed_cache, num_urls_total, result_404, added_new_gallery_pages, stop_reason )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["GALLERY_SEED"] ] = GallerySeed

class GallerySeedLog( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["GALLERY_SEED_LOG"]
	SERIALISABLE_NAME = 'Gallery Log'
	SERIALISABLE_VERSION = 1

	COMPACT_NUMBER = 100

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._gallery_seeds = SerialisableList()

		self._gallery_seeds_to_indices = {}

		self._gallery_seed_log_key = GenerateKey()

		self._status_cache = None
		self._status_cache_generation_time = 0

		self._status_dirty = True

		self._lock = threading.Lock()

	def __len__( self ):
		return len( self._gallery_seeds )

	def _GenerateStatus( self ):
		statuses_to_counts = self._GetStatusesToCounts()

		self._status_cache = GenerateGallerySeedLogStatus( statuses_to_counts )
		self._status_cache_generation_time = GetNow()

		self._status_dirty = False

	def _GetStatusesToCounts( self ):
		statuses_to_counts = collections.Counter()

		for gallery_seed in self._gallery_seeds:
			statuses_to_counts[ gallery_seed.status ] += 1

		return statuses_to_counts

	def _GetGallerySeeds( self, status = None ):
		if status is None:
			return list( self._gallery_seeds )

		else:
			return [ gallery_seed for gallery_seed in self._gallery_seeds if gallery_seed.status == status ]

	def _GetSerialisableInfo( self ):
		return self._gallery_seeds.GetSerialisableTuple()

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		with self._lock:
			self._gallery_seeds = CreateFromSerialisableTuple( serialisable_info )

			self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }

	def _SetStatusDirty( self ):
		self._status_dirty = True

	def AddGallerySeeds( self, gallery_seeds ):
		if len( gallery_seeds ) == 0:
			return 0

		new_gallery_seeds = []

		with self._lock:
			for gallery_seed in gallery_seeds:
				if gallery_seed in self._gallery_seeds_to_indices:
					continue

				new_gallery_seeds.append( gallery_seed )

				self._gallery_seeds.append( gallery_seed )

				self._gallery_seeds_to_indices[ gallery_seed ] = len( self._gallery_seeds ) - 1

			self._SetStatusDirty()

		self.NotifyGallerySeedsUpdated( new_gallery_seeds )

		return len( new_gallery_seeds )

	def AdvanceGallerySeed( self, gallery_seed ):
		with self._lock:
			if gallery_seed in self._gallery_seeds_to_indices:
				index = self._gallery_seeds_to_indices[ gallery_seed ]

				if index > 0:
					self._gallery_seeds.remove( gallery_seed )

					self._gallery_seeds.insert( index - 1, gallery_seed )

				self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }

		self.NotifyGallerySeedsUpdated( ( gallery_seed, ) )

	def CanCompact( self, compact_before_this_source_time ):
		with self._lock:
			if len( self._gallery_seeds ) <= self.COMPACT_NUMBER:
				return False

			for gallery_seed in self._gallery_seeds[:-self.COMPACT_NUMBER]:
				if gallery_seed.status == CC.STATUS_UNKNOWN:
					continue

				if gallery_seed.created < compact_before_this_source_time:
					return True

		return False

	def CanRestartFailedSearch( self ):
		with self._lock:
			if len( self._gallery_seeds ) == 0:
				return False

			last_gallery_seed = self._gallery_seeds[-1]

			if last_gallery_seed.status == CC.STATUS_ERROR:
				return True

	def Compact( self, compact_before_this_source_time ):
		with self._lock:
			if len( self._gallery_seeds ) <= self.COMPACT_NUMBER:
				return

			new_gallery_seeds = SerialisableList()

			for gallery_seed in self._gallery_seeds[:-self.COMPACT_NUMBER]:
				still_to_do = gallery_seed.status == CC.STATUS_UNKNOWN
				still_relevant = gallery_seed.created > compact_before_this_source_time

				if still_to_do or still_relevant:
					new_gallery_seeds.append( gallery_seed )

			new_gallery_seeds.extend( self._gallery_seeds[-self.COMPACT_NUMBER:] )

			self._gallery_seeds = new_gallery_seeds
			self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }

			self._SetStatusDirty()

	def DelayGallerySeed( self, gallery_seed ):
		with self._lock:
			if gallery_seed in self._gallery_seeds_to_indices:
				index = self._gallery_seeds_to_indices[ gallery_seed ]

				if index < len( self._gallery_seeds ) - 1:
					self._gallery_seeds.remove( gallery_seed )

					self._gallery_seeds.insert( index + 1, gallery_seed )

				self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }

		self.NotifyGallerySeedsUpdated( ( gallery_seed, ) )

	def GetNextGallerySeed( self, status ):
		with self._lock:
			for gallery_seed in self._gallery_seeds:
				if gallery_seed.status == status:
					return gallery_seed

		return None

	def GetGallerySeedLogKey( self ):
		return self._gallery_seed_log_key

	def GetGallerySeedCount( self, status = None ):
		result = 0

		with self._lock:
			if status is None:
				result = len( self._gallery_seeds )

			else:
				for gallery_seed in self._gallery_seeds:
					if gallery_seed.status == status:
						result += 1

		return result

	def GetGallerySeeds( self, status = None ):
		with self._lock:
			return self._GetGallerySeeds( status )

	def GetGallerySeedIndex( self, gallery_seed ):
		with self._lock:
			return self._gallery_seeds_to_indices[ gallery_seed ]

	def GetStatus( self ):
		with self._lock:
			if self._status_dirty:
				self._GenerateStatus()

			return self._status_cache

	def GetStatusGenerationTime( self ):
		with self._lock:
			if self._status_dirty:
				return GetNow()

			return self._status_cache_generation_time

	def GetStatusesToCounts( self ):
		with self._lock:
			return self._GetStatusesToCounts()

	def HasGallerySeed( self, gallery_seed ):
		with self._lock:
			return gallery_seed in self._gallery_seeds_to_indices

	def HasGalleryURL( self, url ):
		search_gallery_seed = GallerySeed( url )

		search_url = search_gallery_seed.url

		return search_url in ( gallery_seed.url for gallery_seed in self._gallery_seeds )

	def NotifyGallerySeedsUpdated( self, gallery_seeds ):
		with self._lock:
			self._SetStatusDirty()

		HG.client_controller.pub( 'gallery_seed_log_gallery_seeds_updated', self._gallery_seed_log_key, gallery_seeds )

	def RemoveGallerySeeds( self, gallery_seeds ):
		with self._lock:
			gallery_seeds_to_delete = set( gallery_seeds )

			self._gallery_seeds = SerialisableList( [ gallery_seed for gallery_seed in self._gallery_seeds if gallery_seed not in gallery_seeds_to_delete ] )

			self._gallery_seeds_to_indices = { gallery_seed : index for ( index, gallery_seed ) in enumerate( self._gallery_seeds ) }

			self._SetStatusDirty()

		self.NotifyGallerySeedsUpdated( gallery_seeds_to_delete )

	def RemoveGallerySeedsByStatus( self, statuses_to_remove ):
		with self._lock:
			gallery_seeds_to_delete = [ gallery_seed for gallery_seed in self._gallery_seeds if gallery_seed.status in statuses_to_remove ]

		self.RemoveGallerySeeds( gallery_seeds_to_delete )

	def RemoveAllButUnknownGallerySeeds( self ):
		with self._lock:
			gallery_seeds_to_delete = [ gallery_seed for gallery_seed in self._gallery_seeds if gallery_seed.status != CC.STATUS_UNKNOWN ]

		self.RemoveGallerySeeds( gallery_seeds_to_delete )

	def RestartFailedSearch( self ):
		with self._lock:
			if len( self._gallery_seeds ) == 0:
				return

			last_gallery_seed = self._gallery_seeds[-1]

			if last_gallery_seed.status != CC.STATUS_ERROR:
				return

			can_generate_more_pages = True

			new_gallery_seeds = ( last_gallery_seed.GenerateRestartedDuplicate( can_generate_more_pages ), )

		self.AddGallerySeeds( new_gallery_seeds )
		self.NotifyGallerySeedsUpdated( new_gallery_seeds )

	def RetryFailures( self ):
		with self._lock:
			failed_gallery_seeds = self._GetGallerySeeds( CC.STATUS_ERROR )

			for gallery_seed in failed_gallery_seeds:
				gallery_seed.SetStatus( CC.STATUS_UNKNOWN )

		self.NotifyGallerySeedsUpdated( failed_gallery_seeds )

	def WorkToDo( self ):
		with self._lock:
			if self._status_dirty:
				self._GenerateStatus()

			( status, ( total_processed, total ) ) = self._status_cache

			return total_processed < total

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["GALLERY_SEED_LOG"] ] = GallerySeedLog


code:
import random

from lib.Actions.Import.FileSeeds import FileSeed

from lib.Handling.Parsing import GetURLsFromParseResults

from lib.Hydrus.Data import ConvertValueRangeToPrettyString, DedupeList, Print, \
	PrintException, ToHumanInt
from lib.Hydrus.Threading import JobKey

from lib.Networking.Jobs import NetworkJob

### CONSTANTS ###
import lib.Hydrus.Globals as HG
import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC

from lib.Actions.Import.FileSeeds import FILE_SEED_TYPE_URL

CHECKER_STATUS_OK = 0
CHECKER_STATUS_DEAD = 1
CHECKER_STATUS_404 = 2

DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME = 0.1

REPEATING_JOB_TYPICAL_PERIOD = 30.0

### FUNCTIONS ###

def ConvertAllParseResultsToFileSeeds( all_parse_results, source_url, file_import_options ):
	file_seeds = []

	seen_urls = set()

	for parse_results in all_parse_results:
		parsed_urls = GetURLsFromParseResults( parse_results, ( HC.URL_TYPE_DESIRED, ), only_get_top_priority = True )

		parsed_urls = DedupeList( parsed_urls )

		parsed_urls = [ url for url in parsed_urls if url not in seen_urls ]

		seen_urls.update( parsed_urls )

		# note we do this recursively due to parse_results being appropriate only for these urls--don't move this out again, or tags will be messed up

		for url in parsed_urls:
			file_seed = FileSeed( FILE_SEED_TYPE_URL, url )

			file_seed.SetReferralURL( source_url )

			file_seed.AddParseResults( parse_results, file_import_options )

			file_seeds.append( file_seed )

	return file_seeds

def GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key ):
	def network_job_presentation_context_factory( network_job ):
		def enter_call():
			job_key.SetVariable( 'popup_network_job', network_job )

		def exit_call():
			job_key.SetVariable( 'popup_network_job', None )

		return NetworkJobPresentationContext( enter_call, exit_call )

	return network_job_presentation_context_factory

def GenerateSinglePopupNetworkJobPresentationContextFactory( job_key ):
	def network_job_presentation_context_factory( network_job ):
		def enter_call():
			job_key.SetVariable( 'popup_network_job', network_job )

		def exit_call():
			job_key.DeleteVariable( 'popup_network_job' )

		return NetworkJobPresentationContext( enter_call, exit_call )

	return network_job_presentation_context_factory

def GetRepeatingJobInitialDelay():
	return 0.5 + ( random.random() * 0.5 )

def PageImporterShouldStopWorking( page_key ):
	return HG.view_shutdown or not HG.client_controller.PageAlive( page_key )

def PublishPresentationHashes( publishing_label, hashes, publish_to_popup_button, publish_files_to_page ):
	if publish_to_popup_button:
		files_job_key = JobKey()

		files_job_key.SetVariable( 'popup_files_mergable', True )
		files_job_key.SetVariable( 'popup_files', ( list( hashes ), publishing_label ) )

		HG.client_controller.pub( 'message', files_job_key )

	if publish_files_to_page:
		HG.client_controller.pub( 'imported_files_to_page', list( hashes ), publishing_label )

def THREADDownloadURL( job_key, url, url_string ):
	job_key.SetVariable( 'popup_title', url_string )
	job_key.SetVariable( 'popup_text_1', 'initialising' )

	#

	file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )

	def network_job_factory( *args, **kwargs ):
		network_job = NetworkJob( *args, **kwargs )

		network_job.OverrideBandwidth( 30 )

		return network_job

	def status_hook( text ):
		job_key.SetVariable( 'popup_text_1', text )

	network_job_presentation_context_factory = GenerateSinglePopupNetworkJobPresentationContextFactory( job_key )

	file_seed = FileSeed( FILE_SEED_TYPE_URL, url )

	#

	try:
		file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook )

		status = file_seed.status

		if status in CC.SUCCESSFUL_IMPORT_STATES:
			if status == CC.STATUS_SUCCESSFUL_AND_NEW:
				job_key.SetVariable( 'popup_text_1', 'successful!' )

			elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
				job_key.SetVariable( 'popup_text_1', 'was already in the database!' )

			if file_seed.HasHash():
				hash = file_seed.GetHash()

				job_key.SetVariable( 'popup_files', ( [ hash ], 'download' ) )

		elif status == CC.STATUS_DELETED:
			job_key.SetVariable( 'popup_text_1', 'had already been deleted!' )

	finally:
		job_key.Finish()

def THREADDownloadURLs( job_key, urls, title ):
	job_key.SetVariable( 'popup_title', title )
	job_key.SetVariable( 'popup_text_1', 'initialising' )

	num_successful = 0
	num_redundant = 0
	num_deleted = 0
	num_failed = 0

	presentation_hashes = []
	presentation_hashes_fast = set()

	file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )

	def network_job_factory( *args, **kwargs ):
		network_job = NetworkJob( *args, **kwargs )

		network_job.OverrideBandwidth()

		return network_job

	def status_hook( text ):
		job_key.SetVariable( 'popup_text_2', text )

	network_job_presentation_context_factory = GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key )

	for ( i, url ) in enumerate( urls ):
		( i_paused, should_quit ) = job_key.WaitIfNeeded()

		if should_quit:
			break

		job_key.SetVariable( 'popup_text_1', ConvertValueRangeToPrettyString( i + 1, len( urls ) ) )
		job_key.SetVariable( 'popup_gauge_1', ( i + 1, len( urls ) ) )

		file_seed = FileSeed( FILE_SEED_TYPE_URL, url )

		try:
			file_seed.DownloadAndImportRawFile( url, file_import_options, network_job_factory, network_job_presentation_context_factory, status_hook )

			status = file_seed.status

			if status in CC.SUCCESSFUL_IMPORT_STATES:
				if status == CC.STATUS_SUCCESSFUL_AND_NEW:
					num_successful += 1

				elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
					num_redundant += 1

				if file_seed.HasHash():
					hash = file_seed.GetHash()

					if hash not in presentation_hashes_fast:
						presentation_hashes.append( hash )

					presentation_hashes_fast.add( hash )

			elif status == CC.STATUS_DELETED:
				num_deleted += 1

		except Exception as e:
			num_failed += 1

			Print( url + ' failed to import!' )
			PrintException( e )

		finally:
			job_key.DeleteVariable( 'popup_text_2' )

	job_key.DeleteVariable( 'popup_network_job' )

	text_components = []

	if num_successful > 0:
		text_components.append( ToHumanInt( num_successful ) + ' successful' )

	if num_redundant > 0:
		text_components.append( ToHumanInt( num_redundant ) + ' already in db' )

	if num_deleted > 0:
		text_components.append( ToHumanInt( num_deleted ) + ' deleted' )

	if num_failed > 0:
		text_components.append( ToHumanInt( num_failed ) + ' failed (errors written to log)' )

	job_key.SetVariable( 'popup_text_1', ', '.join( text_components ) )

	if len( presentation_hashes ) > 0:
		job_key.SetVariable( 'popup_files', ( presentation_hashes, 'downloads' ) )

	job_key.DeleteVariable( 'popup_gauge_1' )

	job_key.Finish()

def UpdateFileSeedCacheWithFileSeeds( file_seed_cache, file_seeds, max_new_urls_allowed = None ):
	new_file_seeds = []

	num_urls_added = 0
	num_urls_already_in_file_seed_cache = 0
	can_search_for_more_files = True
	stop_reason = ''

	for file_seed in file_seeds:
		if max_new_urls_allowed is not None and num_urls_added >= max_new_urls_allowed:
			can_search_for_more_files = False

			stop_reason = 'hit file limit'

			break

		if file_seed_cache.HasFileSeed( file_seed ):
			num_urls_already_in_file_seed_cache += 1

		else:
			num_urls_added += 1

			new_file_seeds.append( file_seed )

	file_seed_cache.AddFileSeeds( new_file_seeds )

	return ( num_urls_added, num_urls_already_in_file_seed_cache, can_search_for_more_files, stop_reason )

def WakeRepeatingJob( job ):
	if job is not None:
		job.Wake()

class NetworkJobPresentationContext( object ):
	def __init__( self, enter_call, exit_call ):
		self._enter_call = enter_call
		self._exit_call = exit_call

	def __enter__( self ):
		self._enter_call()

	def __exit__( self, exc_type, exc_val, exc_tb ):
		self._exit_call()

license: wtfpl


code:
import os, threading, time

from lib.Actions.Import import PageImporterShouldStopWorking, PublishPresentationHashes, \
	WakeRepeatingJob, GetRepeatingJobInitialDelay
from lib.Actions.Import.FileSeeds import FileSeed, FileSeedCache
from lib.Actions.Import.Options import FilenameTaggingOptions, TagImportOptions

from lib.Hydrus.Data import ConvertValueRangeToPrettyString, GetNow, Print, \
	ShowException, ShowText, TimeHasPassed, ToHumanInt
from lib.Hydrus.Data.Local import ConvertServiceKeysToTagsToServiceKeysToContentUpdates
from lib.Hydrus.Files.Local import GetAllPaths
from lib.Hydrus.Exceptions import DataMissing
from lib.Hydrus.Paths import AppendPathUntilNoConflicts, FilterFreePaths, MergeFile
from lib.Hydrus.Paths.Local import DeletePath
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, \
	SerialisableBase, SerialisableBaseNamed
from lib.Hydrus.Tags.Local import ServiceKeysToTags
from lib.Hydrus.Threading import JobKey, IsThreadShuttingDown

### CONSTANTS ###

import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Actions.Import import DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME, \
	REPEATING_JOB_TYPICAL_PERIOD
from lib.Actions.Import.FileSeeds import FILE_SEED_TYPE_HDD

from lib.Hydrus.Serialisable import HG.SerialisableType["HDD_IMPORT"], HG.SerialisableType["IMPORT_FOLDER"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES

### CLASSES ###

class HDDImport( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["HDD_IMPORT"]
	SERIALISABLE_NAME = 'Local File Import'
	SERIALISABLE_VERSION = 2

	def __init__( self, paths = None, file_import_options = None, paths_to_service_keys_to_tags = None, delete_after_success = None ):
		SerialisableBase.__init__( self )

		if paths is None:
			self._file_seed_cache = None

		else:
			self._file_seed_cache = FileSeedCache()

			file_seeds = []

			for path in paths:
				file_seed = FileSeed( FILE_SEED_TYPE_HDD, path )

				try:
					s = os.stat( path )

					file_seed.source_time = int( min( s.st_mtime, s.st_ctime ) )

				except Exception:
					pass

				if path in paths_to_service_keys_to_tags:
					file_seed.SetFixedServiceKeysToTags( paths_to_service_keys_to_tags[ path ] )

				file_seeds.append( file_seed )

			self._file_seed_cache.AddFileSeeds( file_seeds )

		self._file_import_options = file_import_options
		self._delete_after_success = delete_after_success

		self._current_action = ''
		self._paused = False

		self._lock = threading.Lock()

		self._files_repeating_job = None

		HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )

	def _GetSerialisableInfo( self ):
		serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
		serialisable_options = self._file_import_options.GetSerialisableTuple()

		return ( serialisable_file_seed_cache, serialisable_options, self._delete_after_success, self._paused )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_file_seed_cache, serialisable_options, self._delete_after_success, self._paused ) = serialisable_info

		self._file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )
		self._file_import_options = CreateFromSerialisableTuple( serialisable_options )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( serialisable_file_seed_cache, serialisable_options, serialisable_paths_to_tags, delete_after_success, paused ) = old_serialisable_info

			file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )

			paths_to_service_keys_to_tags = { path : { bytes.fromhex( service_key ) : tags for ( service_key, tags ) in service_keys_to_tags.items() } for ( path, service_keys_to_tags ) in serialisable_paths_to_tags.items() }

			for file_seed in file_seed_cache.GetFileSeeds():
				path = file_seed.file_seed_data

				if path in paths_to_service_keys_to_tags:
					file_seed.SetFixedServiceKeysToTags( paths_to_service_keys_to_tags[ path ] )

			serialisable_file_seed_cache = file_seed_cache.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_file_seed_cache, serialisable_options, delete_after_success, paused )

			return ( 2, new_serialisable_info )

	def _WorkOnFiles( self, page_key ):
		file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if file_seed is None:
			return

		did_substantial_work = False

		path = file_seed.file_seed_data

		with self._lock:
			self._current_action = 'importing'

		file_seed.ImportPath( self._file_seed_cache, self._file_import_options )

		did_substantial_work = True

		if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
			if file_seed.ShouldPresent( self._file_import_options ):
				file_seed.PresentToPage( page_key )

				did_substantial_work = True

			if self._delete_after_success:
				try:
					DeletePath( path )

				except Exception as e:
					ShowText( 'While attempting to delete ' + path + ', the following error occurred:' )
					ShowException( e )

				txt_path = path + '.txt'

				if os.path.exists( txt_path ):
					try:
						DeletePath( txt_path )

					except Exception as e:
						ShowText( 'While attempting to delete ' + txt_path + ', the following error occurred:' )
						ShowException( e )

		with self._lock:
			self._current_action = ''

		if did_substantial_work:
			time.sleep( DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )

	def CurrentlyWorking( self ):
		with self._lock:
			work_to_do = self._file_seed_cache.WorkToDo()

			return work_to_do and not self._paused

	def GetFileImportOptions( self ):
		with self._lock:
			return self._file_import_options

	def GetFileSeedCache( self ):
		return self._file_seed_cache

	def GetStatus( self ):
		with self._lock:
			return ( self._current_action, self._paused )

	def GetValueRange( self ):
		with self._lock:
			return self._file_seed_cache.GetValueRange()

	def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
		if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
			WakeRepeatingJob( self._files_repeating_job )

	def PausePlay( self ):
		with self._lock:
			self._paused = not self._paused

			WakeRepeatingJob( self._files_repeating_job )

	def SetFileImportOptions( self, file_import_options ):
		with self._lock:
			self._file_import_options = file_import_options

	def Start( self, page_key ):
		self._files_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles, page_key )

		self._files_repeating_job.SetThreadSlotType( 'misc' )

	def REPEATINGWorkOnFiles( self, page_key ):
		with self._lock:
			if PageImporterShouldStopWorking( page_key ):
				self._files_repeating_job.Cancel()

				return

			paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )

			work_to_do = self._file_seed_cache.WorkToDo() and not ( paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )

		while work_to_do:
			try:
				self._WorkOnFiles( page_key )

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( page_key ):
					self._files_repeating_job.Cancel()

					return

				paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )

				work_to_do = self._file_seed_cache.WorkToDo() and not ( paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["HDD_IMPORT"] ] = HDDImport

class ImportFolder( SerialisableBaseNamed ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["IMPORT_FOLDER"]
	SERIALISABLE_NAME = 'Import Folder'
	SERIALISABLE_VERSION = 6

	def __init__( self, name, path = '', file_import_options = None, tag_import_options = None, tag_service_keys_to_filename_tagging_options = None, mimes = None, actions = None, action_locations = None, period = 3600, check_regularly = True, show_working_popup = True, publish_files_to_popup_button = True, publish_files_to_page = False ):
		if mimes is None:
			mimes = HC.ALLOWED_MIMES

		if file_import_options is None:
			file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'quiet' )

		if tag_import_options is None:
			tag_import_options = TagImportOptions()

		if tag_service_keys_to_filename_tagging_options is None:
			tag_service_keys_to_filename_tagging_options = {}

		if actions is None:
			actions = {}

			actions[ CC.STATUS_SUCCESSFUL_AND_NEW ] = CC.IMPORT_FOLDER_IGNORE
			actions[ CC.STATUS_SUCCESSFUL_BUT_REDUNDANT ] = CC.IMPORT_FOLDER_IGNORE
			actions[ CC.STATUS_DELETED ] = CC.IMPORT_FOLDER_IGNORE
			actions[ CC.STATUS_ERROR ] = CC.IMPORT_FOLDER_IGNORE

		if action_locations is None:
			action_locations = {}

		SerialisableBaseNamed.__init__( self, name )

		self._path = path
		self._mimes = mimes
		self._file_import_options = file_import_options
		self._tag_import_options = tag_import_options
		self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
		self._actions = actions
		self._action_locations = action_locations
		self._period = period
		self._check_regularly = check_regularly

		self._file_seed_cache = FileSeedCache()
		self._last_checked = 0
		self._paused = False
		self._check_now = False

		self._show_working_popup = show_working_popup
		self._publish_files_to_popup_button = publish_files_to_popup_button
		self._publish_files_to_page = publish_files_to_page

	def _ActionPaths( self ):
		for status in ( CC.STATUS_SUCCESSFUL_AND_NEW, CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED, CC.STATUS_ERROR ):
			action = self._actions[ status ]

			if action == CC.IMPORT_FOLDER_DELETE:
				while True:
					file_seed = self._file_seed_cache.GetNextFileSeed( status )

					if file_seed is None or HG.view_shutdown:
						break

					path = file_seed.file_seed_data

					try:
						if os.path.exists( path ):
							DeletePath( path )

						txt_path = path + '.txt'

						if os.path.exists( txt_path ):
							DeletePath( txt_path )

						self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )

					except Exception as e:
						ShowText( 'Import folder tried to delete ' + path + ', but could not:' )

						ShowException( e )

						ShowText( 'Import folder has been paused.' )

						self._paused = True

						return

			elif action == CC.IMPORT_FOLDER_MOVE:
				while True:
					file_seed = self._file_seed_cache.GetNextFileSeed( status )

					if file_seed is None or HG.view_shutdown:
						break

					path = file_seed.file_seed_data

					try:
						dest_dir = self._action_locations[ status ]

						if not os.path.exists( dest_dir ):
							raise DataMissing( 'The move location "' + dest_dir + '" does not exist!' )

						if os.path.exists( path ):
							filename = os.path.basename( path )

							dest_path = os.path.join( dest_dir, filename )

							dest_path = AppendPathUntilNoConflicts( dest_path )

							MergeFile( path, dest_path )

						txt_path = path + '.txt'

						if os.path.exists( txt_path ):
							txt_filename = os.path.basename( txt_path )

							txt_dest_path = os.path.join( dest_dir, txt_filename )

							txt_dest_path = AppendPathUntilNoConflicts( txt_dest_path )

							MergeFile( txt_path, txt_dest_path )

						self._file_seed_cache.RemoveFileSeeds( ( file_seed, ) )

					except Exception as e:
						ShowText( 'Import folder tried to move ' + path + ', but could not:' )

						ShowException( e )

						ShowText( 'Import folder has been paused.' )

						self._paused = True

						return

			elif status == CC.IMPORT_FOLDER_IGNORE:
				pass

	def _CheckFolder( self, job_key ):
		filenames = os.listdir( self._path )

		raw_paths = [ os.path.join( self._path, filename ) for filename in filenames ]

		all_paths = GetAllPaths( raw_paths )

		all_paths = FilterFreePaths( all_paths )

		file_seeds = []

		for path in all_paths:
			if job_key.IsCancelled():
				break

			if path.endswith( '.txt' ):
				continue

			file_seed = FileSeed( FILE_SEED_TYPE_HDD, path )

			if not self._file_seed_cache.HasFileSeed( file_seed ):
				file_seeds.append( file_seed )

			job_key.SetVariable( 'popup_text_1', 'checking: found ' + ToHumanInt( len( file_seeds ) ) + ' new files' )

		self._file_seed_cache.AddFileSeeds( file_seeds )

		self._last_checked = GetNow()
		self._check_now = False

	def _GetSerialisableInfo( self ):
		serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
		serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()
		serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(self._tag_service_keys_to_filename_tagging_options.items() ) ]
		serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()

		# json turns int dict keys to strings
		action_pairs = list( self._actions.items() )
		action_location_pairs = list( self._action_locations.items() )

		return ( self._path, self._mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )

	def _ImportFiles( self, job_key ):
		did_work = False

		time_to_save = GetNow() + 600

		num_files_imported = 0
		presentation_hashes = []
		presentation_hashes_fast = set()

		i = 0

		num_total = len( self._file_seed_cache )
		num_total_unknown = self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN )
		num_total_done = num_total - num_total_unknown

		while True:
			file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

			p1 = HC.options[ 'pause_import_folders_sync' ] or self._paused
			p2 = IsThreadShuttingDown()
			p3 = job_key.IsCancelled()

			if file_seed is None or p1 or p2 or p3:
				break

			did_work = True

			if TimeHasPassed( time_to_save ):
				HG.client_controller.WriteSynchronous( 'serialisable', self )

				time_to_save = GetNow() + 600

			gauge_num_done = num_total_done + num_files_imported + 1

			job_key.SetVariable( 'popup_text_1', 'importing file ' + ConvertValueRangeToPrettyString( gauge_num_done, num_total ) )
			job_key.SetVariable( 'popup_gauge_1', ( gauge_num_done, num_total ) )

			path = file_seed.file_seed_data

			file_seed.ImportPath( self._file_seed_cache, self._file_import_options, limited_mimes = self._mimes )

			if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
				if file_seed.HasHash():
					hash = file_seed.GetHash()

					if self._tag_import_options.HasAdditionalTags():
						in_inbox = HG.client_controller.Read( 'in_inbox', hash )

						downloaded_tags = []

						service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, in_inbox, hash, downloaded_tags ) # additional tags

						if len( service_keys_to_content_updates ) > 0:
							HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )

					service_keys_to_tags = ServiceKeysToTags()

					for ( tag_service_key, filename_tagging_options ) in list( self._tag_service_keys_to_filename_tagging_options.items() ):
						if not HG.client_controller.services_manager.ServiceExists( tag_service_key ):
							continue

						try:
							tags = filename_tagging_options.GetTags( tag_service_key, path )

							if len( tags ) > 0:
								service_keys_to_tags[ tag_service_key ] = tags

						except Exception as e:
							ShowText( 'Trying to parse filename tags in the import folder "' + self._name + '" threw an error!' )

							ShowException( e )

					if len( service_keys_to_tags ) > 0:
						service_keys_to_content_updates = ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags )

						HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )

				num_files_imported += 1

				if hash not in presentation_hashes_fast:
					if file_seed.ShouldPresent( self._file_import_options ):
						presentation_hashes.append( hash )

						presentation_hashes_fast.add( hash )

			elif file_seed.status == CC.STATUS_ERROR:
				Print( 'A file failed to import from import folder ' + self._name + ':' + path )

			i += 1

			if i % 10 == 0:
				self._ActionPaths()

		if num_files_imported > 0:
			Print( 'Import folder ' + self._name + ' imported ' + ToHumanInt( num_files_imported ) + ' files.' )

			if len( presentation_hashes ) > 0:
				PublishPresentationHashes( self._name, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page )

		self._ActionPaths()

		return did_work

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._path, self._mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, self._period, self._check_regularly, serialisable_file_seed_cache, self._last_checked, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page ) = serialisable_info

		self._actions = dict( action_pairs )
		self._action_locations = dict( action_location_pairs )

		self._file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )
		self._tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )
		self._tag_service_keys_to_filename_tagging_options = dict( [ ( bytes.fromhex( encoded_service_key ), CreateFromSerialisableTuple( serialisable_filename_tagging_options ) ) for ( encoded_service_key, serialisable_filename_tagging_options ) in serialisable_tag_service_keys_to_filename_tagging_options ] )
		self._file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( path, mimes, serialisable_file_import_options, action_pairs, action_location_pairs, period, open_popup, tag, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info

			# edited out tag carry-over to tio due to bit rot

			tag_import_options = TagImportOptions()

			serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()

			new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused )

			return ( 2, new_serialisable_info )

		if version == 2:
			( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info

			serialisable_txt_parse_tag_service_keys = []

			new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused )

			return ( 3, new_serialisable_info )

		if version == 3:
			( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused ) = old_serialisable_info

			check_now = False

			new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now )

			return ( 4, new_serialisable_info )

		if version == 4:
			( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_txt_parse_tag_service_keys, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now ) = old_serialisable_info

			txt_parse_tag_service_keys = [ bytes.fromhex( service_key ) for service_key in serialisable_txt_parse_tag_service_keys ]

			tag_service_keys_to_filename_tagging_options = {}

			for service_key in txt_parse_tag_service_keys:
				filename_tagging_options = FilenameTaggingOptions()

				filename_tagging_options._load_from_neighbouring_txt_files = True

				tag_service_keys_to_filename_tagging_options[ service_key ] = filename_tagging_options

			serialisable_tag_service_keys_to_filename_tagging_options = [ ( service_key.hex(), filename_tagging_options.GetSerialisableTuple() ) for ( service_key, filename_tagging_options ) in list(tag_service_keys_to_filename_tagging_options.items() ) ]

			new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now )

			return ( 5, new_serialisable_info )

		if version == 5:
			( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, open_popup, serialisable_file_seed_cache, last_checked, paused, check_now ) = old_serialisable_info

			check_regularly = not paused
			show_working_popup = True
			publish_files_to_page = False
			publish_files_to_popup_button = open_popup

			new_serialisable_info = ( path, mimes, serialisable_file_import_options, serialisable_tag_import_options, serialisable_tag_service_keys_to_filename_tagging_options, action_pairs, action_location_pairs, period, check_regularly, serialisable_file_seed_cache, last_checked, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page )

			return ( 6, new_serialisable_info )

	def CheckNow( self ):
		self._paused = False
		self._check_now = True

	def DoWork( self ):
		if HG.view_shutdown:
			return

		if HC.options[ 'pause_import_folders_sync' ] or self._paused:
			return

		checked_folder = False

		did_import_file_work = False

		error_occured = False

		job_key = JobKey( pausable = False, cancellable = True )

		try:
			if not os.path.exists( self._path ) or not os.path.isdir( self._path ):
				raise Exception( 'Path "' + self._path + '" does not seem to exist, or is not a directory.' )

			pubbed_job_key = False

			job_key.SetVariable( 'popup_title', 'import folder - ' + self._name )

			due_by_check_now = self._check_now
			due_by_period = self._check_regularly and TimeHasPassed( self._last_checked + self._period )

			if due_by_check_now or due_by_period:
				if not pubbed_job_key and self._show_working_popup:
					HG.client_controller.pub( 'message', job_key )

					pubbed_job_key = True

				self._CheckFolder( job_key )

				checked_folder = True

			file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

			if file_seed is not None:
				if not pubbed_job_key and self._show_working_popup:
					HG.client_controller.pub( 'message', job_key )

					pubbed_job_key = True

				did_import_file_work = self._ImportFiles( job_key )

		except Exception as e:
			error_occured = True
			self._paused = True

			ShowText( 'The import folder "' + self._name + '" encountered an exception! It has been paused!' )
			ShowException( e )

		if checked_folder or did_import_file_work or error_occured:
			HG.client_controller.WriteSynchronous( 'serialisable', self )

		job_key.Delete()

	def GetFileSeedCache( self ):
		return self._file_seed_cache

	def Paused( self ):
		return self._paused

	def PausePlay( self ):
		self._paused = not self._paused

	def ToListBoxTuple( self ):
		return ( self._name, self._path, self._paused, self._check_regularly, self._period )

	def ToTuple( self ):
		return ( self._name, self._path, self._mimes, self._file_import_options, self._tag_import_options, self._tag_service_keys_to_filename_tagging_options, self._actions, self._action_locations, self._period, self._check_regularly, self._paused, self._check_now, self._show_working_popup, self._publish_files_to_popup_button, self._publish_files_to_page )

	def SetFileSeedCache( self, file_seed_cache ):
		self._file_seed_cache = file_seed_cache

	def SetTuple( self, name, path, mimes, file_import_options, tag_import_options, tag_service_keys_to_filename_tagging_options, actions, action_locations, period, check_regularly, paused, check_now, show_working_popup, publish_files_to_popup_button, publish_files_to_page ):
		if path != self._path:
			self._file_seed_cache = FileSeedCache()

		if set( mimes ) != set( self._mimes ):
			self._file_seed_cache.RemoveFileSeedsByStatus( ( CC.STATUS_VETOED, ) )

		self._name = name
		self._path = path
		self._mimes = mimes
		self._file_import_options = file_import_options
		self._tag_import_options = tag_import_options
		self._tag_service_keys_to_filename_tagging_options = tag_service_keys_to_filename_tagging_options
		self._actions = actions
		self._action_locations = action_locations
		self._period = period
		self._check_regularly = check_regularly
		self._paused = paused
		self._check_now = check_now
		self._show_working_popup = show_working_popup
		self._publish_files_to_popup_button = publish_files_to_popup_button
		self._publish_files_to_page = publish_files_to_page

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["IMPORT_FOLDER"] ] = ImportFolder


code:
import os, re

from lib.Hydrus.Data import ConvertResolutionToPrettyString, GetNow, \
	ShowText, TimeDeltaToPrettyTimeDelta, TimeHasPassed, ToHumanBytes, \
	ToHumanInt
from lib.Hydrus.Data.Local import ConvertServiceKeysToTagsToServiceKeysToContentUpdates
from lib.Hydrus.Exceptions import DataMissing, SizeException, VetoException
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, SerialisableBase
from lib.Hydrus.Tags import CleanTags, SortNumericTags
from lib.Hydrus.Tags.Local import ServiceKeysToTags, TagFilter
from lib.Hydrus.Text import DeserialiseNewlinedTexts

### CONSTANTS ###

import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Hydrus.Serialisable import HG.SerialisableType["CHECKER_OPTIONS"], \
	HG.SerialisableType["FILE_IMPORT_OPTIONS"], HG.SerialisableType["FILENAME_TAGGING_OPTIONS"], \
	HG.SerialisableType["SERVICE_TAG_IMPORT_OPTIONS"], HG.SerialisableType["TAG_IMPORT_OPTIONS"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES
### FUNCTIONS ###

def NewInboxArchiveMatch( new_files, inbox_files, archive_files, status, inbox ):
	if status == CC.STATUS_SUCCESSFUL_AND_NEW and new_files:
		return True

	elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:
		if inbox and inbox_files:
			return True

		elif not inbox and archive_files:
			return True

	return False

def NewInboxArchiveMatchIgnorantOfInbox( new_files, inbox_files, archive_files, status ):
	if status == CC.STATUS_SUCCESSFUL_AND_NEW and new_files:
		return True

	elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT and archive_files and inbox_files:
		return True

	return False

def NewInboxArchiveNonMatchIgnorantOfInbox( new_files, inbox_files, archive_files, status ):
	if status == CC.STATUS_SUCCESSFUL_AND_NEW and not new_files:
		return True

	elif status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT and not ( archive_files or inbox_files ):
		return True

	return False

class CheckerOptions( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["CHECKER_OPTIONS"]
	SERIALISABLE_NAME = 'Checker Timing Options'
	SERIALISABLE_VERSION = 1

	def __init__( self, intended_files_per_check = 8, never_faster_than = 300, never_slower_than = 86400, death_file_velocity = ( 1, 86400 ) ):
		SerialisableBase.__init__( self )

		self._intended_files_per_check = intended_files_per_check
		self._never_faster_than = never_faster_than
		self._never_slower_than = never_slower_than
		self._death_file_velocity = death_file_velocity

	def _GetCurrentFilesVelocity( self, file_seed_cache, last_check_time ):
		( death_files_found, death_time_delta ) = self._death_file_velocity

		since = last_check_time - death_time_delta

		current_files_found = file_seed_cache.GetNumNewFilesSince( since )

		# when a thread is only 30mins old (i.e. first file was posted 30 mins ago), we don't want to calculate based on a longer delete time delta
		# we want next check to be like 30mins from now, not 12 hours
		# so we'll say "5 files in 30 mins" rather than "5 files in 24 hours"

		earliest_source_time = file_seed_cache.GetEarliestSourceTime()

		if earliest_source_time is None:
			current_time_delta = death_time_delta

		else:
			early_time_delta = max( last_check_time - earliest_source_time, 30 )

			current_time_delta = min( early_time_delta, death_time_delta )

		return ( current_files_found, current_time_delta )

	def _GetSerialisableInfo( self ):
		return ( self._intended_files_per_check, self._never_faster_than, self._never_slower_than, self._death_file_velocity )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._intended_files_per_check, self._never_faster_than, self._never_slower_than, self._death_file_velocity ) = serialisable_info

	def GetDeathFileVelocityPeriod( self ):
		( death_files_found, death_time_delta ) = self._death_file_velocity

		return death_time_delta

	def GetNextCheckTime( self, file_seed_cache, last_check_time, last_next_check_time ):
		if len( file_seed_cache ) == 0:
			if last_check_time == 0:
				return 0 # haven't checked yet, so should check immediately

			else:
				return GetNow() + self._never_slower_than

		elif self._never_faster_than == self._never_slower_than:
			if last_next_check_time is None or last_next_check_time == 0:
				next_check_time = last_check_time - 5

			else:
				next_check_time = last_next_check_time

			while TimeHasPassed( next_check_time ):
				next_check_time += self._never_slower_than

			return next_check_time

		else:
			( current_files_found, current_time_delta ) = self._GetCurrentFilesVelocity( file_seed_cache, last_check_time )

			if current_files_found == 0:
				# this shouldn't typically matter, since a dead checker won't care about next check time
				# so let's just have a nice safe value in case this is ever asked legit
				check_period = self._never_slower_than

			else:
				approx_time_per_file = current_time_delta // current_files_found

				ideal_check_period = self._intended_files_per_check * approx_time_per_file

				# if a thread produced lots of files and then stopped completely for whatever reason, we don't want to keep checking fast
				# so, we set a lower limit of time since last file upload, neatly doubling our check period in these situations

				latest_source_time = file_seed_cache.GetLatestSourceTime()

				time_since_latest_file = max( last_check_time - latest_source_time, 30 )

				never_faster_than = max( self._never_faster_than, time_since_latest_file )

				check_period = min( max( never_faster_than, ideal_check_period ), self._never_slower_than )

			return last_check_time + check_period

	def GetPrettyCurrentVelocity( self, file_seed_cache, last_check_time, no_prefix = False ):
		if len( file_seed_cache ) == 0:
			if last_check_time == 0:
				pretty_current_velocity = 'no files yet'

			else:
				pretty_current_velocity = 'no files, unable to determine velocity'

		else:
			if no_prefix:
				pretty_current_velocity = ''

			else:
				pretty_current_velocity = 'at last check, found '

			( current_files_found, current_time_delta ) = self._GetCurrentFilesVelocity( file_seed_cache, last_check_time )

			pretty_current_velocity += ToHumanInt( current_files_found ) + ' files in previous ' + TimeDeltaToPrettyTimeDelta( current_time_delta )

		return pretty_current_velocity

	def GetRawCurrentVelocity( self, file_seed_cache, last_check_time ):
		return self._GetCurrentFilesVelocity( file_seed_cache, last_check_time )

	def GetSummary( self ):
		if self._never_faster_than == self._never_slower_than:
			timing_statement = 'Checking every ' + TimeDeltaToPrettyTimeDelta( self._never_faster_than ) + '.'

		else:
			timing_statement = 'Trying to get ' + ToHumanInt( self._intended_files_per_check ) + ' files per check, never faster than ' + TimeDeltaToPrettyTimeDelta( self._never_faster_than ) + ' and never slower than ' + TimeDeltaToPrettyTimeDelta( self._never_slower_than ) + '.'

		( death_files_found, death_time_delta ) = self._death_file_velocity

		if death_files_found == 0:
			death_statement = 'Never stopping.'

		else:
			death_statement = 'Stopping if file velocity falls below ' + ToHumanInt( death_files_found ) + ' files per ' + TimeDeltaToPrettyTimeDelta( death_time_delta ) + '.'

		return timing_statement + os.linesep * 2 + death_statement

	def IsDead( self, file_seed_cache, last_check_time ):
		if len( file_seed_cache ) == 0 and last_check_time == 0:
			return False

		else:
			( current_files_found, current_time_delta ) = self._GetCurrentFilesVelocity( file_seed_cache, last_check_time )

			( death_files_found, deleted_time_delta ) = self._death_file_velocity

			current_file_velocity_float = current_files_found / current_time_delta
			death_file_velocity_float = death_files_found / deleted_time_delta

			return current_file_velocity_float < death_file_velocity_float

	def ToTuple( self ):
		return ( self._intended_files_per_check, self._never_faster_than, self._never_slower_than, self._death_file_velocity )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["CHECKER_OPTIONS"] ] = CheckerOptions

class FilenameTaggingOptions( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["FILENAME_TAGGING_OPTIONS"]
	SERIALISABLE_NAME = 'Filename Tagging Options'
	SERIALISABLE_VERSION = 2

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._tags_for_all = set()

		self._load_from_neighbouring_txt_files = False

		self._add_filename = ( False, '' )

		self._directories_dict = {}

		for index in ( 0, 1, 2, -3, -2, -1 ):
			self._directories_dict[ index ] = ( False, '' )

		self._quick_namespaces = []
		self._regexes = []

	def _GetSerialisableInfo( self ):
		serialisable_directories_dict = list( self._directories_dict.items() )

		return ( list( self._tags_for_all ), self._load_from_neighbouring_txt_files, self._add_filename, serialisable_directories_dict, self._quick_namespaces, self._regexes )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( tags_for_all_list, self._load_from_neighbouring_txt_files, self._add_filename, serialisable_directories_dict, self._quick_namespaces, self._regexes ) = serialisable_info

		self._directories_dict = dict( serialisable_directories_dict )

		# converting [ namespace, regex ] to ( namespace, regex ) for listctrl et al to handle better
		self._quick_namespaces = [ tuple( item ) for item in self._quick_namespaces ]
		self._tags_for_all = set( tags_for_all_list )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( tags_for_all_list, load_from_neighbouring_txt_files, add_filename, add_first_directory, add_second_directory, add_third_directory, quick_namespaces, regexes ) = old_serialisable_info

			directories_dict = {}

			directories_dict[ 0 ] = add_first_directory
			directories_dict[ 1 ] = add_second_directory
			directories_dict[ 2 ] = add_third_directory

			for index in ( -3, -2, -1 ):
				directories_dict[ index ] = ( False, '' )

			serialisable_directories_dict = list( directories_dict.items() )

			new_serialisable_info = ( tags_for_all_list, load_from_neighbouring_txt_files, add_filename, serialisable_directories_dict, quick_namespaces, regexes )

			return ( 2, new_serialisable_info )

	def AdvancedSetTuple( self, quick_namespaces, regexes ):
		self._quick_namespaces = quick_namespaces
		self._regexes = regexes

	def AdvancedToTuple( self ):
		return ( self._quick_namespaces, self._regexes )

	def GetTags( self, service_key, path ):
		tags = set()

		tags.update( self._tags_for_all )

		if self._load_from_neighbouring_txt_files:
			txt_path = path + '.txt'

			if os.path.exists( txt_path ):
				with open( txt_path, 'r', encoding = 'utf-8'  ) as f:
					txt_tags_string = f.read()

				try:
					txt_tags = [ tag for tag in DeserialiseNewlinedTexts( txt_tags_string ) ]

					if True in ( len( txt_tag ) > 1024 for txt_tag in txt_tags ):
						ShowText( 'Tags were too long--I think this was not a regular text file!' )

						raise Exception()

					tags.update( txt_tags )

				except Exception:
					ShowText( 'Could not parse the tags from ' + txt_path + '!' )

					tags.add( '___had problem parsing .txt file' )

		( base, filename ) = os.path.split( path )

		( filename, any_ext_gumpf ) = os.path.splitext( filename )

		( filename_boolean, filename_namespace ) = self._add_filename

		if filename_boolean:
			if filename_namespace != '':
				tag = filename_namespace + ':' + filename

			else:
				tag = filename

			tags.add( tag )

		( drive, directories ) = os.path.splitdrive( base )

		while directories.startswith( os.path.sep ):
			directories = directories[1:]

		directories = directories.split( os.path.sep )

		for ( index, ( dir_boolean, dir_namespace ) ) in list( self._directories_dict.items() ):
			# we are talking -3 through 2 here

			if not dir_boolean:
				continue

			try:
				directory = directories[ index ]

			except IndexError:
				continue

			if dir_namespace != '':
				tag = dir_namespace + ':' + directory

			else:
				tag = directory

			tags.add( tag )

		#

		for regex in self._regexes:
			try:
				result = re.findall( regex, path )

				for match in result:
					if isinstance( match, tuple ):
						for submatch in match:
							tags.add( submatch )

					else:
						tags.add( match )

			except Exception:
				pass

		for ( namespace, regex ) in self._quick_namespaces:
			try:
				result = re.findall( regex, path )

				for match in result:
					if isinstance( match, tuple ):
						for submatch in match:
							tags.add( namespace + ':' + submatch )

					else:
						tags.add( namespace + ':' + match )

			except Exception:
				pass

		#

		tags = CleanTags( tags )

		siblings_manager = HG.client_controller.tag_siblings_manager
		parents_manager = HG.client_controller.tag_parents_manager

		tags = siblings_manager.CollapseTags( service_key, tags )
		tags = parents_manager.ExpandTags( service_key, tags )
		tags = HG.client_controller.tag_censorship_manager.FilterTags( service_key, tags )

		return tags

	def SimpleSetTuple( self, tags_for_all, load_from_neighbouring_txt_files, add_filename, directories_dict ):
		self._tags_for_all = tags_for_all
		self._load_from_neighbouring_txt_files = load_from_neighbouring_txt_files
		self._add_filename = add_filename
		self._directories_dict = directories_dict

	def SimpleToTuple( self ):
		return ( self._tags_for_all, self._load_from_neighbouring_txt_files, self._add_filename, self._directories_dict )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["FILENAME_TAGGING_OPTIONS"] ] = FilenameTaggingOptions

class FileImportOptions( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["FILE_IMPORT_OPTIONS"]
	SERIALISABLE_NAME = 'File Import Options'
	SERIALISABLE_VERSION = 4

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._exclude_deleted = True
		self._do_not_check_known_urls_before_importing = False
		self._do_not_check_hashes_before_importing = False
		self._allow_decompression_bombs = False
		self._min_size = None
		self._max_size = None
		self._max_gif_size = None
		self._min_resolution = None
		self._max_resolution = None
		self._automatic_archive = False
		self._associate_source_urls = True
		self._present_new_files = True
		self._present_already_in_inbox_files = True
		self._present_already_in_archive_files = True

	def _GetSerialisableInfo( self ):
		pre_import_options = ( self._exclude_deleted, self._do_not_check_known_urls_before_importing, self._do_not_check_hashes_before_importing, self._allow_decompression_bombs, self._min_size, self._max_size, self._max_gif_size, self._min_resolution, self._max_resolution )
		post_import_options = ( self._automatic_archive, self._associate_source_urls )
		presentation_options = ( self._present_new_files, self._present_already_in_inbox_files, self._present_already_in_archive_files )

		return ( pre_import_options, post_import_options, presentation_options )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( pre_import_options, post_import_options, presentation_options ) = serialisable_info

		( self._exclude_deleted, self._do_not_check_known_urls_before_importing, self._do_not_check_hashes_before_importing, self._allow_decompression_bombs, self._min_size, self._max_size, self._max_gif_size, self._min_resolution, self._max_resolution ) = pre_import_options
		( self._automatic_archive, self._associate_source_urls ) = post_import_options
		( self._present_new_files, self._present_already_in_inbox_files, self._present_already_in_archive_files ) = presentation_options

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( automatic_archive, exclude_deleted, min_size, min_resolution ) = old_serialisable_info

			present_new_files = True
			present_already_in_inbox_files = False
			present_already_in_archive_files = False

			new_serialisable_info = ( automatic_archive, exclude_deleted, present_new_files, present_already_in_inbox_files, present_already_in_archive_files, min_size, min_resolution )

			return ( 2, new_serialisable_info )

		if version == 2:
			( automatic_archive, exclude_deleted, present_new_files, present_already_in_inbox_files, present_already_in_archive_files, min_size, min_resolution ) = old_serialisable_info

			max_size = None
			max_resolution = None

			allow_decompression_bombs = False
			max_gif_size = 32 * 1048576

			pre_import_options = ( exclude_deleted, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )
			post_import_options = automatic_archive
			presentation_options = ( present_new_files, present_already_in_inbox_files, present_already_in_archive_files )

			new_serialisable_info = ( pre_import_options, post_import_options, presentation_options )

			return ( 3, new_serialisable_info )

		if version == 3:
			( pre_import_options, post_import_options, presentation_options ) = old_serialisable_info

			( exclude_deleted, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution ) = pre_import_options

			automatic_archive = post_import_options

			do_not_check_known_urls_before_importing = False
			do_not_check_hashes_before_importing = False
			associate_source_urls = True

			pre_import_options = ( exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution )

			post_import_options = ( automatic_archive, associate_source_urls )

			new_serialisable_info = ( pre_import_options, post_import_options, presentation_options )

			return ( 4, new_serialisable_info )

	def AllowsDecompressionBombs( self ):
		return self._allow_decompression_bombs

	def AutomaticallyArchives( self ):
		return self._automatic_archive

	def CheckFileIsValid( self, size, mime, width, height ):
		if self._min_size is not None and size < self._min_size:
			raise SizeException( 'File was ' + ToHumanBytes( size ) + ' but the lower limit is ' + ToHumanBytes( self._min_size ) + '.' )

		if self._max_size is not None and size > self._max_size:
			raise SizeException( 'File was ' + ToHumanBytes( size ) + ' but the upper limit is ' + ToHumanBytes( self._max_size ) + '.' )

		if mime == HC.IMAGE_GIF and self._max_gif_size is not None and size > self._max_gif_size:
			raise SizeException( 'File was ' + ToHumanBytes( size ) + ' but the upper limit for gifs is ' + ToHumanBytes( self._max_gif_size ) + '.' )

		if self._min_resolution is not None:
			( min_width, min_height ) = self._min_resolution

			too_thin = width is not None and width < min_width
			too_short = height is not None and height < min_height

			if too_thin or too_short:
				raise SizeException( 'File had resolution ' + ConvertResolutionToPrettyString( ( width, height ) ) + ' but the lower limit is ' + ConvertResolutionToPrettyString( self._min_resolution ) )

		if self._max_resolution is not None:
			( max_width, max_height ) = self._max_resolution

			too_wide = width is not None and width > max_width
			too_tall = height is not None and height > max_height

			if too_wide or too_tall:
				raise SizeException( 'File had resolution ' + ConvertResolutionToPrettyString( ( width, height ) ) + ' but the upper limit is ' + ConvertResolutionToPrettyString( self._max_resolution ) )

	def CheckNetworkDownload( self, possible_mime, num_bytes, is_complete_file_size ):
		if is_complete_file_size:
			error_prefix = 'Download was apparently '

		else:
			error_prefix = 'Download was at least '

		if possible_mime is not None:
			if possible_mime == HC.IMAGE_GIF and self._max_gif_size is not None and num_bytes > self._max_gif_size:
				raise SizeException( error_prefix + ToHumanBytes( num_bytes ) + ' but the upper limit for gifs is ' + ToHumanBytes( self._max_gif_size ) + '.' )

		if self._max_size is not None and num_bytes > self._max_size:
			raise SizeException( error_prefix + ToHumanBytes( num_bytes ) + ' but the upper limit is ' + ToHumanBytes( self._max_size ) + '.' )

		if is_complete_file_size:
			if self._min_size is not None and num_bytes < self._min_size:
				raise SizeException( error_prefix + ToHumanBytes( num_bytes ) + ' but the lower limit is ' + ToHumanBytes( self._min_size ) + '.' )

	def ExcludesDeleted( self ):
		return self._exclude_deleted

	def GetPostImportOptions( self ):
		post_import_options = ( self._automatic_archive, self._associate_source_urls )

		return post_import_options

	def GetPresentationOptions( self ):
		presentation_options = ( self._present_new_files, self._present_already_in_inbox_files, self._present_already_in_archive_files )

		return presentation_options

	def GetPreImportOptions( self ):
		pre_import_options = ( self._exclude_deleted, self._do_not_check_known_urls_before_importing, self._do_not_check_hashes_before_importing, self._allow_decompression_bombs, self._min_size, self._max_size, self._max_gif_size, self._min_resolution, self._max_resolution )

		return pre_import_options

	def GetSummary( self ):
		statements = []

		if self._exclude_deleted:
			statements.append( 'excluding previously deleted' )

		if not self._allow_decompression_bombs:
			statements.append( 'excluding decompression bombs' )

		if self._min_size is not None:
			statements.append( 'excluding < ' + ToHumanBytes( self._min_size ) )

		if self._max_size is not None:
			statements.append( 'excluding > ' + ToHumanBytes( self._max_size ) )

		if self._max_gif_size is not None:
			statements.append( 'excluding gifs > ' + ToHumanBytes( self._max_gif_size ) )

		if self._min_resolution is not None:
			( width, height ) = self._min_resolution

			statements.append( 'excluding < ( ' + ToHumanInt( width ) + ' x ' + ToHumanInt( height ) + ' )' )

		if self._max_resolution is not None:
			( width, height ) = self._max_resolution

			statements.append( 'excluding > ( ' + ToHumanInt( width ) + ' x ' + ToHumanInt( height ) + ' )' )

		#

		if self._automatic_archive:
			statements.append( 'automatically archiving' )

		#

		presentation_statements = []

		if self._present_new_files:
			presentation_statements.append( 'new' )

		if self._present_already_in_inbox_files:
			presentation_statements.append( 'already in inbox' )

		if self._present_already_in_archive_files:
			presentation_statements.append( 'already in archive' )

		if len( presentation_statements ) == 0:
			statements.append( 'not presenting any files' )

		elif len( presentation_statements ) == 3:
			statements.append( 'presenting all files' )

		else:
			statements.append( 'presenting ' + ', '.join( presentation_statements ) + ' files' )

		summary = os.linesep.join( statements )

		return summary

	def SetPostImportOptions( self, automatic_archive, associate_source_urls ):
		self._automatic_archive = automatic_archive
		self._associate_source_urls = associate_source_urls

	def SetPresentationOptions( self, present_new_files, present_already_in_inbox_files, present_already_in_archive_files ):
		self._present_new_files = present_new_files
		self._present_already_in_inbox_files = present_already_in_inbox_files
		self._present_already_in_archive_files = present_already_in_archive_files

	def SetPreImportOptions( self, exclude_deleted, do_not_check_known_urls_before_importing, do_not_check_hashes_before_importing, allow_decompression_bombs, min_size, max_size, max_gif_size, min_resolution, max_resolution ):
		self._exclude_deleted = exclude_deleted
		self._do_not_check_known_urls_before_importing = do_not_check_known_urls_before_importing
		self._do_not_check_hashes_before_importing = do_not_check_hashes_before_importing
		self._allow_decompression_bombs = allow_decompression_bombs
		self._min_size = min_size
		self._max_size = max_size
		self._max_gif_size = max_gif_size
		self._min_resolution = min_resolution
		self._max_resolution = max_resolution

	def ShouldAssociateSourceURLs( self ):
		return self._associate_source_urls

	def DoNotCheckHashesBeforeImporting( self ):
		return self._do_not_check_hashes_before_importing

	def DoNotCheckKnownURLsBeforeImporting( self ):
		return self._do_not_check_known_urls_before_importing

	def ShouldNotPresentIgnorantOfInbox( self, status ):
		return NewInboxArchiveNonMatchIgnorantOfInbox( self._present_new_files, self._present_already_in_inbox_files, self._present_already_in_archive_files, status )

	def ShouldPresent( self, status, inbox ):
		return NewInboxArchiveMatch( self._present_new_files, self._present_already_in_inbox_files, self._present_already_in_archive_files, status, inbox )

	def ShouldPresentIgnorantOfInbox( self, status ):
		return NewInboxArchiveMatchIgnorantOfInbox( self._present_new_files, self._present_already_in_inbox_files, self._present_already_in_archive_files, status )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["FILE_IMPORT_OPTIONS"] ] = FileImportOptions

class TagImportOptions( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["TAG_IMPORT_OPTIONS"]
	SERIALISABLE_NAME = 'Tag Import Options'
	SERIALISABLE_VERSION = 7

	def __init__( self, fetch_tags_even_if_url_recognised_and_file_already_in_db = False, fetch_tags_even_if_hash_recognised_and_file_already_in_db = False, tag_blacklist = None, service_keys_to_service_tag_import_options = None, is_default = False ):
		SerialisableBase.__init__( self )

		if tag_blacklist is None:
			tag_blacklist = TagFilter()

		if service_keys_to_service_tag_import_options is None:
			service_keys_to_service_tag_import_options = {}

		self._fetch_tags_even_if_url_recognised_and_file_already_in_db = fetch_tags_even_if_url_recognised_and_file_already_in_db
		self._fetch_tags_even_if_hash_recognised_and_file_already_in_db = fetch_tags_even_if_hash_recognised_and_file_already_in_db
		self._tag_blacklist = tag_blacklist
		self._service_keys_to_service_tag_import_options = service_keys_to_service_tag_import_options
		self._is_default = is_default

	def _GetSerialisableInfo( self ):
		if HG.client_controller.IsBooted():
			services_manager = HG.client_controller.services_manager

			test_func = services_manager.ServiceExists

		else:
			def test_func( service_key ):
				return True

		serialisable_tag_blacklist = self._tag_blacklist.GetSerialisableTuple()

		serialisable_service_keys_to_service_tag_import_options = [ ( service_key.hex(), service_tag_import_options.GetSerialisableTuple() ) for ( service_key, service_tag_import_options ) in list(self._service_keys_to_service_tag_import_options.items() ) if test_func( service_key ) ]

		return ( self._fetch_tags_even_if_url_recognised_and_file_already_in_db, self._fetch_tags_even_if_hash_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_service_keys_to_service_tag_import_options, self._is_default )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._fetch_tags_even_if_url_recognised_and_file_already_in_db, self._fetch_tags_even_if_hash_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_service_keys_to_service_tag_import_options, self._is_default ) = serialisable_info

		self._tag_blacklist = CreateFromSerialisableTuple( serialisable_tag_blacklist )

		self._service_keys_to_service_tag_import_options = { bytes.fromhex( encoded_service_key ) : CreateFromSerialisableTuple( serialisable_service_tag_import_options ) for ( encoded_service_key, serialisable_service_tag_import_options ) in serialisable_service_keys_to_service_tag_import_options }

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			safe_service_keys_to_namespaces = old_serialisable_info

			safe_service_keys_to_additional_tags = {}

			new_serialisable_info = ( safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags )

			return ( 2, new_serialisable_info )

		if version == 2:
			( safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags ) = old_serialisable_info

			fetch_tags_even_if_url_recognised_and_file_already_in_db = False

			new_serialisable_info = ( fetch_tags_even_if_url_recognised_and_file_already_in_db, safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags )

			return ( 3, new_serialisable_info )

		if version == 3:
			( fetch_tags_even_if_url_recognised_and_file_already_in_db, safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags ) = old_serialisable_info

			tag_blacklist = TagFilter()

			serialisable_tag_blacklist = tag_blacklist.GetSerialisableTuple()

			new_serialisable_info = ( fetch_tags_even_if_url_recognised_and_file_already_in_db, serialisable_tag_blacklist, safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags )

			return ( 4, new_serialisable_info )

		if version == 4:
			( fetch_tags_even_if_url_recognised_and_file_already_in_db, serialisable_tag_blacklist, safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags ) = old_serialisable_info

			serialisable_get_all_service_keys = []

			new_serialisable_info = ( fetch_tags_even_if_url_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_get_all_service_keys, safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags )

			return ( 5, new_serialisable_info )

		if version == 5:
			( fetch_tags_even_if_url_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_get_all_service_keys, safe_service_keys_to_namespaces, safe_service_keys_to_additional_tags ) = old_serialisable_info

			fetch_tags_even_if_hash_recognised_and_file_already_in_db = fetch_tags_even_if_url_recognised_and_file_already_in_db

			get_all_service_keys = { bytes.fromhex( encoded_service_key ) for encoded_service_key in serialisable_get_all_service_keys }
			service_keys_to_namespaces = { bytes.fromhex( service_key ) : set( namespaces ) for ( service_key, namespaces ) in list( safe_service_keys_to_namespaces.items() ) }
			service_keys_to_additional_tags = { bytes.fromhex( service_key ) : set( tags ) for ( service_key, tags ) in list( safe_service_keys_to_additional_tags.items() ) }

			service_keys_to_service_tag_import_options = {}

			service_keys = set()

			service_keys.update( get_all_service_keys )
			service_keys.update( list( service_keys_to_namespaces.keys() ) )
			service_keys.update( list( service_keys_to_additional_tags.keys() ) )

			for service_key in service_keys:
				get_tags = False
				namespaces = []
				additional_tags = []

				if service_key in service_keys_to_namespaces:
					namespaces = service_keys_to_namespaces[ service_key ]

				if service_key in get_all_service_keys or 'all namespaces' in namespaces:
					get_tags = True

				if service_key in service_keys_to_additional_tags:
					additional_tags = service_keys_to_additional_tags[ service_key ]

				( to_new_files, to_already_in_inbox, to_already_in_archive, only_add_existing_tags ) = ( True, True, True, False )

				service_tag_import_options = ServiceTagImportOptions( get_tags = get_tags, additional_tags = additional_tags, to_new_files = to_new_files, to_already_in_inbox = to_already_in_inbox, to_already_in_archive = to_already_in_archive, only_add_existing_tags = only_add_existing_tags )

				service_keys_to_service_tag_import_options[ service_key ] = service_tag_import_options

			serialisable_service_keys_to_service_tag_import_options = [ ( service_key.hex(), service_tag_import_options.GetSerialisableTuple() ) for ( service_key, service_tag_import_options ) in list(service_keys_to_service_tag_import_options.items() ) ]

			new_serialisable_info = ( fetch_tags_even_if_url_recognised_and_file_already_in_db, fetch_tags_even_if_hash_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_service_keys_to_service_tag_import_options )

			return ( 6, new_serialisable_info )

		if version == 6:
			( fetch_tags_even_if_url_recognised_and_file_already_in_db, fetch_tags_even_if_hash_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_service_keys_to_service_tag_import_options ) = old_serialisable_info

			is_default = False

			new_serialisable_info = ( fetch_tags_even_if_url_recognised_and_file_already_in_db, fetch_tags_even_if_hash_recognised_and_file_already_in_db, serialisable_tag_blacklist, serialisable_service_keys_to_service_tag_import_options, is_default )

			return ( 7, new_serialisable_info )

	def CheckBlacklist( self, tags ):
		ok_tags = self._tag_blacklist.Filter( tags )

		if len( ok_tags ) < len( tags ):
			bad_tags = set( tags ).difference( ok_tags )

			bad_tags = SortNumericTags( bad_tags )

			raise VetoException( ', '.join( bad_tags ) + ' is blacklisted!' )

	def GetServiceKeysToContentUpdates( self, status, in_inbox, hash, parsed_tags ):
		siblings_manager = HG.client_controller.tag_siblings_manager
		parents_manager = HG.client_controller.tag_parents_manager

		parsed_tags = CleanTags( parsed_tags )

		service_keys_to_tags = ServiceKeysToTags()

		for ( service_key, service_tag_import_options ) in list( self._service_keys_to_service_tag_import_options.items() ):
			service_parsed_tags = siblings_manager.CollapseTags( service_key, parsed_tags )
			service_parsed_tags = parents_manager.ExpandTags( service_key, service_parsed_tags )

			service_tags = service_tag_import_options.GetTags( service_key, status, in_inbox, hash, service_parsed_tags )

			if len( service_tags ) > 0:
				service_tags = siblings_manager.CollapseTags( service_key, service_tags )
				service_tags = parents_manager.ExpandTags( service_key, service_tags )

				service_keys_to_tags[ service_key ] = service_tags

		service_keys_to_content_updates = ConvertServiceKeysToTagsToServiceKeysToContentUpdates( { hash }, service_keys_to_tags )

		return service_keys_to_content_updates

	def GetServiceTagImportOptions( self, service_key ):
		if service_key not in self._service_keys_to_service_tag_import_options:
			self._service_keys_to_service_tag_import_options[ service_key ] = ServiceTagImportOptions()

		return self._service_keys_to_service_tag_import_options[ service_key ]

	def GetSummary( self, show_downloader_options ):
		if self._is_default:
			return 'Using whatever the default tag import options is at at time of import.'

		statements = []

		for ( service_key, service_tag_import_options ) in list( self._service_keys_to_service_tag_import_options.items() ):
			sub_statements = service_tag_import_options.GetSummaryStatements()

			if len( sub_statements ) > 0:
				try:
					name = HG.client_controller.services_manager.GetName( service_key )

				except DataMissing:
					continue

				service_statement = name + ':' + os.linesep * 2 + os.linesep.join( sub_statements )

				statements.append( service_statement )

		if len( statements ) > 0:
			if show_downloader_options:
				pre_statements = []

				pre_statements.append( self._tag_blacklist.ToBlacklistString() )

				if self._fetch_tags_even_if_url_recognised_and_file_already_in_db:
					s = 'fetching tags even if url is recognised and file already in db'

				else:
					s = 'not fetching tags if url is recognised and file already in db'

				pre_statements.append( s )

				if self._fetch_tags_even_if_hash_recognised_and_file_already_in_db:
					s = 'fetching tags even if hash is recognised and file already in db'

				else:
					s = 'not fetching tags if hash is recognised and file already in db'

				pre_statements.append( s )

				statements = pre_statements + [ '---' ] + statements

			separator = os.linesep * 2

			summary = separator.join( statements )

		else:
			summary = 'not adding any tags'

		return summary

	def GetTagBlacklist( self ):
		return self._tag_blacklist

	def HasAdditionalTags( self ):
		return True in ( service_tag_import_options.HasAdditionalTags() for service_tag_import_options in list( self._service_keys_to_service_tag_import_options.values() ) )

	def IsDefault( self ):
		return self._is_default

	def SetDefault( self ):
		self._is_default = True

	def ShouldFetchTagsEvenIfHashKnownAndFileAlreadyInDB( self ):
		return self._fetch_tags_even_if_hash_recognised_and_file_already_in_db

	def ShouldFetchTagsEvenIfURLKnownAndFileAlreadyInDB( self ):
		return self._fetch_tags_even_if_url_recognised_and_file_already_in_db

	def WorthFetchingTags( self ):
		return True in ( service_tag_import_options.WorthFetchingTags() for service_tag_import_options in list( self._service_keys_to_service_tag_import_options.values() ) )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["TAG_IMPORT_OPTIONS"] ] = TagImportOptions

class ServiceTagImportOptions( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["SERVICE_TAG_IMPORT_OPTIONS"]
	SERIALISABLE_NAME = 'Service Tag Import Options'
	SERIALISABLE_VERSION = 3

	def __init__( self, get_tags = False, get_tags_filter = None, additional_tags = None, to_new_files = True, to_already_in_inbox = True, to_already_in_archive = True, only_add_existing_tags = False, only_add_existing_tags_filter = None ):
		if get_tags_filter is None:
			get_tags_filter = TagFilter()

		if additional_tags is None:
			additional_tags = []

		if only_add_existing_tags_filter is None:
			only_add_existing_tags_filter = TagFilter()

		SerialisableBase.__init__( self )

		self._get_tags = get_tags
		self._get_tags_filter = get_tags_filter
		self._additional_tags = additional_tags
		self._to_new_files = to_new_files
		self._to_already_in_inbox = to_already_in_inbox
		self._to_already_in_archive = to_already_in_archive
		self._only_add_existing_tags = only_add_existing_tags
		self._only_add_existing_tags_filter = only_add_existing_tags_filter

	def _GetSerialisableInfo( self ):
		serialisable_get_tags_filter = self._get_tags_filter.GetSerialisableTuple()
		serialisable_only_add_existing_tags_filter = self._only_add_existing_tags_filter.GetSerialisableTuple()

		return ( self._get_tags, serialisable_get_tags_filter, list( self._additional_tags ), self._to_new_files, self._to_already_in_inbox, self._to_already_in_archive, self._only_add_existing_tags, serialisable_only_add_existing_tags_filter )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._get_tags, serialisable_get_tags_filter, self._additional_tags, self._to_new_files, self._to_already_in_inbox, self._to_already_in_archive, self._only_add_existing_tags, serialisable_only_add_existing_tags_filter ) = serialisable_info

		self._get_tags_filter = CreateFromSerialisableTuple( serialisable_get_tags_filter )
		self._only_add_existing_tags_filter = CreateFromSerialisableTuple( serialisable_only_add_existing_tags_filter )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( get_tags, namespaces, additional_tags, to_new_files, to_already_in_inbox, to_already_in_archive, only_add_existing_tags ) = old_serialisable_info

			get_tags_filter = TagFilter()
			only_add_existing_tags_filter = TagFilter()

			serialisable_get_tags_filter = get_tags_filter.GetSerialisableTuple()
			serialisable_only_add_existing_tags_filter = only_add_existing_tags_filter.GetSerialisableTuple()

			new_serialisable_info = ( get_tags, serialisable_get_tags_filter, namespaces, additional_tags, to_new_files, to_already_in_inbox, to_already_in_archive, only_add_existing_tags, serialisable_only_add_existing_tags_filter )

			return ( 2, new_serialisable_info )

		if version == 2:
			( get_tags, serialisable_get_tags_filter, namespaces, additional_tags, to_new_files, to_already_in_inbox, to_already_in_archive, only_add_existing_tags, serialisable_only_add_existing_tags_filter ) = old_serialisable_info

			if not get_tags and len( namespaces ) > 0:
				get_tags = True
				get_tags_filter = TagFilter()

				namespaces = list( namespaces )

				get_tags_filter.SetRule( ':', CC.FILTER_BLACKLIST )

				if '' in namespaces: # if unnamespaced in original checkboxes, then leave it unblocked

					namespaces.remove( '' )

				else: # else block it

					get_tags_filter.SetRule( '', CC.FILTER_BLACKLIST )

				for namespace in namespaces:
					get_tags_filter.SetRule( namespace + ':', CC.FILTER_WHITELIST )

				serialisable_get_tags_filter = get_tags_filter.GetSerialisableTuple()

			new_serialisable_info = ( get_tags, serialisable_get_tags_filter, additional_tags, to_new_files, to_already_in_inbox, to_already_in_archive, only_add_existing_tags, serialisable_only_add_existing_tags_filter )

			return ( 3, new_serialisable_info )

	def GetSummaryStatements( self ):
		statements = []

		if self._get_tags:
			statements.append( self._get_tags_filter.ToPermittedString() )

		if len( self._additional_tags ) > 0:
			pretty_additional_tags = list( self._additional_tags )

			pretty_additional_tags.sort()

			statements.append( 'additional tags: ' + ', '.join( pretty_additional_tags ) )

		return statements

	def GetTags( self, service_key, status, in_inbox, hash, parsed_tags ):
		tags = set()

		if NewInboxArchiveMatch( self._to_new_files, self._to_already_in_inbox, self._to_already_in_archive, status, in_inbox ):
			if self._get_tags:
				filtered_tags = self._get_tags_filter.Filter( parsed_tags )

				tags.update( filtered_tags )

			tags.update( CleanTags( self._additional_tags ) )

			if self._only_add_existing_tags:
				applicable_tags = self._only_add_existing_tags_filter.Filter( tags )

				tags.difference_update( applicable_tags )

				existing_applicable_tags = HG.client_controller.Read( 'filter_existing_tags', service_key, tags )

				tags.update( existing_applicable_tags )

		return tags

	def HasAdditionalTags( self ):
		return len( self._additional_tags ) > 0

	def ToTuple( self ):
		return ( self._get_tags, self._get_tags_filter, self._additional_tags, self._to_new_files, self._to_already_in_inbox, self._to_already_in_archive, self._only_add_existing_tags, self._only_add_existing_tags_filter )

	def WorthFetchingTags( self ):
		return self._get_tags

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["SERVICE_TAG_IMPORT_OPTIONS"] ] = ServiceTagImportOptions