code:
import threading, time, urllib.parse

from lib.Actions.Import import DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME, \
	GetRepeatingJobInitialDelay, NetworkJobPresentationContext, PageImporterShouldStopWorking, \
	REPEATING_JOB_TYPICAL_PERIOD, UpdateFileSeedCacheWithFileSeeds, WakeRepeatingJob
from lib.Actions.Import.FileSeeds import FILE_SEED_TYPE_URL, FileSeed, FileSeedCache
from lib.Actions.Import.GallerySeeds import GallerySeed, GallerySeedLog
from lib.Actions.Import.Options import TagImportOptions

from lib.Hydrus.Data import GenerateKey, ShowException, ToHumanInt
from lib.Hydrus.Exceptions import NotFoundException, ShutdownException
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, SerialisableBase
from lib.Hydrus.Tags.Local import ServiceKeysToTags

from lib.Networking.Jobs import NetworkJobDownloader

import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Hydrus.Serialisable import HG.SerialisableType["SIMPLE_DOWNLOADER_IMPORT"], \
	HG.SerialisableType["URLS_IMPORT"], SERIALISABLE_TYPES_TO_OBJECT_TYPES

### CLASSES ###

class SimpleDownloaderImport( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["SIMPLE_DOWNLOADER_IMPORT"]
	SERIALISABLE_NAME = 'Simple Downloader Import'
	SERIALISABLE_VERSION = 5

	def __init__( self ):
		SerialisableBase.__init__( self )

		file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )

		self._pending_jobs = []
		self._gallery_seed_log = GallerySeedLog()
		self._file_seed_cache = FileSeedCache()
		self._file_import_options = file_import_options
		self._formula_name = 'all files linked by images in page'
		self._queue_paused = False
		self._files_paused = False

		self._downloader_key = GenerateKey()

		self._parser_status = ''
		self._current_action = ''

		self._lock = threading.Lock()

		self._files_network_job = None
		self._page_network_job = None

		self._files_repeating_job = None
		self._queue_repeating_job = None

		HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )

	def _FileNetworkJobPresentationContextFactory( self, network_job ):
		def enter_call():
			with self._lock:
				self._files_network_job = network_job

		def exit_call():
			with self._lock:
				self._files_network_job = None

		return NetworkJobPresentationContext( enter_call, exit_call )

	def _GetSerialisableInfo( self ):
		serialisable_pending_jobs = [ ( url, simple_downloader_formula.GetSerialisableTuple() ) for ( url, simple_downloader_formula ) in self._pending_jobs ]

		serialisable_gallery_seed_log = self._gallery_seed_log.GetSerialisableTuple()
		serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
		serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()

		return ( serialisable_pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, self._formula_name, self._queue_paused, self._files_paused )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, self._formula_name, self._queue_paused, self._files_paused ) = serialisable_info

		self._pending_jobs = [ ( url, CreateFromSerialisableTuple( serialisable_simple_downloader_formula ) ) for ( url, serialisable_simple_downloader_formula ) in serialisable_pending_jobs ]

		self._gallery_seed_log = CreateFromSerialisableTuple( serialisable_gallery_seed_log )
		self._file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )
		self._file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )

	def _NetworkJobFactory( self, *args, **kwargs ):
		network_job = NetworkJobDownloader( self._downloader_key, *args, **kwargs )

		return network_job

	def _PageNetworkJobPresentationContextFactory( self, network_job ):
		def enter_call():
			with self._lock:
				self._page_network_job = network_job

		def exit_call():
			with self._lock:
				self._page_network_job = None

		return NetworkJobPresentationContext( enter_call, exit_call )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( pending_page_urls, serialisable_file_seed_cache, serialisable_file_import_options, download_image_links, download_unlinked_images, paused ) = old_serialisable_info

			queue_paused = paused
			files_paused = paused

			new_serialisable_info = ( pending_page_urls, serialisable_file_seed_cache, serialisable_file_import_options, download_image_links, download_unlinked_images, queue_paused, files_paused )

			return ( 2, new_serialisable_info )

		if version == 2:
			( pending_page_urls, serialisable_file_seed_cache, serialisable_file_import_options, download_image_links, download_unlinked_images, queue_paused, files_paused ) = old_serialisable_info

			pending_jobs = []

			new_serialisable_info = ( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, queue_paused, files_paused )

			return ( 3, new_serialisable_info )

		if version == 3:
			( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, queue_paused, files_paused ) = old_serialisable_info

			pending_jobs = []

			formula_name = 'all files linked by images in page'

			new_serialisable_info = ( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, formula_name, queue_paused, files_paused )

			return ( 4, new_serialisable_info )

		if version == 4:
			( pending_jobs, serialisable_file_seed_cache, serialisable_file_import_options, formula_name, queue_paused, files_paused ) = old_serialisable_info

			gallery_seed_log = GallerySeedLog()

			serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()

			new_serialisable_info = ( pending_jobs, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, formula_name, queue_paused, files_paused )

			return ( 5, new_serialisable_info )

	def _WorkOnFiles( self, page_key ):
		file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if file_seed is None:
			return

		did_substantial_work = False

		def status_hook( text ):
			with self._lock:
				self._current_action = text

		tag_import_options = TagImportOptions( is_default = True )

		did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, tag_import_options )

		if file_seed.ShouldPresent( self._file_import_options ):
			file_seed.PresentToPage( page_key )

			did_substantial_work = True

		with self._lock:
			self._current_action = ''

		if did_substantial_work:
			time.sleep( DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )

	def _WorkOnQueue( self, page_key ):
		if len( self._pending_jobs ) > 0:
			with self._lock:
				( url, simple_downloader_formula ) = self._pending_jobs.pop( 0 )

				self._parser_status = 'checking ' + url

			error_occurred = False

			try:
				gallery_seed = GallerySeed( url, can_generate_more_pages = False )

				self._gallery_seed_log.AddGallerySeeds( ( gallery_seed, ) )

				network_job = self._NetworkJobFactory( 'GET', url )

				network_job.OverrideBandwidth( 30 )

				HG.client_controller.network_engine.AddJob( network_job )

				with self._PageNetworkJobPresentationContextFactory( network_job ):
					network_job.WaitUntilDone()

				parsing_text = network_job.GetContentText()

				#

				parsing_context = {}

				parsing_context[ 'url' ] = url

				parsing_formula = simple_downloader_formula.GetFormula()

				file_seeds = []

				for parsed_text in parsing_formula.Parse( parsing_context, parsing_text ):
					try:
						file_url = urllib.parse.urljoin( url, parsed_text )

						file_seed = FileSeed( FILE_SEED_TYPE_URL, file_url )

						file_seed.SetReferralURL( url )

						file_seeds.append( file_seed )

					except Exception:
						continue

				num_new = self._file_seed_cache.AddFileSeeds( file_seeds )

				if num_new > 0:
					WakeRepeatingJob( self._files_repeating_job )

				parser_status = 'page checked OK with formula "' + simple_downloader_formula.GetName() + '" - ' + ToHumanInt( num_new ) + ' new urls'

				num_already_in_file_seed_cache = len( file_seeds ) - num_new

				if num_already_in_file_seed_cache > 0:
					parser_status += ' (' + ToHumanInt( num_already_in_file_seed_cache ) + ' already in queue)'

				gallery_seed_status = CC.STATUS_SUCCESSFUL_AND_NEW

			except ShutdownException:
				gallery_seed_status = CC.STATUS_VETOED
				parser_status = 'program is shutting down'

				return

			except NotFoundException:
				gallery_seed_status = CC.STATUS_VETOED

				error_occurred = True

				parser_status = 'page 404'

			except Exception as e:
				gallery_seed_status = CC.STATUS_ERROR

				error_occurred = True

				parser_status = str( e )

			finally:
				gallery_seed_note = parser_status

				gallery_seed.SetStatus( gallery_seed_status, note = gallery_seed_note )

				self._gallery_seed_log.NotifyGallerySeedsUpdated( ( gallery_seed, ) )

			with self._lock:
				self._parser_status = parser_status

			if error_occurred:
				time.sleep( 5 )

			return True

		else:
			with self._lock:
				self._parser_status = ''

			return False

	def AdvanceJob( self, job ):
		with self._lock:
			if job in self._pending_jobs:
				index = self._pending_jobs.index( job )

				if index - 1 >= 0:
					self._pending_jobs.remove( job )

					self._pending_jobs.insert( index - 1, job )

	def CurrentlyWorking( self ):
		with self._lock:
			finished = not self._file_seed_cache.WorkToDo() or len( self._pending_jobs ) > 0

			return not finished and not self._files_paused

	def DelayJob( self, job ):
		with self._lock:
			if job in self._pending_jobs:
				index = self._pending_jobs.index( job )

				if index + 1 < len( self._pending_jobs ):
					self._pending_jobs.remove( job )

					self._pending_jobs.insert( index + 1, job )

	def DeleteJob( self, job ):
		with self._lock:
			if job in self._pending_jobs:
				self._pending_jobs.remove( job )

	def GetFileSeedCache( self ):
		with self._lock:
			return self._file_seed_cache

	def GetFileImportOptions( self ):
		with self._lock:
			return self._file_import_options

	def GetFormulaName( self ):
		with self._lock:
			return self._formula_name

	def GetGallerySeedLog( self ):
		with self._lock:
			return self._gallery_seed_log

	def GetNetworkJobs( self ):
		with self._lock:
			return ( self._files_network_job, self._page_network_job )

	def GetStatus( self ):
		with self._lock:
			return ( list( self._pending_jobs ), self._parser_status, self._current_action, self._queue_paused, self._files_paused )

	def GetValueRange( self ):
		with self._lock:
			return self._file_seed_cache.GetValueRange()

	def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
		if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
			WakeRepeatingJob( self._files_repeating_job )

	def PausePlayFiles( self ):
		with self._lock:
			self._files_paused = not self._files_paused

			WakeRepeatingJob( self._files_repeating_job )

	def PausePlayQueue( self ):
		with self._lock:
			self._queue_paused = not self._queue_paused

			WakeRepeatingJob( self._queue_repeating_job )

	def PendJob( self, job ):
		with self._lock:
			if job not in self._pending_jobs:
				self._pending_jobs.append( job )

				WakeRepeatingJob( self._queue_repeating_job )

	def SetFileImportOptions( self, file_import_options ):
		with self._lock:
			self._file_import_options = file_import_options

	def SetFormulaName( self, formula_name ):
		with self._lock:
			self._formula_name = formula_name

	def Start( self, page_key ):
		self._files_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles, page_key )
		self._queue_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnQueue, page_key )

		self._files_repeating_job.SetThreadSlotType( 'misc' )
		self._queue_repeating_job.SetThreadSlotType( 'misc' )

	def REPEATINGWorkOnFiles( self, page_key ):
		with self._lock:
			if PageImporterShouldStopWorking( page_key ):
				self._files_repeating_job.Cancel()

				return

			files_paused = self._files_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )

			work_to_do = self._file_seed_cache.WorkToDo() and not ( files_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )
			network_engine_good = not HG.client_controller.network_engine.IsBusy()

			ok_to_work = work_to_do and network_engine_good

		while ok_to_work:
			try:
				self._WorkOnFiles( page_key )

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( page_key ):
					self._files_repeating_job.Cancel()

					return

				files_paused = self._files_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )

				work_to_do = self._file_seed_cache.WorkToDo() and not ( files_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )
				network_engine_good = not HG.client_controller.network_engine.IsBusy()

				ok_to_work = work_to_do and network_engine_good

	def REPEATINGWorkOnQueue( self, page_key ):
		with self._lock:
			if PageImporterShouldStopWorking( page_key ):
				self._queue_repeating_job.Cancel()

				return

			queue_paused = self._queue_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )

			queue_good = not queue_paused
			page_shown = not HG.client_controller.PageClosedButNotDestroyed( page_key )
			network_engine_good = not HG.client_controller.network_engine.IsBusy()

			ok_to_work = queue_good and page_shown and network_engine_good

		while ok_to_work:
			try:
				did_work = self._WorkOnQueue( page_key )

				if did_work:
					time.sleep( DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )

				else:
					return

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( page_key ):
					self._queue_repeating_job.Cancel()

					return

				queue_paused = self._queue_paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )

				queue_good = not queue_paused
				page_shown = not HG.client_controller.PageClosedButNotDestroyed( page_key )
				network_engine_good = not HG.client_controller.network_engine.IsBusy()

				ok_to_work = queue_good and page_shown and network_engine_good

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["SIMPLE_DOWNLOADER_IMPORT"] ] = SimpleDownloaderImport

class URLsImport( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["URLS_IMPORT"]
	SERIALISABLE_NAME = 'URL Import'
	SERIALISABLE_VERSION = 3

	def __init__( self ):
		SerialisableBase.__init__( self )

		self._gallery_seed_log = GallerySeedLog()
		self._file_seed_cache = FileSeedCache()
		self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'loud' )
		self._tag_import_options = TagImportOptions( is_default = True )
		self._paused = False

		self._downloader_key = GenerateKey()

		self._lock = threading.Lock()

		self._files_network_job = None
		self._gallery_network_job = None

		self._files_repeating_job = None
		self._gallery_repeating_job = None

		HG.client_controller.sub( self, 'NotifyFileSeedsUpdated', 'file_seed_cache_file_seeds_updated' )
		HG.client_controller.sub( self, 'NotifyGallerySeedsUpdated', 'gallery_seed_log_gallery_seeds_updated' )

	def _FileNetworkJobPresentationContextFactory( self, network_job ):
		def enter_call():
			with self._lock:
				self._files_network_job = network_job

		def exit_call():
			with self._lock:
				self._files_network_job = None

		return NetworkJobPresentationContext( enter_call, exit_call )

	def _GalleryNetworkJobPresentationContextFactory( self, network_job ):
		def enter_call():
			with self._lock:
				self._gallery_network_job = network_job

		def exit_call():
			with self._lock:
				self._gallery_network_job = None

		return NetworkJobPresentationContext( enter_call, exit_call )

	def _GetSerialisableInfo( self ):
		serialisable_gallery_seed_log = self._gallery_seed_log.GetSerialisableTuple()
		serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
		serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
		serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()

		return ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, self._paused )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, self._paused ) = serialisable_info

		self._gallery_seed_log = CreateFromSerialisableTuple( serialisable_gallery_seed_log )
		self._file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )
		self._file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )
		self._tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )

	def _NetworkJobFactory( self, *args, **kwargs ):
		network_job = NetworkJobDownloader( self._downloader_key, *args, **kwargs )

		return network_job

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( serialisable_file_seed_cache, serialisable_file_import_options, paused ) = old_serialisable_info

			gallery_seed_log = GallerySeedLog()

			serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, paused )

			return ( 2, new_serialisable_info )

		if version == 2:
			( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, paused ) = old_serialisable_info

			tag_import_options = TagImportOptions( is_default = True )

			serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_file_import_options, serialisable_tag_import_options, paused )

			return ( 3, new_serialisable_info )

	def _WorkOnFiles( self, page_key ):
		file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if file_seed is None:
			return

		did_substantial_work = False

		# url = file_seed.file_seed_data

		try:
			status_hook = lambda s: s # do nothing for now

			did_substantial_work = file_seed.WorkOnURL( self._file_seed_cache, status_hook, self._NetworkJobFactory, self._FileNetworkJobPresentationContextFactory, self._file_import_options, self._tag_import_options )

			if file_seed.ShouldPresent( self._file_import_options ):
				file_seed.PresentToPage( page_key )

				did_substantial_work = True

		except Exception as e:
			status = CC.STATUS_ERROR

			file_seed.SetStatus( status, exception = e )

			time.sleep( 3 )

		if did_substantial_work:
			time.sleep( DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )

	def _WorkOnGallery( self, page_key ):
		gallery_seed = self._gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )

		if gallery_seed is None:
			return

		try:
			status_hook = lambda s: s
			title_hook = lambda s: s

			def file_seeds_callable( file_seeds ):
				return UpdateFileSeedCacheWithFileSeeds( self._file_seed_cache, file_seeds )

			gallery_seed.WorkOnURL( 'download page', self._gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._NetworkJobFactory, self._GalleryNetworkJobPresentationContextFactory, self._file_import_options )

		except Exception as e:
			status = CC.STATUS_ERROR

			gallery_seed.SetStatus( status, exception = e )

			time.sleep( 3 )

		time.sleep( 1 )

	def CurrentlyWorking( self ):
		with self._lock:
			finished = not self._file_seed_cache.WorkToDo()

			return not finished and not self._paused

	def GetFileSeedCache( self ):
		with self._lock:
			return self._file_seed_cache

	def GetGallerySeedLog( self ):
		with self._lock:
			return self._gallery_seed_log

	def GetNetworkJobs( self ):
		with self._lock:
			return ( self._files_network_job, self._gallery_network_job )

	def GetOptions( self ):
		with self._lock:
			return ( self._file_import_options, self._tag_import_options )

	def GetStatus( self ):
		with self._lock:
			return ( self._file_seed_cache.GetStatus(), self._paused )

	def GetValueRange( self ):
		with self._lock:
			return self._file_seed_cache.GetValueRange()

	def IsPaused( self ):
		with self._lock:
			return self._paused

	def NotifyFileSeedsUpdated( self, file_seed_cache_key, file_seeds ):
		if file_seed_cache_key == self._file_seed_cache.GetFileSeedCacheKey():
			WakeRepeatingJob( self._files_repeating_job )

	def NotifyGallerySeedsUpdated( self, gallery_seed_log_key, gallery_seeds ):
		if gallery_seed_log_key == self._gallery_seed_log.GetGallerySeedLogKey():
			WakeRepeatingJob( self._gallery_repeating_job )

	def PausePlay( self ):
		with self._lock:
			self._paused = not self._paused

			WakeRepeatingJob( self._files_repeating_job )
			WakeRepeatingJob( self._gallery_repeating_job )

	def PendURLs( self, urls, service_keys_to_tags = None ):
		if service_keys_to_tags is None:
			service_keys_to_tags = ServiceKeysToTags()

		with self._lock:
			urls = [u for u in urls if len( u ) > 1] # > _1_ to take out the occasional whitespace

			file_seeds = []

			gallery_seeds = []

			for url in urls:
				url_match = HG.client_controller.network_engine.domain_manager.GetURLMatch( url )

				if url_match is None or url_match.GetURLType() in ( HC.URL_TYPE_FILE, HC.URL_TYPE_POST ):
					file_seed = FileSeed( FILE_SEED_TYPE_URL, url )

					file_seed.SetFixedServiceKeysToTags( service_keys_to_tags )

					file_seeds.append( file_seed )

				else:
					can_generate_more_pages = False

					gallery_seed = GallerySeed( url, can_generate_more_pages = can_generate_more_pages )

					gallery_seed.SetFixedServiceKeysToTags( service_keys_to_tags )

					gallery_seeds.append( gallery_seed )

			if len( gallery_seeds ) > 0:
				self._gallery_seed_log.AddGallerySeeds( gallery_seeds )

				WakeRepeatingJob( self._gallery_repeating_job )

			if len( file_seeds ) > 0:
				self._file_seed_cache.AddFileSeeds( file_seeds )

				WakeRepeatingJob( self._files_repeating_job )

	def SetFileImportOptions( self, file_import_options ):
		with self._lock:
			self._file_import_options = file_import_options

	def SetTagImportOptions( self, tag_import_options ):
		with self._lock:
			self._tag_import_options = tag_import_options

	def Start( self, page_key ):
		self._files_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnFiles, page_key )
		self._gallery_repeating_job = HG.client_controller.CallRepeating( GetRepeatingJobInitialDelay(), REPEATING_JOB_TYPICAL_PERIOD, self.REPEATINGWorkOnGallery, page_key )

		self._files_repeating_job.SetThreadSlotType( 'misc' )
		self._gallery_repeating_job.SetThreadSlotType( 'misc' )

	def REPEATINGWorkOnFiles( self, page_key ):
		with self._lock:
			if PageImporterShouldStopWorking( page_key ):
				self._files_repeating_job.Cancel()

				return

			files_paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )

			work_to_do = self._file_seed_cache.WorkToDo() and not ( files_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )
			network_engine_good = not HG.client_controller.network_engine.IsBusy()

			ok_to_work = work_to_do and network_engine_good

		while ok_to_work:
			try:
				self._WorkOnFiles( page_key )

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( page_key ):
					self._files_repeating_job.Cancel()

					return

				files_paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_file_queues' )

				work_to_do = self._file_seed_cache.WorkToDo() and not ( files_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )
				network_engine_good = not HG.client_controller.network_engine.IsBusy()

				ok_to_work = work_to_do and network_engine_good

	def REPEATINGWorkOnGallery( self, page_key ):
		with self._lock:
			if PageImporterShouldStopWorking( page_key ):
				self._gallery_repeating_job.Cancel()

				return

			gallery_paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )

			work_to_do = self._gallery_seed_log.WorkToDo() and not ( gallery_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )
			network_engine_good = not HG.client_controller.network_engine.IsBusy()

			ok_to_work = work_to_do and network_engine_good

		while ok_to_work:
			try:
				self._WorkOnGallery( page_key )

				HG.client_controller.WaitUntilViewFree()

			except Exception as e:
				ShowException( e )

			with self._lock:
				if PageImporterShouldStopWorking( page_key ):
					self._gallery_repeating_job.Cancel()

					return

				gallery_paused = self._paused or HG.client_controller.new_options.GetBoolean( 'pause_all_gallery_searches' )

				work_to_do = self._gallery_seed_log.WorkToDo() and not ( gallery_paused or HG.client_controller.PageClosedButNotDestroyed( page_key ) )
				network_engine_good = not HG.client_controller.network_engine.IsBusy()

				ok_to_work = work_to_do and network_engine_good

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["URLS_IMPORT"] ] = URLsImport


code:
import os, random, time

from lib.Actions.Import import GenerateMultiplePopupNetworkJobPresentationContextFactory, \
	PublishPresentationHashes
from lib.Actions.Import.FileSeeds import FileSeedCache
from lib.Actions.Import.GallerySeeds import GallerySeed, GallerySeedLog
from lib.Actions.Import.Options import CheckerOptions, TagImportOptions

from lib.Networking.Contexts import NetworkContext
from lib.Networking.Downloading import ConvertGalleryIdentifierToGUGKeyAndName
from lib.Networking.Jobs import NetworkJobSubscription

from lib.Hydrus.Data import ConvertValueRangeToPrettyString, GenerateKey, \
	GetNow, Print, PrintException, ShowException, ShowText, TimeHasPassed, \
	TimestampToPrettyTimeDelta, ToHumanInt
from lib.Hydrus.Exceptions import CancelledException, DataMissing, NetworkException, \
	NotFoundException, VetoException
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, SerialisableBase, \
	SerialisableBaseNamed
from lib.Hydrus.Threading import IsThreadShuttingDown, JobKey

### CONSTANTS ###

import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Actions.Import import CHECKER_STATUS_DEAD, CHECKER_STATUS_OK, \
	DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME

from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, HG.SerialisableType["SUBSCRIPTION"], \
	HG.SerialisableType["SUBSCRIPTION_QUERY"], SERIALISABLE_TYPES_TO_OBJECT_TYPES

from lib.Networking.Contexts import GLOBAL_NETWORK_CONTEXT

### CLASSES ###

class Subscription( SerialisableBaseNamed ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["SUBSCRIPTION"]
	SERIALISABLE_NAME = 'Subscription'
	SERIALISABLE_VERSION = 10

	def __init__( self, name, gug_key_and_name = None ):
		SerialisableBaseNamed.__init__( self, name )

		if gug_key_and_name is None:
			gug_key_and_name = ( GenerateKey(), 'unknown source' )

		self._gug_key_and_name = gug_key_and_name

		self._queries = []

		new_options = HG.client_controller.new_options

		self._checker_options = HG.client_controller.new_options.GetDefaultSubscriptionCheckerOptions()

		if HC.options[ 'gallery_file_limit' ] is None:
			self._initial_file_limit = 100

		else:
			self._initial_file_limit = min( 100, HC.options[ 'gallery_file_limit' ] )

		self._periodic_file_limit = 100
		self._paused = False

		self._file_import_options = HG.client_controller.new_options.GetDefaultFileImportOptions( 'quiet' )

		# new_options = HG.client_controller.new_options

		self._tag_import_options = TagImportOptions( is_default = True )

		self._no_work_until = 0
		self._no_work_until_reason = ''

		self._show_a_popup_while_working = True
		self._publish_files_to_popup_button = True
		self._publish_files_to_page = False
		self._publish_label_override = None
		self._merge_query_publish_events = True

	def _DelayWork( self, time_delta, reason ):
		self._no_work_until = GetNow() + time_delta
		self._no_work_until_reason = reason

	def _GetExampleNetworkContexts( self, query ):
		file_seed_cache = query.GetFileSeedCache()

		file_seed = file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if file_seed is None:
			return [ NetworkContext( CC.NETWORK_CONTEXT_SUBSCRIPTION, self._GetNetworkJobSubscriptionKey( query ) ), GLOBAL_NETWORK_CONTEXT ]

		url = file_seed.file_seed_data

		example_nj = NetworkJobSubscription( self._GetNetworkJobSubscriptionKey( query ), 'GET', url )
		example_network_contexts = example_nj.GetNetworkContexts()

		return example_network_contexts

	def _GetNetworkJobSubscriptionKey( self, query ):
		return self._name + ': ' + query.GetHumanName()

	def _GetPublishingLabel( self, query ):
		if self._publish_label_override is None:
			label = self._name

		else:
			label = self._publish_label_override

		if not self._merge_query_publish_events:
			label += ': ' + query.GetHumanName()

		return label

	def _GetQueriesForProcessing( self ):
		queries = list( self._queries )

		if HG.client_controller.new_options.GetBoolean( 'process_subs_in_random_order' ):
			random.shuffle( queries )

		else:
			def key( q ):
				return q.GetHumanName()

			queries.sort( key = key )

		return queries

	def _GetSerialisableInfo( self ):
		( gug_key, gug_name ) = self._gug_key_and_name

		serialisable_gug_key_and_name = ( gug_key.hex(), gug_name )
		serialisable_queries = [ query.GetSerialisableTuple() for query in self._queries ]
		serialisable_checker_options = self._checker_options.GetSerialisableTuple()
		serialisable_file_import_options = self._file_import_options.GetSerialisableTuple()
		serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()

		return ( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, self._initial_file_limit, self._periodic_file_limit, self._paused, serialisable_file_import_options, serialisable_tag_import_options, self._no_work_until, self._no_work_until_reason, self._show_a_popup_while_working, self._publish_files_to_popup_button, self._publish_files_to_page, self._publish_label_override, self._merge_query_publish_events )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, self._initial_file_limit, self._periodic_file_limit, self._paused, serialisable_file_import_options, serialisable_tag_import_options, self._no_work_until, self._no_work_until_reason, self._show_a_popup_while_working, self._publish_files_to_popup_button, self._publish_files_to_page, self._publish_label_override, self._merge_query_publish_events ) = serialisable_info

		( serialisable_gug_key, gug_name ) = serialisable_gug_key_and_name

		self._gug_key_and_name = ( bytes.fromhex( serialisable_gug_key ), gug_name )
		self._queries = [ CreateFromSerialisableTuple( serialisable_query ) for serialisable_query in serialisable_queries ]
		self._checker_options = CreateFromSerialisableTuple( serialisable_checker_options )
		self._file_import_options = CreateFromSerialisableTuple( serialisable_file_import_options )
		self._tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )

	def _GenerateNetworkJobFactory( self, query ):
		subscription_key = self._GetNetworkJobSubscriptionKey( query )

		def network_job_factory( *args, **kwargs ):
			network_job = NetworkJobSubscription( subscription_key, *args, **kwargs )

			network_job.OverrideBandwidth( 30 )

			return network_job

		return network_job_factory

	def _NoDelays( self ):
		return TimeHasPassed( self._no_work_until )

	def _QueryBandwidthIsOK( self, query ):
		example_network_contexts = self._GetExampleNetworkContexts( query )

		threshold = 90

		result = HG.client_controller.network_engine.bandwidth_manager.CanDoWork( example_network_contexts, threshold = threshold )

		if HG.subscription_report_mode:
			ShowText( 'Query "' + query.GetHumanName() + '" pre-work bandwidth test. Bandwidth ok: ' + str( result ) + '.' )

		return result

	def _QueryFileLoginIsOK( self, query ):
		file_seed_cache = query.GetFileSeedCache()

		file_seed = file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if file_seed is None:
			result = True

		else:
			nj = file_seed.GetExampleNetworkJob( self._GenerateNetworkJobFactory( query ) )

			nj.engine = HG.client_controller.network_engine

			if nj.NeedsLogin():
				try:
					nj.CheckCanLogin()

					result = True

				except Exception as e:
					result = False

					if not self._paused:
						login_fail_reason = str( e )

						message = 'Query "' + query.GetHumanName() + '" for subscription "' + self._name + '" seemed to have an invalid login for one of its file imports. The reason was:'
						message += os.linesep * 2
						message += login_fail_reason
						message += os.linesep * 2
						message += 'The subscription has paused. Please see if you can fix the problem and then unpause. Hydrus dev would like feedback on this process.'

						ShowText( message )

						self._DelayWork( 300, login_fail_reason )

						self._paused = True

			else:
				result = True

		if HG.subscription_report_mode:
			ShowText( 'Query "' + query.GetHumanName() + '" pre-work file login test. Login ok: ' + str( result ) + '.' )

		return result

	def _QuerySyncLoginIsOK( self, query ):
		gallery_seed_log = query.GetGallerySeedLog()

		gallery_seed = gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )

		if gallery_seed is None:
			result = True

		else:
			nj = gallery_seed.GetExampleNetworkJob( self._GenerateNetworkJobFactory( query ) )

			nj.engine = HG.client_controller.network_engine

			if nj.NeedsLogin():
				try:
					nj.CheckCanLogin()

					result = True

				except Exception as e:
					result = False

					if not self._paused:
						login_fail_reason = str( e )

						message = 'Query "' + query.GetHumanName() + '" for subscription "' + self._name + '" seemed to have an invalid login. The reason was:'
						message += os.linesep * 2
						message += login_fail_reason
						message += os.linesep * 2
						message += 'The subscription has paused. Please see if you can fix the problem and then unpause. Hydrus dev would like feedback on this process.'

						ShowText( message )

						self._DelayWork( 300, login_fail_reason )

						self._paused = True

			else:
				result = True

		if HG.subscription_report_mode:
			ShowText( 'Query "' + query.GetHumanName() + '" pre-work sync login test. Login ok: ' + str( result ) + '.' )

		return result

	def _ShowHitPeriodicFileLimitMessage( self, query_text ):
		message = 'The query "' + query_text + '" for subscription "' + self._name + '" hit its periodic file limit without seeing any already-seen files.'

		ShowText( message )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, query, period, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, last_checked, last_error, serialisable_file_seed_cache ) = old_serialisable_info

			check_now = False

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, query, period, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, last_checked, check_now, last_error, serialisable_file_seed_cache )

			return ( 2, new_serialisable_info )

		if version == 2:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, query, period, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, last_checked, check_now, last_error, serialisable_file_seed_cache ) = old_serialisable_info

			no_work_until = 0
			no_work_until_reason = ''

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, query, period, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, last_checked, check_now, last_error, no_work_until, no_work_until_reason, serialisable_file_seed_cache )

			return ( 3, new_serialisable_info )

		if version == 3:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, query, period, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, last_checked, check_now, last_error, no_work_until, no_work_until_reason, serialisable_file_seed_cache ) = old_serialisable_info

			checker_options = CheckerOptions( 5, period // 5, period * 10, ( 1, period * 10 ) )

			file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )

			query = SubscriptionQuery( query )

			query._file_seed_cache = file_seed_cache
			query._last_check_time = last_checked

			query.UpdateNextCheckTime( checker_options )

			queries = [ query ]

			serialisable_queries = [ query.GetSerialisableTuple() for query in queries ]
			serialisable_checker_options = checker_options.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason )

			return ( 4, new_serialisable_info )

		if version == 4:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, get_tags_if_url_recognised_and_file_redundant, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason ) = old_serialisable_info

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason )

			return ( 5, new_serialisable_info )

		if version == 5:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason ) = old_serialisable_info

			publish_files_to_popup_button = True
			publish_files_to_page = False
			merge_query_publish_events = True

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events )

			return ( 6, new_serialisable_info )

		if version == 6:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events ) = old_serialisable_info

			if initial_file_limit is None or initial_file_limit > 1000:
				initial_file_limit = 1000

			if periodic_file_limit is None or periodic_file_limit > 1000:
				periodic_file_limit = 1000

			new_serialisable_info = ( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events )

			return ( 7, new_serialisable_info )

		if version == 7:
			( serialisable_gallery_identifier, serialisable_gallery_stream_identifiers, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events ) = old_serialisable_info

			gallery_identifier = CreateFromSerialisableTuple( serialisable_gallery_identifier )

			( gug_key, gug_name ) = ConvertGalleryIdentifierToGUGKeyAndName( gallery_identifier )

			serialisable_gug_key_and_name = ( gug_key.hex(), gug_name )

			new_serialisable_info = ( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events )

			return ( 8, new_serialisable_info )

		if version == 8:
			( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events ) = old_serialisable_info

			show_a_popup_while_working = True

			new_serialisable_info = ( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, show_a_popup_while_working, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events )

			return ( 9, new_serialisable_info )

		if version == 9:
			( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, show_a_popup_while_working, publish_files_to_popup_button, publish_files_to_page, merge_query_publish_events ) = old_serialisable_info

			publish_label_override = None

			new_serialisable_info = ( serialisable_gug_key_and_name, serialisable_queries, serialisable_checker_options, initial_file_limit, periodic_file_limit, paused, serialisable_file_import_options, serialisable_tag_import_options, no_work_until, no_work_until_reason, show_a_popup_while_working, publish_files_to_popup_button, publish_files_to_page, publish_label_override, merge_query_publish_events )

			return ( 10, new_serialisable_info )

	def _WorkOnFiles( self, job_key ):
		error_count = 0

		all_presentation_hashes = []
		all_presentation_hashes_fast = set()

		queries = self._GetQueriesForProcessing()

		num_queries = len( queries )

		for ( i, query ) in enumerate( queries ):
			this_query_has_done_work = False

			query_name = query.GetHumanName()
			file_seed_cache = query.GetFileSeedCache()

			text_1 = 'downloading files'
			query_summary_name = self._name

			if query_name != self._name:
				text_1 += ' for "' + query_name + '"'
				query_summary_name += ': ' + query_name

			if num_queries > 1:
				text_1 += ' (' + ConvertValueRangeToPrettyString( i + 1, num_queries ) + ')'

			job_key.SetVariable( 'popup_text_1', text_1 )

			presentation_hashes = []
			presentation_hashes_fast = set()

			while True:
				num_urls = file_seed_cache.GetFileSeedCount()
				num_unknown = file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN )
				num_done = num_urls - num_unknown

				file_seed = file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

				if file_seed is None:
					if HG.subscription_report_mode:
						ShowText( 'Query "' + query_name + '" can do no more file work due to running out of unknown urls.' )

					break

				if job_key.IsCancelled():
					self._DelayWork( 300, 'recently cancelled' )

					break

				p1 = HC.options[ 'pause_subs_sync' ]
				p2 = IsThreadShuttingDown()
				p3 = HG.view_shutdown
				p4 = not self._QueryBandwidthIsOK( query )
				p5 = not self._QueryFileLoginIsOK( query )

				if p1 or p2 or p3 or p4 or p5:
					if p4 and this_query_has_done_work:
						job_key.SetVariable( 'popup_text_2', 'no more bandwidth to download files, will do some more later' )

						time.sleep( 5 )

					break

				try:
					x_out_of_y = 'file ' + ConvertValueRangeToPrettyString( num_done + 1, num_urls ) + ': '

					job_key.SetVariable( 'popup_gauge_2', ( num_done, num_urls ) )

					def status_hook( text ):
						job_key.SetVariable( 'popup_text_2', x_out_of_y + text )

					file_seed.WorkOnURL( file_seed_cache, status_hook, self._GenerateNetworkJobFactory( query ), GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key ), self._file_import_options, self._tag_import_options )

					query_tag_import_options = query.GetTagImportOptions()

					if query_tag_import_options.HasAdditionalTags() and file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:
						if file_seed.HasHash():
							hash = file_seed.GetHash()

							in_inbox = HG.client_controller.Read( 'in_inbox', hash )

							downloaded_tags = []

							service_keys_to_content_updates = query_tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, in_inbox, hash, downloaded_tags ) # additional tags

							if len( service_keys_to_content_updates ) > 0:
								HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates )

					if file_seed.ShouldPresent( self._file_import_options ):
						hash = file_seed.GetHash()

						if hash not in presentation_hashes_fast:
							if hash not in all_presentation_hashes_fast:
								all_presentation_hashes.append( hash )

								all_presentation_hashes_fast.add( hash )

							presentation_hashes.append( hash )

							presentation_hashes_fast.add( hash )

				except CancelledException as e:
					self._DelayWork( 300, str( e ) )

					break

				except VetoException as e:
					status = CC.STATUS_VETOED

					note = str( e )

					file_seed.SetStatus( status, note = note )

				except NotFoundException:
					status = CC.STATUS_VETOED

					note = '404'

					file_seed.SetStatus( status, note = note )

				except Exception as e:
					status = CC.STATUS_ERROR

					job_key.SetVariable( 'popup_text_2', x_out_of_y + 'file failed' )

					file_seed.SetStatus( status, exception = e )

					if isinstance( e, DataMissing ):
						# DataMissing is a quick thing to avoid subscription abandons when lots of deleted files in e621 (or any other booru)
						# this should be richer in any case in the new system

						pass

					else:
						error_count += 1

						time.sleep( 10 )

					if error_count > 4:
						raise Exception( 'The subscription ' + self._name + ' encountered several errors when downloading files, so it abandoned its sync.' )

				this_query_has_done_work = True

				if len( presentation_hashes ) > 0:
					job_key.SetVariable( 'popup_files', ( list( presentation_hashes ), query_summary_name ) )

				time.sleep( DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME )

				HG.client_controller.WaitUntilViewFree()

			if not self._merge_query_publish_events and len( presentation_hashes ) > 0:
				publishing_label = self._GetPublishingLabel( query )

				PublishPresentationHashes( publishing_label, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page )

		if self._merge_query_publish_events and len( all_presentation_hashes ) > 0:
			publishing_label = self._GetPublishingLabel( query )

			PublishPresentationHashes( publishing_label, all_presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page )

		job_key.DeleteVariable( 'popup_files' )
		job_key.DeleteVariable( 'popup_text_1' )
		job_key.DeleteVariable( 'popup_text_2' )
		job_key.DeleteVariable( 'popup_gauge_2' )

	def _WorkOnFilesCanDoWork( self ):
		for query in self._queries:
			if query.CanWorkOnFiles():
				if self._QueryBandwidthIsOK( query ):
					return True

		return False

	def _SyncQuery( self, job_key ):
		have_made_an_initial_sync_bandwidth_notification = False

		gug = HG.client_controller.network_engine.domain_manager.GetGUG( self._gug_key_and_name )

		if gug is None:
			self._paused = True

			ShowText( 'The subscription "' + self._name + '" could not find a Gallery URL Generator for "' + self._gug_key_and_name[1] + '"! The sub has paused!' )

			return

		if not gug.IsFunctional():
			self._paused = True

			ShowText( 'The subscription "' + self._name + '"\'s Gallery URL Generator, "' + self._gug_key_and_name[1] + '" seems not to be functional! Maybe it needs a gallery url class or a gallery parser? The sub has paused!' )

			return

		self._gug_key_and_name = gug.GetGUGKeyAndName() # just a refresher, to keep up with any changes

		queries = self._GetQueriesForProcessing()

		num_queries = len( queries )

		for ( i, query ) in enumerate( queries ):
			can_sync = query.CanSync()

			if HG.subscription_report_mode:
				ShowText( 'Query "' + query.GetHumanName() + '" started. Current can_sync is ' + str( can_sync ) + '.' )

			if not can_sync:
				continue

			query_text = query.GetQueryText()
			query_name = query.GetHumanName()
			file_seed_cache = query.GetFileSeedCache()
			gallery_seed_log = query.GetGallerySeedLog()

			this_is_initial_sync = query.IsInitialSync()
			total_new_urls_for_this_sync = 0
			total_already_in_urls_for_this_sync = 0

			gallery_urls_seen_this_sync = set()

			if this_is_initial_sync:
				file_limit_for_this_sync = self._initial_file_limit

			else:
				file_limit_for_this_sync = self._periodic_file_limit

			file_seeds_to_add = set()
			file_seeds_to_add_ordered = []

			stop_reason = 'unknown stop reason'

			prefix = 'synchronising'

			if query_name != self._name:
				prefix += ' "' + query_name + '"'

			if num_queries > 1:
				prefix += ' (' + ConvertValueRangeToPrettyString( i + 1, num_queries ) + ')'

			job_key.SetVariable( 'popup_text_1', prefix )

			initial_search_urls = gug.GenerateGalleryURLs( query_text )

			if len( initial_search_urls ) == 0:
				self._paused = True

				ShowText( 'The subscription "' + self._name + '"\'s Gallery URL Generator, "' + self._gug_key_and_name[1] + '" did not generate any URLs! The sub has paused!' )

				return

			gallery_seeds = [ GallerySeed( url, can_generate_more_pages = True ) for url in initial_search_urls ]

			gallery_seed_log.AddGallerySeeds( gallery_seeds )

			try:
				while gallery_seed_log.WorkToDo():
					p1 = HC.options[ 'pause_subs_sync' ]
					p2 = HG.view_shutdown
					p3 = not self._QuerySyncLoginIsOK( query )
					p4 = IsThreadShuttingDown()

					if p1 or p2 or p3 or p4:
						if p3:
							stop_reason = 'Login was invalid!'

						return

					if job_key.IsCancelled():
						stop_reason = 'gallery parsing cancelled, likely by user'

						self._DelayWork( 600, stop_reason )

						return

					gallery_seed = gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )

					if gallery_seed is None:
						stop_reason = 'thought there was a page to check, but apparently there was not!'

						break

					def status_hook( text ):
						job_key.SetVariable( 'popup_text_1', prefix + ': ' + text )

					def title_hook( text ):
						pass

					def file_seeds_callable( file_seeds ):
						num_urls_added = 0
						num_urls_already_in_file_seed_cache = 0
						can_search_for_more_files = True
						stop_reason = 'unknown stop reason'
						current_contiguous_num_urls_already_in_file_seed_cache = 0

						for file_seed in file_seeds:
							if file_seed in file_seeds_to_add:
								# this catches the occasional overflow when a new file is uploaded while gallery parsing is going on
								# we don't want to count these 'seen before this run' urls in the 'caught up to last time' count

								continue

							# When are we caught up? This is not a trivial problem. Tags are not always added when files are uploaded, so the order we find files is not completely reliable.
							# Ideally, we want to search a _bit_ deeper than the first already-seen.
							# And since we have a page of urls here and now, there is no point breaking early if there might be some new ones at the end.
							# Current rule is "We are caught up if the final X contiguous files are 'already in'". X is 5 for now.

							if file_seed_cache.HasFileSeed( file_seed ):
								num_urls_already_in_file_seed_cache += 1
								current_contiguous_num_urls_already_in_file_seed_cache += 1

							else:
								num_urls_added += 1
								current_contiguous_num_urls_already_in_file_seed_cache = 0

								file_seeds_to_add.add( file_seed )
								file_seeds_to_add_ordered.append( file_seed )

							if file_limit_for_this_sync is not None and total_new_urls_for_this_sync + num_urls_added >= file_limit_for_this_sync:
								# we have found enough new files this sync, so should stop adding files and new gallery pages

								if this_is_initial_sync:
									stop_reason = 'hit initial file limit'

								else:
									if total_already_in_urls_for_this_sync + num_urls_already_in_file_seed_cache > 0:
										# this sync produced some knowns, so it is likely we have stepped through a mix of old and tagged-late new files
										# we might also be on the second sync with a periodic limit greater than the initial limit
										# either way, this is no reason to go crying to the user

										stop_reason = 'hit periodic file limit after seeing several already-seen files'

									else:
										# this page had all entirely new files

										self._ShowHitPeriodicFileLimitMessage( query_name )

										stop_reason = 'hit periodic file limit without seeing any already-seen files!'

								can_search_for_more_files = False

								break

						WE_HIT_OLD_GROUND_THRESHOLD = 5

						if current_contiguous_num_urls_already_in_file_seed_cache >= WE_HIT_OLD_GROUND_THRESHOLD:
							# this gallery page has caught up to before, so it should not spawn any more gallery pages

							can_search_for_more_files = False

							stop_reason = 'saw ' + ToHumanInt( WE_HIT_OLD_GROUND_THRESHOLD ) + ' previously seen urls, so assuming we caught up'

						if num_urls_added == 0:
							can_search_for_more_files = False
							stop_reason = 'no new urls found'

						return ( num_urls_added, num_urls_already_in_file_seed_cache, can_search_for_more_files, stop_reason )

					job_key.SetVariable( 'popup_text_1', prefix + ': found ' + ToHumanInt( total_new_urls_for_this_sync ) + ' new urls, checking next page' )

					try:
						( num_urls_added, num_urls_already_in_file_seed_cache, num_urls_total, result_404, added_new_gallery_pages, stop_reason ) = gallery_seed.WorkOnURL( 'subscription', gallery_seed_log, file_seeds_callable, status_hook, title_hook, self._GenerateNetworkJobFactory( query ), GenerateMultiplePopupNetworkJobPresentationContextFactory( job_key ), self._file_import_options, gallery_urls_seen_before = gallery_urls_seen_this_sync )

					except CancelledException:
						stop_reason = 'gallery network job cancelled, likely by user'

						self._DelayWork( 600, stop_reason )

						return

					except Exception as e:
						stop_reason = str( e )

						raise

					total_new_urls_for_this_sync += num_urls_added
					total_already_in_urls_for_this_sync += num_urls_already_in_file_seed_cache

					if file_limit_for_this_sync is not None and total_new_urls_for_this_sync >= file_limit_for_this_sync:
						# we have found enough new files this sync, so stop and cancel any outstanding gallery urls

						if this_is_initial_sync:
							stop_reason = 'hit initial file limit'

						else:
							stop_reason = 'hit periodic file limit'

						break

			finally:
				while gallery_seed_log.WorkToDo():
					gallery_seed = gallery_seed_log.GetNextGallerySeed( CC.STATUS_UNKNOWN )

					if gallery_seed is None:
						break

					gallery_seed.SetStatus( CC.STATUS_VETOED, note = stop_reason )

			file_seeds_to_add_ordered.reverse()

			# 'first' urls are now at the end, so the file_seed_cache should stay roughly in oldest->newest order

			file_seed_cache.AddFileSeeds( file_seeds_to_add_ordered )

			query.RegisterSyncComplete()
			query.UpdateNextCheckTime( self._checker_options )

			if query.CanCompact( self._checker_options ):
				query.Compact( self._checker_options )

			if query.IsDead():
				if this_is_initial_sync:
					ShowText( 'The query "' + query_name + '" for subscription "' + self._name + '" did not find any files on its first sync! Could the query text have a typo, like a missing underscore?' )

				else:
					ShowText( 'The query "' + query_name + '" for subscription "' + self._name + '" appears to be dead!' )

			else:
				if this_is_initial_sync:
					if not self._QueryBandwidthIsOK( query ) and not have_made_an_initial_sync_bandwidth_notification:
						ShowText( 'FYI: The query "' + query_name + '" for subscription "' + self._name + '" performed its initial sync ok, but that domain is short on bandwidth right now, so no files will be downloaded yet. The subscription will catch up in future as bandwidth becomes available. You can review the estimated time until bandwidth is available under the manage subscriptions dialog. If more queries are performing initial syncs in this run, they may be the same.' )

						have_made_an_initial_sync_bandwidth_notification = True

	def _SyncQueryCanDoWork( self ):
		return True in ( query.CanSync() for query in self._queries )

	def CanCheckNow( self ):
		return True in ( query.CanCheckNow() for query in self._queries )

	def CanReset( self ):
		return True in ( not query.IsInitialSync() for query in self._queries )

	def CanRetryFailures( self ):
		return True in ( query.CanRetryFailed() for query in self._queries )

	def CanRetryIgnored( self ):
		return True in ( query.CanRetryIgnored() for query in self._queries )

	def CanScrubDelay( self ):
		return not TimeHasPassed( self._no_work_until )

	def CheckNow( self ):
		for query in self._queries:
			query.CheckNow()

		self.ScrubDelay()

	def GetBandwidthWaitingEstimate( self, query ):
		example_network_contexts = self._GetExampleNetworkContexts( query )

		( estimate, bandwidth_network_context ) = HG.client_controller.network_engine.bandwidth_manager.GetWaitingEstimateAndContext( example_network_contexts )

		return estimate

	def GetBandwidthWaitingEstimateMinMax( self ):
		if len( self._queries ) == 0:
			return ( 0, 0 )

		estimates = []

		for query in self._queries:
			example_network_contexts = self._GetExampleNetworkContexts( query )

			( estimate, bandwidth_network_context ) = HG.client_controller.network_engine.bandwidth_manager.GetWaitingEstimateAndContext( example_network_contexts )

			estimates.append( estimate )

		min_estimate = min( estimates )
		max_estimate = max( estimates )

		return ( min_estimate, max_estimate )

	def GetGUGKeyAndName( self ):
		return self._gug_key_and_name

	def GetQueries( self ):
		return self._queries

	def GetMergeable( self, potential_mergees ):
		mergeable = []
		unmergeable = []

		for subscription in potential_mergees:
			if subscription._gug_key_and_name[1] == self._gug_key_and_name[1]:
				mergeable.append( subscription )

			else:
				unmergeable.append( subscription )

		return ( mergeable, unmergeable )

	def GetPresentationOptions( self ):
		return ( self._show_a_popup_while_working, self._publish_files_to_popup_button, self._publish_files_to_page, self._publish_label_override, self._merge_query_publish_events )

	def GetTagImportOptions( self ):
		return self._tag_import_options

	def HasQuerySearchText( self, search_text ):
		for query in self._queries:
			query_text = query.GetQueryText()

			if search_text in query_text:
				return True

		return False

	def Merge( self, mergees ):
		for subscription in mergees:
			if subscription._gug_key_and_name[1] == self._gug_key_and_name[1]:
				my_new_queries = [ query.Duplicate() for query in subscription._queries ]

				self._queries.extend( my_new_queries )

			else:
				raise Exception( self._name + ' was told to merge an unmergeable subscription, ' + subscription.GetName() + '!' )

	def PauseResume( self ):
		self._paused = not self._paused

	def Reset( self ):
		for query in self._queries:
			query.Reset()

		self.ScrubDelay()

	def RetryFailures( self ):
		for query in self._queries:
			query.RetryFailures()

	def RetryIgnored( self ):
		for query in self._queries:
			query.RetryIgnored()

	def ReviveDead( self ):
		for query in self._queries:
			if query.IsDead():
				query.CheckNow()

	def Separate( self, base_name, only_these_queries = None ):
		if only_these_queries is None:
			only_these_queries = set( self._queries )

		else:
			only_these_queries = set( only_these_queries )

		subscriptions = []

		for query in self._queries:
			if query not in only_these_queries:
				continue

			subscription = self.Duplicate()

			subscription._queries = [ query.Duplicate() ]

			subscription.SetName( base_name + ': ' + query.GetHumanName() )

			subscriptions.append( subscription )

		self._queries = [ query for query in self._queries if query not in only_these_queries ]

		return subscriptions

	def SetCheckerOptions( self, checker_options ):
		self._checker_options = checker_options

		for query in self._queries:
			query.UpdateNextCheckTime( self._checker_options )

	def SetPresentationOptions( self, show_a_popup_while_working, publish_files_to_popup_button, publish_files_to_page, publish_label_override, merge_query_publish_events ):
		self._show_a_popup_while_working = show_a_popup_while_working
		self._publish_files_to_popup_button = publish_files_to_popup_button
		self._publish_files_to_page = publish_files_to_page
		self._publish_label_override = publish_label_override
		self._merge_query_publish_events = merge_query_publish_events

	def SetTagImportOptions( self, tag_import_options ):
		self._tag_import_options = tag_import_options.Duplicate()

	def SetTuple( self, gug_key_and_name, queries, checker_options, initial_file_limit, periodic_file_limit, paused, file_import_options, tag_import_options, no_work_until ):
		self._gug_key_and_name = gug_key_and_name
		self._queries = queries
		self._checker_options = checker_options
		self._initial_file_limit = initial_file_limit
		self._periodic_file_limit = periodic_file_limit
		self._paused = paused

		self._file_import_options = file_import_options
		self._tag_import_options = tag_import_options

		self._no_work_until = no_work_until

	def ScrubDelay( self ):
		self._no_work_until = 0
		self._no_work_until_reason = ''

	def Sync( self ):
		p1 = not self._paused
		p2 = not HG.view_shutdown
		p3 = self._NoDelays()
		p4 = self._SyncQueryCanDoWork()
		p5 = self._WorkOnFilesCanDoWork()

		if HG.subscription_report_mode:
			message = 'Subscription "' + self._name + '" entered sync.'
			message += os.linesep
			message += 'Unpaused: ' + str( p1 )
			message += os.linesep
			message += 'No delays: ' + str( p3 )
			message += os.linesep
			message += 'Sync can do work: ' + str( p4 )
			message += os.linesep
			message += 'Files can do work: ' + str( p5 )

			ShowText( message )

		if p1 and p2 and p3 and ( p4 or p5 ):
			job_key = JobKey( pausable = False, cancellable = True )

			try:
				job_key.SetVariable( 'popup_title', 'subscriptions - ' + self._name )

				if self._show_a_popup_while_working:
					HG.client_controller.pub( 'message', job_key )

				self._SyncQuery( job_key )

				self._WorkOnFiles( job_key )

			except NetworkException as e:
				delay = HG.client_controller.new_options.GetInteger( 'subscription_network_error_delay' )

				Print( 'The subscription ' + self._name + ' encountered an exception when trying to sync:' )
				PrintException( e )

				job_key.SetVariable( 'popup_text_1', 'Encountered a network error, will retry again later' )

				self._DelayWork( delay, 'network error: ' + str( e ) )

				time.sleep( 5 )

			except Exception as e:
				ShowText( 'The subscription ' + self._name + ' encountered an exception when trying to sync:' )
				ShowException( e )

				delay = HG.client_controller.new_options.GetInteger( 'subscription_other_error_delay' )

				self._DelayWork( delay, 'error: ' + str( e ) )

			finally:
				job_key.DeleteVariable( 'popup_network_job' )

			HG.client_controller.WriteSynchronous( 'serialisable', self )

			if job_key.HasVariable( 'popup_files' ):
				job_key.Finish()

			else:
				job_key.Delete()

	def ToTuple( self ):
		return ( self._name, self._gug_key_and_name, self._queries, self._checker_options, self._initial_file_limit, self._periodic_file_limit, self._paused, self._file_import_options, self._tag_import_options, self._no_work_until, self._no_work_until_reason )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["SUBSCRIPTION"] ] = Subscription

class SubscriptionQuery( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["SUBSCRIPTION_QUERY"]
	SERIALISABLE_NAME = 'Subscription Query'
	SERIALISABLE_VERSION = 3

	def __init__( self, query = 'query text' ):
		SerialisableBase.__init__( self )

		self._query = query
		self._display_name = None
		self._check_now = False
		self._last_check_time = 0
		self._next_check_time = 0
		self._paused = False
		self._status = CHECKER_STATUS_OK
		self._gallery_seed_log = GallerySeedLog()
		self._file_seed_cache = FileSeedCache()
		self._tag_import_options = TagImportOptions()

	def _GetSerialisableInfo( self ):
		serialisable_gallery_seed_log = self._gallery_seed_log.GetSerialisableTuple()
		serialisable_file_seed_cache = self._file_seed_cache.GetSerialisableTuple()
		serialisable_tag_import_options = self._tag_import_options.GetSerialisableTuple()

		return ( self._query, self._display_name, self._check_now, self._last_check_time, self._next_check_time, self._paused, self._status, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_tag_import_options )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._query, self._display_name, self._check_now, self._last_check_time, self._next_check_time, self._paused, self._status, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_tag_import_options ) = serialisable_info

		self._gallery_seed_log = CreateFromSerialisableTuple( serialisable_gallery_seed_log )
		self._file_seed_cache = CreateFromSerialisableTuple( serialisable_file_seed_cache )
		self._tag_import_options = CreateFromSerialisableTuple( serialisable_tag_import_options )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( query, check_now, last_check_time, next_check_time, paused, status, serialisable_file_seed_cache ) = old_serialisable_info

			gallery_seed_log = GallerySeedLog()

			serialisable_gallery_seed_log = gallery_seed_log.GetSerialisableTuple()

			new_serialisable_info = ( query, check_now, last_check_time, next_check_time, paused, status, serialisable_gallery_seed_log, serialisable_file_seed_cache )

			return ( 2, new_serialisable_info )

		if version == 2:
			( query, check_now, last_check_time, next_check_time, paused, status, serialisable_gallery_seed_log, serialisable_file_seed_cache ) = old_serialisable_info

			display_name = None
			tag_import_options = TagImportOptions()

			serialisable_tag_import_options = tag_import_options.GetSerialisableTuple()

			new_serialisable_info = ( query, display_name, check_now, last_check_time, next_check_time, paused, status, serialisable_gallery_seed_log, serialisable_file_seed_cache, serialisable_tag_import_options )

			return ( 3, new_serialisable_info )

	def CanWorkOnFiles( self ):
		file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )

		if HG.subscription_report_mode:
			ShowText( 'Query "' + self._query + '" CanWorkOnFiles test. Next import is ' + repr( file_seed ) + '.' )

		return file_seed is not None

	def CanCheckNow( self ):
		return not self._check_now

	def CanCompact( self, checker_options ):
		death_period = checker_options.GetDeathFileVelocityPeriod()

		compact_before_this_source_time = self._last_check_time - ( death_period * 2 )

		return self._file_seed_cache.CanCompact( compact_before_this_source_time ) or self._gallery_seed_log.CanCompact( compact_before_this_source_time )

	def CanRetryFailed( self ):
		return self._file_seed_cache.GetFileSeedCount( CC.STATUS_ERROR ) > 0

	def CanRetryIgnored( self ):
		return self._file_seed_cache.GetFileSeedCount( CC.STATUS_VETOED ) > 0

	def CanSync( self ):
		if HG.subscription_report_mode:
			ShowText( 'Query "' + self._query + '" CanSync test. Paused status is ' + str( self._paused ) + ' and check time due is ' + str( TimeHasPassed( self._next_check_time ) ) + ' and check_now is ' + str( self._check_now ) + '.' )

		if self._paused:
			return False

		return TimeHasPassed( self._next_check_time ) or self._check_now

	def CheckNow( self ):
		self._check_now = True
		self._paused = False

		self._next_check_time = 0
		self._status = CHECKER_STATUS_OK

	def Compact( self, checker_options ):
		death_period = checker_options.GetDeathFileVelocityPeriod()

		compact_before_this_time = self._last_check_time - ( death_period * 2 )

		self._file_seed_cache.Compact( compact_before_this_time )
		self._gallery_seed_log.Compact( compact_before_this_time )

	def GetDisplayName( self ):
		return self._display_name

	def GetFileSeedCache( self ):
		return self._file_seed_cache

	def GetGallerySeedLog( self ):
		return self._gallery_seed_log

	def GetHumanName( self ):
		if self._display_name is None:
			return self._query

		else:
			return self._display_name

	def GetLastChecked( self ):
		return self._last_check_time

	def GetLatestAddedTime( self ):
		return self._file_seed_cache.GetLatestAddedTime()

	def GetNextCheckStatusString( self ):
		if self._check_now:
			return 'checking on dialog ok'

		elif self._status == CHECKER_STATUS_DEAD:
			return 'dead, so not checking'

		else:
			if TimeHasPassed( self._next_check_time ):
				s = 'imminent'

			else:
				s = TimestampToPrettyTimeDelta( self._next_check_time )

			if self._paused:
				s = 'paused, but would be ' + s

			return s

	def GetNumURLsAndFailed( self ):
		return ( self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN ), len( self._file_seed_cache ), self._file_seed_cache.GetFileSeedCount( CC.STATUS_ERROR ) )

	def GetQueryText( self ):
		return self._query

	def GetTagImportOptions( self ):
		return self._tag_import_options

	def IsDead( self ):
		return self._status == CHECKER_STATUS_DEAD

	def IsInitialSync( self ):
		return self._last_check_time == 0

	def IsPaused( self ):
		return self._paused

	def PausePlay( self ):
		self._paused = not self._paused

	def RegisterSyncComplete( self ):
		self._last_check_time = GetNow()

		self._check_now = False

	def Reset( self ):
		self._last_check_time = 0
		self._next_check_time = 0
		self._status = CHECKER_STATUS_OK
		self._paused = False

		self._file_seed_cache = FileSeedCache()

	def RetryFailures( self ):
		self._file_seed_cache.RetryFailures()

	def RetryIgnored( self ):
		self._file_seed_cache.RetryIgnored()

	def SetCheckNow( self, check_now ):
		self._check_now = check_now

	def SetDisplayName( self, display_name ):
		self._display_name = display_name

	def SetPaused( self, paused ):
		self._paused = paused

	def SetQueryAndSeeds( self, query, file_seed_cache, gallery_seed_log ):
		self._query = query
		self._file_seed_cache = file_seed_cache
		self._gallery_seed_log = gallery_seed_log

	def SetTagImportOptions( self, tag_import_options ):
		self._tag_import_options = tag_import_options

	def UpdateNextCheckTime( self, checker_options ):
		if self._check_now:
			self._next_check_time = 0

			self._status = CHECKER_STATUS_OK

		else:
			if checker_options.IsDead( self._file_seed_cache, self._last_check_time ):
				self._status = CHECKER_STATUS_DEAD

				self._paused = True

			last_next_check_time = self._next_check_time

			self._next_check_time = checker_options.GetNextCheckTime( self._file_seed_cache, self._last_check_time, last_next_check_time )

	def ToTuple( self ):
		return ( self._query, self._check_now, self._last_check_time, self._next_check_time, self._paused, self._status )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["SUBSCRIPTION_QUERY"] ] = SubscriptionQuery

license: wtfpl