code:
from lib.Hydrus.Data import GenerateKey
from lib.Hydrus.Serialisable import SerialisableBase

from lib.Networking.Domain import GalleryURLGenerator

from lib.Handling.Parsing import ContentParser, \
	PageParser, ParseFormulaHTML, ParseRuleHTML, StringMatch

### CONSTANTS ###

import lib.Hydrus.Constants as HC

from lib.Handling.Parsing import HTML_CONTENT_ATTRIBUTE, HTML_CONTENT_STRING, HTML_RULE_TYPE_DESCENDING, \
	STRING_MATCH_FIXED

from lib.Hydrus.Serialisable import SERIALISABLE_TYPES_TO_OBJECT_TYPES, HG.SerialisableType["GALLERY_IDENTIFIER"]

### FUNCTIONS ###

def ConvertBooruToNewObjects( booru ):
	name = booru.GetName()

	name = 'zzz - auto-generated from legacy booru system - ' + name

	( search_url, search_separator, advance_by_page_num, thumb_classname, image_id, image_data, tag_classnames_to_namespaces ) = booru.GetData()

	if advance_by_page_num:
		search_url = search_url.replace( '%index%', '1' )

	else:
		search_url = search_url.replace( '%index%', '0' )

	gug = GalleryURLGenerator( name + ' search', url_template = search_url, replacement_phrase = '%tags%', search_terms_separator = search_separator, initial_search_text = 'tag search', example_search_text = 'blonde_hair blue_eyes' )

	#

	tag_rules = []

	rule_type = HTML_RULE_TYPE_DESCENDING
	tag_name = None
	tag_attributes = { 'class' : thumb_classname }
	tag_index = None

	tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index ) )

	rule_type = HTML_RULE_TYPE_DESCENDING
	tag_name = 'a'
	tag_attributes = None
	tag_index = None

	tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index ) )

	formula = ParseFormulaHTML( tag_rules = tag_rules, content_to_fetch = HTML_CONTENT_ATTRIBUTE, attribute_to_fetch = 'href' )

	url_type = HC.URL_TYPE_DESIRED
	priority = 50

	additional_info = ( url_type, priority )

	thumb_content_parser = ContentParser( name = 'get post urls (based on old booru thumb search)', content_type = HC.CONTENT_TYPE_URLS, formula = formula, additional_info = additional_info )

	gallery_parser = PageParser( name + ' gallery page parser', content_parsers = [ thumb_content_parser ], example_urls = [ gug.GetExampleURL() ] )

	#

	content_parsers = []

	if image_id is not None:
		tag_rules = []

		rule_type = HTML_RULE_TYPE_DESCENDING
		tag_name = 'a'
		tag_attributes = { 'id' : image_id }
		tag_index = None

		tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index ) )

		formula = ParseFormulaHTML( tag_rules = tag_rules, content_to_fetch = HTML_CONTENT_ATTRIBUTE, attribute_to_fetch = 'href' )

		url_type = HC.URL_TYPE_DESIRED
		priority = 75

		additional_info = ( url_type, priority )

		image_link_content_parser = ContentParser( name = 'get image file link url (based on old booru parser)', content_type = HC.CONTENT_TYPE_URLS, formula = formula, additional_info = additional_info )

		content_parsers.append( image_link_content_parser )

		#

		tag_rules = []

		rule_type = HTML_RULE_TYPE_DESCENDING
		tag_name = 'img'
		tag_attributes = { 'id' : image_id }
		tag_index = None

		tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index ) )

		formula = ParseFormulaHTML( tag_rules = tag_rules, content_to_fetch = HTML_CONTENT_ATTRIBUTE, attribute_to_fetch = 'src' )

		url_type = HC.URL_TYPE_DESIRED
		priority = 50

		additional_info = ( url_type, priority )

		image_src_content_parser = ContentParser( name = 'get image file src url (based on old booru parser)', content_type = HC.CONTENT_TYPE_URLS, formula = formula, additional_info = additional_info )

		content_parsers.append( image_src_content_parser )

	elif image_data is not None:
		tag_rules = []

		rule_type = HTML_RULE_TYPE_DESCENDING
		tag_name = 'a'
		tag_attributes = None
		tag_index = None

		string_match = StringMatch( match_type = STRING_MATCH_FIXED, match_value = image_data, example_string = image_data )

		tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index, should_test_tag_string = True, tag_string_string_match = string_match ) )

		formula = ParseFormulaHTML( tag_rules = tag_rules, content_to_fetch = HTML_CONTENT_ATTRIBUTE, attribute_to_fetch = 'href' )

		url_type = HC.URL_TYPE_DESIRED
		priority = 50

		additional_info = ( url_type, priority )

		image_link_content_parser = ContentParser( name = 'get image file url (based on old booru parser)', content_type = HC.CONTENT_TYPE_URLS, formula = formula, additional_info = additional_info )

		content_parsers.append( image_link_content_parser )

	for ( classname, namespace ) in list( tag_classnames_to_namespaces.items() ):
		tag_rules = []

		rule_type = HTML_RULE_TYPE_DESCENDING
		tag_name = None
		tag_attributes = { 'class' : classname }
		tag_index = None

		tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index ) )

		rule_type = HTML_RULE_TYPE_DESCENDING
		tag_name = 'a'
		tag_attributes = None
		tag_index = None

		tag_rules.append( ParseRuleHTML( rule_type = rule_type, tag_name = tag_name, tag_attributes = tag_attributes, tag_index = tag_index ) )

		formula = ParseFormulaHTML( tag_rules = tag_rules, content_to_fetch = HTML_CONTENT_STRING )

		additional_info = namespace

		tag_content_parser = ContentParser( name = 'get "' + namespace + '" tags', content_type = HC.CONTENT_TYPE_MAPPINGS, formula = formula, additional_info = additional_info )

		content_parsers.append( tag_content_parser )

	post_parser = PageParser( name + ' post page parser', content_parsers = content_parsers, example_urls = [] )

	#

	return ( gug, gallery_parser, post_parser )

def ConvertGalleryIdentifierToGUGKeyAndName( gallery_identifier ):
	gug_name = ConvertGalleryIdentifierToGUGName( gallery_identifier )

	import lib.Hydrus.Defaults

	gugs = lib.Hydrus.Defaults.GetDefaultGUGs()

	for gug in gugs:
		if gug.GetName() == gug_name:
			return gug.GetGUGKeyAndName()

	return ( GenerateKey(), gug_name )

def ConvertGalleryIdentifierToGUGName( gallery_identifier ):
	site_type = gallery_identifier.GetSiteType()

	if site_type == HC.SITE_TYPE_DEVIANT_ART:
		return 'deviant art artist lookup'

	elif site_type == HC.SITE_TYPE_TUMBLR:
		return 'tumblr username lookup'

	elif site_type == HC.SITE_TYPE_NEWGROUNDS:
		return 'newgrounds artist lookup'

	elif site_type == HC.SITE_TYPE_HENTAI_FOUNDRY_ARTIST:
		return 'hentai foundry artist lookup'

	elif site_type == HC.SITE_TYPE_HENTAI_FOUNDRY_TAGS:
		return 'hentai foundry tag search'

	elif site_type == HC.SITE_TYPE_PIXIV_ARTIST_ID:
		return 'pixiv artist lookup'

	elif site_type == HC.SITE_TYPE_PIXIV_TAG:
		return 'pixiv tag search'

	elif site_type == HC.SITE_TYPE_BOORU:
		booru_name_converter = {}

		booru_name_converter[ 'gelbooru' ] = 'gelbooru tag search'
		booru_name_converter[ 'safebooru' ] = 'safebooru tag search'
		booru_name_converter[ 'e621' ] = 'e621 tag search'
		booru_name_converter[ 'rule34@paheal' ] = 'rule34.paheal tag search'
		booru_name_converter[ 'danbooru' ] = 'danbooru tag search'
		booru_name_converter[ 'mishimmie' ] = 'mishimmie tag search'
		booru_name_converter[ 'rule34@booru.org' ] = 'rule34.xxx tag search'
		booru_name_converter[ 'furry@booru.org' ] = 'furry.booru.org tag search'
		booru_name_converter[ 'xbooru' ] = 'xbooru tag search'
		booru_name_converter[ 'konachan' ] = 'konachan tag search'
		booru_name_converter[ 'yande.re' ] = 'yande.re tag search'
		booru_name_converter[ 'tbib' ] = 'tbib tag search'
		booru_name_converter[ 'sankaku chan' ] = 'sankaku channel tag search'
		booru_name_converter[ 'sankaku idol' ] = 'sankaku idol tag search'
		booru_name_converter[ 'rule34hentai' ] = 'rule34hentai tag search'

		booru_name = gallery_identifier.GetAdditionalInfo()

		if booru_name in booru_name_converter:
			return booru_name_converter[ booru_name ]

		else:
			return booru_name

	else:
		return 'unknown site'

class GalleryIdentifier( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["GALLERY_IDENTIFIER"]
	SERIALISABLE_NAME = 'Gallery Identifier'
	SERIALISABLE_VERSION = 1

	def __init__( self, site_type = None, additional_info = None ):
		SerialisableBase.__init__( self )

		self._site_type = site_type
		self._additional_info = additional_info

	def __eq__( self, other ):
		return self.__hash__() == other.__hash__()

	def __hash__( self ):
		return ( self._site_type, self._additional_info ).__hash__()

	def __ne__( self, other ):
		return self.__hash__() != other.__hash__()

	def __repr__( self ):
		text = 'Gallery Identifier: ' + HC.site_type_string_lookup[ self._site_type ]

		if self._site_type == HC.SITE_TYPE_BOORU:
			text += ': ' + str( self._additional_info )

		return text

	def _GetSerialisableInfo( self ):
		return ( self._site_type, self._additional_info )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._site_type, self._additional_info ) = serialisable_info

	def GetAdditionalInfo( self ):
		return self._additional_info

	def GetSiteType( self ):
		return self._site_type

	def ToString( self ):
		text = HC.site_type_string_lookup[ self._site_type ]

		if self._site_type == HC.SITE_TYPE_BOORU and self._additional_info is not None:
			booru_name = self._additional_info

			text = booru_name

		return text

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["GALLERY_IDENTIFIER"] ] = GalleryIdentifier


code:
import os, requests, threading, traceback, time, urllib, io

from lib.Hydrus.Data import DebugPrint, GetNow, Print, \
	ShowException, ShowText, TimeHasPassed, TimestampToPrettyTimeDelta, ToHumanBytes
from lib.Hydrus.Exceptions import BadRequestException, BandwidthException, CancelledException, \
	ConnectionException, InsufficientCredentialsException, MissingCredentialsException, NetworkException, \
	NetworkVersionException, NotFoundException, NotModifiedException, ServerBusyException, ServerException, \
	SessionException, ShouldReattemptNetworkException, ShutdownException, SizeException, ValidationException, \
	WrongServiceTypeException
from lib.Hydrus.Threading import IsThreadShuttingDown
from lib.Hydrus.Text import NonFailingUnicodeDecode

from lib.Networking import BandwidthTracker
from lib.Networking.Contexts import NetworkContext
from lib.Networking.Domain import ConvertDomainIntoAllApplicableDomains, \
	ConvertURLIntoDomain, ConvertURLIntoSecondLevelDomain

### CONTEXTS ###

import lib.Hydrus.Constants as HC
import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Networking.Contexts import GLOBAL_NETWORK_CONTEXT

### FUNCTIONS ###

def ConvertStatusCodeAndDataIntoExceptionInfo( status_code, data, is_hydrus_service = False ):
	( error_text, encoding ) = NonFailingUnicodeDecode( data, 'utf-8' )

	print_long_error_text = True

	if status_code == 304:
		print_long_error_text = False

		eclass = NotModifiedException

	elif status_code == 400:
		eclass = BadRequestException

	elif status_code == 401:
		eclass = MissingCredentialsException

	elif status_code == 403:
		eclass = InsufficientCredentialsException

	elif status_code == 404:
		print_long_error_text = False

		eclass = NotFoundException

	elif status_code == 419:
		eclass = SessionException

	elif status_code == 426:
		eclass = NetworkVersionException

	elif status_code == 509:
		eclass = BandwidthException

	elif status_code >= 500:
		if is_hydrus_service and status_code == 503:
			eclass = ServerBusyException

		else:
			eclass = ServerException

	else:
		eclass = NetworkException

	if len( error_text ) > 1024 and print_long_error_text:
		large_chunk = error_text[ : 512 * 1024 ]

		smaller_chunk = large_chunk[:256]

		DebugPrint( large_chunk )

		error_text = 'The server\'s error text was too long to display. The first part follows, while a larger chunk has been written to the log.'
		error_text += os.linesep
		error_text += smaller_chunk

	e = eclass( error_text )

	return ( e, error_text )

class NetworkJob( object ):
	WILLING_TO_WAIT_ON_INVALID_LOGIN = True
	IS_HYDRUS_SERVICE = False

	def __init__( self, method, url, body = None, referral_url = None, temp_path = None ):
		if body is not None and isinstance( body, str ):
			body = bytes( body, 'utf-8' )

		self.engine = None

		self._lock = threading.Lock()

		self._method = method
		self._url = url

		self._domain = ConvertURLIntoDomain( self._url )
		self._second_level_domain = ConvertURLIntoSecondLevelDomain( self._url )

		self._body = body
		self._referral_url = referral_url
		self._temp_path = temp_path

		self._files = None
		self._for_login = False

		self._current_connection_attempt_number = 1

		self._additional_headers = {}

		self._creation_time = GetNow()

		self._bandwidth_tracker = BandwidthTracker()

		self._wake_time = 0

		self._content_type = None

		self._encoding = 'utf-8'
		self._encoding_confirmed = False

		self._stream_io = io.BytesIO()

		self._error_exception = Exception( 'Exception not initialised.' ) # PyLint hint, wew
		self._error_exception = None
		self._error_text = None

		self._is_done_event = threading.Event()

		self._is_started = False
		self._is_done = False
		self._is_cancelled = False

		self._gallery_token_name = None
		self._gallery_token_consumed = False
		self._bandwidth_manual_override = False
		self._bandwidth_manual_override_delayed_timestamp = None

		self._last_time_ongoing_bandwidth_failed = 0

		self._status_text = 'initialising\u2026'
		self._num_bytes_read = 0
		self._num_bytes_to_read = 1

		self._death_time = None

		self._file_import_options = None

		self._network_contexts = self._GenerateNetworkContexts()

		( self._session_network_context, self._login_network_context ) = self._GenerateSpecificNetworkContexts()

	def _CanReattemptConnection( self ):
		max_attempts_allowed = 3

		return self._current_connection_attempt_number <= max_attempts_allowed

	def _CanReattemptRequest( self ):
		if self._method == 'GET':
			max_attempts_allowed = 5

		elif self._method == 'POST':
			max_attempts_allowed = 1

		return self._current_connection_attempt_number <= max_attempts_allowed

	def _GenerateNetworkContexts( self ):
		network_contexts = []

		network_contexts.append( GLOBAL_NETWORK_CONTEXT )

		domains = ConvertDomainIntoAllApplicableDomains( self._domain )

		network_contexts.extend( ( NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, domain ) for domain in domains ) )

		return network_contexts

	def _GenerateSpecificNetworkContexts( self ):
		# we always store cookies in the larger session (even if the cookie itself refers to a subdomain in the session object)
		# but we can login to a specific subdomain

		session_network_context = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, self._second_level_domain )
		login_network_context = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, self._domain )

		return ( session_network_context, login_network_context )

	def _SendRequestAndGetResponse( self ):
		with self._lock:
			ncs = list( self._network_contexts )

		headers = self.engine.domain_manager.GetHeaders( ncs )

		with self._lock:
			method = self._method
			url = self._url
			data = self._body
			files = self._files

			if self.IS_HYDRUS_SERVICE:
				headers[ 'User-Agent' ] = 'hydrus client/' + str( HC.NETWORK_VERSION )

			if self._referral_url is not None:
				headers[ 'referer' ] = urllib.parse.quote( self._referral_url, "!#$%&'( )*+,/:;=?@[]~" ) # quick and dirty way to quote this url when it comes here with full unicode chars. not perfect, but does the job

			for ( key, value ) in list( self._additional_headers.items() ):
				headers[ key ] = value

			self._status_text = 'sending request\u2026'

			snc = self._session_network_context

		session = self.engine.session_manager.GetSession( snc )

		connect_timeout = HG.client_controller.new_options.GetInteger( 'network_timeout' )

		read_timeout = connect_timeout * 6

		response = session.request( method, url, data = data, files = files, headers = headers, stream = True, timeout = ( connect_timeout, read_timeout ) )

		return response

	def _IsCancelled( self ):
		if self._is_cancelled:
			return True

		if self.engine.controller.ModelIsShutdown():
			return True

		return False

	def _IsDone( self ):
		if self._is_done:
			return True

		if self.engine.controller.ModelIsShutdown() or IsThreadShuttingDown():
			return True

		return False

	def _ObeysBandwidth( self ):
		if self._bandwidth_manual_override:
			return False

		if self._bandwidth_manual_override_delayed_timestamp is not None and TimeHasPassed( self._bandwidth_manual_override_delayed_timestamp ):
			return False

		if self._method == 'POST':
			return False

		if self._for_login:
			return False

		return True

	def _OngoingBandwidthOK( self ):
		now = GetNow()

		if now == self._last_time_ongoing_bandwidth_failed: # it won't have changed, so no point spending any cpu checking

			return False

		else:
			result = self.engine.bandwidth_manager.CanContinueDownload( self._network_contexts )

			if not result:
				self._last_time_ongoing_bandwidth_failed = now

			return result

	def _ReadResponse( self, response, stream_dest, max_allowed = None ):
		with self._lock:
			if self._content_type is not None and self._content_type in HC.mime_enum_lookup:
				mime = HC.mime_enum_lookup[ self._content_type ]

			else:
				mime = None

			if 'content-length' in response.headers:
				self._num_bytes_to_read = int( response.headers[ 'content-length' ] )

				if max_allowed is not None and self._num_bytes_to_read > max_allowed:
					raise NetworkException( 'The url was apparently ' + ToHumanBytes( self._num_bytes_to_read ) + ' but the max network size for this type of job is ' + ToHumanBytes( max_allowed ) + '!' )

				if self._file_import_options is not None:
					is_complete_file_size = True

					self._file_import_options.CheckNetworkDownload( mime, self._num_bytes_to_read, is_complete_file_size )

			else:
				self._num_bytes_to_read = None

		for chunk in response.iter_content( chunk_size = 65536 ):
			if self._IsCancelled():
				return

			stream_dest.write( chunk )

			chunk_length = len( chunk )

			with self._lock:
				self._num_bytes_read += chunk_length

				if max_allowed is not None and self._num_bytes_read > max_allowed:
					raise NetworkException( 'The url exceeded the max network size for this type of job, which is ' + ToHumanBytes( max_allowed ) + '!' )

				if self._file_import_options is not None:
					is_complete_file_size = False

					self._file_import_options.CheckNetworkDownload( mime, self._num_bytes_read, is_complete_file_size )

			self._ReportDataUsed( chunk_length )
			self._WaitOnOngoingBandwidth()

			if HG.view_shutdown:
				raise ShutdownException()

		if self._num_bytes_to_read is not None and self._num_bytes_read < self._num_bytes_to_read * 0.8:
			raise ShouldReattemptNetworkException( 'Was expecting ' + ToHumanBytes( self._num_bytes_to_read ) + ' but only got ' + ToHumanBytes( self._num_bytes_read ) + '.' )

	def _ReportDataUsed( self, num_bytes ):
		self._bandwidth_tracker.ReportDataUsed( num_bytes )

		self.engine.bandwidth_manager.ReportDataUsed( self._network_contexts, num_bytes )

	def _SetCancelled( self ):
		self._is_cancelled = True

		self._SetDone()

	def _SetError( self, e, error ):
		self._error_exception = e
		self._error_text = error

		if HG.network_report_mode:
			ShowText( 'Network error should follow:' )
			ShowException( e )
			ShowText( error )

		self._SetDone()

	def _SetDone( self ):
		self._is_done = True

		self._is_done_event.set()

	def _Sleep( self, seconds ):
		self._wake_time = GetNow() + seconds

	def _WaitOnOngoingBandwidth( self ):
		while not self._OngoingBandwidthOK() and not self._IsCancelled():
			time.sleep( 0.1 )

	def AddAdditionalHeader( self, key, value ):
		with self._lock:
			self._additional_headers[ key ] = value

	def BandwidthOK( self ):
		with self._lock:
			if self._ObeysBandwidth():
				result = self.engine.bandwidth_manager.TryToStartRequest( self._network_contexts )

				if result:
					self._bandwidth_tracker.ReportRequestUsed()

				else:
					( bandwidth_waiting_duration, bandwidth_network_context ) = self.engine.bandwidth_manager.GetWaitingEstimateAndContext( self._network_contexts )

					will_override = self._bandwidth_manual_override_delayed_timestamp is not None

					override_coming_first = False

					if will_override:
						override_waiting_duration = self._bandwidth_manual_override_delayed_timestamp - GetNow()

						override_coming_first = override_waiting_duration < bandwidth_waiting_duration

					just_now_threshold = 2

					if override_coming_first:
						waiting_duration = override_waiting_duration

						waiting_str = 'overriding bandwidth ' + TimestampToPrettyTimeDelta( self._bandwidth_manual_override_delayed_timestamp, just_now_string = 'imminently', just_now_threshold = just_now_threshold )

					else:
						waiting_duration = bandwidth_waiting_duration

						waiting_str = 'bandwidth free ' + TimestampToPrettyTimeDelta( GetNow() + waiting_duration, just_now_string = 'imminently', just_now_threshold = just_now_threshold )

					waiting_str += '\u2026 (' + bandwidth_network_context.ToHumanString() + ')'

					self._status_text = waiting_str

					if waiting_duration > 1200:
						self._Sleep( 30 )

					elif waiting_duration > 120:
						self._Sleep( 10 )

					elif waiting_duration > 10:
						self._Sleep( 1 )

				return result

			else:
				self._bandwidth_tracker.ReportRequestUsed()

				self.engine.bandwidth_manager.ReportRequestUsed( self._network_contexts )

				return True

	def Cancel( self, status_text = None ):
		with self._lock:
			if status_text is None:
				status_text = 'cancelled!'

			self._status_text = status_text

			self._SetCancelled()

	def CanValidateInPopup( self ):
		with self._lock:
			return self.engine.domain_manager.CanValidateInPopup( self._network_contexts )

	def CheckCanLogin( self ):
		with self._lock:
			if self._for_login:
				raise ValidationException( 'Login jobs should not be asked if they can login!' )

			else:
				return self.engine.login_manager.CheckCanLogin( self._login_network_context )

	def GenerateLoginProcess( self ):
		with self._lock:
			if self._for_login:
				raise Exception( 'Login jobs should not be asked to generate login processes!' )

			else:
				return self.engine.login_manager.GenerateLoginProcess( self._login_network_context )

	def GenerateValidationPopupProcess( self ):
		with self._lock:
			return self.engine.domain_manager.GenerateValidationPopupProcess( self._network_contexts )

	def GetContentBytes( self ):
		with self._lock:
			self._stream_io.seek( 0 )

			return self._stream_io.read()

	def GetContentText( self ):
		data = self.GetContentBytes()

		( text, self._encoding ) = NonFailingUnicodeDecode( data, self._encoding )

		return text

	def GetContentType( self ):
		with self._lock:
			return self._content_type

	def GetCreationTime( self ):
		with self._lock:
			return self._creation_time

	def GetDomain( self ):
		with self._lock:
			return self._domain

	def GetErrorException( self ):
		with self._lock:
			return self._error_exception

	def GetErrorText( self ):
		with self._lock:
			return self._error_text

	def GetLoginNetworkContext( self ):
		with self._lock:
			return self._login_network_context

	def GetNetworkContexts( self ):
		with self._lock:
			return list( self._network_contexts )

	def GetSecondLevelDomain( self ):
		with self._lock:
			return self._second_level_domain

	def GetSession( self ):
		with self._lock:
			snc = self._session_network_context

		session = self.engine.session_manager.GetSession( snc )

		return session

	def GetStatus( self ):
		with self._lock:
			return ( self._status_text, self._bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, 1 ), self._num_bytes_read, self._num_bytes_to_read )

	def GetTotalDataUsed( self ):
		with self._lock:
			return self._bandwidth_tracker.GetUsage( HC.BANDWIDTH_TYPE_DATA, None )

	def GetURL( self ):
		with self._lock:
			return self._url

	def HasError( self ):
		with self._lock:
			return self._error_exception is not None

	def IsAsleep( self ):
		with self._lock:
			return not TimeHasPassed( self._wake_time )

	def IsCancelled( self ):
		with self._lock:
			return self._IsCancelled()

	def IsDone( self ):
		with self._lock:
			return self._IsDone()

	def IsHydrusJob( self ):
		with self._lock:
			return False

	def IsValid( self ):
		with self._lock:
			return self.engine.domain_manager.IsValid( self._network_contexts )

	def NeedsLogin( self ):
		with self._lock:
			if self._for_login:
				return False

			else:
				return self.engine.login_manager.NeedsLogin( self._login_network_context )

	def NoEngineYet( self ):
		return self.engine is None

	def ObeysBandwidth( self ):
		return self._ObeysBandwidth()

	def OverrideBandwidth( self, delay = None ):
		with self._lock:
			if delay is None:
				self._bandwidth_manual_override = True

				self._wake_time = 0

			else:
				self._bandwidth_manual_override_delayed_timestamp = GetNow() + delay

				self._wake_time = min( self._wake_time, self._bandwidth_manual_override_delayed_timestamp + 1 )

	def OverrideToken( self ):
		with self._lock:
			self._gallery_token_consumed = True

			self._wake_time = 0

	def SetDeathTime( self, death_time ):
		self._death_time = death_time

	def SetError( self, e, error ):
		with self._lock:
			self._SetError( e, error )

	def SetFiles( self, files ):
		with self._lock:
			self._files = files

	def SetFileImportOptions( self, file_import_options ):
		with self._lock:
			self._file_import_options = file_import_options

	def SetForLogin( self, for_login ):
		with self._lock:
			self._for_login = for_login

	def SetGalleryToken( self, token_name ):
		with self._lock:
			self._gallery_token_name = token_name

	def SetStatus( self, text ):
		with self._lock:
			self._status_text = text

	def Sleep( self, seconds ):
		with self._lock:
			self._Sleep( seconds )

	def Start( self ):
		try:
			with self._lock:
				self._is_started = True
				self._status_text = 'job started'

			request_completed = False

			while not request_completed:
				try:
					response = self._SendRequestAndGetResponse()

					with self._lock:
						if self._body is not None:
							self._ReportDataUsed( len( self._body ) )

					if 'Content-Type' in response.headers:
						self._content_type = response.headers[ 'Content-Type' ]

					if response.ok:
						with self._lock:
							self._status_text = 'downloading\u2026'

						if response.encoding is not None:
							encoding = response.encoding

							# we'll default to utf-8 rather than ISO-8859-1
							we_got_lame_iso_default_from_requests = encoding == 'ISO-8859-1' and ( self._content_type is None or encoding not in self._content_type )

							if not we_got_lame_iso_default_from_requests:
								self._encoding = encoding

						if self._temp_path is None:
							self._ReadResponse( response, self._stream_io, 104857600 )

						else:
							with open( self._temp_path, 'wb' ) as f:
								self._ReadResponse( response, f )

						with self._lock:
							self._status_text = 'done!'

					else:
						with self._lock:
							self._status_text = str( response.status_code ) + ' - ' + str( response.reason )

						self._ReadResponse( response, self._stream_io, 104857600 )

						with self._lock:
							self._stream_io.seek( 0 )

							data = self._stream_io.read()

							( e, error_text ) = ConvertStatusCodeAndDataIntoExceptionInfo( response.status_code, data, self.IS_HYDRUS_SERVICE )

							self._SetError( e, error_text )

					request_completed = True

				except ShouldReattemptNetworkException as e:
					self._current_connection_attempt_number += 1

					if not self._CanReattemptRequest():
						raise NetworkException( 'Ran out of reattempts on this error: ' + str( e ) )

					with self._lock:
						self._status_text = str( e ) + '--retrying'

					time.sleep( 3 )

				except requests.exceptions.ChunkedEncodingError:
					self._current_connection_attempt_number += 1

					if not self._CanReattemptRequest():
						raise ConnectionException( 'Unable to complete request--it broke mid-way!' )

					with self._lock:
						self._status_text = 'connection broke mid-request--retrying'

					time.sleep( 3 )

				except ( requests.exceptions.ConnectionError, requests.exceptions.ConnectTimeout ):
					self._current_connection_attempt_number += 1

					if not self._CanReattemptConnection():
						raise ConnectionException( 'Could not connect!' )

					with self._lock:
						self._status_text = 'connection failed--retrying'

					time.sleep( 3 )

				except requests.exceptions.ReadTimeout:
					self._current_connection_attempt_number += 1

					if not self._CanReattemptRequest():
						raise ConnectionException( 'Connection successful, but reading response timed out!' )

					with self._lock:
						self._status_text = 'read timed out--retrying'

					time.sleep( 3 )

		except Exception as e:
			with self._lock:
				trace = traceback.format_exc()

				if not isinstance( e, ( ConnectionException, SizeException ) ):
					Print( trace )

				self._status_text = 'Error: ' + str( e )

				self._SetError( e, trace )

		finally:
			with self._lock:
				self._SetDone()

	def TokensOK( self ):
		with self._lock:
			need_token = self._gallery_token_name is not None and not self._gallery_token_consumed

			sld = self._second_level_domain
			gtn = self._gallery_token_name

		if need_token:
			( consumed, next_timestamp ) = self.engine.bandwidth_manager.TryToConsumeAGalleryToken( sld, gtn )

			with self._lock:
				if consumed:
					self._status_text = 'slot consumed, starting soon'

					self._gallery_token_consumed = True

				else:
					self._status_text = 'waiting for a ' + self._gallery_token_name + ' slot:
 	next ' + TimestampToPrettyTimeDelta( next_timestamp, just_now_threshold = 1 )

					self._Sleep( 1 )

					return False

		return True

	def WaitUntilDone( self ):
		while True:
			self._is_done_event.wait( 5 )

			with self._lock:
				if not self._is_started and self._death_time is not None and TimeHasPassed( self._death_time ):
					raise Exception( 'Network job death time reached--not sure what the error was. Maybe a paused service?' )

			if self.IsDone():
				break

		with self._lock:
			if self.engine.controller.ModelIsShutdown() or IsThreadShuttingDown():
				raise ShutdownException()

			elif self._error_exception is not None:
				if isinstance( self._error_exception, Exception ):
					raise self._error_exception

				else:
					raise Exception( 'Problem in network error handling.' )

			elif self._IsCancelled():
				if self._method == 'POST':
					message = 'Upload cancelled: ' + self._status_text

				else:
					message = 'Download cancelled: ' + self._status_text

				raise CancelledException( message )

	def WillingToWaitOnInvalidLogin( self ):
		return self.WILLING_TO_WAIT_ON_INVALID_LOGIN

class NetworkJobDownloader( NetworkJob ):
	def __init__( self, downloader_page_key, method, url, body = None, referral_url = None, temp_path = None ):
		self._downloader_page_key = downloader_page_key

		NetworkJob.__init__( self, method, url, body = body, referral_url = referral_url, temp_path = temp_path )

	def _GenerateNetworkContexts( self ):
		network_contexts = NetworkJob._GenerateNetworkContexts( self )

		network_contexts.append( NetworkContext( CC.NETWORK_CONTEXT_DOWNLOADER_PAGE, self._downloader_page_key ) )

		return network_contexts

class NetworkJobSubscription( NetworkJob ):
	WILLING_TO_WAIT_ON_INVALID_LOGIN = False

	def __init__( self, subscription_key, method, url, body = None, referral_url = None, temp_path = None ):
		self._subscription_key = subscription_key

		NetworkJob.__init__( self, method, url, body = body, referral_url = referral_url, temp_path = temp_path )

	def _GenerateNetworkContexts( self ):
		network_contexts = NetworkJob._GenerateNetworkContexts( self )

		network_contexts.append( NetworkContext( CC.NETWORK_CONTEXT_SUBSCRIPTION, self._subscription_key ) )

		return network_contexts

class NetworkJobHydrus( NetworkJob ):
	WILLING_TO_WAIT_ON_INVALID_LOGIN = False
	IS_HYDRUS_SERVICE = True

	def __init__( self, service_key, method, url, body = None, referral_url = None, temp_path = None ):
		self._service_key = service_key

		NetworkJob.__init__( self, method, url, body = body, referral_url = referral_url, temp_path = temp_path )

	def _CheckHydrusVersion( self, service_type, response ):
		service_string = HC.service_string_lookup[ service_type ]

		headers = response.headers

		if 'server' not in headers or service_string not in headers[ 'server' ]:
			raise WrongServiceTypeException( 'Target was not a ' + service_string + '!' )

		server_header = headers[ 'server' ]

		( service_string_gumpf, network_version ) = server_header.split( '/' )

		network_version = int( network_version )

		if network_version != HC.NETWORK_VERSION:
			if network_version > HC.NETWORK_VERSION:
				message = 'Your client is out of date; please download the latest release.'

			else:
				message = 'The server is out of date; please ask its admin to update to the latest release.'

			raise NetworkVersionException( 'Network version mismatch! The server\'s network version was ' + str( network_version ) + ', whereas your client\'s is ' + str( HC.NETWORK_VERSION ) + '! ' + message )

	def _GenerateNetworkContexts( self ):
		network_contexts = []

		network_contexts.append( GLOBAL_NETWORK_CONTEXT )
		network_contexts.append( NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, self._service_key ) )

		return network_contexts

	def _GenerateSpecificNetworkContexts( self ):
		# we store cookies on and login to the same hydrus-specific context

		session_network_context = NetworkContext( CC.NETWORK_CONTEXT_HYDRUS, self._service_key )
		login_network_context = session_network_context

		return ( session_network_context, login_network_context )

	def _ReportDataUsed( self, num_bytes ):
		service = self.engine.controller.services_manager.GetService( self._service_key )

		service_type = service.GetServiceType()

		if service_type in HC.RESTRICTED_SERVICES:
			account = service.GetAccount()

			account.ReportDataUsed( num_bytes )

		NetworkJob._ReportDataUsed( self, num_bytes )

	def _SendRequestAndGetResponse( self ):
		service = self.engine.controller.services_manager.GetService( self._service_key )

		service_type = service.GetServiceType()

		if service_type in HC.RESTRICTED_SERVICES:
			account = service.GetAccount()

			account.ReportRequestUsed()

		response = NetworkJob._SendRequestAndGetResponse( self )

		if service_type in HC.RESTRICTED_SERVICES:
			self._CheckHydrusVersion( service_type, response )

		return response

	def IsHydrusJob( self ):
		with self._lock:
			return True

class NetworkJobWatcherPage( NetworkJob ):
	def __init__( self, watcher_key, method, url, body = None, referral_url = None, temp_path = None ):
		self._watcher_key = watcher_key

		NetworkJob.__init__( self, method, url, body = body, referral_url = referral_url, temp_path = temp_path )

	def _GenerateNetworkContexts( self ):
		network_contexts = NetworkJob._GenerateNetworkContexts( self )

		network_contexts.append( NetworkContext( CC.NETWORK_CONTEXT_WATCHER_PAGE, self._watcher_key ) )

		return network_contexts

license: wtfpl


code:
import threading, time, collections

from lib.Hydrus.Data import DedupeList, ShowException, ShowText
from lib.Hydrus.Exceptions import ValidationException

### CONSTANTS ###
import lib.Hydrus.Globals as HG

JOB_STATUS_AWAITING_VALIDITY = 0
JOB_STATUS_AWAITING_BANDWIDTH = 1
JOB_STATUS_AWAITING_LOGIN = 2
JOB_STATUS_AWAITING_SLOT = 3
JOB_STATUS_RUNNING = 4

job_status_str_lookup = {}

job_status_str_lookup[ JOB_STATUS_AWAITING_VALIDITY ] = 'waiting for validation'
job_status_str_lookup[ JOB_STATUS_AWAITING_BANDWIDTH ] = 'waiting for bandwidth'
job_status_str_lookup[ JOB_STATUS_AWAITING_LOGIN ] = 'waiting for login'
job_status_str_lookup[ JOB_STATUS_AWAITING_SLOT ] = 'waiting for slot'
job_status_str_lookup[ JOB_STATUS_RUNNING ] = 'running'

### CLASSES ###

class NetworkEngine( object ):
	def __init__( self, controller, bandwidth_manager, session_manager, domain_manager, login_manager ):
		self.controller = controller

		self.bandwidth_manager = bandwidth_manager
		self.session_manager = session_manager
		self.domain_manager = domain_manager
		self.login_manager = login_manager

		self.bandwidth_manager.engine = self
		self.session_manager.engine = self
		self.domain_manager.engine = self
		self.login_manager.engine = self

		self._lock = threading.Lock()

		self.RefreshOptions()

		self._new_work_to_do = threading.Event()

		self._domains_to_login = []

		self._active_domains_counter = collections.Counter()

		self._jobs_awaiting_validity = []
		self._current_validation_process = None
		self._jobs_awaiting_bandwidth = []
		self._jobs_awaiting_login = []
		self._current_login_process = None
		self._jobs_awaiting_slot = []
		self._jobs_running = []

		self._pause_all_new_network_traffic = self.controller.new_options.GetBoolean( 'pause_all_new_network_traffic' )

		self._is_running = False
		self._is_shutdown = False
		self._local_shutdown = False

		self.controller.sub( self, 'RefreshOptions', 'notify_new_options' )

	def AddJob( self, job ):
		if HG.network_report_mode:
			ShowText( 'Network Job Added: ' + job._method + ' ' + job._url )

		with self._lock:
			job.engine = self

			self._jobs_awaiting_validity.append( job )

		self._new_work_to_do.set()

	def ForceLogins( self, domains_to_login ):
		with self._lock:
			self._domains_to_login.extend( domains_to_login )

			self._domains_to_login = DedupeList( self._domains_to_login )

	def GetJobsSnapshot( self ):
		with self._lock:
			jobs = []

			jobs.extend( ( ( JOB_STATUS_AWAITING_VALIDITY, j ) for j in self._jobs_awaiting_validity ) )
			jobs.extend( ( ( JOB_STATUS_AWAITING_BANDWIDTH, j ) for j in self._jobs_awaiting_bandwidth ) )
			jobs.extend( ( ( JOB_STATUS_AWAITING_LOGIN, j ) for j in self._jobs_awaiting_login ) )
			jobs.extend( ( ( JOB_STATUS_AWAITING_SLOT, j ) for j in self._jobs_awaiting_slot ) )
			jobs.extend( ( ( JOB_STATUS_RUNNING, j ) for j in self._jobs_running ) )

			return jobs

	def IsBusy( self ):
		with self._lock:
			return len( self._jobs_awaiting_validity ) + len( self._jobs_awaiting_bandwidth ) + len( self._jobs_awaiting_login ) + len( self._jobs_awaiting_slot ) + len( self._jobs_running ) > 50

	def IsRunning( self ):
		with self._lock:
			return self._is_running

	def IsShutdown( self ):
		with self._lock:
			return self._is_shutdown

	def MainLoop( self ):
		def ProcessValidationJob( job ):
			if job.IsDone():
				return False

			elif job.IsAsleep():
				return True

			elif not job.IsValid():
				if job.CanValidateInPopup():
					if self._current_validation_process is None:
						validation_process = job.GenerateValidationPopupProcess()

						self.controller.CallToThread( validation_process.Start )

						self._current_validation_process = validation_process

						job.SetStatus( 'validation presented to user\u2026' )

					else:
						job.SetStatus( 'waiting in user validation queue\u2026' )

						job.Sleep( 5 )

					return True

				else:
					error_text = 'network context not currently valid!'

					job.SetError( ValidationException( error_text ), error_text )

					return False

			else:
				self._jobs_awaiting_bandwidth.append( job )

				return False

		def ProcessCurrentValidationJob():
			if self._current_validation_process is not None:
				if self._current_validation_process.IsDone():
					self._current_validation_process = None

		def ProcessBandwidthJob( job ):
			if job.IsDone():
				return False

			elif job.IsAsleep():
				return True

			elif not job.BandwidthOK():
				return True

			else:
				self._jobs_awaiting_login.append( job )

				return False

		def ProcessForceLogins():
			if len( self._domains_to_login ) > 0 and self._current_login_process is None:
				try:
					login_domain = self._domains_to_login.pop( 0 )

					login_process = self.login_manager.GenerateLoginProcessForDomain( login_domain )

				except Exception as e:
					ShowException( e )

					return

				self.controller.CallToThread( login_process.Start )

				self._current_login_process = login_process

		def ProcessLoginJob( job ):
			if job.IsDone():
				return False

			elif job.IsAsleep():
				return True

			elif job.NeedsLogin():
				try:
					job.CheckCanLogin()

				except Exception as e:
					if job.WillingToWaitOnInvalidLogin():
						job.SetStatus( str( e ) )

						job.Sleep( 60 )

						return True

					else:
						if job.IsHydrusJob():
							message = 'This hydrus service (' + job.GetLoginNetworkContext().ToString() + ') recently failed to log in. Please hit its \'refresh account\' under \'review services\' and try again.'

						else:
							message = 'This job\'s network context (' + job.GetLoginNetworkContext().ToString() + ') seems to have an invalid login, and it is not willing to wait! Please review its login details and then try again.'

						job.Cancel( message )

						return False

				if self._current_login_process is None:
					try:
						login_process = job.GenerateLoginProcess()

					except Exception as e:
						ShowException( e )

						job.SetStatus( str( e ) )

						job.Sleep( 60 )

						return True

					self.controller.CallToThread( login_process.Start )

					self._current_login_process = login_process

					job.SetStatus( 'logging in\u2026' )

				else:
					job.SetStatus( 'waiting in login queue\u2026' )

					job.Sleep( 5 )

				return True

			else:
				self._jobs_awaiting_slot.append( job )

				return False

		def ProcessCurrentLoginJob():
			if self._current_login_process is not None:
				if self._current_login_process.IsDone():
					self._current_login_process = None

		def ProcessReadyJob( job ):
			if job.IsDone():
				return False

			elif job.IsAsleep():
				return True

			elif len( self._jobs_running ) < self.MAX_JOBS:
				if self._pause_all_new_network_traffic:
					job.SetStatus( 'all new network traffic is paused\u2026' )

					job.Sleep( 2 )

					return True

				elif self.controller.JustWokeFromSleep():
					job.SetStatus( 'looks like computer just woke up, waiting a bit' )

					job.Sleep( 5 )

					return True

				elif self._active_domains_counter[ job.GetSecondLevelDomain() ] >= self.MAX_JOBS_PER_DOMAIN:
					job.SetStatus( 'waiting for a slot on this domain' )

					job.Sleep( 2 )

					return True

				elif not job.TokensOK():
					return True

				else:
					if HG.network_report_mode:
						ShowText( 'Network Job Starting: ' + job._method + ' ' + job._url )

					self._active_domains_counter[ job.GetSecondLevelDomain() ] += 1

					self.controller.CallToThread( job.Start )

					self._jobs_running.append( job )

					return False

			else:
				job.SetStatus( 'waiting for a slot\u2026' )

				return True

		def ProcessRunningJob( job ):
			if job.IsDone():
				if HG.network_report_mode:
					ShowText( 'Network Job Done: ' + job._method + ' ' + job._url )

				second_level_domain = job.GetSecondLevelDomain()

				self._active_domains_counter[ second_level_domain ] -= 1

				if self._active_domains_counter[ second_level_domain ] == 0:
					del self._active_domains_counter[ second_level_domain ]

				return False

			else:
				return True

		self._is_running = True

		while not ( self._local_shutdown or self.controller.ModelIsShutdown() ):
			with self._lock:
				self._jobs_awaiting_validity = list( filter( ProcessValidationJob, self._jobs_awaiting_validity ) )

				ProcessCurrentValidationJob()

				self._jobs_awaiting_bandwidth = list( filter( ProcessBandwidthJob, self._jobs_awaiting_bandwidth ) )

				ProcessForceLogins()

				self._jobs_awaiting_login = list( filter( ProcessLoginJob, self._jobs_awaiting_login ) )

				ProcessCurrentLoginJob()

				self._jobs_awaiting_slot = list( filter( ProcessReadyJob, self._jobs_awaiting_slot ) )

				self._jobs_running = list( filter( ProcessRunningJob, self._jobs_running ) )

			# we want to catch the rollover of the second for bandwidth jobs

			now_with_subsecond = time.time()
			subsecond_part = now_with_subsecond % 1

			time_until_next_second = 1.0 - subsecond_part

			self._new_work_to_do.wait( time_until_next_second )

			self._new_work_to_do.clear()

		self._is_running = False

		self._is_shutdown = True

	def PausePlayNewJobs( self ):
		self._pause_all_new_network_traffic = not self._pause_all_new_network_traffic

		self.controller.new_options.SetBoolean( 'pause_all_new_network_traffic', self._pause_all_new_network_traffic )

	def RefreshOptions( self ):
		with self._lock:
			self.MAX_JOBS = self.controller.new_options.GetInteger( 'max_network_jobs' )
			self.MAX_JOBS_PER_DOMAIN = self.controller.new_options.GetInteger( 'max_network_jobs_per_domain' )

	def Shutdown( self ):
		self._local_shutdown = True

		self._new_work_to_do.set()


code:
import itertools, os, re, threading, time, urllib.parse

from lib.Networking.Contexts import NetworkContext
from lib.Networking.Domain import ConvertDomainIntoAllApplicableDomains, ConvertQueryDictToText, GetCookie
from lib.Networking.Jobs import NetworkJob, NetworkJobHydrus

from lib.Handling.Parsing import GetVariableFromParseResults, \
	ParseFormulaHTML, ParseRuleHTML, StringMatch

from lib.Hydrus.Data import ConvertTimestampToPrettyExpires, GenerateKey, GetNow, Print, ShowException, \
	ShowText, TimeHasPassed
from lib.Hydrus.Defaults import GetDefaultLoginScripts
from lib.Hydrus.Exceptions import DataMissing, NetworkException, ParseException, \
	StringMatchException, ValidationException, VetoException
from lib.Hydrus.Serialisable import CreateFromSerialisableTuple, SerialisableBase, \
	SerialisableBaseNamed, SerialisableDictionary, SerialisableList
from lib.Hydrus.Threading import JobKey

### CONSTANTS ###
import lib.Hydrus.Constants.Local as CC
import lib.Hydrus.Globals as HG

from lib.Handling.Parsing import HTML_CONTENT_ATTRIBUTE, HTML_RULE_TYPE_DESCENDING, \
	STRING_MATCH_FIXED

from lib.Hydrus.Serialisable import HG.SerialisableType["LOGIN_CREDENTIAL_DEFINITION"], \
	HG.SerialisableType["LOGIN_SCRIPT_DOMAIN"], HG.SerialisableType["LOGIN_STEP"], HG.SerialisableType["NETWORK_LOGIN_MANAGER"], \
	SERIALISABLE_TYPES_TO_OBJECT_TYPES

VALIDITY_VALID = 0
VALIDITY_UNTESTED = 1
VALIDITY_INVALID = 2

validity_str_lookup = {}

validity_str_lookup[ VALIDITY_VALID ] = 'valid'
validity_str_lookup[ VALIDITY_UNTESTED ] = 'untested'
validity_str_lookup[ VALIDITY_INVALID ] = 'invalid'

LOGIN_ACCESS_TYPE_EVERYTHING = 0
LOGIN_ACCESS_TYPE_NSFW = 1
LOGIN_ACCESS_TYPE_SPECIAL = 2
LOGIN_ACCESS_TYPE_USER_PREFS_ONLY = 3

login_access_type_str_lookup = {}

login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_EVERYTHING ] = 'Everything'
login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_NSFW ] = 'NSFW'
login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_SPECIAL ] = 'Special'
login_access_type_str_lookup[ LOGIN_ACCESS_TYPE_USER_PREFS_ONLY ] = 'User prefs'

login_access_type_default_description_lookup = {}

login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_EVERYTHING ] = 'Login required to access any content.'
login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_NSFW ] = 'Login required to access NSFW content.'
login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_SPECIAL ] = 'Login required to access special content.'
login_access_type_default_description_lookup[ LOGIN_ACCESS_TYPE_USER_PREFS_ONLY ] = 'Login only required to access user preferences.'

PIXIV_NETWORK_CONTEXT = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'pixiv.net' )
HENTAI_FOUNDRY_NETWORK_CONTEXT = NetworkContext( CC.NETWORK_CONTEXT_DOMAIN, 'hentai-foundry.com' )

### CLASSES ###

class NetworkLoginManager( SerialisableBase ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["NETWORK_LOGIN_MANAGER"]
	SERIALISABLE_NAME = 'Login Manager'
	SERIALISABLE_VERSION = 1

	SESSION_TIMEOUT = 60 * 45

	def __init__( self ):
		SerialisableBase.__init__( self )

		# needs _dirty and setdirty and be on that serialisation check and so on

		self.engine = None

		self._dirty = False

		self._lock = threading.Lock()

		self._login_scripts = SerialisableList()
		self._domains_to_login_info = {}

		self._login_script_keys_to_login_scripts = {}
		self._login_script_names_to_login_scripts = {}

		self._hydrus_login_script = LoginScriptHydrus()

		self._error_names = set()

	def _GetBestLoginScript( self, login_domain ):
		self._login_scripts.sort( key = lambda ls: len( ls.GetCredentialDefinitions() ) )

		for login_script in self._login_scripts:
			if login_domain in login_script.GetExampleDomains():
				return login_script

		return None

	def _GetLoginDomainStatus( self, network_context ):
		login_domain = None
		login_expected = False
		login_possible = True
		login_error_text = ''

		domain = network_context.context_data

		potential_login_domains = ConvertDomainIntoAllApplicableDomains( domain, discard_www = False )

		for potential_login_domain in potential_login_domains:
			if potential_login_domain in self._domains_to_login_info:
				login_domain = potential_login_domain

				( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

				if active or login_access_type == LOGIN_ACCESS_TYPE_EVERYTHING:
					login_expected = True

				if not active:
					login_possible = False
					login_error_text = 'Not active - ' + login_access_text

				elif validity == VALIDITY_INVALID:
					login_possible = False
					login_error_text = validity_error_text

				elif not TimeHasPassed( no_work_until ):
					login_possible = False
					login_error_text = no_work_until_reason

				break

		return ( login_domain, login_expected, login_possible, login_error_text )

	def _GetLoginScriptAndCredentials( self, login_domain ):
		if login_domain in self._domains_to_login_info:
			( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

			( login_script_key, login_script_name ) = login_script_key_and_name

			if login_script_key in self._login_script_keys_to_login_scripts:
				login_script = self._login_script_keys_to_login_scripts[ login_script_key ]

			elif login_script_name in self._login_script_names_to_login_scripts:
				login_script = self._login_script_names_to_login_scripts[ login_script_name ]

				login_script_key_and_name = login_script.GetLoginScriptKeyAndName()

				self._SetDirty()

				self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			else:
				validity = VALIDITY_INVALID
				validity_error_text = 'Could not find the login script for "' + login_domain + '"!'

				self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

				self._SetDirty()

				raise ValidationException( validity_error_text )

			try:
				login_script.CheckCanLogin( credentials )

			except ValidationException as e:
				validity = VALIDITY_INVALID
				validity_error_text = str( e )

				self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

				self._SetDirty()

				raise

			if validity == VALIDITY_UNTESTED and validity_error_text != '':
				# cleaning up the 'restart dialog to test validity in cases where it is valid

				validity_error_text = ''

				self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			return ( login_script, credentials )

		else:
			raise ValidationException( 'Could not find any login entry for "' + login_domain + '"!' )

	def _GetSerialisableInfo( self ):
		serialisable_login_scripts = self._login_scripts.GetSerialisableTuple()

		serialisable_domains_to_login_info = {}

		for ( login_domain, ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) ) in list( self._domains_to_login_info.items() ):
			( login_script_key, login_script_name ) = login_script_key_and_name

			serialisable_login_script_key_and_name = ( login_script_key.hex(), login_script_name )

			serialisable_domains_to_login_info[ login_domain ] = ( serialisable_login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

		return ( serialisable_login_scripts, serialisable_domains_to_login_info )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_login_scripts, serialisable_domains_to_login_info ) = serialisable_info

		self._login_scripts = CreateFromSerialisableTuple( serialisable_login_scripts )

		self._domains_to_login_info = {}

		for ( login_domain, ( serialisable_login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) ) in list( serialisable_domains_to_login_info.items() ):
			( serialisable_login_script_key, login_script_name ) = serialisable_login_script_key_and_name

			login_script_key_and_name = ( bytes.fromhex( serialisable_login_script_key ), login_script_name )

			self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

	def _RecalcCache( self ):
		self._login_script_keys_to_login_scripts = { login_script.GetLoginScriptKey() : login_script for login_script in self._login_scripts }
		self._login_script_names_to_login_scripts = { login_script.GetName() : login_script for login_script in self._login_scripts }

		self._RevalidateCache()

	def _RevalidateCache( self ):
		for login_domain in list( self._domains_to_login_info.keys() ):
			try:
				self._GetLoginScriptAndCredentials( login_domain )

			except ValidationException:
				pass

	def _SetDirty( self ):
		self._dirty = True

	def AlreadyHaveExactlyThisLoginScript( self, new_login_script ):
		with self._lock:
			# absent irrelevant variables, do we have the exact same object already in?

			login_script_key_and_name = new_login_script.GetLoginScriptKeyAndName()

			dupe_login_scripts = [ login_script.Duplicate() for login_script in self._login_scripts ]

			for dupe_login_script in dupe_login_scripts:
				dupe_login_script.SetLoginScriptKeyAndName( login_script_key_and_name )

				if dupe_login_script.DumpToString() == new_login_script.DumpToString():
					return True

		return False

	def AutoAddLoginScripts( self, login_scripts ):
		with self._lock:
			next_login_scripts = list( self._login_scripts )

			for login_script in login_scripts:
				login_script.RegenerateLoginScriptKey()

			next_login_scripts.extend( login_scripts )

		self.SetLoginScripts( next_login_scripts )

	def CheckCanLogin( self, network_context ):
		with self._lock:
			if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
				( login_domain, login_expected, login_possible, login_error_text ) = self._GetLoginDomainStatus( network_context )

				if login_domain is None or not login_expected:
					raise ValidationException( 'The domain ' + login_domain + ' has no active login script--has it just been turned off?' )

				elif not login_possible:
					raise ValidationException( 'The domain ' + login_domain + ' cannot log in: ' + login_error_text )

			elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
				service_key = network_context.context_data

				services_manager = self.engine.controller.services_manager

				if not services_manager.ServiceExists( service_key ):
					raise ValidationException( 'Service does not exist!' )

				service = services_manager.GetService( service_key )

				try:
					service.CheckFunctional( including_bandwidth = False, including_account = False )

				except Exception as e:
					message = 'Service has had a recent error or is otherwise not functional! Specific error was:'
					message += os.linesep * 2
					message += str( e )
					message += os.linesep * 2
					message += 'You might like to try refreshing its account in \'review services\'.'

					raise ValidationException( message )

	def DelayLoginScript( self, login_domain, login_script_key, reason ):
		with self._lock:
			if login_domain not in self._domains_to_login_info:
				return

			( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

			if login_script_key != login_script_key_and_name[0]:
				return

			no_work_until = GetNow() + 3600 * 4
			no_work_until_reason = reason

			self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			self._SetDirty()

	def DeleteLoginDomain( self, login_domain ):
		with self._lock:
			if login_domain in self._domains_to_login_info:
				del self._domains_to_login_info[ login_domain ]

				self._RecalcCache()

				self._SetDirty()

	def DeleteLoginScripts( self, login_script_names ):
		with self._lock:
			login_scripts = [ login_script for login_script in self._login_scripts if login_script.GetName() not in login_script_names ]

		self.SetLoginScripts( login_scripts )

	def GenerateLoginProcess( self, network_context ):
		with self._lock:
			if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
				( login_domain, login_expected, login_possible, login_error_text ) = self._GetLoginDomainStatus( network_context )

				if login_domain is None or not login_expected:
					raise ValidationException( 'The domain ' + login_domain + ' has no active login script--has it just been turned off?' )

				elif not login_possible:
					raise ValidationException( 'The domain ' + login_domain + ' cannot log in: ' + login_error_text )

				else:
					login_network_context = NetworkContext( context_type = CC.NETWORK_CONTEXT_DOMAIN, context_data = login_domain )

					( login_script, credentials ) = self._GetLoginScriptAndCredentials( login_domain )

					login_process = LoginProcessDomain( self.engine, login_network_context, login_script, credentials )

					return login_process

			elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
				login_process = LoginProcessHydrus( self.engine, network_context, self._hydrus_login_script )

				return login_process

	def GenerateLoginProcessForDomain( self, login_domain ):
		network_context = NetworkContext.STATICGenerateForDomain( login_domain )

		return self.GenerateLoginProcess( network_context )

	def GetDomainsToLoginInfo( self ):
		with self._lock:
			self._RevalidateCache()

			return dict( self._domains_to_login_info )

	def GetLoginScripts( self ):
		with self._lock:
			return list( self._login_scripts )

	def Initialise( self ):
		self._RecalcCache()

	def InvalidateLoginScript( self, login_domain, login_script_key, reason ):
		with self._lock:
			if login_domain not in self._domains_to_login_info:
				return

			( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

			if login_script_key != login_script_key_and_name[0]:
				return

			validity = VALIDITY_INVALID
			validity_error_text = reason

			self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			ShowText( 'The login for "' + login_domain + '" failed! It will not be reattempted until the problem is fixed. The failure reason was:' + os.linesep * 2 + validity_error_text )

			self._SetDirty()

	def IsDirty( self ):
		with self._lock:
			return self._dirty

	def NeedsLogin( self, network_context ):
		with self._lock:
			if network_context.context_type == CC.NETWORK_CONTEXT_DOMAIN:
				( login_domain, login_expected, login_possible, login_error_text ) = self._GetLoginDomainStatus( network_context )

				if login_domain is None or not login_expected:
					return False # no login required, no problem

				else:
					try:
						( login_script, credentials ) = self._GetLoginScriptAndCredentials( login_domain )

					except ValidationException:
						# couldn't find the script or something. assume we need a login to move errors forward to checkcanlogin trigger phase

						return True

					login_network_context = NetworkContext( context_type = CC.NETWORK_CONTEXT_DOMAIN, context_data = login_domain )

					return not login_script.IsLoggedIn( self.engine, login_network_context )

			elif network_context.context_type == CC.NETWORK_CONTEXT_HYDRUS:
				return not self._hydrus_login_script.IsLoggedIn( self.engine, network_context )

	def OverwriteDefaultLoginScripts( self, login_script_names ):
		with self._lock:
			default_login_scripts = GetDefaultLoginScripts()

			for login_script in default_login_scripts:
				login_script.RegenerateLoginScriptKey()

			existing_login_scripts = list( self._login_scripts )

			new_login_scripts = [ login_script for login_script in existing_login_scripts if login_script.GetName() not in login_script_names ]
			new_login_scripts.extend( [ login_script for login_script in default_login_scripts if login_script.GetName() in login_script_names ] )

		self.SetLoginScripts( new_login_scripts )

	def SetClean( self ):
		with self._lock:
			self._dirty = False

	def SetCredentialsAndActivate( self, login_domain, new_credentials ):
		with self._lock:
			if login_domain not in self._domains_to_login_info:
				return

			( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

			credentials = new_credentials
			active = True

			validity = VALIDITY_UNTESTED
			validity_error_text = ''

			self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			self._SetDirty()

	def SetDomainsToLoginInfo( self, domains_to_login_info ):
		with self._lock:
			self._domains_to_login_info = dict( domains_to_login_info )

			self._RecalcCache()

			self._SetDirty()

	def SetLoginScripts( self, login_scripts ):
		with self._lock:
			self._login_scripts = SerialisableList( login_scripts )

			# start with simple stuff first
			self._login_scripts.sort( key = lambda ls: len( ls.GetCredentialDefinitions() ) )

			for login_script in self._login_scripts:
				login_script_key_and_name = login_script.GetLoginScriptKeyAndName()

				example_domains_info = login_script.GetExampleDomainsInfo()

				for ( login_domain, login_access_type, login_access_text ) in example_domains_info:
					if '.' in login_domain:
						# looks good, so let's see if we can update/add some info

						if login_domain in self._domains_to_login_info:
							( old_login_script_key_and_name, credentials, old_login_access_type, old_login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

							if old_login_script_key_and_name[1] == login_script_key_and_name[1]:
								self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

						else:
							credentials = {}

							# if there is nothing to enter, turn it on by default, like HF click-through
							active = len( login_script.GetCredentialDefinitions() ) == 0

							validity = VALIDITY_UNTESTED
							validity_error_text = ''

							no_work_until = 0
							no_work_until_reason = ''

							self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			self._RecalcCache()

			self._SetDirty()

	def ValidateLoginScript( self, login_domain, login_script_key ):
		with self._lock:
			if login_domain not in self._domains_to_login_info:
				return

			( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

			if login_script_key != login_script_key_and_name[0]:
				return

			validity = VALIDITY_VALID
			validity_error_text = ''

			self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			self._SetDirty()

	def TryToLinkMissingLoginScripts( self, login_domains ):
		with self._lock:
			for login_domain in login_domains:
				try:
					( existing_login_script, existing_credentials ) = self._GetLoginScriptAndCredentials( login_domain )

					continue # already seems to have a good login script, so nothing to fix

				except ValidationException:
					pass

				( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason ) = self._domains_to_login_info[ login_domain ]

				login_script = self._GetBestLoginScript( login_domain )

				if login_script is None:
					continue

				login_script_key_and_name = login_script.GetLoginScriptKeyAndName()

				self._domains_to_login_info[ login_domain ] = ( login_script_key_and_name, credentials, login_access_type, login_access_text, active, validity, validity_error_text, no_work_until, no_work_until_reason )

			self._SetDirty()

	def LoginTumblrGDPR( self ):
		# t-thanks, EU
		# this is cribbed from poking around here https://github.com/johanneszab/TumblThree/commit/3563d6cebf1a467151d6b8d6eee9806ddd6e6364

		network_job = NetworkJob( 'GET', 'http://www.tumblr.com/' )

		network_job.SetForLogin( True )

		self.engine.AddJob( network_job )

		network_job.WaitUntilDone()

		html = network_job.GetContentText()

		formula = ParseFormulaHTML( tag_rules = [ ParseRuleHTML( rule_type = HTML_RULE_TYPE_DESCENDING, tag_name = 'meta', tag_attributes = { 'id' :
			'tumblr_form_key' } ) ], content_to_fetch = HTML_CONTENT_ATTRIBUTE, attribute_to_fetch = "content" )

		results = formula.Parse( {}, html )

		if len( results ) != 1:
			raise ParseException( 'Could not figure out the tumblr form key for the GDPR click-through.' )

		tumblr_form_key = results[0]

		#

		body = '{\"eu_resident\":true,\"gdpr_is_acceptable_age\":true,\"gdpr_consent_core\":true,\"gdpr_consent_first_party_ads\":true,\"gdpr_consent_third_party_ads\":true,\"gdpr_consent_search_history\":true,\"redirect_to\":\"\"}'
		referral_url = 'https://www.tumblr.com/privacy/consent?redirect='

		network_job = NetworkJob( 'POST', 'https://www.tumblr.com/svc/privacy/consent', body = body, referral_url = referral_url )

		network_job.SetForLogin( True )

		network_job.AddAdditionalHeader( 'Accept', 'application/json, text/javascript, */*; q=0.01')
		network_job.AddAdditionalHeader( 'Content-Type', 'application/json' )
		network_job.AddAdditionalHeader( 'X-Requested-With', 'XMLHttpRequest' )
		network_job.AddAdditionalHeader( 'X-tumblr-form-key', tumblr_form_key )

		self.engine.AddJob( network_job )

		network_job.WaitUntilDone()

		# test cookies here or something

		ShowText( 'Looks like tumblr GDPR click-through worked! You should be good for a year, at which point we should have an automatic solution for this!' )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["NETWORK_LOGIN_MANAGER"] ] = NetworkLoginManager

CREDENTIAL_TYPE_TEXT = 0
CREDENTIAL_TYPE_PASS = 1

credential_type_str_lookup = {}

credential_type_str_lookup[ CREDENTIAL_TYPE_TEXT ] = 'normal'
credential_type_str_lookup[ CREDENTIAL_TYPE_PASS ] = 'hidden (password)'

class LoginCredentialDefinition( SerialisableBaseNamed ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["LOGIN_CREDENTIAL_DEFINITION"]
	SERIALISABLE_NAME = 'Login Credential Definition'
	SERIALISABLE_VERSION = 1

	def __init__( self, name = 'username', credential_type = CREDENTIAL_TYPE_TEXT, string_match = None ):
		if string_match is None:
			string_match = StringMatch()

		SerialisableBaseNamed.__init__( self, name )

		self._credential_type = credential_type
		self._string_match = string_match

	def _GetSerialisableInfo( self ):
		serialisable_string_match = self._string_match.GetSerialisableTuple()

		return ( self._credential_type, serialisable_string_match )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._credential_type, serialisable_string_match ) = serialisable_info

		self._string_match = CreateFromSerialisableTuple( serialisable_string_match )

	def GetStringMatch( self ):
		return self._string_match

	def GetType( self ):
		return self._credential_type

	def SetStringMatch( self, string_match ):
		self._string_match = string_match

	def SetType( self, credential_type ):
		self._credential_type = credential_type

	def ShouldHide( self ):
		return self._credential_type == CREDENTIAL_TYPE_PASS

	def Test( self, text ):
		if self._string_match is not None:
			try:
				self._string_match.Test( text )

			except StringMatchException as e:
				raise ValidationException( 'Could not validate "' + self._name + '" credential: ' + str( e ) )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["LOGIN_CREDENTIAL_DEFINITION"] ] = LoginCredentialDefinition

class LoginProcess( object ):
	def __init__( self, engine, network_context, login_script ):
		self.engine = engine
		self.network_context = network_context
		self.login_script = login_script

		self._done = False

	def _Start( self ):
		raise NotImplementedError()

	def IsDone( self ):
		return self._done

	def Start( self ):
		try:
			self._Start()

		finally:
			self._done = True

class LoginProcessDomain( LoginProcess ):
	def __init__( self, engine, network_context, login_script, credentials ):
		LoginProcess.__init__( self, engine, network_context, login_script )

		self.credentials = credentials

	def _Start( self ):
		login_domain = self.network_context.context_data

		job_key = JobKey( cancellable = True )

		job_key.SetVariable( 'popup_title', 'Logging in ' + login_domain )

		HG.client_controller.pub( 'message', job_key )

		Print( 'Starting login for ' + login_domain )

		result = self.login_script.Start( self.engine, self.network_context, self.credentials, job_key = job_key )

		Print( 'Finished login for ' + self.network_context.context_data + '. Result was:
 	' + result )

		job_key.SetVariable( 'popup_text_1', result )

		job_key.Finish()

		job_key.Delete( 4 )

class LoginProcessHydrus( LoginProcess ):
	def _Start( self ):
		self.login_script.Start( self.engine, self.network_context )

class LoginScriptHydrus( object ):
	def _IsLoggedIn( self, engine, network_context ):
		session = engine.session_manager.GetSession( network_context )

		cookies = session.cookies

		cookies.clear_expired_cookies()

		return 'session_key' in cookies

	def IsLoggedIn( self, engine, network_context ):
		return self._IsLoggedIn( engine, network_context )

	def Start( self, engine, network_context ):
		service_key = network_context.context_data

		service = engine.controller.services_manager.GetService( service_key )

		base_url = service.GetBaseURL()

		url = base_url + 'session_key'

		access_key = service.GetCredentials().GetAccessKey()

		network_job = NetworkJobHydrus( service_key, 'GET', url )

		network_job.SetForLogin( True )

		network_job.AddAdditionalHeader( 'Hydrus-Key', access_key.hex() )

		engine.AddJob( network_job )

		try:
			network_job.WaitUntilDone()

			if self._IsLoggedIn( engine, network_context ):
				Print( 'Successfully logged into ' + service.GetName() + '.' )

			else:
				service.DelayFutureRequests( 'Could not log in for unknown reason.' )

		except Exception as e:
			e_string = str( e )

			service.DelayFutureRequests( e_string )

class LoginScriptDomain( SerialisableBaseNamed ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["LOGIN_SCRIPT_DOMAIN"]
	SERIALISABLE_NAME = 'Login Script - Domain'
	SERIALISABLE_VERSION = 2

	def __init__( self, name = 'login script', login_script_key = None, required_cookies_info = None, credential_definitions = None, login_steps = None, example_domains_info = None ):
		if required_cookies_info is None:
			required_cookies_info = {}

		required_cookies_info = SerialisableDictionary( required_cookies_info )

		if credential_definitions is None:
			credential_definitions = []

		credential_definitions = SerialisableList( credential_definitions )

		if login_steps is None:
			login_steps = []

		login_steps = SerialisableList( login_steps )

		if example_domains_info is None:
			example_domains_info = []

		SerialisableBaseNamed.__init__( self, name )

		self._login_script_key = GenerateKey()
		self._required_cookies_info = required_cookies_info # string match : string match
		self._credential_definitions = credential_definitions
		self._login_steps = login_steps
		self._example_domains_info = example_domains_info # domain | login_access_type | login_access_text

	def _GetSerialisableInfo( self ):
		serialisable_login_script_key = self._login_script_key.hex()
		serialisable_required_cookies = self._required_cookies_info.GetSerialisableTuple()
		serialisable_credential_definitions = self._credential_definitions.GetSerialisableTuple()
		serialisable_login_steps = self._login_steps.GetSerialisableTuple()

		return ( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, self._example_domains_info )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, self._example_domains_info ) = serialisable_info

		self._login_script_key = bytes.fromhex( serialisable_login_script_key )
		self._required_cookies_info = CreateFromSerialisableTuple( serialisable_required_cookies )
		self._credential_definitions = CreateFromSerialisableTuple( serialisable_credential_definitions )
		self._login_steps = CreateFromSerialisableTuple( serialisable_login_steps )

		# convert lists to tups for listctrl data hashing
		self._example_domains_info = [ tuple( l ) for l in self._example_domains_info ]

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, example_domains_info ) = old_serialisable_info

			old_required_cookies_info = CreateFromSerialisableTuple( serialisable_required_cookies )
			new_required_cookies_info = SerialisableDictionary()

			for ( name, value_string_match ) in list( old_required_cookies_info.items() ):
				key_string_match = StringMatch( match_type = STRING_MATCH_FIXED, match_value = name, example_string = name )

				new_required_cookies_info[ key_string_match ] = value_string_match

			serialisable_required_cookies = new_required_cookies_info.GetSerialisableTuple()

			new_serialisable_info = ( serialisable_login_script_key, serialisable_required_cookies, serialisable_credential_definitions, serialisable_login_steps, example_domains_info )

			return ( 2, new_serialisable_info )

	def _IsLoggedIn( self, engine, network_context, validation_check = False ):
		session = engine.session_manager.GetSession( network_context )

		cookies = session.cookies

		cookies.clear_expired_cookies()

		search_domain = network_context.context_data

		for ( cookie_name_string_match, value_string_match ) in list( self._required_cookies_info.items() ):
			try:
				cookie = GetCookie( cookies, search_domain, cookie_name_string_match )

			except DataMissing:
				if validation_check:
					raise ValidationException( 'Missing cookie "' + cookie_name_string_match.ToString() + '"!' )

				return False

			cookie_text = cookie.value

			try:
				value_string_match.Test( cookie_text )

			except StringMatchException as e:
				if validation_check:
					raise ValidationException( 'Cookie "' + cookie_name_string_match.ToString() + '" failed: ' + str( e ) + '!' )

				return False

		return True

	def CheckCanLogin( self, given_credentials ):
		self.CheckIsValid()

		given_cred_names = set( given_credentials.keys() )
		required_cred_names = { name for name in itertools.chain.from_iterable( ( step.GetRequiredCredentials() for step in self._login_steps ) ) }

		missing_givens = required_cred_names.difference( given_cred_names )

		if len( missing_givens ) > 0:
			missing_givens = list( missing_givens )
			missing_givens.sort()

			raise ValidationException( 'Missing required credentials: ' + ', '.join( missing_givens ) )

		#

		cred_names_to_definitions = { credential_definition.GetName() : credential_definition for credential_definition in self._credential_definitions }

		for ( pretty_name, text ) in given_credentials.items():
			if pretty_name not in cred_names_to_definitions:
				continue

			credential_definition = cred_names_to_definitions[ pretty_name ]

			credential_definition.Test( text )

	def CheckIsValid( self ):
		defined_cred_names = { credential_definition.GetName() for credential_definition in self._credential_definitions }
		required_cred_names = { name for name in itertools.chain.from_iterable( ( step.GetRequiredCredentials() for step in self._login_steps ) ) }

		missing_definitions = required_cred_names.difference( defined_cred_names )

		if len( missing_definitions ) > 0:
			missing_definitions = list( missing_definitions )
			missing_definitions.sort()

			raise ValidationException( 'Missing required credential definitions: ' + ', '.join( missing_definitions ) )

		#

		temp_vars = set()

		for login_step in self._login_steps:
			( required_vars, set_vars ) = login_step.GetRequiredAndSetTempVariables()

			missing_vars = required_vars.difference( temp_vars )

			if len( missing_vars ) > 0:
				missing_vars = list( missing_vars )
				missing_vars.sort()

				raise ValidationException( 'Missing temp variables for login step "' + login_step.GetName() + '":
 	' + ', '.join( missing_vars ) )

			temp_vars.update( set_vars )

	def GetCredentialDefinitions( self ):
		return self._credential_definitions

	def GetExampleDomains( self ):
		return [ domain for ( domain, login_access_type, login_access_text ) in self._example_domains_info ]

	def GetExampleDomainsInfo( self ):
		return self._example_domains_info

	def GetExampleDomainInfo( self, given_domain ):
		for ( domain, login_access_type, login_access_text ) in self._example_domains_info:
			if domain == given_domain:
				return ( login_access_type, login_access_text )

		raise DataMissing( 'Could not find that domain!' )

	def GetRequiredCookiesInfo( self ):
		return self._required_cookies_info

	def GetLoginExpiry( self, engine, network_context ):
		session = engine.session_manager.GetSession( network_context )

		cookies = session.cookies

		cookies.clear_expired_cookies()

		search_domain = network_context.context_data

		session_cookies = False
		expiry_timestamps = []

		for cookie_name_string_match in list( self._required_cookies_info.keys() ):
			try:
				cookie = GetCookie( cookies, search_domain, cookie_name_string_match )

			except DataMissing:
				return None

			expiry = cookie.expires

			if expiry is None:
				session_cookies = True

			else:
				expiry_timestamps.append( expiry )

		if session_cookies:
			return None

		else:
			return min( expiry_timestamps )

	def GetLoginScriptKey( self ):
		return self._login_script_key

	def GetLoginScriptKeyAndName( self ):
		return ( self._login_script_key, self._name )

	def GetLoginSteps( self ):
		return self._login_steps

	def GetRequiredCredentials( self ):
		required_creds = []

		for login_step in self._login_steps:
			required_creds.extend( login_step.GetRequiredCredentials() ) # name with an order

		return required_creds

	def GetSafeSummary( self ):
		return 'Login Script "' + self._name + '" - ' + ', '.join( self.GetExampleDomains() )

	def IsLoggedIn( self, engine, network_context ):
		return self._IsLoggedIn( engine, network_context )

	def RegenerateLoginScriptKey( self ):
		self._login_script_key = GenerateKey()

	def SetLoginScriptKey( self, login_script_key ):
		self._login_script_key = login_script_key

	def SetLoginScriptKeyAndName( self, login_script_key_and_name ):
		( login_script_key, name ) = login_script_key_and_name

		self._login_script_key = login_script_key
		self._name = name

	def Start( self, engine, network_context, given_credentials, network_job_presentation_context_factory = None, test_result_callable = None, job_key = None ):
		# don't mess with the domain--assume that we are given precisely the right domain

		login_domain = network_context.context_data

		temp_variables = {}

		last_url_used = None

		for login_step in self._login_steps:
			if job_key is not None:
				if job_key.IsCancelled():
					message = 'User cancelled the login process.'

					engine.login_manager.DelayLoginScript( login_domain, self._login_script_key, message )

					return message

				job_key.SetVariable( 'popup_text_1', login_step.GetName() )

			try:
				last_url_used = login_step.Start( engine, login_domain, given_credentials, temp_variables, referral_url = last_url_used, network_job_presentation_context_factory = network_job_presentation_context_factory, test_result_callable = test_result_callable )

			except ValidationException as e:
				if test_result_callable is not None:
					ShowException( e )

				message = str( e )

				engine.login_manager.InvalidateLoginScript( login_domain, self._login_script_key, message )

				return 'Verification error: ' + message

			except NetworkException as e:
				if test_result_callable is not None:
					ShowException( e )

				message = str( e )

				engine.login_manager.DelayLoginScript( login_domain, self._login_script_key, message )

				return 'Network error: ' + message

			except Exception as e:
				if test_result_callable is not None:
					ShowException( e )

				message = str( e )

				engine.login_manager.InvalidateLoginScript( login_domain, self._login_script_key, message )

				return 'Unusual error: ' + message

			time.sleep( 2 )

		try:
			self._IsLoggedIn( engine, network_context, validation_check = True )

		except Exception as e:
			if test_result_callable is not None:
				ShowException( e )

			message = str( e )

			engine.login_manager.InvalidateLoginScript( login_domain, self._login_script_key, message )

			return 'Final cookie check failed: ' + message

		engine.login_manager.ValidateLoginScript( login_domain, self._login_script_key )

		return 'Login OK!'

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["LOGIN_SCRIPT_DOMAIN"] ] = LoginScriptDomain

LOGIN_PARAMETER_TYPE_PARAMETER = 0
LOGIN_PARAMETER_TYPE_COOKIE = 1
LOGIN_PARAMETER_TYPE_HEADER = 2

class LoginStep( SerialisableBaseNamed ):
	SERIALISABLE_TYPE_NUM = HG.SerialisableType["LOGIN_STEP"]
	SERIALISABLE_NAME = 'Login Step'
	SERIALISABLE_VERSION = 2

	def __init__( self, name = 'hit home page to establish session', scheme = 'https', method = 'GET', subdomain = None, path = '/' ):
		SerialisableBaseNamed.__init__( self, name )

		self._scheme = scheme
		self._method = method
		self._subdomain = subdomain
		self._path = path

		self._CleanseSubdomainAndPath()

		self._required_credentials = {} # pretty_name : arg name

		self._static_args = {} # arg name : string

		self._temp_args = {} # temp arg name : arg name

		self._required_cookies_info = SerialisableDictionary() # string match : string match

		self._content_parsers = SerialisableList()

	def _CleanseSubdomainAndPath( self ):
		if self._subdomain is not None:
			self._subdomain = re.sub( '[^a-z\\.]+', '', self._subdomain )

		if not self._path.startswith( '/' ):
			self._path = '/' + self._path

	def _GetSerialisableInfo( self ):
		serialisable_required_cookies = self._required_cookies_info.GetSerialisableTuple()
		serialisable_content_parsers = self._content_parsers.GetSerialisableTuple()

		return ( self._scheme, self._method, self._subdomain, self._path, self._required_credentials, self._static_args, self._temp_args, serialisable_required_cookies, serialisable_content_parsers )

	def _InitialiseFromSerialisableInfo( self, serialisable_info ):
		( self._scheme, self._method, self._subdomain, self._path, self._required_credentials, self._static_args, self._temp_args, serialisable_required_cookies, serialisable_content_parsers ) = serialisable_info

		self._CleanseSubdomainAndPath()

		self._required_cookies_info = CreateFromSerialisableTuple( serialisable_required_cookies )
		self._content_parsers = CreateFromSerialisableTuple( serialisable_content_parsers )

	def _UpdateSerialisableInfo( self, version, old_serialisable_info ):
		if version == 1:
			( scheme, method, subdomain, path, required_credentials, static_args, temp_args, serialisable_required_cookies, serialisable_content_parsers ) = old_serialisable_info

			old_required_cookies_info = CreateFromSerialisableTuple( serialisable_required_cookies )
			new_required_cookies_info = SerialisableDictionary()

			for ( name, value_string_match ) in list( old_required_cookies_info.items() ):
				key_string_match = StringMatch( match_type = STRING_MATCH_FIXED, match_value = name, example_string = name )

				new_required_cookies_info[ key_string_match ] = value_string_match

			serialisable_required_cookies = new_required_cookies_info.GetSerialisableTuple()

			new_serialisable_info = ( scheme, method, subdomain, path, required_credentials, static_args, temp_args, serialisable_required_cookies, serialisable_content_parsers )

			return ( 2, new_serialisable_info )

	def GetRequiredCredentials( self ):
		return [ pretty_name for ( pretty_name, arg_name ) in list( self._required_credentials.items() ) ]

	def GetRequiredAndSetTempVariables( self ):
		required_temp_variables = set( self._temp_args.keys() )

		set_temp_variables = { additional_info for [ ( name, content_type, additional_info ) ] in [ content_parser.GetParsableContent() for content_parser in self._content_parsers ] }

		return ( required_temp_variables, set_temp_variables )

	def SetComplicatedVariables( self, required_credentials, static_args, temp_args, required_cookies_info, content_parsers ):
		self._required_credentials = required_credentials

		self._static_args = static_args

		self._temp_args = temp_args

		self._required_cookies_info = SerialisableDictionary( required_cookies_info )

		self._content_parsers = SerialisableList( content_parsers )

	def Start( self, engine, domain, given_credentials, temp_variables, referral_url = None, network_job_presentation_context_factory = None, test_result_callable = None ):
		def session_to_cookie_strings( sess ):
			cookie_strings = set()

			for cookie in sess.cookies:
				s = cookie.name + ': ' + cookie.value + ' | ' + cookie.domain + ' | '

				expiry = cookie.expires

				if expiry is None:
					expiry = -1
					pretty_expiry = 'session'

				else:
					pretty_expiry = ConvertTimestampToPrettyExpires( expiry )

				s += pretty_expiry

				cookie_strings.add( s )

			return cookie_strings

		url = 'Did not make a url.'
		test_result_body = None
		downloaded_data = 'Did not download data.'
		new_temp_variables = {}
		original_cookie_strings = session_to_cookie_strings( engine.session_manager.GetSessionForDomain( domain ) )
		test_script_result = 'Did not start.'

		try:
			domain_to_hit = domain

			if self._subdomain is not None:
				if domain.startswith( 'www.' ):
					domain = domain[4:]

				domain_to_hit = self._subdomain + '.' + domain

			query_dict = {}

			query_dict.update( self._static_args )

			for ( pretty_name, arg_name ) in list( self._required_credentials.items() ):
				query_dict[ arg_name ] = given_credentials[ pretty_name ]

			for ( temp_name, arg_name ) in list( self._temp_args.items() ):
				if temp_name not in temp_variables:
					raise ValidationException( 'The temporary variable \'' + temp_name + '\' was not found!' )

				query_dict[ arg_name ] = temp_variables[ temp_name ]

			scheme = self._scheme
			netloc = domain_to_hit
			path = self._path
			params = ''
			fragment = ''

			if self._method == 'GET':
				query = ConvertQueryDictToText( query_dict )
				body = None
				test_result_body = ''

			elif self._method == 'POST':
				query = ''
				body = query_dict
				test_result_body = ConvertQueryDictToText( query_dict )

			r = urllib.parse.ParseResult( scheme, netloc, path, params, query, fragment )

			url = r.geturl()

			network_job = NetworkJob( self._method, url, body = body, referral_url = referral_url )

			if self._method == 'POST' and referral_url is not None:
				p = urllib.parse.urlparse( referral_url )

				r = urllib.parse.ParseResult( p.scheme, p.netloc, '', '', '', '' )

				origin = r.geturl() # https://accounts.pixiv.net

				network_job.AddAdditionalHeader( 'origin', origin ) # GET/POST forms are supposed to have this for CSRF. we'll try it just with POST for now

			network_job.SetForLogin( True )

			engine.AddJob( network_job )

			if network_job_presentation_context_factory is not None:
				with network_job_presentation_context_factory( network_job ):
					network_job.WaitUntilDone()

			else:
				network_job.WaitUntilDone()

			session = network_job.GetSession()

			cookies = session.cookies

			for ( cookie_name_string_match, string_match ) in list( self._required_cookies_info.items() ):
				try:
					cookie = GetCookie( cookies, domain, cookie_name_string_match )

				except DataMissing:
					raise ValidationException( 'Missing cookie "' + cookie_name_string_match.ToString() + '" on step "' + self._name + '"!' )

				cookie_text = cookie.value

				try:
					string_match.Test( cookie_text )

				except StringMatchException as e:
					raise ValidationException( 'Cookie "' + cookie_name_string_match.ToString() + '" failed on step "' + self._name + '": ' + str( e ) + '!' )

			downloaded_text = network_job.GetContentText()

			parsing_context = {}

			parsing_context[ 'url' ] = url

			for content_parser in self._content_parsers:
				try:
					parse_results = content_parser.Parse( parsing_context, downloaded_text )

				except VetoException as e:
					raise ValidationException( str( e ) )

				result = GetVariableFromParseResults( parse_results )

				if result is not None:
					( temp_name, value ) = result

					new_temp_variables[ temp_name ] = value

			temp_variables.update( new_temp_variables )

			test_script_result = 'OK!'

			return url

		except Exception as e:
			test_script_result = str( e )

			raise

		finally:
			if test_result_callable is not None:
				current_cookie_strings = session_to_cookie_strings( engine.session_manager.GetSessionForDomain( domain ) )

				new_cookie_strings = tuple( current_cookie_strings.difference( original_cookie_strings ) )

				new_temp_strings = tuple( ( key + ': ' + value for ( key, value ) in list( new_temp_variables.items() ) ) )

				test_result = ( self._name, url, test_result_body, downloaded_data, new_temp_strings, new_cookie_strings, test_script_result )

				test_result_callable( test_result )

	def ToTuple( self ):
		return ( self._scheme, self._method, self._subdomain, self._path, self._required_credentials, self._static_args, self._temp_args, self._required_cookies_info, self._content_parsers )

HG.LookupTables[ "SERIALISABLE_TYPES_TO_OBJECT_TYPES" ][ HG.SerialisableType["LOGIN_STEP"] ] = LoginStep